fxreader.online/freelance-project-34-marketing-blog
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

[+] reformat with ruff

Browse Source
This commit is contained in:
Siarhei Siniak 2025-05-20 11:13:17 +03:00
parent 8510d49015
commit 0f17070c62
20 changed files with 7886 additions and 7960 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
python/_m.py
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
#vim: set filetype=python # vim: set filetype=python
import logging import logging
import json import json
@ -7,27 +7,35 @@ import enum
import pathlib import pathlib
import sys import sys
import argparse import argparse
#import optparse
# import optparse
import dataclasses import dataclasses
import subprocess import subprocess
import os import os
from typing import ( from typing import (
Optional, Any, TypeAlias, Literal, cast, BinaryIO, Generator, Optional,
ClassVar, Self, Any,
TypeAlias,
Literal,
cast,
BinaryIO,
Generator,
ClassVar,
Self,
) )
logger = logging.getLogger() logger = logging.getLogger()
@dataclasses.dataclass @dataclasses.dataclass
class Settings: class Settings:
project_root : pathlib.Path = pathlib.Path.cwd() project_root: pathlib.Path = pathlib.Path.cwd()
env_path : pathlib.Path = project_root / 'tmp' / 'env3' env_path: pathlib.Path = project_root / 'tmp' / 'env3'
_settings : ClassVar[Optional['Settings']] = None _settings: ClassVar[Optional['Settings']] = None
@classmethod @classmethod
def settings(cls) -> Self: def settings(cls) -> Self:
@ -36,8 +44,10 @@ class Settings:
return cls._settings return cls._settings
def js(argv: list[str]) -> int: def js(argv: list[str]) -> int:
return subprocess.check_call([ return subprocess.check_call(
[
'sudo', 'sudo',
'docker-compose', 'docker-compose',
'--project-directory', '--project-directory',
@ -45,7 +55,9 @@ def js(argv: list[str]) -> int:
'-f', '-f',
Settings.settings().project_root / 'docker' / 'js' / 'docker-compose.yml', Settings.settings().project_root / 'docker' / 'js' / 'docker-compose.yml',
*argv, *argv,
]) ]
)
def env( def env(
argv: Optional[list[str]] = None, argv: Optional[list[str]] = None,
@ -55,17 +67,18 @@ def env(
env_path = Settings.settings().env_path env_path = Settings.settings().env_path
if not env_path.exists(): if not env_path.exists():
subprocess.check_call([ subprocess.check_call([sys.executable, '-m', 'venv', '--system-site-packages', str(env_path)])
sys.executable, '-m', 'venv',
'--system-site-packages',
str(env_path)
])
subprocess.check_call([ subprocess.check_call(
[
env_path / 'bin' / 'python3', env_path / 'bin' / 'python3',
'-m', 'pip', '-m',
'install', '-r', 'requirements.txt', 'pip',
]) 'install',
'-r',
'requirements.txt',
]
)
if not argv is None: if not argv is None:
python_path = str(env_path / 'bin' / 'python3') python_path = str(env_path / 'bin' / 'python3')
@ -80,15 +93,19 @@ def env(
) )
return None return None
elif mode == 'subprocess': elif mode == 'subprocess':
return subprocess.run([ return subprocess.run(
[
python_path, python_path,
*argv, *argv,
], **kwargs) ],
**kwargs,
)
else: else:
raise NotImplementedError raise NotImplementedError
return None return None
def ruff(argv: list[str]) -> None: def ruff(argv: list[str]) -> None:
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
@ -109,28 +126,38 @@ def ruff(argv: list[str]) -> None:
options, args = parser.parse_known_args(argv) options, args = parser.parse_known_args(argv)
if len(options.paths) == 0: if len(options.paths) == 0:
options.paths.extend([ options.paths.extend(
[
'.', '.',
'dotfiles/.local/bin/commands', 'dotfiles/.local/bin/commands',
]) ]
)
if len(options.exclude) == 0: if len(options.exclude) == 0:
options.exclude.extend([ options.exclude.extend(
[
'E731', 'E731',
'E713', 'E713',
'E714', 'E714',
'E703', 'E703',
]) ]
)
res = env([ res = env(
[
'-m', '-m',
'ruff', 'ruff',
'check', 'check',
*args, *args,
'--output-format', 'json', '--output-format',
'--ignore', ','.join(options.exclude), 'json',
'--ignore',
','.join(options.exclude),
*options.paths, *options.paths,
], stdout=subprocess.PIPE, stderr=subprocess.PIPE) ],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
assert not res is None assert not res is None
@ -142,10 +169,7 @@ def ruff(argv: list[str]) -> None:
g[o['filename']] = [] g[o['filename']] = []
g[o['filename']].append(o) g[o['filename']].append(o)
h = { h = {k: len(v) for k, v in g.items()}
k : len(v)
for k, v in g.items()
}
logger.info(json.dumps(errors, indent=4)) logger.info(json.dumps(errors, indent=4))
logger.info(json.dumps(h, indent=4)) logger.info(json.dumps(h, indent=4))
@ -154,11 +178,13 @@ def ruff(argv: list[str]) -> None:
def inside_env() -> bool: def inside_env() -> bool:
try: try:
import numpy import numpy
return True return True
except Exception: except Exception:
return False return False
#class Commands(enum.StrEnum):
# class Commands(enum.StrEnum):
# js = 'js' # js = 'js'
# mypy = 'mypy' # mypy = 'mypy'
# env = 'env' # env = 'env'
@ -172,42 +198,53 @@ def inside_env() -> bool:
# argv, # argv,
# ) # )
def host_deps(argv: list[str]) -> None: def host_deps(argv: list[str]) -> None:
if sys.platform in ['linux']: if sys.platform in ['linux']:
subprocess.check_call(r''' subprocess.check_call(
r"""
exec yay -S $(cat requirements-archlinux.txt) exec yay -S $(cat requirements-archlinux.txt)
''', shell=True,) """,
shell=True,
)
else: else:
raise NotImplementedError raise NotImplementedError
Command_args = ['js', 'mypy', 'env', 'ruff', 'm2', 'host_deps',]
Command : TypeAlias = Literal['js', 'mypy', 'env', 'ruff', 'm2', 'host_deps',] Command_args = [
'js',
'mypy',
'env',
'ruff',
'm2',
'host_deps',
]
Command: TypeAlias = Literal[
'js',
'mypy',
'env',
'ruff',
'm2',
'host_deps',
]
def run(argv: Optional[list[str]] = None) -> None: def run(argv: Optional[list[str]] = None) -> None:
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format=( format=('%(levelname)s:%(name)s:%(message)s:%(process)d:%(asctime)s:%(pathname)s:%(funcName)s:%(lineno)s'),
'%(levelname)s:%(name)s:%(message)s'
':%(process)d'
':%(asctime)s'
':%(pathname)s:%(funcName)s:%(lineno)s'
),
) )
if argv is None: if argv is None:
argv = sys.argv[:] argv = sys.argv[:]
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
'command', 'command',
#'_command', #'_command',
choices=[ choices=[o for o in Command_args],
o # required=True,
for o in Command_args
],
#required=True,
) )
options, args = parser.parse_known_args(argv[1:]) options, args = parser.parse_known_args(argv[1:])
@ -217,14 +254,17 @@ def run(argv: Optional[list[str]] = None) -> None:
if len(args) > 0 and args[0] == '--': if len(args) > 0 and args[0] == '--':
del args[0] del args[0]
#options.command = Commands(options._command) # options.command = Commands(options._command)
if options.command == 'js': if options.command == 'js':
js(args) js(args)
elif options.command == 'host_deps': elif options.command == 'host_deps':
host_deps(args) host_deps(args)
elif options.command == 'env': elif options.command == 'env':
env(args, mode='exec',) env(
args,
mode='exec',
)
# elif options.command == 'mypy': # elif options.command == 'mypy':
# if not inside_env(): # if not inside_env():
# env( # env(
@ -244,11 +284,11 @@ def run(argv: Optional[list[str]] = None) -> None:
return return
import python.tasks.cython import python.tasks.cython
python.tasks.cython.mypyc_build(
pathlib.Path('_m.py') python.tasks.cython.mypyc_build(pathlib.Path('_m.py'))
)
else: else:
raise NotImplementedError raise NotImplementedError
if __name__ == '__main__': if __name__ == '__main__':
run() run()

46
python/cli.py
View File

@ -10,7 +10,10 @@ import enum
import argparse import argparse
import dataclasses import dataclasses
from typing import (Optional, override,) from typing import (
Optional,
override,
)
from online.fxreader.pr34.commands_typed.logging import setup as logging_setup from online.fxreader.pr34.commands_typed.logging import setup as logging_setup
@ -28,6 +31,7 @@ class Command(enum.StrEnum):
deploy_wheel = 'deploy:wheel' deploy_wheel = 'deploy:wheel'
tests = 'tests' tests = 'tests'
@dataclasses.dataclass @dataclasses.dataclass
class Settings( class Settings(
_cli.DistSettings, _cli.DistSettings,
@ -50,7 +54,7 @@ class CLI(_cli.CLI):
) )
} }
self._dependencies : dict[str, _cli.Dependency] = dict() self._dependencies: dict[str, _cli.Dependency] = dict()
@override @override
@property @property
@ -74,22 +78,22 @@ class CLI(_cli.CLI):
argv, argv,
settings=_mypy.MypySettings( settings=_mypy.MypySettings(
paths=[ paths=[
#Settings.settings().project_root / 'dotfiles/.local/bin/commands', # Settings.settings().project_root / 'dotfiles/.local/bin/commands',
# project.source_dir / 'm.py', # project.source_dir / 'm.py',
project.source_dir / '_m.py', project.source_dir / '_m.py',
project.source_dir / 'online', project.source_dir / 'online',
project.source_dir / 'cli.py', project.source_dir / 'cli.py',
self.settings.base_dir / 'm.py', self.settings.base_dir / 'm.py',
# Settings.settings().project_root / 'deps/com.github.aiortc.aiortc/src', # Settings.settings().project_root / 'deps/com.github.aiortc.aiortc/src',
#Settings.settings().project_root / 'm.py', # Settings.settings().project_root / 'm.py',
], ],
max_errors={ max_errors={
'python/online/fxreader/pr34/commands_typed': 0, 'python/online/fxreader/pr34/commands_typed': 0,
'python/cli.py': 0, 'python/cli.py': 0,
'm.py': 0, 'm.py': 0,
'deps/com.github.aiortc.aiortc/src/online_fxreader': 0, 'deps/com.github.aiortc.aiortc/src/online_fxreader': 0,
'deps/com.github.aiortc.aiortc/src/aiortc/contrib/signaling': 0 'deps/com.github.aiortc.aiortc/src/aiortc/contrib/signaling': 0,
} },
), ),
) )
@ -103,27 +107,17 @@ class CLI(_cli.CLI):
argv = copy.deepcopy(sys.argv) argv = copy.deepcopy(sys.argv)
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('command', choices=[o.value for o in Command])
parser.add_argument('-p', '--project', choices=[o for o in self.projects])
parser.add_argument( parser.add_argument(
'command', '-o',
choices=[ '--output_dir',
o.value
for o in Command
]
)
parser.add_argument(
'-p', '--project',
choices=[
o
for o in self.projects
]
)
parser.add_argument(
'-o', '--output_dir',
default=None, default=None,
help='wheel output dir for deploy:wheel', help='wheel output dir for deploy:wheel',
) )
parser.add_argument( parser.add_argument(
'-f', '--force', '-f',
'--force',
default=False, default=False,
action='store_true', action='store_true',
help='remove install dir, before installing, default = false', help='remove install dir, before installing, default = false',
@ -148,15 +142,19 @@ class CLI(_cli.CLI):
) )
elif options.command is Command.tests: elif options.command is Command.tests:
for k, v in self.projects.items(): for k, v in self.projects.items():
subprocess.check_call([ subprocess.check_call(
[
sys.executable, sys.executable,
'-m', '-m',
'unittest', 'unittest',
'online.fxreader.pr34.tests.test_crypto', 'online.fxreader.pr34.tests.test_crypto',
*args, *args,
], cwd=str(v.source_dir)) ],
cwd=str(v.source_dir),
)
else: else:
raise NotImplementedError raise NotImplementedError
if __name__ == '__main__': if __name__ == '__main__':
CLI().run() CLI().run()

1705
python/online/fxreader/pr34/commands.py
View File

File diff suppressed because it is too large Load Diff

13
python/online/fxreader/pr34/commands_typed/argparse.py
View File

@ -1,11 +1,12 @@
__all__ = ( __all__ = ('parse_args',)
'parse_args',
)
import sys import sys
import argparse import argparse
from typing import (Optional,) from typing import (
Optional,
)
def parse_args( def parse_args(
parser: argparse.ArgumentParser, parser: argparse.ArgumentParser,
@ -14,11 +15,11 @@ def parse_args(
if args is None: if args is None:
args = sys.argv[1:] args = sys.argv[1:]
argv : list[str] = [] argv: list[str] = []
for i, o in enumerate(args): for i, o in enumerate(args):
if o == '--': if o == '--':
argv.extend(args[i + 1:]) argv.extend(args[i + 1 :])
del args[i:] del args[i:]

15
python/online/fxreader/pr34/commands_typed/asyncio.py
View File

@ -1,14 +1,23 @@
import logging import logging
import asyncio import asyncio
from typing import (Any,) from typing import (
Any,
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def handle_task_result(fut: asyncio.Future[Any]) -> None: def handle_task_result(fut: asyncio.Future[Any]) -> None:
try: try:
fut.result() fut.result()
logger.debug(dict(fut=fut, msg='done'), stacklevel=2,) logger.debug(
dict(fut=fut, msg='done'),
stacklevel=2,
)
except: except:
logger.exception('', stacklevel=2,) logger.exception(
'',
stacklevel=2,
)

185
python/online/fxreader/pr34/commands_typed/cli.py
View File

@ -19,26 +19,30 @@ from typing import (
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@dataclasses.dataclass @dataclasses.dataclass
class Project: class Project:
source_dir : pathlib.Path source_dir: pathlib.Path
build_dir : pathlib.Path build_dir: pathlib.Path
dest_dir : pathlib.Path dest_dir: pathlib.Path
meson_path: Optional[pathlib.Path] = None meson_path: Optional[pathlib.Path] = None
@dataclasses.dataclass @dataclasses.dataclass
class Dependency: class Dependency:
name: str name: str
mode : Literal['pyproject', 'meson', 'meson-python', 'm'] mode: Literal['pyproject', 'meson', 'meson-python', 'm']
source_path : pathlib.Path source_path: pathlib.Path
args: Optional[list[str]] = None args: Optional[list[str]] = None
@dataclasses.dataclass @dataclasses.dataclass
class DistSettings: class DistSettings:
wheel_dir : pathlib.Path wheel_dir: pathlib.Path
python_path: pathlib.Path python_path: pathlib.Path
env_path: pathlib.Path env_path: pathlib.Path
class CLI(abc.ABC): class CLI(abc.ABC):
@property @property
@abc.abstractmethod @abc.abstractmethod
@ -55,10 +59,7 @@ class CLI(abc.ABC):
def dependencies(self) -> dict[str, Dependency]: def dependencies(self) -> dict[str, Dependency]:
raise NotImplementedError raise NotImplementedError
def mypy( def mypy(self, argv: list[str]) -> None:
self,
argv: list[str]
) -> None:
from . import mypy as _mypy from . import mypy as _mypy
_mypy.run( _mypy.run(
@ -73,15 +74,21 @@ class CLI(abc.ABC):
project = self.projects[project_name] project = self.projects[project_name]
if len(argv) == 0: if len(argv) == 0:
argv = ['check', '.',] argv = [
'check',
'.',
]
subprocess.check_call([ subprocess.check_call(
[
self.dist_settings.python_path, self.dist_settings.python_path,
'-m', '-m',
'ruff', 'ruff',
'--config', str(project.source_dir / 'pyproject.toml'), '--config',
str(project.source_dir / 'pyproject.toml'),
*argv, *argv,
]) ]
)
def pyright( def pyright(
self, self,
@ -97,8 +104,10 @@ class CLI(abc.ABC):
str(self.dist_settings.python_path), str(self.dist_settings.python_path),
'-m', '-m',
'pyright', 'pyright',
'--pythonpath', str(self.dist_settings.python_path), '--pythonpath',
'-p', str(project.source_dir / 'pyproject.toml'), str(self.dist_settings.python_path),
'-p',
str(project.source_dir / 'pyproject.toml'),
*argv, *argv,
] ]
@ -113,38 +122,44 @@ class CLI(abc.ABC):
) -> None: ) -> None:
from . import cli_bootstrap from . import cli_bootstrap
pyproject = cli_bootstrap.pyproject_load( pyproject = cli_bootstrap.pyproject_load(self.projects[project].source_dir / 'pyproject.toml')
self.projects[project].source_dir / 'pyproject.toml'
)
dependencies = sum([ dependencies = sum([pyproject.dependencies[o] for o in features], [])
pyproject.dependencies[o]
for o in features
], [])
pip_find_links : list[pathlib.Path] = [] pip_find_links: list[pathlib.Path] = []
if not pyproject.pip_find_links is None: if not pyproject.pip_find_links is None:
pip_find_links.extend(pyproject.pip_find_links) pip_find_links.extend(pyproject.pip_find_links)
logger.info(
logger.info(dict( dict(
dependencies=dependencies, dependencies=dependencies,
)) )
)
if len(dependencies) > 0: if len(dependencies) > 0:
subprocess.check_call([ subprocess.check_call(
[
self.dist_settings.python_path, self.dist_settings.python_path,
'-m', '-m',
'uv', 'pip', 'install', 'uv',
*sum([ 'pip',
['-f', str(o),] 'install',
*sum(
[
[
'-f',
str(o),
]
for o in pip_find_links for o in pip_find_links
], []), ],
[],
),
# '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'), # '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'),
'--offline', '--offline',
*dependencies, *dependencies,
]) ]
)
def deploy_fetch_dist( def deploy_fetch_dist(
self, self,
@ -152,19 +167,16 @@ class CLI(abc.ABC):
) -> None: ) -> None:
for k, d in self.dependencies.items(): for k, d in self.dependencies.items():
whl_glob = self.dist_settings.wheel_dir / ('*%s*.whl' % d.name.replace('.', '_')) whl_glob = self.dist_settings.wheel_dir / ('*%s*.whl' % d.name.replace('.', '_'))
if len(glob.glob( if len(glob.glob(str(whl_glob))) == 0 or force:
str(whl_glob)
)) == 0 or force:
if d.source_path.exists(): if d.source_path.exists():
def whl_files_get() -> list[dict[str, Any]]: def whl_files_get() -> list[dict[str, Any]]:
return [ return [
dict( dict(
path=o, path=o,
stat=os.stat(o).st_mtime, stat=os.stat(o).st_mtime,
) )
for o in glob.glob( for o in glob.glob(str(whl_glob))
str(whl_glob)
)
] ]
present_files = whl_files_get() present_files = whl_files_get()
@ -194,12 +206,9 @@ class CLI(abc.ABC):
def index_get(o: dict[str, Any]) -> tuple[Any, ...]: def index_get(o: dict[str, Any]) -> tuple[Any, ...]:
return (o['path'], o['stat']) return (o['path'], o['stat'])
present_files_index = { present_files_index = {index_get(o): o for o in present_files}
index_get(o) : o
for o in present_files
}
new_files : list[dict[str, Any]] = [] new_files: list[dict[str, Any]] = []
for o in updated_files: for o in updated_files:
entry_index = index_get(o) entry_index = index_get(o)
@ -210,20 +219,22 @@ class CLI(abc.ABC):
if len(new_files) == 0: if len(new_files) == 0:
raise NotImplementedError raise NotImplementedError
latest_file = sorted( latest_file = sorted(new_files, key=lambda x: x['stat'])[-1]
new_files,
key=lambda x: x['stat']
)[-1]
subprocess.check_call([ subprocess.check_call(
[
self.dist_settings.python_path, self.dist_settings.python_path,
'-m', 'pip', '-m',
'pip',
'install', 'install',
latest_file['path'], latest_file['path'],
]) ]
)
@property @property
def pkg_config_path(self,) -> set[pathlib.Path]: def pkg_config_path(
self,
) -> set[pathlib.Path]:
return { return {
pathlib.Path(o) pathlib.Path(o)
for o in glob.glob( for o in glob.glob(
@ -276,18 +287,14 @@ class CLI(abc.ABC):
extra_args: list[str] = [] extra_args: list[str] = []
if len(self.third_party_roots) > 0: if len(self.third_party_roots) > 0:
extra_args.extend([ extra_args.extend(['-Csetup-args=%s' % ('-Dthird_party_roots=%s' % str(o.absolute())) for o in self.third_party_roots])
'-Csetup-args=%s' % (
'-Dthird_party_roots=%s' % str(o.absolute())
)
for o in self.third_party_roots
])
cmd = [ cmd = [
sys.executable, sys.executable,
'-m', '-m',
'build', 'build',
'-w', '-n', '-w',
'-n',
*extra_args, *extra_args,
'-Csetup-args=-Dmodes=pyproject', '-Csetup-args=-Dmodes=pyproject',
'-Cbuild-dir=%s' % str(project.build_dir / 'pyproject'), '-Cbuild-dir=%s' % str(project.build_dir / 'pyproject'),
@ -335,31 +342,38 @@ class CLI(abc.ABC):
if force and project.dest_dir.exists(): if force and project.dest_dir.exists():
shutil.rmtree(project.dest_dir) shutil.rmtree(project.dest_dir)
subprocess.check_call([ subprocess.check_call(
shutil_which('meson', True,), [
shutil_which(
'meson',
True,
),
'install', 'install',
'-C', '-C',
project.build_dir / 'meson', project.build_dir / 'meson',
'--destdir', project.dest_dir, '--destdir',
project.dest_dir,
*argv, *argv,
]) ]
)
for o in glob.glob( for o in glob.glob(
str(project.dest_dir / 'lib' / 'pkgconfig' / '*.pc'), str(project.dest_dir / 'lib' / 'pkgconfig' / '*.pc'),
recursive=True, recursive=True,
): ):
logger.info(dict( logger.info(
dict(
path=o, path=o,
action='patch prefix', action='patch prefix',
)) )
)
with io.open(o, 'r') as f: with io.open(o, 'r') as f:
content = f.read() content = f.read()
with io.open(o, 'w') as f: with io.open(o, 'w') as f:
f.write( f.write(content.replace('prefix=/', 'prefix=${pcfiledir}/../../'))
content.replace('prefix=/', 'prefix=${pcfiledir}/../../')
)
def ninja( def ninja(
self, self,
project_name: str, project_name: str,
@ -396,14 +410,18 @@ class CLI(abc.ABC):
if argv is None: if argv is None:
argv = [] argv = []
subprocess.check_call([ subprocess.check_call(
shutil_which('meson', True,), [
shutil_which(
'meson',
True,
),
'test', 'test',
'-C', '-C',
project.build_dir / 'meson', project.build_dir / 'meson',
*argv, *argv,
]) ]
)
def meson_compile( def meson_compile(
self, self,
@ -415,13 +433,18 @@ class CLI(abc.ABC):
if argv is None: if argv is None:
argv = [] argv = []
subprocess.check_call([ subprocess.check_call(
shutil_which('meson', True,), [
shutil_which(
'meson',
True,
),
'compile', 'compile',
'-C', '-C',
project.build_dir / 'meson', project.build_dir / 'meson',
*argv, *argv,
]) ]
)
@property @property
def third_party_roots(self) -> list[pathlib.Path]: def third_party_roots(self) -> list[pathlib.Path]:
@ -450,16 +473,16 @@ class CLI(abc.ABC):
logger.info(dict(action='removing build dir', path=project.build_dir / 'meson')) logger.info(dict(action='removing build dir', path=project.build_dir / 'meson'))
shutil.rmtree(project.build_dir / 'meson') shutil.rmtree(project.build_dir / 'meson')
extra_args : list[str] = [] extra_args: list[str] = []
if len(self.third_party_roots) > 0: if len(self.third_party_roots) > 0:
extra_args.extend([ extra_args.extend(['-Dthird_party_roots=%s' % str(o.absolute()) for o in self.third_party_roots])
'-Dthird_party_roots=%s' % str(o.absolute())
for o in self.third_party_roots
])
cmd = [ cmd = [
shutil_which('meson', True,), shutil_which(
'meson',
True,
),
'setup', 'setup',
str(project.source_dir), str(project.source_dir),
str(project.build_dir / 'meson'), str(project.build_dir / 'meson'),

159
python/online/fxreader/pr34/commands_typed/cli_bootstrap.py
View File

@ -10,28 +10,36 @@ import os
import logging import logging
from typing import (Optional, Any,) from typing import (
Optional,
Any,
)
from typing_extensions import ( from typing_extensions import (
Self, BinaryIO, Self,
BinaryIO,
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def toml_load(f: BinaryIO) -> Any: def toml_load(f: BinaryIO) -> Any:
try: try:
import tomllib import tomllib
return tomllib.load(f) return tomllib.load(f)
except: except:
pass pass
try: try:
import tomli import tomli
return tomli.load(f) return tomli.load(f)
except: except:
pass pass
raise NotImplementedError raise NotImplementedError
@dataclasses.dataclass @dataclasses.dataclass
class PyProject: class PyProject:
path: pathlib.Path path: pathlib.Path
@ -40,7 +48,8 @@ class PyProject:
pip_find_links: Optional[list[pathlib.Path]] = None pip_find_links: Optional[list[pathlib.Path]] = None
runtime_libdirs: Optional[list[pathlib.Path]] = None runtime_libdirs: Optional[list[pathlib.Path]] = None
runtime_preload: Optional[list[pathlib.Path]] = None runtime_preload: Optional[list[pathlib.Path]] = None
requirements: dict[str, pathlib.Path] = dataclasses.field(default_factory=lambda : dict()) requirements: dict[str, pathlib.Path] = dataclasses.field(default_factory=lambda: dict())
def pyproject_load( def pyproject_load(
d: pathlib.Path, d: pathlib.Path,
@ -50,17 +59,12 @@ def pyproject_load(
assert isinstance(content, dict) assert isinstance(content, dict)
dependencies : dict[str, list[str]] = dict() dependencies: dict[str, list[str]] = dict()
dependencies['default'] = content['project']['dependencies'] dependencies['default'] = content['project']['dependencies']
if ( if 'optional-dependencies' in content['project']:
'optional-dependencies' in content['project'] assert isinstance(content['project']['optional-dependencies'], dict)
):
assert isinstance(
content['project']['optional-dependencies'],
dict
)
for k, v in content['project']['optional-dependencies'].items(): for k, v in content['project']['optional-dependencies'].items():
assert isinstance(v, list) assert isinstance(v, list)
@ -68,7 +72,6 @@ def pyproject_load(
dependencies[k] = v dependencies[k] = v
res = PyProject( res = PyProject(
path=d, path=d,
dependencies=dependencies, dependencies=dependencies,
@ -76,25 +79,12 @@ def pyproject_load(
tool_name = 'online.fxreader.pr34'.replace('.', '-') tool_name = 'online.fxreader.pr34'.replace('.', '-')
if ( if 'tool' in content and isinstance(content['tool'], dict) and tool_name in content['tool'] and isinstance(content['tool'][tool_name], dict):
'tool' in content and
isinstance(
content['tool'], dict
) and
tool_name in content['tool'] and
isinstance(
content['tool'][tool_name],
dict
)
):
if 'early_features' in content['tool'][tool_name]: if 'early_features' in content['tool'][tool_name]:
res.early_features = content['tool'][tool_name]['early_features'] res.early_features = content['tool'][tool_name]['early_features']
if 'pip_find_links' in content['tool'][tool_name]: if 'pip_find_links' in content['tool'][tool_name]:
res.pip_find_links = [ res.pip_find_links = [d.parent / pathlib.Path(o) for o in content['tool'][tool_name]['pip_find_links']]
d.parent / pathlib.Path(o)
for o in content['tool'][tool_name]['pip_find_links']
]
if 'runtime_libdirs' in content['tool'][tool_name]: if 'runtime_libdirs' in content['tool'][tool_name]:
res.runtime_libdirs = [ res.runtime_libdirs = [
@ -114,29 +104,31 @@ def pyproject_load(
assert isinstance(content['tool'][tool_name]['requirements'], dict) assert isinstance(content['tool'][tool_name]['requirements'], dict)
res.requirements = { res.requirements = {
k : d.parent / pathlib.Path(v) k: d.parent / pathlib.Path(v)
# pathlib.Path(o) # pathlib.Path(o)
for k, v in content['tool'][tool_name]['requirements'].items() for k, v in content['tool'][tool_name]['requirements'].items()
} }
return res return res
@dataclasses.dataclass @dataclasses.dataclass
class BootstrapSettings: class BootstrapSettings:
env_path: pathlib.Path env_path: pathlib.Path
python_path: pathlib.Path python_path: pathlib.Path
base_dir: pathlib.Path base_dir: pathlib.Path
python_version: Optional[str] = dataclasses.field( python_version: Optional[str] = dataclasses.field(
default_factory=lambda : os.environ.get( default_factory=lambda: os.environ.get(
'PYTHON_VERSION', 'PYTHON_VERSION',
'%d.%d' % ( '%d.%d'
% (
sys.version_info.major, sys.version_info.major,
sys.version_info.minor, sys.version_info.minor,
), ),
).strip() ).strip()
) )
uv_args: list[str] = dataclasses.field( uv_args: list[str] = dataclasses.field(
default_factory=lambda : os.environ.get( default_factory=lambda: os.environ.get(
'UV_ARGS', 'UV_ARGS',
'--offline', '--offline',
).split(), ).split(),
@ -159,21 +151,28 @@ class BootstrapSettings:
python_path=python_path, python_path=python_path,
) )
def env_bootstrap( def env_bootstrap(
bootstrap_settings: BootstrapSettings, bootstrap_settings: BootstrapSettings,
pyproject: PyProject, pyproject: PyProject,
) -> None: ) -> None:
pip_find_links : list[pathlib.Path] = [] pip_find_links: list[pathlib.Path] = []
if not pyproject.pip_find_links is None: if not pyproject.pip_find_links is None:
pip_find_links.extend(pyproject.pip_find_links) pip_find_links.extend(pyproject.pip_find_links)
pip_find_links_args = sum([ pip_find_links_args = sum(
['-f', str(o),] [
[
'-f',
str(o),
]
for o in pip_find_links for o in pip_find_links
], []) ],
[],
)
features : list[str] = [] features: list[str] = []
if pyproject.early_features: if pyproject.early_features:
features.extend(pyproject.early_features) features.extend(pyproject.early_features)
@ -182,35 +181,31 @@ def env_bootstrap(
if not bootstrap_settings.python_version is None: if not bootstrap_settings.python_version is None:
requirements_python_version = bootstrap_settings.python_version.replace('.', '_') requirements_python_version = bootstrap_settings.python_version.replace('.', '_')
requirements_name = '_'.join(sorted(features)) requirements_name = '_'.join(sorted(features))
if requirements_python_version: if requirements_python_version:
requirements_name += '_' + requirements_python_version requirements_name += '_' + requirements_python_version
requirements_path : Optional[pathlib.Path] = None requirements_path: Optional[pathlib.Path] = None
if requirements_name in pyproject.requirements: if requirements_name in pyproject.requirements:
requirements_path = pyproject.requirements[requirements_name] requirements_path = pyproject.requirements[requirements_name]
else: else:
requirements_path = pyproject.path.parent / 'requirements.txt' requirements_path = pyproject.path.parent / 'requirements.txt'
requirements_in : list[str] = [] requirements_in: list[str] = []
requirements_in.extend([ requirements_in.extend(['uv', 'pip', 'build', 'setuptools', 'meson-python', 'pybind11'])
'uv', 'pip', 'build', 'setuptools', 'meson-python', 'pybind11'
])
if pyproject.early_features: if pyproject.early_features:
early_dependencies = sum([ early_dependencies = sum([pyproject.dependencies[o] for o in pyproject.early_features], [])
pyproject.dependencies[o]
for o in pyproject.early_features
], [])
logger.info(dict( logger.info(
dict(
requirements_name=requirements_name, requirements_name=requirements_name,
early_dependencies=early_dependencies, early_dependencies=early_dependencies,
)) )
)
requirements_in.extend(early_dependencies) requirements_in.extend(early_dependencies)
# if len(early_dependencies) > 0: # if len(early_dependencies) > 0:
@ -230,12 +225,11 @@ def env_bootstrap(
prefix='requirements', prefix='requirements',
suffix='.in', suffix='.in',
) as f: ) as f:
f.write( f.write('\n'.join(requirements_in))
'\n'.join(requirements_in)
)
f.flush() f.flush()
subprocess.check_call([ subprocess.check_call(
[
'uv', 'uv',
'pip', 'pip',
'compile', 'compile',
@ -244,27 +238,36 @@ def env_bootstrap(
# '-p', # '-p',
# bootstrap_settings.python_path, # bootstrap_settings.python_path,
*bootstrap_settings.uv_args, *bootstrap_settings.uv_args,
'-o', str(requirements_path), '-o',
str(requirements_path),
f.name, f.name,
]) ]
)
uv_python_version: list[str] = [] uv_python_version: list[str] = []
if not bootstrap_settings.python_version is None: if not bootstrap_settings.python_version is None:
uv_python_version.extend([ uv_python_version.extend(
'-p', bootstrap_settings.python_version, [
]) '-p',
bootstrap_settings.python_version,
]
)
subprocess.check_call([ subprocess.check_call(
'uv', 'venv', [
'uv',
'venv',
*uv_python_version, *uv_python_version,
*pip_find_links_args, *pip_find_links_args,
# '--seed', # '--seed',
*bootstrap_settings.uv_args, *bootstrap_settings.uv_args,
str(bootstrap_settings.env_path) str(bootstrap_settings.env_path),
]) ]
)
subprocess.check_call([ subprocess.check_call(
[
'uv', 'uv',
'pip', 'pip',
'install', 'install',
@ -273,19 +276,16 @@ def env_bootstrap(
bootstrap_settings.python_path, bootstrap_settings.python_path,
'--require-hashes', '--require-hashes',
*bootstrap_settings.uv_args, *bootstrap_settings.uv_args,
'-r', str(requirements_path), '-r',
]) str(requirements_path),
]
def paths_equal(
a: pathlib.Path | str,
b: pathlib.Path | str
) -> bool:
return (
os.path.abspath(str(a)) ==
os.path.abspath(str(b))
) )
def paths_equal(a: pathlib.Path | str, b: pathlib.Path | str) -> bool:
return os.path.abspath(str(a)) == os.path.abspath(str(b))
def run( def run(
d: Optional[pathlib.Path] = None, d: Optional[pathlib.Path] = None,
cli_path: Optional[pathlib.Path] = None, cli_path: Optional[pathlib.Path] = None,
@ -298,9 +298,7 @@ def run(
bootstrap_settings = BootstrapSettings.get() bootstrap_settings = BootstrapSettings.get()
pyproject : PyProject = pyproject_load( pyproject: PyProject = pyproject_load(d)
d
)
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
@ -318,19 +316,18 @@ def run(
[ [
str(bootstrap_settings.python_path), str(bootstrap_settings.python_path),
*sys.argv, *sys.argv,
] ],
) )
os.execv( os.execv(
str(bootstrap_settings.python_path), str(bootstrap_settings.python_path),
[ [
str(bootstrap_settings.python_path), str(bootstrap_settings.python_path),
str( str(cli_path),
cli_path
),
*sys.argv[1:], *sys.argv[1:],
] ],
) )
if __name__ == '__main__': if __name__ == '__main__':
run() run()

23
python/online/fxreader/pr34/commands_typed/crypto.py
View File

@ -3,7 +3,12 @@ import os
import cryptography.hazmat.primitives.kdf.scrypt import cryptography.hazmat.primitives.kdf.scrypt
from typing import (Literal, overload, Optional,) from typing import (
Literal,
overload,
Optional,
)
class PasswordUtils: class PasswordUtils:
@overload @overload
@ -44,19 +49,21 @@ class PasswordUtils:
if mode == 'bytes': if mode == 'bytes':
return (salt, hashed_secret) return (salt, hashed_secret)
elif mode == 'base64': elif mode == 'base64':
res_tuple = tuple(( res_tuple = tuple(
(
base64.b64encode(o).decode('utf-8') base64.b64encode(o).decode('utf-8')
for o in (salt, hashed_secret,) for o in (
)) salt,
hashed_secret,
)
)
)
return (res_tuple[0], res_tuple[1]) return (res_tuple[0], res_tuple[1])
else: else:
raise NotImplementedError raise NotImplementedError
@classmethod @classmethod
def _scrypt_init( def _scrypt_init(cls, salt: bytes) -> cryptography.hazmat.primitives.kdf.scrypt.Scrypt:
cls,
salt: bytes
) -> cryptography.hazmat.primitives.kdf.scrypt.Scrypt:
return cryptography.hazmat.primitives.kdf.scrypt.Scrypt( return cryptography.hazmat.primitives.kdf.scrypt.Scrypt(
salt=salt, salt=salt,
length=32, length=32,

6
python/online/fxreader/pr34/commands_typed/debug.py
View File

@ -1,10 +1,13 @@
import os import os
import logging import logging
from typing import (Optional,) from typing import (
Optional,
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class DebugPy: class DebugPy:
@classmethod @classmethod
def set_trace( def set_trace(
@ -25,6 +28,7 @@ class DebugPy:
if os.environ.get('DEBUGPY_RUNNING') != 'true': if os.environ.get('DEBUGPY_RUNNING') != 'true':
logger.info('debugpy init') logger.info('debugpy init')
import debugpy import debugpy
debugpy.listen((host, port)) debugpy.listen((host, port))
os.environ['DEBUGPY_RUNNING'] = 'true' os.environ['DEBUGPY_RUNNING'] = 'true'

12
python/online/fxreader/pr34/commands_typed/logging.py
View File

@ -1,5 +1,8 @@
import logging import logging
from typing import (Optional,) from typing import (
Optional,
)
def setup(level: Optional[int] = None) -> None: def setup(level: Optional[int] = None) -> None:
if level is None: if level is None:
@ -7,10 +10,5 @@ def setup(level: Optional[int] = None) -> None:
logging.basicConfig( logging.basicConfig(
level=level, level=level,
format=( format=('%(levelname)s:%(name)s:%(message)s:%(process)d:%(asctime)s:%(pathname)s:%(funcName)s:%(lineno)s'),
'%(levelname)s:%(name)s:%(message)s'
':%(process)d'
':%(asctime)s'
':%(pathname)s:%(funcName)s:%(lineno)s'
),
) )

104
python/online/fxreader/pr34/commands_typed/mypy.py
View File

@ -9,17 +9,26 @@ import logging
import sys import sys
import argparse import argparse
from pydantic import (Field,) from pydantic import (
Field,
)
from typing import (ClassVar, Generator, Annotated, Optional, Any,) from typing import (
ClassVar,
Generator,
Annotated,
Optional,
Any,
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@pydantic.dataclasses.dataclass @pydantic.dataclasses.dataclass
class MypyFormatEntry: class MypyFormatEntry:
name : str name: str
value : str value: str
def __eq__(self, other: object) -> bool: def __eq__(self, other: object) -> bool:
if not isinstance(other, type(self)): if not isinstance(other, type(self)):
@ -27,10 +36,10 @@ class MypyFormatEntry:
return self.value == other.value return self.value == other.value
class MypyFormat:
vscode : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='vscode', value='vscode')
json : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='json', value='json')
class MypyFormat:
vscode: ClassVar[MypyFormatEntry] = MypyFormatEntry(name='vscode', value='vscode')
json: ClassVar[MypyFormatEntry] = MypyFormatEntry(name='json', value='json')
@classmethod @classmethod
def from_value(cls, value: str) -> MypyFormatEntry: def from_value(cls, value: str) -> MypyFormatEntry:
@ -41,7 +50,13 @@ class MypyFormat:
raise NotImplementedError raise NotImplementedError
@classmethod @classmethod
def entries(cls) -> Generator[MypyFormatEntry, None, None,]: def entries(
cls,
) -> Generator[
MypyFormatEntry,
None,
None,
]:
for o in dir(cls): for o in dir(cls):
e = getattr(cls, o) e = getattr(cls, o)
if not isinstance(e, MypyFormatEntry): if not isinstance(e, MypyFormatEntry):
@ -49,15 +64,17 @@ class MypyFormat:
yield e yield e
class MypySettings(pydantic_settings.BaseSettings): class MypySettings(pydantic_settings.BaseSettings):
model_config = pydantic_settings.SettingsConfigDict( model_config = pydantic_settings.SettingsConfigDict(
env_prefix='online_fxreader_pr34_mypy_', env_prefix='online_fxreader_pr34_mypy_',
case_sensitive=False, case_sensitive=False,
) )
config_path : pathlib.Path = pathlib.Path.cwd() / '.mypy.ini' config_path: pathlib.Path = pathlib.Path.cwd() / '.mypy.ini'
max_errors : dict[str, int] = dict() max_errors: dict[str, int] = dict()
paths : Annotated[list[pathlib.Path], Field(default_factory=lambda : ['.'])] paths: Annotated[list[pathlib.Path], Field(default_factory=lambda: ['.'])]
def run( def run(
argv: Optional[list[str]] = None, argv: Optional[list[str]] = None,
@ -71,7 +88,8 @@ def run(
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
'-q', '--quiet', '-q',
'--quiet',
dest='quiet', dest='quiet',
action='store_true', action='store_true',
help='do not print anything if the program is correct according to max_errors limits', help='do not print anything if the program is correct according to max_errors limits',
@ -85,14 +103,12 @@ def run(
action='append', action='append',
) )
parser.add_argument( parser.add_argument(
'-f', '--format', '-f',
'--format',
dest='_format', dest='_format',
help='output format of errors', help='output format of errors',
default=MypyFormat.json.value, default=MypyFormat.json.value,
choices=[ choices=[o.value for o in MypyFormat.entries()],
o.value
for o in MypyFormat.entries()
],
) )
options, args = parser.parse_known_args(argv) options, args = parser.parse_known_args(argv)
@ -110,7 +126,8 @@ def run(
sys.executable, sys.executable,
'-m', '-m',
'mypy', 'mypy',
'--config-file', str(settings.config_path), '--config-file',
str(settings.config_path),
'--strict', '--strict',
'-O', '-O',
'json', 'json',
@ -118,7 +135,6 @@ def run(
*options.paths, *options.paths,
] ]
logger.info(dict(cmd=mypy_cmd)) logger.info(dict(cmd=mypy_cmd))
res = subprocess.run( res = subprocess.run(
@ -132,14 +148,13 @@ def run(
try: try:
assert not res.returncode is None assert not res.returncode is None
errors = sorted([ errors = sorted(
json.loads(o) [json.loads(o) for o in res.stdout.decode('utf-8').splitlines() if not o.strip() == ''],
for o in res.stdout.decode('utf-8').splitlines() key=lambda x: (
if not o.strip() == ''
], key=lambda x: (
x.get('file', ''), x.get('file', ''),
x.get('line', 0), x.get('line', 0),
)) ),
)
if not options.quiet: if not options.quiet:
if (len(res.stderr)) > 0: if (len(res.stderr)) > 0:
@ -150,15 +165,14 @@ def run(
logger.error(res.stderr.decode('utf-8')) logger.error(res.stderr.decode('utf-8'))
sys.exit(res.returncode) sys.exit(res.returncode)
g: dict[str, Any] = dict()
g : dict[str, Any] = dict()
for o in errors: for o in errors:
if not o['file'] in g: if not o['file'] in g:
g[o['file']] = [] g[o['file']] = []
g[o['file']].append(o) g[o['file']].append(o)
h = { h = {
k : len(v) k: len(v)
for k, v in sorted( for k, v in sorted(
list(g.items()), list(g.items()),
key=lambda x: x[0], key=lambda x: x[0],
@ -167,24 +181,26 @@ def run(
mentioned_paths = marisa_trie.Trie(list(h)) mentioned_paths = marisa_trie.Trie(list(h))
violated_limits : dict[str, str] = dict() violated_limits: dict[str, str] = dict()
for k, v in settings.max_errors.items(): for k, v in settings.max_errors.items():
matching_paths = mentioned_paths.keys(k) matching_paths = mentioned_paths.keys(k)
total_errors = sum([ total_errors = sum([h[o] for o in matching_paths], 0)
h[o]
for o in matching_paths
], 0)
if total_errors > v: if total_errors > v:
violated_limits[k] = '%s - [%s]: has %d errors > %d' % ( violated_limits[k] = '%s - [%s]: has %d errors > %d' % (
k, ', '.join(matching_paths), total_errors, v, k,
', '.join(matching_paths),
total_errors,
v,
) )
if len(violated_limits) > 0 or not options.quiet: if len(violated_limits) > 0 or not options.quiet:
if options.format == MypyFormat.vscode: if options.format == MypyFormat.vscode:
for o in errors: for o in errors:
sys.stdout.write('[%s] %s:%d,%d %s - %s - %s\n' % ( sys.stdout.write(
'[%s] %s:%d,%d %s - %s - %s\n'
% (
o['severity'], o['severity'],
o['file'], o['file'],
o['line'], o['line'],
@ -192,25 +208,33 @@ def run(
o['message'], o['message'],
o['hint'], o['hint'],
o['code'], o['code'],
)) )
)
sys.stdout.flush() sys.stdout.flush()
#logger.info(json.dumps(errors, indent=4)) # logger.info(json.dumps(errors, indent=4))
else: else:
logger.info(json.dumps(errors, indent=4)) logger.info(json.dumps(errors, indent=4))
#if len(violated_limits) > 0: # if len(violated_limits) > 0:
# logger.info(json.dumps(violated_limits, indent=4)) # logger.info(json.dumps(violated_limits, indent=4))
logger.info(json.dumps(dict( logger.info(
json.dumps(
dict(
max_errors=settings.max_errors, max_errors=settings.max_errors,
violated_limits=violated_limits, violated_limits=violated_limits,
histogram=h, histogram=h,
elapsed=(done_at - started_at).total_seconds(), elapsed=(done_at - started_at).total_seconds(),
), indent=4)) ),
indent=4,
)
)
if len(violated_limits) > 0: if len(violated_limits) > 0:
sys.exit(1) sys.exit(1)
if __name__ == '__main__': if __name__ == '__main__':
from . import logging as _logging from . import logging as _logging
_logging.setup() _logging.setup()
run(sys.argv[1:]) run(sys.argv[1:])

85
python/online/fxreader/pr34/commands_typed/os.py
View File

@ -11,22 +11,31 @@ import dataclasses
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from typing import (overload, Optional, Literal, Any, Annotated,) from typing import (
overload,
Optional,
Literal,
Any,
Annotated,
)
from .cli_bootstrap import PyProject from .cli_bootstrap import PyProject
@overload @overload
def shutil_which( def shutil_which(
name: str, name: str,
raise_on_failure: Literal[True], raise_on_failure: Literal[True],
) -> str: ... ) -> str: ...
@overload @overload
def shutil_which( def shutil_which(
name: str, name: str,
raise_on_failure: bool, raise_on_failure: bool,
) -> Optional[str]: ... ) -> Optional[str]: ...
def shutil_which( def shutil_which(
name: str, name: str,
raise_on_failure: bool, raise_on_failure: bool,
@ -37,63 +46,50 @@ def shutil_which(
else: else:
return res return res
def runtime_libdirs_init( def runtime_libdirs_init(
project: PyProject, project: PyProject,
) -> None: ) -> None:
if sys.platform == 'linux': if sys.platform == 'linux':
ld_library_path : list[pathlib.Path] = [ ld_library_path: list[pathlib.Path] = [
o o
for o in [ for o in [
*[ *[o.absolute() for o in (project.runtime_libdirs if project.runtime_libdirs else [])],
o.absolute() *[pathlib.Path(o) for o in os.environ.get('LD_LIBRARY_PATH', '').split(os.path.pathsep) if o != ''],
for o in (
project.runtime_libdirs
if project.runtime_libdirs
else []
)
],
*[
pathlib.Path(o)
for o in os.environ.get(
'LD_LIBRARY_PATH',
''
).split(os.path.pathsep)
if o != ''
]
] ]
] ]
ld_library_path_present : list[pathlib.Path] = [] ld_library_path_present: list[pathlib.Path] = []
for o in ld_library_path: for o in ld_library_path:
if not o.exists(): if not o.exists():
logger.warning(dict( logger.warning(
dict(
ld_library_path=o, ld_library_path=o,
msg='not found', msg='not found',
)) )
)
ld_library_path_present.append(o) ld_library_path_present.append(o)
os.environ.update( os.environ.update(LD_LIBRARY_PATH=os.path.pathsep.join([str(o) for o in ld_library_path_present]))
LD_LIBRARY_PATH=os.path.pathsep.join([
str(o) for o in ld_library_path_present
])
)
for preload_path in (project.runtime_preload or []): for preload_path in project.runtime_preload or []:
for preload_found in glob.glob(str( for preload_found in glob.glob(str(preload_path.parent / ('lib%s.so' % preload_path.name))):
preload_path.parent / ('lib%s.so' % preload_path.name) logger.info(
)): dict(
logger.info(dict( preload_path=preload_path,
preload_path=preload_path, preload_found=preload_found, preload_found=preload_found,
# lib_path=o, # lib_path=o,
msg='load_library', msg='load_library',
)) )
)
ctypes.cdll.LoadLibrary(preload_found) ctypes.cdll.LoadLibrary(preload_found)
else: else:
raise NotImplementedError raise NotImplementedError
class interfaces_index_t: class interfaces_index_t:
@dataclasses.dataclass @dataclasses.dataclass
class Interface: class Interface:
@ -106,17 +102,24 @@ class interfaces_index_t:
str, str,
pydantic.Field( pydantic.Field(
alias='ifname', alias='ifname',
) ),
] ]
addr_info: list[AddrInfo] addr_info: list[AddrInfo]
def interfaces_index() -> list[interfaces_index_t.Interface]: def interfaces_index() -> list[interfaces_index_t.Interface]:
res = pydantic.RootModel[ res = (
list[interfaces_index_t.Interface] pydantic.RootModel[list[interfaces_index_t.Interface]]
].model_validate_json( .model_validate_json(
subprocess.check_output([ subprocess.check_output(
'ip', '-j', 'addr', [
]).decode('utf-8') 'ip',
).root '-j',
'addr',
]
).decode('utf-8')
)
.root
)
return res return res

166
python/online/fxreader/pr34/commands_typed/pip.py
View File

@ -23,7 +23,10 @@ if typing.TYPE_CHECKING:
import pip._internal.operations.prepare import pip._internal.operations.prepare
from typing import ( from typing import (
Literal, Optional, Iterable, Any, Literal,
Optional,
Iterable,
Any,
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -33,6 +36,7 @@ def pip_show(
argv: list[str], argv: list[str],
) -> list['pip._internal.commands.show._PackageInfo']: ) -> list['pip._internal.commands.show._PackageInfo']:
import pip._internal.commands.show import pip._internal.commands.show
return list( return list(
pip._internal.commands.show.search_packages_info( pip._internal.commands.show.search_packages_info(
argv, argv,
@ -43,10 +47,10 @@ def pip_show(
class pip_resolve_t: class pip_resolve_t:
class kwargs_t: class kwargs_t:
class mode_t(enum.StrEnum): class mode_t(enum.StrEnum):
copy_paste = "copy_paste" copy_paste = 'copy_paste'
monkey_patch = "monkey_patch" monkey_patch = 'monkey_patch'
uv_pip_freeze = "uv_pip_freeze" uv_pip_freeze = 'uv_pip_freeze'
uv_pip_compile = "uv_pip_compile" uv_pip_compile = 'uv_pip_compile'
@dataclasses.dataclass @dataclasses.dataclass
class res_t: class res_t:
@ -60,20 +64,9 @@ class pip_resolve_t:
entries: Optional[list[download_info_t]] = None entries: Optional[list[download_info_t]] = None
def pip_resolve_entries_to_txt( def pip_resolve_entries_to_txt(entries: list[pip_resolve_t.res_t.download_info_t]) -> str:
entries: list[pip_resolve_t.res_t.download_info_t] return '\n'.join(['#%s\n%s %s' % (o.url, o.constraint, ' '.join(['--hash=sha256:%s' % o2 for o2 in o.sha256])) for o in entries])
) -> str:
return '\n'.join([
'#%s\n%s %s' % (
o.url,
o.constraint,
' '.join([
'--hash=sha256:%s' % o2
for o2 in o.sha256
])
)
for o in entries
])
def pip_resolve( def pip_resolve(
argv: list[str], argv: list[str],
@ -102,8 +95,8 @@ def pip_resolve(
t2 = pip._internal.cli.main_parser.create_main_parser() t2 = pip._internal.cli.main_parser.create_main_parser()
t3 = t2.parse_args(["download"]) t3 = t2.parse_args(['download'])
t1 = pip._internal.commands.download.DownloadCommand("blah", "shit") t1 = pip._internal.commands.download.DownloadCommand('blah', 'shit')
stack.enter_context(t1.main_context()) stack.enter_context(t1.main_context())
@ -143,9 +136,7 @@ def pip_resolve(
target_python=target_python, target_python=target_python,
ignore_requires_python=options.ignore_requires_python, ignore_requires_python=options.ignore_requires_python,
) )
build_tracker = t1.enter_context( build_tracker = t1.enter_context(pip._internal.operations.build.build_tracker.get_build_tracker())
pip._internal.operations.build.build_tracker.get_build_tracker()
)
reqs = t1.get_requirements( reqs = t1.get_requirements(
[ [
#'pip', 'uv', 'ipython', #'pip', 'uv', 'ipython',
@ -156,9 +147,7 @@ def pip_resolve(
session, session,
) )
pip._internal.req.req_install.check_legacy_setup_py_options(options, reqs) pip._internal.req.req_install.check_legacy_setup_py_options(options, reqs)
directory = pip._internal.utils.temp_dir.TempDirectory( directory = pip._internal.utils.temp_dir.TempDirectory(delete=True, kind='download', globally_managed=True)
delete=True, kind="download", globally_managed=True
)
preparer = t1.make_requirement_preparer( preparer = t1.make_requirement_preparer(
temp_build_dir=directory, temp_build_dir=directory,
options=options, options=options,
@ -195,14 +184,12 @@ def pip_resolve(
res.entries.append( res.entries.append(
pip_resolve_t.res_t.download_info_t( pip_resolve_t.res_t.download_info_t(
constraint=k, constraint=k,
sha256=v.download_info.info.hashes["sha256"], sha256=v.download_info.info.hashes['sha256'],
url=v.download_info.url, url=v.download_info.url,
) )
) )
res.txt = pip_resolve_entries_to_txt( res.txt = pip_resolve_entries_to_txt(res.entries)
res.entries
)
return res return res
elif mode is pip_resolve_t.kwargs_t.mode_t.monkey_patch: elif mode is pip_resolve_t.kwargs_t.mode_t.monkey_patch:
@ -248,23 +235,17 @@ def pip_resolve(
return downloader_call_def( return downloader_call_def(
_self, _self,
link, location, link,
location,
) )
batch_downloader_call_def = ( batch_downloader_call_def = pip._internal.network.download.BatchDownloader.__call__
pip._internal.network.download.BatchDownloader.__call__
)
def batch_downloader_call( def batch_downloader_call(
_self: pip._internal.network.download.BatchDownloader, _self: pip._internal.network.download.BatchDownloader,
links: Iterable[pip._internal.models.link.Link], links: Iterable[pip._internal.models.link.Link],
location: str, location: str,
) -> Iterable[ ) -> Iterable[tuple[pip._internal.models.link.Link, tuple[str, str]]]:
tuple[
pip._internal.models.link.Link,
tuple[str, str]
]
]:
# print(args) # print(args)
logger.info( logger.info(
@ -274,34 +255,21 @@ def pip_resolve(
) )
) )
return [ return [(o, ('/dev/null', '')) for o in links]
(o, ("/dev/null", ''))
for o in links
]
# base_resolver_resolve_def = pip._internal.resolution.base.BaseResolver.resolve # base_resolver_resolve_def = pip._internal.resolution.base.BaseResolver.resolve
base_resolver_resolve_def = ( base_resolver_resolve_def = pip._internal.resolution.resolvelib.resolver.Resolver.resolve
pip._internal.resolution.resolvelib.resolver.Resolver.resolve
)
result_requirements : list[ result_requirements: list[RequirementSet | InstallRequirement] = []
RequirementSet | InstallRequirement
] = []
def base_resolver_resolve( def base_resolver_resolve(
_self: pip._internal.resolution.resolvelib.resolver.Resolver, _self: pip._internal.resolution.resolvelib.resolver.Resolver,
root_reqs: list[ root_reqs: list[InstallRequirement,],
InstallRequirement,
],
check_supported_wheels: bool, check_supported_wheels: bool,
) -> RequirementSet: ) -> RequirementSet:
# print(args, kwargs) # print(args, kwargs)
res = base_resolver_resolve_def( res = base_resolver_resolve_def(_self, root_reqs, check_supported_wheels)
_self,
root_reqs,
check_supported_wheels
)
result_requirements.append(res) result_requirements.append(res)
raise NotImplementedError raise NotImplementedError
@ -322,21 +290,16 @@ def pip_resolve(
) )
) )
if link.url.endswith(".whl"): if link.url.endswith('.whl'):
print("blah") print('blah')
hashes = None hashes = None
return File( return File(
"/dev/null", '/dev/null',
'', '',
) )
else: else:
return get_http_url_def( return get_http_url_def(link, download, download_dir, hashes)
link,
download,
download_dir,
hashes
)
prepare_linked_requirements_more_def = pip._internal.operations.prepare.RequirementPreparer.prepare_linked_requirements_more prepare_linked_requirements_more_def = pip._internal.operations.prepare.RequirementPreparer.prepare_linked_requirements_more
@ -345,9 +308,7 @@ def pip_resolve(
reqs: Iterable[InstallRequirement], reqs: Iterable[InstallRequirement],
parallel_builds: bool = False, parallel_builds: bool = False,
) -> None: ) -> None:
result_requirements.extend( result_requirements.extend(reqs)
reqs
)
raise NotImplementedError raise NotImplementedError
_complete_partial_requirements_def = pip._internal.operations.prepare.RequirementPreparer._complete_partial_requirements _complete_partial_requirements_def = pip._internal.operations.prepare.RequirementPreparer._complete_partial_requirements
@ -357,18 +318,12 @@ def pip_resolve(
partially_downloaded_reqs: Iterable[InstallRequirement], partially_downloaded_reqs: Iterable[InstallRequirement],
parallel_builds: bool = False, parallel_builds: bool = False,
) -> None: ) -> None:
result_requirements.extend( result_requirements.extend(partially_downloaded_reqs)
partially_downloaded_reqs
)
raise NotImplementedError raise NotImplementedError
patches : list[Any] = [] patches: list[Any] = []
patches.append( patches.append(unittest.mock.patch.object(pip._internal.network.download.Downloader, '__call__', downloader_call))
unittest.mock.patch.object(
pip._internal.network.download.Downloader, "__call__", downloader_call
)
)
# patches.append( # patches.append(
# unittest.mock.patch.object( # unittest.mock.patch.object(
# pip._internal.network.download.BatchDownloader, # pip._internal.network.download.BatchDownloader,
@ -383,21 +338,21 @@ def pip_resolve(
patches.append( patches.append(
unittest.mock.patch.object( unittest.mock.patch.object(
pip._internal.resolution.resolvelib.resolver.Resolver, pip._internal.resolution.resolvelib.resolver.Resolver,
"resolve", 'resolve',
base_resolver_resolve, base_resolver_resolve,
) )
) )
patches.append( patches.append(
unittest.mock.patch.object( unittest.mock.patch.object(
pip._internal.operations.prepare, pip._internal.operations.prepare,
"get_http_url", 'get_http_url',
get_http_url, get_http_url,
) )
) )
patches.append( patches.append(
unittest.mock.patch.object( unittest.mock.patch.object(
pip._internal.operations.prepare.RequirementPreparer, pip._internal.operations.prepare.RequirementPreparer,
"prepare_linked_requirements_more", 'prepare_linked_requirements_more',
prepare_linked_requirements_more, prepare_linked_requirements_more,
) )
) )
@ -415,11 +370,11 @@ def pip_resolve(
pip._internal.cli.main.main( pip._internal.cli.main.main(
[ [
"download", 'download',
"-q", '-q',
"--no-cache", '--no-cache',
"-d", '-d',
"/dev/null", '/dev/null',
*argv, *argv,
# 'numpy', # 'numpy',
] ]
@ -445,7 +400,7 @@ def pip_resolve(
for o in result_requirements: for o in result_requirements:
assert isinstance(o, InstallRequirement) assert isinstance(o, InstallRequirement)
sha256_hashes = o.hashes()._allowed["sha256"] sha256_hashes = o.hashes()._allowed['sha256']
assert len(sha256_hashes) == 1 assert len(sha256_hashes) == 1
assert not o.link is None assert not o.link is None
@ -457,9 +412,7 @@ def pip_resolve(
) )
) )
res.txt = pip_resolve_entries_to_txt( res.txt = pip_resolve_entries_to_txt(res.entries)
res.entries
)
return res return res
elif mode is pip_resolve_t.kwargs_t.mode_t.uv_pip_freeze: elif mode is pip_resolve_t.kwargs_t.mode_t.uv_pip_freeze:
@ -468,21 +421,23 @@ def pip_resolve(
pip_freeze = subprocess.check_output( pip_freeze = subprocess.check_output(
[ [
sys.executable, sys.executable,
"-m", '-m',
"uv", 'uv',
"pip", 'pip',
"freeze", 'freeze',
], ],
).decode('utf-8') ).decode('utf-8')
pip_compile = subprocess.check_output( pip_compile = subprocess.check_output(
[ [
sys.executable, '-m', sys.executable,
'uv', 'pip', 'compile', '-m',
'uv',
'pip',
'compile',
'--generate-hashes', '--generate-hashes',
'-', '-',
], ],
input=pip_freeze.encode('utf-8') input=pip_freeze.encode('utf-8'),
).decode('utf-8') ).decode('utf-8')
return pip_resolve_t.res_t( return pip_resolve_t.res_t(
@ -498,9 +453,7 @@ def pip_resolve(
suffix='.txt', suffix='.txt',
) )
) )
f.write( f.write(('\n'.join(requirements)).encode('utf-8'))
('\n'.join(requirements)).encode('utf-8')
)
f.flush() f.flush()
argv.append(f.name) argv.append(f.name)
@ -510,8 +463,11 @@ def pip_resolve(
pip_compile = subprocess.check_output( pip_compile = subprocess.check_output(
[ [
sys.executable, '-m', sys.executable,
'uv', 'pip', 'compile', '-m',
'uv',
'pip',
'compile',
'--generate-hashes', '--generate-hashes',
*argv, *argv,
], ],

5
python/online/fxreader/pr34/commands_typed/typing.py
View File

@ -6,7 +6,8 @@ from typing import Any
from typing_extensions import Protocol from typing_extensions import Protocol
from abc import abstractmethod from abc import abstractmethod
C = typing.TypeVar("C", bound="Comparable") C = typing.TypeVar('C', bound='Comparable')
class Comparable(Protocol): class Comparable(Protocol):
@abstractmethod @abstractmethod
@ -24,4 +25,4 @@ class Comparable(Protocol):
return self < other or self == other return self < other or self == other
def __ge__(self: C, other: C) -> bool: def __ge__(self: C, other: C) -> bool:
return (not self < other) return not self < other

28
python/online/fxreader/pr34/tasks/ble.py
View File

@ -8,6 +8,7 @@ async def f1():
devices = await bleak.BleakScanner.discover() devices = await bleak.BleakScanner.discover()
return devices return devices
async def f2(device, timeout=None): async def f2(device, timeout=None):
if timeout is None: if timeout is None:
timeout = 1.0 timeout = 1.0
@ -20,19 +21,12 @@ async def f2(device, timeout=None):
).__aenter__() ).__aenter__()
return p return p
async def f3(client): async def f3(client):
t1 = [ t1 = [dict(service=o.__dict__, characteristics=[o2.__dict__ for o2 in o.characteristics]) for o in client.services]
dict(
service=o.__dict__,
characteristics=[
o2.__dict__
for o2 in o.characteristics
]
)
for o in client.services
]
return t1 return t1
async def f5( async def f5(
name_check=None, name_check=None,
): ):
@ -47,18 +41,9 @@ async def f5(
if not name_check is None: if not name_check is None:
assert inspect.isfunction(name_check) assert inspect.isfunction(name_check)
t5 = { t5 = {i: o.details[0].name() for i, o in enumerate(t1)}
i : o.details[0].name()
for i, o in enumerate(t1)
}
t2.extend( t2.extend([t1[k] for k, v in t5.items() if isinstance(v, str) and name_check(v)])
[
t1[k]
for k, v in t5.items()
if isinstance(v, str) and name_check(v)
]
)
else: else:
t2.extend(t1) t2.extend(t1)
@ -70,6 +55,7 @@ async def f5(
return t2 return t2
async def f4( async def f4(
timeout=None, timeout=None,
characteristics=None, characteristics=None,

93
python/online/fxreader/pr34/tasks/cython.py
View File

@ -10,30 +10,35 @@ import threading
import cython import cython
import datetime import datetime
from typing import (Any, Optional, TypeVar, Type, cast) from typing import Any, Optional, TypeVar, Type, cast
# from scoping import scoping as s # from scoping import scoping as s
def test( def test(
_id: int, _id: int,
T: float, T: float,
a: numpy.ndarray[Any, numpy.dtype[numpy.int32]], a: numpy.ndarray[Any, numpy.dtype[numpy.int32]],
) -> None: ) -> None:
with cython.nogil: with cython.nogil:
#if True: # if True:
started_at = datetime.datetime.now() started_at = datetime.datetime.now()
print('started') print('started')
def elapsed() -> float: def elapsed() -> float:
return (datetime.datetime.now() - started_at).total_seconds() return (datetime.datetime.now() - started_at).total_seconds()
#a = 0
# a = 0
while elapsed() < T: while elapsed() < T:
#a += 1 # a += 1
for k in range(1024 * 1024): for k in range(1024 * 1024):
a[_id] += 1 a[_id] += 1
print(['done', started_at, elapsed(), a[_id]]) print(['done', started_at, elapsed(), a[_id]])
M = TypeVar('M', bound=Type[Any]) M = TypeVar('M', bound=Type[Any])
def build(content: str, module: M) -> M: def build(content: str, module: M) -> M:
import pathlib import pathlib
import tempfile import tempfile
@ -44,7 +49,6 @@ def build(content: str, module: M) -> M:
output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute() output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute()
if not output_dir.exists() or True: if not output_dir.exists() or True:
os.makedirs(str(output_dir), exist_ok=True) os.makedirs(str(output_dir), exist_ok=True)
@ -57,23 +61,16 @@ def build(content: str, module: M) -> M:
t1.extensions = Cython.Build.cythonize(str(source_path)) t1.extensions = Cython.Build.cythonize(str(source_path))
t1.build_temp = str(pathlib.Path('/')) t1.build_temp = str(pathlib.Path('/'))
t1.build_lib = str(output_dir) t1.build_lib = str(output_dir)
#t2 = Cython.Build.Inline.Extension( # t2 = Cython.Build.Inline.Extension(
# name=sha256sum, # name=sha256sum,
#) # )
t1.run() t1.run()
return cast( return cast(M, Cython.Build.Inline.load_dynamic('_%s' % sha256sum, glob.glob(str(output_dir / ('_%s*.so' % sha256sum)))[0]))
M,
Cython.Build.Inline.load_dynamic(
'_%s' % sha256sum,
glob.glob(
str(output_dir / ('_%s*.so' % sha256sum))
)[0]
)
)
raise NotImplementedError raise NotImplementedError
def mypyc_build(file_path: pathlib.Path) -> Any: def mypyc_build(file_path: pathlib.Path) -> Any:
import pathlib import pathlib
import tempfile import tempfile
@ -83,26 +80,25 @@ def mypyc_build(file_path: pathlib.Path) -> Any:
assert isinstance(file_path, pathlib.Path) assert isinstance(file_path, pathlib.Path)
#sha256sum = hashlib.sha256(content.encode('utf-8')).digest().hex() # sha256sum = hashlib.sha256(content.encode('utf-8')).digest().hex()
#output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute() # output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute()
output_dir = pathlib.Path('.') / 'tmp' / 'mypyc' output_dir = pathlib.Path('.') / 'tmp' / 'mypyc'
sha256sum = file_path.stem sha256sum = file_path.stem
lib_pattern = file_path.parent / ('%s.cpython*.so' % sha256sum) lib_pattern = file_path.parent / ('%s.cpython*.so' % sha256sum)
lib_dir = pathlib.Path('.') lib_dir = pathlib.Path('.')
def lib_path_glob(path: str | pathlib.Path) -> Optional[pathlib.Path]: def lib_path_glob(path: str | pathlib.Path) -> Optional[pathlib.Path]:
res : list[str] = glob.glob(str(path)) res: list[str] = glob.glob(str(path))
if len(res) == 0: if len(res) == 0:
return None return None
else: else:
return pathlib.Path(res[0]) return pathlib.Path(res[0])
need_build : bool = False need_build: bool = False
lib_path : Optional[pathlib.Path] = None lib_path: Optional[pathlib.Path] = None
lib_path = lib_path_glob(lib_pattern) lib_path = lib_path_glob(lib_pattern)
@ -117,55 +113,46 @@ def mypyc_build(file_path: pathlib.Path) -> Any:
else: else:
need_build = True need_build = True
if need_build: if need_build:
for o in [ for o in [
output_dir, output_dir,
output_dir / 'build' / file_path.parent, output_dir / 'build' / file_path.parent,
]: ]:
os.makedirs( os.makedirs(str(o), exist_ok=True)
str(o), # source_path = output_dir / ('_%s.py' % sha256sum)
exist_ok=True
)
#source_path = output_dir / ('_%s.py' % sha256sum)
source_path = file_path source_path = file_path
#with io.open(str(source_path), 'w') as f: # with io.open(str(source_path), 'w') as f:
# f.write(content) # f.write(content)
t1 = Cython.Build.Inline._get_build_extension() t1 = Cython.Build.Inline._get_build_extension()
t1.extensions = mypyc.build.mypycify( t1.extensions = mypyc.build.mypycify([str(source_path)], target_dir=str(output_dir / 'build'))
[str(source_path)],
target_dir=str(output_dir / 'build')
)
t1.build_temp = str(output_dir) t1.build_temp = str(output_dir)
t1.build_lib = str(lib_dir) t1.build_lib = str(lib_dir)
#t2 = Cython.Build.Inline.Extension( # t2 = Cython.Build.Inline.Extension(
# name=sha256sum, # name=sha256sum,
#) # )
t1.run() t1.run()
lib_path = lib_path_glob(lib_pattern) lib_path = lib_path_glob(lib_pattern)
return Cython.Build.Inline.load_dynamic( return Cython.Build.Inline.load_dynamic(
#'_%s' % sha256sum, #'_%s' % sha256sum,
#t1.extensions[0].name, # t1.extensions[0].name,
file_path.stem, file_path.stem,
str(lib_path), str(lib_path),
) )
raise NotImplementedError raise NotImplementedError
class Source: class Source:
@staticmethod @staticmethod
def test2( def test2(_a: numpy.ndarray[Any, numpy.dtype[numpy.int64]], _id: numpy.dtype[numpy.int32] | int, T: float = 16) -> int:
_a : numpy.ndarray[Any, numpy.dtype[numpy.int64]],
_id : numpy.dtype[numpy.int32] | int,
T : float=16
) -> int:
raise NotImplementedError raise NotImplementedError
source = build(r''' source = build(
r"""
cimport cython cimport cython
@cython.boundscheck(False) @cython.boundscheck(False)
@ -226,10 +213,13 @@ def test2(long long [:] _a, int _id, double T=16) -> int:
return _a[_id] return _a[_id]
''', Source) """,
Source,
)
def test_cython(N: int=4, T:int=16) -> None:
#a = [0] * N def test_cython(N: int = 4, T: int = 16) -> None:
# a = [0] * N
a = numpy.zeros((N,), dtype=numpy.int64) a = numpy.zeros((N,), dtype=numpy.int64)
t = [ t = [
@ -249,18 +239,15 @@ def test_cython(N: int=4, T:int=16) -> None:
for o in t: for o in t:
o.join() o.join()
#cython_module['test2'](a, 0) # cython_module['test2'](a, 0)
def test_mypyc(N: int=4, W:int=35) -> None:
cython2 = mypyc_build( def test_mypyc(N: int = 4, W: int = 35) -> None:
(pathlib.Path(__file__).parent / 'cython2.py').relative_to( cython2 = mypyc_build((pathlib.Path(__file__).parent / 'cython2.py').relative_to(pathlib.Path.cwd()))
pathlib.Path.cwd()
)
)
# from .cython2 import fib # from .cython2 import fib
#a = [0] * N # a = [0] * N
t = [ t = [
threading.Thread( threading.Thread(
target=functools.partial( target=functools.partial(

2
python/online/fxreader/pr34/tasks/cython2.py
View File

@ -1,11 +1,13 @@
import time import time
def fib(n: int) -> int: def fib(n: int) -> int:
if n <= 1: if n <= 1:
return n return n
else: else:
return fib(n - 2) + fib(n - 1) return fib(n - 2) + fib(n - 1)
t0 = time.time() t0 = time.time()
fib(32) fib(32)
print(time.time() - t0) print(time.time() - t0)

108
python/online/fxreader/pr34/tasks/jigsaw_toxic.py
View File

@ -16,15 +16,9 @@ def kernel_1_sample_scrap(
t10 = p.content.decode('utf-8') t10 = p.content.decode('utf-8')
t11 = pyquery.PyQuery(t10) t11 = pyquery.PyQuery(t10)
t13 = t11('.crayons-story__title > a') t13 = t11('.crayons-story__title > a')
t12 = [ t12 = [pyquery.PyQuery(o).attr('href') for o in t13]
pyquery.PyQuery(o).attr('href')
for o in t13
]
pprint.pprint(t12) pprint.pprint(t12)
t14 = [ t14 = ['https://dev.to/%s' % o for o in t12]
'https://dev.to/%s' % o
for o in t12
]
t8 = [] t8 = []
for t7 in t14[:max_articles]: for t7 in t14[:max_articles]:
@ -46,7 +40,7 @@ def kernel_1_sample_scrap(
) )
) )
#pprint.pprint(t3) # pprint.pprint(t3)
pprint.pprint(t6) pprint.pprint(t6)
t8.append( t8.append(
dict( dict(
@ -66,6 +60,7 @@ def kernel_1_sample_scrap(
t12=t12, t12=t12,
) )
def kernel_2(): def kernel_2():
import numpy as np # linear algebra import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
@ -73,7 +68,7 @@ def kernel_2():
from sklearn.model_selection import train_test_split from sklearn.model_selection import train_test_split
import tensorflow as tf import tensorflow as tf
from keras.models import Sequential from keras.models import Sequential
from keras.layers.recurrent import LSTM, GRU,SimpleRNN from keras.layers.recurrent import LSTM, GRU, SimpleRNN
from keras.layers.core import Dense, Activation, Dropout from keras.layers.core import Dense, Activation, Dropout
from keras.layers.embeddings import Embedding from keras.layers.embeddings import Embedding
from keras.layers.normalization import BatchNormalization from keras.layers.normalization import BatchNormalization
@ -83,10 +78,10 @@ def kernel_2():
from keras.preprocessing import sequence, text from keras.preprocessing import sequence, text
from keras.callbacks import EarlyStopping from keras.callbacks import EarlyStopping
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import seaborn as sns import seaborn as sns
#%matplotlib inline
# %matplotlib inline
from plotly import graph_objs as go from plotly import graph_objs as go
import plotly.express as px import plotly.express as px
import plotly.figure_factory as ff import plotly.figure_factory as ff
@ -114,7 +109,7 @@ def kernel_2():
# Default distribution strategy in Tensorflow. Works on CPU and single GPU. # Default distribution strategy in Tensorflow. Works on CPU and single GPU.
strategy = tf.distribute.get_strategy() strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync) print('REPLICAS: ', strategy.num_replicas_in_sync)
# %% [code] # %% [code]
train = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv') train = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')
@ -125,27 +120,25 @@ def kernel_2():
# We will drop the other columns and approach this problem as a Binary Classification Problem and also we will have our exercise done on a smaller subsection of the dataset(only 12000 data points) to make it easier to train the models # We will drop the other columns and approach this problem as a Binary Classification Problem and also we will have our exercise done on a smaller subsection of the dataset(only 12000 data points) to make it easier to train the models
# %% [code] # %% [code]
train.drop(['severe_toxic','obscene','threat','insult','identity_hate'],axis=1,inplace=True) train.drop(['severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'], axis=1, inplace=True)
# %% [code] # %% [code]
train = train.loc[:12000,:] train = train.loc[:12000, :]
train.shape train.shape
# %% [markdown] # %% [markdown]
# We will check the maximum number of words that can be present in a comment , this will help us in padding later # We will check the maximum number of words that can be present in a comment , this will help us in padding later
# %% [code] # %% [code]
train['comment_text'].apply(lambda x:len(str(x).split())).max() train['comment_text'].apply(lambda x: len(str(x).split())).max()
# %% [markdown] # %% [markdown]
# ### Data Preparation # ### Data Preparation
# %% [code] # %% [code]
xtrain, xvalid, ytrain, yvalid = train_test_split(train.comment_text.values, train.toxic.values, xtrain, xvalid, ytrain, yvalid = train_test_split(
stratify=train.toxic.values, train.comment_text.values, train.toxic.values, stratify=train.toxic.values, random_state=42, test_size=0.2, shuffle=True
random_state=42, )
test_size=0.2, shuffle=True)
# %% [markdown] # %% [markdown]
# # Before We Begin # # Before We Begin
@ -199,20 +192,18 @@ def kernel_2():
xtrain_seq = token.texts_to_sequences(xtrain) xtrain_seq = token.texts_to_sequences(xtrain)
xvalid_seq = token.texts_to_sequences(xvalid) xvalid_seq = token.texts_to_sequences(xvalid)
#zero pad the sequences # zero pad the sequences
xtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len) xtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len)
xvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len) xvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len)
word_index = token.word_index word_index = token.word_index
# %% [code] # %% [code]
#%%time # %%time
with strategy.scope(): with strategy.scope():
# A simpleRNN without any pretrained embeddings and one dense layer # A simpleRNN without any pretrained embeddings and one dense layer
model = Sequential() model = Sequential()
model.add(Embedding(len(word_index) + 1, model.add(Embedding(len(word_index) + 1, 300, input_length=max_len))
300,
input_length=max_len))
model.add(SimpleRNN(100)) model.add(SimpleRNN(100))
model.add(Dense(1, activation='sigmoid')) model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
@ -245,12 +236,13 @@ def kernel_3(
# Writing a function for getting auc score for validation # Writing a function for getting auc score for validation
# %% [code] # %% [code]
def roc_auc(predictions,target): def roc_auc(predictions, target):
import sklearn.metrics import sklearn.metrics
'''
"""
This methods returns the AUC Score when given the Predictions This methods returns the AUC Score when given the Predictions
and Labels and Labels
''' """
fpr, tpr, thresholds = sklearn.metrics.roc_curve(target, predictions) fpr, tpr, thresholds = sklearn.metrics.roc_curve(target, predictions)
roc_auc = sklearn.metrics.auc(fpr, tpr) roc_auc = sklearn.metrics.auc(fpr, tpr)
@ -261,35 +253,17 @@ def kernel_3(
o_2['model'].load_weights('model.h5') o_2['model'].load_weights('model.h5')
else: else:
o_2['model'].fit( o_2['model'].fit(
o_2['xtrain_pad'], o_2['xtrain_pad'], o_2['ytrain'], nb_epoch=nb_epochs, batch_size=64 * o_2['strategy'].num_replicas_in_sync
o_2['ytrain'], ) # Multiplying by Strategy to run on TPU's
nb_epoch=nb_epochs,
batch_size=64*o_2['strategy'].num_replicas_in_sync
) #Multiplying by Strategy to run on TPU's
o_2['model'].save_weights('model.h5') o_2['model'].save_weights('model.h5')
# %% [code] # %% [code]
scores = o_2['model'].predict(o_2['xvalid_pad']) scores = o_2['model'].predict(o_2['xvalid_pad'])
print( print('Auc: %.2f%%' % (roc_auc(scores, o_2['yvalid'])))
"Auc: %.2f%%" % (
roc_auc(
scores,
o_2['yvalid']
)
)
)
# %% [code] # %% [code]
scores_model = [] scores_model = []
scores_model.append( scores_model.append({'Model': 'SimpleRNN', 'AUC_Score': roc_auc(scores, o_2['yvalid'])})
{
'Model': 'SimpleRNN',
'AUC_Score': roc_auc(
scores,
o_2['yvalid']
)
}
)
# %% [markdown] # %% [markdown]
# ## Code Explanantion # ## Code Explanantion
@ -301,6 +275,7 @@ def kernel_3(
# %% [code] # %% [code]
o_2['xtrain_seq'][:1] o_2['xtrain_seq'][:1]
def kernel_4( def kernel_4(
o_2, o_2,
input_texts=None, input_texts=None,
@ -308,12 +283,7 @@ def kernel_4(
import keras.preprocessing.sequence import keras.preprocessing.sequence
if input_texts is None: if input_texts is None:
input_texts = [ input_texts = ['blahb blahb blah', 'Hello World!', 'This is very good!', 'A very non toxic comment! This is so polite and polished one!']
'blahb blahb blah',
'Hello World!',
'This is very good!',
'A very non toxic comment! This is so polite and polished one!'
]
t6 = [] t6 = []
for o in input_texts: for o in input_texts:
@ -321,10 +291,7 @@ def kernel_4(
t2 = o_2['token'].texts_to_sequences( t2 = o_2['token'].texts_to_sequences(
[t1], [t1],
) )
t3 = keras.preprocessing.sequence.pad_sequences( t3 = keras.preprocessing.sequence.pad_sequences(t2, maxlen=o_2['max_len'])
t2,
maxlen=o_2['max_len']
)
t4 = o_2['model'].predict( t4 = o_2['model'].predict(
t3, t3,
) )
@ -348,6 +315,7 @@ def kernel_4(
t6=t6, t6=t6,
) )
def kernel_5( def kernel_5(
o_1=None, o_1=None,
o_2=None, o_2=None,
@ -357,23 +325,11 @@ def kernel_5(
if o_2 is None: if o_2 is None:
o_2 = kernel_2() o_2 = kernel_2()
o_3 = kernel_3( o_3 = kernel_3(o_2=o_2, nb_epochs=1)
o_2=o_2,
nb_epochs=1
)
t1 = sum( t1 = sum([[o['text'] for o in o2['comments']] for o2 in o_1['t8']], [])
[
[
o['text'] for o in o2['comments']
] for o2 in o_1['t8']
], []
)
t2 = kernel_4( t2 = kernel_4(o_2=o_2, input_texts=t1)
o_2=o_2,
input_texts=t1
)
t3 = sorted( t3 = sorted(
t2['t6'], t2['t6'],

833
python/online/fxreader/pr34/tasks/mlb_player.py
View File

File diff suppressed because it is too large Load Diff