[+] reformat with ruff
This commit is contained in:
parent
8510d49015
commit
0f17070c62
150
python/_m.py
150
python/_m.py
@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
#vim: set filetype=python
|
||||
# vim: set filetype=python
|
||||
|
||||
import logging
|
||||
import json
|
||||
@ -7,27 +7,35 @@ import enum
|
||||
import pathlib
|
||||
import sys
|
||||
import argparse
|
||||
#import optparse
|
||||
|
||||
# import optparse
|
||||
import dataclasses
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
|
||||
|
||||
from typing import (
|
||||
Optional, Any, TypeAlias, Literal, cast, BinaryIO, Generator,
|
||||
ClassVar, Self,
|
||||
Optional,
|
||||
Any,
|
||||
TypeAlias,
|
||||
Literal,
|
||||
cast,
|
||||
BinaryIO,
|
||||
Generator,
|
||||
ClassVar,
|
||||
Self,
|
||||
)
|
||||
|
||||
logger = logging.getLogger()
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Settings:
|
||||
project_root : pathlib.Path = pathlib.Path.cwd()
|
||||
project_root: pathlib.Path = pathlib.Path.cwd()
|
||||
|
||||
env_path : pathlib.Path = project_root / 'tmp' / 'env3'
|
||||
env_path: pathlib.Path = project_root / 'tmp' / 'env3'
|
||||
|
||||
_settings : ClassVar[Optional['Settings']] = None
|
||||
_settings: ClassVar[Optional['Settings']] = None
|
||||
|
||||
@classmethod
|
||||
def settings(cls) -> Self:
|
||||
@ -36,8 +44,10 @@ class Settings:
|
||||
|
||||
return cls._settings
|
||||
|
||||
|
||||
def js(argv: list[str]) -> int:
|
||||
return subprocess.check_call([
|
||||
return subprocess.check_call(
|
||||
[
|
||||
'sudo',
|
||||
'docker-compose',
|
||||
'--project-directory',
|
||||
@ -45,7 +55,9 @@ def js(argv: list[str]) -> int:
|
||||
'-f',
|
||||
Settings.settings().project_root / 'docker' / 'js' / 'docker-compose.yml',
|
||||
*argv,
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def env(
|
||||
argv: Optional[list[str]] = None,
|
||||
@ -55,17 +67,18 @@ def env(
|
||||
env_path = Settings.settings().env_path
|
||||
|
||||
if not env_path.exists():
|
||||
subprocess.check_call([
|
||||
sys.executable, '-m', 'venv',
|
||||
'--system-site-packages',
|
||||
str(env_path)
|
||||
])
|
||||
subprocess.check_call([sys.executable, '-m', 'venv', '--system-site-packages', str(env_path)])
|
||||
|
||||
subprocess.check_call([
|
||||
subprocess.check_call(
|
||||
[
|
||||
env_path / 'bin' / 'python3',
|
||||
'-m', 'pip',
|
||||
'install', '-r', 'requirements.txt',
|
||||
])
|
||||
'-m',
|
||||
'pip',
|
||||
'install',
|
||||
'-r',
|
||||
'requirements.txt',
|
||||
]
|
||||
)
|
||||
|
||||
if not argv is None:
|
||||
python_path = str(env_path / 'bin' / 'python3')
|
||||
@ -80,15 +93,19 @@ def env(
|
||||
)
|
||||
return None
|
||||
elif mode == 'subprocess':
|
||||
return subprocess.run([
|
||||
return subprocess.run(
|
||||
[
|
||||
python_path,
|
||||
*argv,
|
||||
], **kwargs)
|
||||
],
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def ruff(argv: list[str]) -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
@ -109,28 +126,38 @@ def ruff(argv: list[str]) -> None:
|
||||
options, args = parser.parse_known_args(argv)
|
||||
|
||||
if len(options.paths) == 0:
|
||||
options.paths.extend([
|
||||
options.paths.extend(
|
||||
[
|
||||
'.',
|
||||
'dotfiles/.local/bin/commands',
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
if len(options.exclude) == 0:
|
||||
options.exclude.extend([
|
||||
options.exclude.extend(
|
||||
[
|
||||
'E731',
|
||||
'E713',
|
||||
'E714',
|
||||
'E703',
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
res = env([
|
||||
res = env(
|
||||
[
|
||||
'-m',
|
||||
'ruff',
|
||||
'check',
|
||||
*args,
|
||||
'--output-format', 'json',
|
||||
'--ignore', ','.join(options.exclude),
|
||||
'--output-format',
|
||||
'json',
|
||||
'--ignore',
|
||||
','.join(options.exclude),
|
||||
*options.paths,
|
||||
], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
assert not res is None
|
||||
|
||||
@ -142,10 +169,7 @@ def ruff(argv: list[str]) -> None:
|
||||
g[o['filename']] = []
|
||||
g[o['filename']].append(o)
|
||||
|
||||
h = {
|
||||
k : len(v)
|
||||
for k, v in g.items()
|
||||
}
|
||||
h = {k: len(v) for k, v in g.items()}
|
||||
|
||||
logger.info(json.dumps(errors, indent=4))
|
||||
logger.info(json.dumps(h, indent=4))
|
||||
@ -154,11 +178,13 @@ def ruff(argv: list[str]) -> None:
|
||||
def inside_env() -> bool:
|
||||
try:
|
||||
import numpy
|
||||
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
#class Commands(enum.StrEnum):
|
||||
|
||||
# class Commands(enum.StrEnum):
|
||||
# js = 'js'
|
||||
# mypy = 'mypy'
|
||||
# env = 'env'
|
||||
@ -172,42 +198,53 @@ def inside_env() -> bool:
|
||||
# argv,
|
||||
# )
|
||||
|
||||
|
||||
def host_deps(argv: list[str]) -> None:
|
||||
if sys.platform in ['linux']:
|
||||
subprocess.check_call(r'''
|
||||
subprocess.check_call(
|
||||
r"""
|
||||
exec yay -S $(cat requirements-archlinux.txt)
|
||||
''', shell=True,)
|
||||
""",
|
||||
shell=True,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
Command_args = ['js', 'mypy', 'env', 'ruff', 'm2', 'host_deps',]
|
||||
|
||||
Command : TypeAlias = Literal['js', 'mypy', 'env', 'ruff', 'm2', 'host_deps',]
|
||||
Command_args = [
|
||||
'js',
|
||||
'mypy',
|
||||
'env',
|
||||
'ruff',
|
||||
'm2',
|
||||
'host_deps',
|
||||
]
|
||||
|
||||
Command: TypeAlias = Literal[
|
||||
'js',
|
||||
'mypy',
|
||||
'env',
|
||||
'ruff',
|
||||
'm2',
|
||||
'host_deps',
|
||||
]
|
||||
|
||||
|
||||
def run(argv: Optional[list[str]] = None) -> None:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format=(
|
||||
'%(levelname)s:%(name)s:%(message)s'
|
||||
':%(process)d'
|
||||
':%(asctime)s'
|
||||
':%(pathname)s:%(funcName)s:%(lineno)s'
|
||||
),
|
||||
format=('%(levelname)s:%(name)s:%(message)s:%(process)d:%(asctime)s:%(pathname)s:%(funcName)s:%(lineno)s'),
|
||||
)
|
||||
|
||||
if argv is None:
|
||||
argv = sys.argv[:]
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'command',
|
||||
#'_command',
|
||||
choices=[
|
||||
o
|
||||
for o in Command_args
|
||||
],
|
||||
#required=True,
|
||||
choices=[o for o in Command_args],
|
||||
# required=True,
|
||||
)
|
||||
|
||||
options, args = parser.parse_known_args(argv[1:])
|
||||
@ -217,14 +254,17 @@ def run(argv: Optional[list[str]] = None) -> None:
|
||||
if len(args) > 0 and args[0] == '--':
|
||||
del args[0]
|
||||
|
||||
#options.command = Commands(options._command)
|
||||
# options.command = Commands(options._command)
|
||||
|
||||
if options.command == 'js':
|
||||
js(args)
|
||||
elif options.command == 'host_deps':
|
||||
host_deps(args)
|
||||
elif options.command == 'env':
|
||||
env(args, mode='exec',)
|
||||
env(
|
||||
args,
|
||||
mode='exec',
|
||||
)
|
||||
# elif options.command == 'mypy':
|
||||
# if not inside_env():
|
||||
# env(
|
||||
@ -244,11 +284,11 @@ def run(argv: Optional[list[str]] = None) -> None:
|
||||
return
|
||||
|
||||
import python.tasks.cython
|
||||
python.tasks.cython.mypyc_build(
|
||||
pathlib.Path('_m.py')
|
||||
)
|
||||
|
||||
python.tasks.cython.mypyc_build(pathlib.Path('_m.py'))
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
@ -10,7 +10,10 @@ import enum
|
||||
import argparse
|
||||
import dataclasses
|
||||
|
||||
from typing import (Optional, override,)
|
||||
from typing import (
|
||||
Optional,
|
||||
override,
|
||||
)
|
||||
|
||||
from online.fxreader.pr34.commands_typed.logging import setup as logging_setup
|
||||
|
||||
@ -28,6 +31,7 @@ class Command(enum.StrEnum):
|
||||
deploy_wheel = 'deploy:wheel'
|
||||
tests = 'tests'
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Settings(
|
||||
_cli.DistSettings,
|
||||
@ -50,7 +54,7 @@ class CLI(_cli.CLI):
|
||||
)
|
||||
}
|
||||
|
||||
self._dependencies : dict[str, _cli.Dependency] = dict()
|
||||
self._dependencies: dict[str, _cli.Dependency] = dict()
|
||||
|
||||
@override
|
||||
@property
|
||||
@ -74,22 +78,22 @@ class CLI(_cli.CLI):
|
||||
argv,
|
||||
settings=_mypy.MypySettings(
|
||||
paths=[
|
||||
#Settings.settings().project_root / 'dotfiles/.local/bin/commands',
|
||||
# Settings.settings().project_root / 'dotfiles/.local/bin/commands',
|
||||
# project.source_dir / 'm.py',
|
||||
project.source_dir / '_m.py',
|
||||
project.source_dir / 'online',
|
||||
project.source_dir / 'cli.py',
|
||||
self.settings.base_dir / 'm.py',
|
||||
# Settings.settings().project_root / 'deps/com.github.aiortc.aiortc/src',
|
||||
#Settings.settings().project_root / 'm.py',
|
||||
# Settings.settings().project_root / 'm.py',
|
||||
],
|
||||
max_errors={
|
||||
'python/online/fxreader/pr34/commands_typed': 0,
|
||||
'python/cli.py': 0,
|
||||
'm.py': 0,
|
||||
'deps/com.github.aiortc.aiortc/src/online_fxreader': 0,
|
||||
'deps/com.github.aiortc.aiortc/src/aiortc/contrib/signaling': 0
|
||||
}
|
||||
'deps/com.github.aiortc.aiortc/src/aiortc/contrib/signaling': 0,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
@ -103,27 +107,17 @@ class CLI(_cli.CLI):
|
||||
argv = copy.deepcopy(sys.argv)
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('command', choices=[o.value for o in Command])
|
||||
parser.add_argument('-p', '--project', choices=[o for o in self.projects])
|
||||
parser.add_argument(
|
||||
'command',
|
||||
choices=[
|
||||
o.value
|
||||
for o in Command
|
||||
]
|
||||
)
|
||||
parser.add_argument(
|
||||
'-p', '--project',
|
||||
choices=[
|
||||
o
|
||||
for o in self.projects
|
||||
]
|
||||
)
|
||||
parser.add_argument(
|
||||
'-o', '--output_dir',
|
||||
'-o',
|
||||
'--output_dir',
|
||||
default=None,
|
||||
help='wheel output dir for deploy:wheel',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-f', '--force',
|
||||
'-f',
|
||||
'--force',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='remove install dir, before installing, default = false',
|
||||
@ -148,15 +142,19 @@ class CLI(_cli.CLI):
|
||||
)
|
||||
elif options.command is Command.tests:
|
||||
for k, v in self.projects.items():
|
||||
subprocess.check_call([
|
||||
subprocess.check_call(
|
||||
[
|
||||
sys.executable,
|
||||
'-m',
|
||||
'unittest',
|
||||
'online.fxreader.pr34.tests.test_crypto',
|
||||
*args,
|
||||
], cwd=str(v.source_dir))
|
||||
],
|
||||
cwd=str(v.source_dir),
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
CLI().run()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,11 +1,12 @@
|
||||
__all__ = (
|
||||
'parse_args',
|
||||
)
|
||||
__all__ = ('parse_args',)
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
|
||||
from typing import (Optional,)
|
||||
from typing import (
|
||||
Optional,
|
||||
)
|
||||
|
||||
|
||||
def parse_args(
|
||||
parser: argparse.ArgumentParser,
|
||||
@ -14,11 +15,11 @@ def parse_args(
|
||||
if args is None:
|
||||
args = sys.argv[1:]
|
||||
|
||||
argv : list[str] = []
|
||||
argv: list[str] = []
|
||||
|
||||
for i, o in enumerate(args):
|
||||
if o == '--':
|
||||
argv.extend(args[i + 1:])
|
||||
argv.extend(args[i + 1 :])
|
||||
|
||||
del args[i:]
|
||||
|
||||
|
||||
@ -1,14 +1,23 @@
|
||||
import logging
|
||||
import asyncio
|
||||
|
||||
from typing import (Any,)
|
||||
from typing import (
|
||||
Any,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def handle_task_result(fut: asyncio.Future[Any]) -> None:
|
||||
try:
|
||||
fut.result()
|
||||
|
||||
logger.debug(dict(fut=fut, msg='done'), stacklevel=2,)
|
||||
logger.debug(
|
||||
dict(fut=fut, msg='done'),
|
||||
stacklevel=2,
|
||||
)
|
||||
except:
|
||||
logger.exception('', stacklevel=2,)
|
||||
logger.exception(
|
||||
'',
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
@ -19,26 +19,30 @@ from typing import (
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Project:
|
||||
source_dir : pathlib.Path
|
||||
build_dir : pathlib.Path
|
||||
dest_dir : pathlib.Path
|
||||
source_dir: pathlib.Path
|
||||
build_dir: pathlib.Path
|
||||
dest_dir: pathlib.Path
|
||||
meson_path: Optional[pathlib.Path] = None
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Dependency:
|
||||
name: str
|
||||
mode : Literal['pyproject', 'meson', 'meson-python', 'm']
|
||||
source_path : pathlib.Path
|
||||
mode: Literal['pyproject', 'meson', 'meson-python', 'm']
|
||||
source_path: pathlib.Path
|
||||
args: Optional[list[str]] = None
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DistSettings:
|
||||
wheel_dir : pathlib.Path
|
||||
wheel_dir: pathlib.Path
|
||||
python_path: pathlib.Path
|
||||
env_path: pathlib.Path
|
||||
|
||||
|
||||
class CLI(abc.ABC):
|
||||
@property
|
||||
@abc.abstractmethod
|
||||
@ -55,10 +59,7 @@ class CLI(abc.ABC):
|
||||
def dependencies(self) -> dict[str, Dependency]:
|
||||
raise NotImplementedError
|
||||
|
||||
def mypy(
|
||||
self,
|
||||
argv: list[str]
|
||||
) -> None:
|
||||
def mypy(self, argv: list[str]) -> None:
|
||||
from . import mypy as _mypy
|
||||
|
||||
_mypy.run(
|
||||
@ -73,15 +74,21 @@ class CLI(abc.ABC):
|
||||
project = self.projects[project_name]
|
||||
|
||||
if len(argv) == 0:
|
||||
argv = ['check', '.',]
|
||||
argv = [
|
||||
'check',
|
||||
'.',
|
||||
]
|
||||
|
||||
subprocess.check_call([
|
||||
subprocess.check_call(
|
||||
[
|
||||
self.dist_settings.python_path,
|
||||
'-m',
|
||||
'ruff',
|
||||
'--config', str(project.source_dir / 'pyproject.toml'),
|
||||
'--config',
|
||||
str(project.source_dir / 'pyproject.toml'),
|
||||
*argv,
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
def pyright(
|
||||
self,
|
||||
@ -97,8 +104,10 @@ class CLI(abc.ABC):
|
||||
str(self.dist_settings.python_path),
|
||||
'-m',
|
||||
'pyright',
|
||||
'--pythonpath', str(self.dist_settings.python_path),
|
||||
'-p', str(project.source_dir / 'pyproject.toml'),
|
||||
'--pythonpath',
|
||||
str(self.dist_settings.python_path),
|
||||
'-p',
|
||||
str(project.source_dir / 'pyproject.toml'),
|
||||
*argv,
|
||||
]
|
||||
|
||||
@ -113,38 +122,44 @@ class CLI(abc.ABC):
|
||||
) -> None:
|
||||
from . import cli_bootstrap
|
||||
|
||||
pyproject = cli_bootstrap.pyproject_load(
|
||||
self.projects[project].source_dir / 'pyproject.toml'
|
||||
)
|
||||
pyproject = cli_bootstrap.pyproject_load(self.projects[project].source_dir / 'pyproject.toml')
|
||||
|
||||
dependencies = sum([
|
||||
pyproject.dependencies[o]
|
||||
for o in features
|
||||
], [])
|
||||
dependencies = sum([pyproject.dependencies[o] for o in features], [])
|
||||
|
||||
pip_find_links : list[pathlib.Path] = []
|
||||
pip_find_links: list[pathlib.Path] = []
|
||||
|
||||
if not pyproject.pip_find_links is None:
|
||||
pip_find_links.extend(pyproject.pip_find_links)
|
||||
|
||||
|
||||
logger.info(dict(
|
||||
logger.info(
|
||||
dict(
|
||||
dependencies=dependencies,
|
||||
))
|
||||
)
|
||||
)
|
||||
|
||||
if len(dependencies) > 0:
|
||||
subprocess.check_call([
|
||||
subprocess.check_call(
|
||||
[
|
||||
self.dist_settings.python_path,
|
||||
'-m',
|
||||
'uv', 'pip', 'install',
|
||||
*sum([
|
||||
['-f', str(o),]
|
||||
'uv',
|
||||
'pip',
|
||||
'install',
|
||||
*sum(
|
||||
[
|
||||
[
|
||||
'-f',
|
||||
str(o),
|
||||
]
|
||||
for o in pip_find_links
|
||||
], []),
|
||||
],
|
||||
[],
|
||||
),
|
||||
# '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'),
|
||||
'--offline',
|
||||
*dependencies,
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
def deploy_fetch_dist(
|
||||
self,
|
||||
@ -152,19 +167,16 @@ class CLI(abc.ABC):
|
||||
) -> None:
|
||||
for k, d in self.dependencies.items():
|
||||
whl_glob = self.dist_settings.wheel_dir / ('*%s*.whl' % d.name.replace('.', '_'))
|
||||
if len(glob.glob(
|
||||
str(whl_glob)
|
||||
)) == 0 or force:
|
||||
if len(glob.glob(str(whl_glob))) == 0 or force:
|
||||
if d.source_path.exists():
|
||||
|
||||
def whl_files_get() -> list[dict[str, Any]]:
|
||||
return [
|
||||
dict(
|
||||
path=o,
|
||||
stat=os.stat(o).st_mtime,
|
||||
)
|
||||
for o in glob.glob(
|
||||
str(whl_glob)
|
||||
)
|
||||
for o in glob.glob(str(whl_glob))
|
||||
]
|
||||
|
||||
present_files = whl_files_get()
|
||||
@ -194,12 +206,9 @@ class CLI(abc.ABC):
|
||||
def index_get(o: dict[str, Any]) -> tuple[Any, ...]:
|
||||
return (o['path'], o['stat'])
|
||||
|
||||
present_files_index = {
|
||||
index_get(o) : o
|
||||
for o in present_files
|
||||
}
|
||||
present_files_index = {index_get(o): o for o in present_files}
|
||||
|
||||
new_files : list[dict[str, Any]] = []
|
||||
new_files: list[dict[str, Any]] = []
|
||||
|
||||
for o in updated_files:
|
||||
entry_index = index_get(o)
|
||||
@ -210,20 +219,22 @@ class CLI(abc.ABC):
|
||||
if len(new_files) == 0:
|
||||
raise NotImplementedError
|
||||
|
||||
latest_file = sorted(
|
||||
new_files,
|
||||
key=lambda x: x['stat']
|
||||
)[-1]
|
||||
latest_file = sorted(new_files, key=lambda x: x['stat'])[-1]
|
||||
|
||||
subprocess.check_call([
|
||||
subprocess.check_call(
|
||||
[
|
||||
self.dist_settings.python_path,
|
||||
'-m', 'pip',
|
||||
'-m',
|
||||
'pip',
|
||||
'install',
|
||||
latest_file['path'],
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
@property
|
||||
def pkg_config_path(self,) -> set[pathlib.Path]:
|
||||
def pkg_config_path(
|
||||
self,
|
||||
) -> set[pathlib.Path]:
|
||||
return {
|
||||
pathlib.Path(o)
|
||||
for o in glob.glob(
|
||||
@ -276,18 +287,14 @@ class CLI(abc.ABC):
|
||||
extra_args: list[str] = []
|
||||
|
||||
if len(self.third_party_roots) > 0:
|
||||
extra_args.extend([
|
||||
'-Csetup-args=%s' % (
|
||||
'-Dthird_party_roots=%s' % str(o.absolute())
|
||||
)
|
||||
for o in self.third_party_roots
|
||||
])
|
||||
extra_args.extend(['-Csetup-args=%s' % ('-Dthird_party_roots=%s' % str(o.absolute())) for o in self.third_party_roots])
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
'-m',
|
||||
'build',
|
||||
'-w', '-n',
|
||||
'-w',
|
||||
'-n',
|
||||
*extra_args,
|
||||
'-Csetup-args=-Dmodes=pyproject',
|
||||
'-Cbuild-dir=%s' % str(project.build_dir / 'pyproject'),
|
||||
@ -335,31 +342,38 @@ class CLI(abc.ABC):
|
||||
if force and project.dest_dir.exists():
|
||||
shutil.rmtree(project.dest_dir)
|
||||
|
||||
subprocess.check_call([
|
||||
shutil_which('meson', True,),
|
||||
subprocess.check_call(
|
||||
[
|
||||
shutil_which(
|
||||
'meson',
|
||||
True,
|
||||
),
|
||||
'install',
|
||||
'-C',
|
||||
project.build_dir / 'meson',
|
||||
'--destdir', project.dest_dir,
|
||||
'--destdir',
|
||||
project.dest_dir,
|
||||
*argv,
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
for o in glob.glob(
|
||||
str(project.dest_dir / 'lib' / 'pkgconfig' / '*.pc'),
|
||||
recursive=True,
|
||||
):
|
||||
logger.info(dict(
|
||||
logger.info(
|
||||
dict(
|
||||
path=o,
|
||||
action='patch prefix',
|
||||
))
|
||||
)
|
||||
)
|
||||
|
||||
with io.open(o, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
with io.open(o, 'w') as f:
|
||||
f.write(
|
||||
content.replace('prefix=/', 'prefix=${pcfiledir}/../../')
|
||||
)
|
||||
f.write(content.replace('prefix=/', 'prefix=${pcfiledir}/../../'))
|
||||
|
||||
def ninja(
|
||||
self,
|
||||
project_name: str,
|
||||
@ -396,14 +410,18 @@ class CLI(abc.ABC):
|
||||
if argv is None:
|
||||
argv = []
|
||||
|
||||
subprocess.check_call([
|
||||
shutil_which('meson', True,),
|
||||
subprocess.check_call(
|
||||
[
|
||||
shutil_which(
|
||||
'meson',
|
||||
True,
|
||||
),
|
||||
'test',
|
||||
'-C',
|
||||
project.build_dir / 'meson',
|
||||
*argv,
|
||||
])
|
||||
|
||||
]
|
||||
)
|
||||
|
||||
def meson_compile(
|
||||
self,
|
||||
@ -415,13 +433,18 @@ class CLI(abc.ABC):
|
||||
if argv is None:
|
||||
argv = []
|
||||
|
||||
subprocess.check_call([
|
||||
shutil_which('meson', True,),
|
||||
subprocess.check_call(
|
||||
[
|
||||
shutil_which(
|
||||
'meson',
|
||||
True,
|
||||
),
|
||||
'compile',
|
||||
'-C',
|
||||
project.build_dir / 'meson',
|
||||
*argv,
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
@property
|
||||
def third_party_roots(self) -> list[pathlib.Path]:
|
||||
@ -450,16 +473,16 @@ class CLI(abc.ABC):
|
||||
logger.info(dict(action='removing build dir', path=project.build_dir / 'meson'))
|
||||
shutil.rmtree(project.build_dir / 'meson')
|
||||
|
||||
extra_args : list[str] = []
|
||||
extra_args: list[str] = []
|
||||
|
||||
if len(self.third_party_roots) > 0:
|
||||
extra_args.extend([
|
||||
'-Dthird_party_roots=%s' % str(o.absolute())
|
||||
for o in self.third_party_roots
|
||||
])
|
||||
extra_args.extend(['-Dthird_party_roots=%s' % str(o.absolute()) for o in self.third_party_roots])
|
||||
|
||||
cmd = [
|
||||
shutil_which('meson', True,),
|
||||
shutil_which(
|
||||
'meson',
|
||||
True,
|
||||
),
|
||||
'setup',
|
||||
str(project.source_dir),
|
||||
str(project.build_dir / 'meson'),
|
||||
|
||||
@ -10,28 +10,36 @@ import os
|
||||
import logging
|
||||
|
||||
|
||||
from typing import (Optional, Any,)
|
||||
from typing import (
|
||||
Optional,
|
||||
Any,
|
||||
)
|
||||
from typing_extensions import (
|
||||
Self, BinaryIO,
|
||||
Self,
|
||||
BinaryIO,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def toml_load(f: BinaryIO) -> Any:
|
||||
try:
|
||||
import tomllib
|
||||
|
||||
return tomllib.load(f)
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
import tomli
|
||||
|
||||
return tomli.load(f)
|
||||
except:
|
||||
pass
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class PyProject:
|
||||
path: pathlib.Path
|
||||
@ -40,7 +48,8 @@ class PyProject:
|
||||
pip_find_links: Optional[list[pathlib.Path]] = None
|
||||
runtime_libdirs: Optional[list[pathlib.Path]] = None
|
||||
runtime_preload: Optional[list[pathlib.Path]] = None
|
||||
requirements: dict[str, pathlib.Path] = dataclasses.field(default_factory=lambda : dict())
|
||||
requirements: dict[str, pathlib.Path] = dataclasses.field(default_factory=lambda: dict())
|
||||
|
||||
|
||||
def pyproject_load(
|
||||
d: pathlib.Path,
|
||||
@ -50,17 +59,12 @@ def pyproject_load(
|
||||
|
||||
assert isinstance(content, dict)
|
||||
|
||||
dependencies : dict[str, list[str]] = dict()
|
||||
dependencies: dict[str, list[str]] = dict()
|
||||
|
||||
dependencies['default'] = content['project']['dependencies']
|
||||
|
||||
if (
|
||||
'optional-dependencies' in content['project']
|
||||
):
|
||||
assert isinstance(
|
||||
content['project']['optional-dependencies'],
|
||||
dict
|
||||
)
|
||||
if 'optional-dependencies' in content['project']:
|
||||
assert isinstance(content['project']['optional-dependencies'], dict)
|
||||
|
||||
for k, v in content['project']['optional-dependencies'].items():
|
||||
assert isinstance(v, list)
|
||||
@ -68,7 +72,6 @@ def pyproject_load(
|
||||
|
||||
dependencies[k] = v
|
||||
|
||||
|
||||
res = PyProject(
|
||||
path=d,
|
||||
dependencies=dependencies,
|
||||
@ -76,25 +79,12 @@ def pyproject_load(
|
||||
|
||||
tool_name = 'online.fxreader.pr34'.replace('.', '-')
|
||||
|
||||
if (
|
||||
'tool' in content and
|
||||
isinstance(
|
||||
content['tool'], dict
|
||||
) and
|
||||
tool_name in content['tool'] and
|
||||
isinstance(
|
||||
content['tool'][tool_name],
|
||||
dict
|
||||
)
|
||||
):
|
||||
if 'tool' in content and isinstance(content['tool'], dict) and tool_name in content['tool'] and isinstance(content['tool'][tool_name], dict):
|
||||
if 'early_features' in content['tool'][tool_name]:
|
||||
res.early_features = content['tool'][tool_name]['early_features']
|
||||
|
||||
if 'pip_find_links' in content['tool'][tool_name]:
|
||||
res.pip_find_links = [
|
||||
d.parent / pathlib.Path(o)
|
||||
for o in content['tool'][tool_name]['pip_find_links']
|
||||
]
|
||||
res.pip_find_links = [d.parent / pathlib.Path(o) for o in content['tool'][tool_name]['pip_find_links']]
|
||||
|
||||
if 'runtime_libdirs' in content['tool'][tool_name]:
|
||||
res.runtime_libdirs = [
|
||||
@ -114,29 +104,31 @@ def pyproject_load(
|
||||
assert isinstance(content['tool'][tool_name]['requirements'], dict)
|
||||
|
||||
res.requirements = {
|
||||
k : d.parent / pathlib.Path(v)
|
||||
k: d.parent / pathlib.Path(v)
|
||||
# pathlib.Path(o)
|
||||
for k, v in content['tool'][tool_name]['requirements'].items()
|
||||
}
|
||||
|
||||
return res
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class BootstrapSettings:
|
||||
env_path: pathlib.Path
|
||||
python_path: pathlib.Path
|
||||
base_dir: pathlib.Path
|
||||
python_version: Optional[str] = dataclasses.field(
|
||||
default_factory=lambda : os.environ.get(
|
||||
default_factory=lambda: os.environ.get(
|
||||
'PYTHON_VERSION',
|
||||
'%d.%d' % (
|
||||
'%d.%d'
|
||||
% (
|
||||
sys.version_info.major,
|
||||
sys.version_info.minor,
|
||||
),
|
||||
).strip()
|
||||
)
|
||||
uv_args: list[str] = dataclasses.field(
|
||||
default_factory=lambda : os.environ.get(
|
||||
default_factory=lambda: os.environ.get(
|
||||
'UV_ARGS',
|
||||
'--offline',
|
||||
).split(),
|
||||
@ -159,21 +151,28 @@ class BootstrapSettings:
|
||||
python_path=python_path,
|
||||
)
|
||||
|
||||
|
||||
def env_bootstrap(
|
||||
bootstrap_settings: BootstrapSettings,
|
||||
pyproject: PyProject,
|
||||
) -> None:
|
||||
pip_find_links : list[pathlib.Path] = []
|
||||
pip_find_links: list[pathlib.Path] = []
|
||||
|
||||
if not pyproject.pip_find_links is None:
|
||||
pip_find_links.extend(pyproject.pip_find_links)
|
||||
|
||||
pip_find_links_args = sum([
|
||||
['-f', str(o),]
|
||||
pip_find_links_args = sum(
|
||||
[
|
||||
[
|
||||
'-f',
|
||||
str(o),
|
||||
]
|
||||
for o in pip_find_links
|
||||
], [])
|
||||
],
|
||||
[],
|
||||
)
|
||||
|
||||
features : list[str] = []
|
||||
features: list[str] = []
|
||||
|
||||
if pyproject.early_features:
|
||||
features.extend(pyproject.early_features)
|
||||
@ -182,35 +181,31 @@ def env_bootstrap(
|
||||
if not bootstrap_settings.python_version is None:
|
||||
requirements_python_version = bootstrap_settings.python_version.replace('.', '_')
|
||||
|
||||
|
||||
requirements_name = '_'.join(sorted(features))
|
||||
|
||||
if requirements_python_version:
|
||||
requirements_name += '_' + requirements_python_version
|
||||
|
||||
requirements_path : Optional[pathlib.Path] = None
|
||||
requirements_path: Optional[pathlib.Path] = None
|
||||
|
||||
if requirements_name in pyproject.requirements:
|
||||
requirements_path = pyproject.requirements[requirements_name]
|
||||
else:
|
||||
requirements_path = pyproject.path.parent / 'requirements.txt'
|
||||
|
||||
requirements_in : list[str] = []
|
||||
requirements_in: list[str] = []
|
||||
|
||||
requirements_in.extend([
|
||||
'uv', 'pip', 'build', 'setuptools', 'meson-python', 'pybind11'
|
||||
])
|
||||
requirements_in.extend(['uv', 'pip', 'build', 'setuptools', 'meson-python', 'pybind11'])
|
||||
|
||||
if pyproject.early_features:
|
||||
early_dependencies = sum([
|
||||
pyproject.dependencies[o]
|
||||
for o in pyproject.early_features
|
||||
], [])
|
||||
early_dependencies = sum([pyproject.dependencies[o] for o in pyproject.early_features], [])
|
||||
|
||||
logger.info(dict(
|
||||
logger.info(
|
||||
dict(
|
||||
requirements_name=requirements_name,
|
||||
early_dependencies=early_dependencies,
|
||||
))
|
||||
)
|
||||
)
|
||||
|
||||
requirements_in.extend(early_dependencies)
|
||||
# if len(early_dependencies) > 0:
|
||||
@ -230,12 +225,11 @@ def env_bootstrap(
|
||||
prefix='requirements',
|
||||
suffix='.in',
|
||||
) as f:
|
||||
f.write(
|
||||
'\n'.join(requirements_in)
|
||||
)
|
||||
f.write('\n'.join(requirements_in))
|
||||
f.flush()
|
||||
|
||||
subprocess.check_call([
|
||||
subprocess.check_call(
|
||||
[
|
||||
'uv',
|
||||
'pip',
|
||||
'compile',
|
||||
@ -244,27 +238,36 @@ def env_bootstrap(
|
||||
# '-p',
|
||||
# bootstrap_settings.python_path,
|
||||
*bootstrap_settings.uv_args,
|
||||
'-o', str(requirements_path),
|
||||
'-o',
|
||||
str(requirements_path),
|
||||
f.name,
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
uv_python_version: list[str] = []
|
||||
|
||||
if not bootstrap_settings.python_version is None:
|
||||
uv_python_version.extend([
|
||||
'-p', bootstrap_settings.python_version,
|
||||
])
|
||||
uv_python_version.extend(
|
||||
[
|
||||
'-p',
|
||||
bootstrap_settings.python_version,
|
||||
]
|
||||
)
|
||||
|
||||
subprocess.check_call([
|
||||
'uv', 'venv',
|
||||
subprocess.check_call(
|
||||
[
|
||||
'uv',
|
||||
'venv',
|
||||
*uv_python_version,
|
||||
*pip_find_links_args,
|
||||
# '--seed',
|
||||
*bootstrap_settings.uv_args,
|
||||
str(bootstrap_settings.env_path)
|
||||
])
|
||||
str(bootstrap_settings.env_path),
|
||||
]
|
||||
)
|
||||
|
||||
subprocess.check_call([
|
||||
subprocess.check_call(
|
||||
[
|
||||
'uv',
|
||||
'pip',
|
||||
'install',
|
||||
@ -273,19 +276,16 @@ def env_bootstrap(
|
||||
bootstrap_settings.python_path,
|
||||
'--require-hashes',
|
||||
*bootstrap_settings.uv_args,
|
||||
'-r', str(requirements_path),
|
||||
])
|
||||
|
||||
|
||||
def paths_equal(
|
||||
a: pathlib.Path | str,
|
||||
b: pathlib.Path | str
|
||||
) -> bool:
|
||||
return (
|
||||
os.path.abspath(str(a)) ==
|
||||
os.path.abspath(str(b))
|
||||
'-r',
|
||||
str(requirements_path),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def paths_equal(a: pathlib.Path | str, b: pathlib.Path | str) -> bool:
|
||||
return os.path.abspath(str(a)) == os.path.abspath(str(b))
|
||||
|
||||
|
||||
def run(
|
||||
d: Optional[pathlib.Path] = None,
|
||||
cli_path: Optional[pathlib.Path] = None,
|
||||
@ -298,9 +298,7 @@ def run(
|
||||
|
||||
bootstrap_settings = BootstrapSettings.get()
|
||||
|
||||
pyproject : PyProject = pyproject_load(
|
||||
d
|
||||
)
|
||||
pyproject: PyProject = pyproject_load(d)
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
@ -318,19 +316,18 @@ def run(
|
||||
[
|
||||
str(bootstrap_settings.python_path),
|
||||
*sys.argv,
|
||||
]
|
||||
],
|
||||
)
|
||||
|
||||
os.execv(
|
||||
str(bootstrap_settings.python_path),
|
||||
[
|
||||
str(bootstrap_settings.python_path),
|
||||
str(
|
||||
cli_path
|
||||
),
|
||||
str(cli_path),
|
||||
*sys.argv[1:],
|
||||
]
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
|
||||
@ -3,7 +3,12 @@ import os
|
||||
|
||||
import cryptography.hazmat.primitives.kdf.scrypt
|
||||
|
||||
from typing import (Literal, overload, Optional,)
|
||||
from typing import (
|
||||
Literal,
|
||||
overload,
|
||||
Optional,
|
||||
)
|
||||
|
||||
|
||||
class PasswordUtils:
|
||||
@overload
|
||||
@ -44,19 +49,21 @@ class PasswordUtils:
|
||||
if mode == 'bytes':
|
||||
return (salt, hashed_secret)
|
||||
elif mode == 'base64':
|
||||
res_tuple = tuple((
|
||||
res_tuple = tuple(
|
||||
(
|
||||
base64.b64encode(o).decode('utf-8')
|
||||
for o in (salt, hashed_secret,)
|
||||
))
|
||||
for o in (
|
||||
salt,
|
||||
hashed_secret,
|
||||
)
|
||||
)
|
||||
)
|
||||
return (res_tuple[0], res_tuple[1])
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def _scrypt_init(
|
||||
cls,
|
||||
salt: bytes
|
||||
) -> cryptography.hazmat.primitives.kdf.scrypt.Scrypt:
|
||||
def _scrypt_init(cls, salt: bytes) -> cryptography.hazmat.primitives.kdf.scrypt.Scrypt:
|
||||
return cryptography.hazmat.primitives.kdf.scrypt.Scrypt(
|
||||
salt=salt,
|
||||
length=32,
|
||||
|
||||
@ -1,10 +1,13 @@
|
||||
import os
|
||||
import logging
|
||||
|
||||
from typing import (Optional,)
|
||||
from typing import (
|
||||
Optional,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DebugPy:
|
||||
@classmethod
|
||||
def set_trace(
|
||||
@ -25,6 +28,7 @@ class DebugPy:
|
||||
if os.environ.get('DEBUGPY_RUNNING') != 'true':
|
||||
logger.info('debugpy init')
|
||||
import debugpy
|
||||
|
||||
debugpy.listen((host, port))
|
||||
os.environ['DEBUGPY_RUNNING'] = 'true'
|
||||
|
||||
|
||||
@ -1,5 +1,8 @@
|
||||
import logging
|
||||
from typing import (Optional,)
|
||||
from typing import (
|
||||
Optional,
|
||||
)
|
||||
|
||||
|
||||
def setup(level: Optional[int] = None) -> None:
|
||||
if level is None:
|
||||
@ -7,10 +10,5 @@ def setup(level: Optional[int] = None) -> None:
|
||||
|
||||
logging.basicConfig(
|
||||
level=level,
|
||||
format=(
|
||||
'%(levelname)s:%(name)s:%(message)s'
|
||||
':%(process)d'
|
||||
':%(asctime)s'
|
||||
':%(pathname)s:%(funcName)s:%(lineno)s'
|
||||
),
|
||||
format=('%(levelname)s:%(name)s:%(message)s:%(process)d:%(asctime)s:%(pathname)s:%(funcName)s:%(lineno)s'),
|
||||
)
|
||||
@ -9,17 +9,26 @@ import logging
|
||||
import sys
|
||||
import argparse
|
||||
|
||||
from pydantic import (Field,)
|
||||
from pydantic import (
|
||||
Field,
|
||||
)
|
||||
|
||||
from typing import (ClassVar, Generator, Annotated, Optional, Any,)
|
||||
from typing import (
|
||||
ClassVar,
|
||||
Generator,
|
||||
Annotated,
|
||||
Optional,
|
||||
Any,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@pydantic.dataclasses.dataclass
|
||||
class MypyFormatEntry:
|
||||
name : str
|
||||
value : str
|
||||
name: str
|
||||
value: str
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if not isinstance(other, type(self)):
|
||||
@ -27,10 +36,10 @@ class MypyFormatEntry:
|
||||
|
||||
return self.value == other.value
|
||||
|
||||
class MypyFormat:
|
||||
vscode : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='vscode', value='vscode')
|
||||
json : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='json', value='json')
|
||||
|
||||
class MypyFormat:
|
||||
vscode: ClassVar[MypyFormatEntry] = MypyFormatEntry(name='vscode', value='vscode')
|
||||
json: ClassVar[MypyFormatEntry] = MypyFormatEntry(name='json', value='json')
|
||||
|
||||
@classmethod
|
||||
def from_value(cls, value: str) -> MypyFormatEntry:
|
||||
@ -41,7 +50,13 @@ class MypyFormat:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def entries(cls) -> Generator[MypyFormatEntry, None, None,]:
|
||||
def entries(
|
||||
cls,
|
||||
) -> Generator[
|
||||
MypyFormatEntry,
|
||||
None,
|
||||
None,
|
||||
]:
|
||||
for o in dir(cls):
|
||||
e = getattr(cls, o)
|
||||
if not isinstance(e, MypyFormatEntry):
|
||||
@ -49,15 +64,17 @@ class MypyFormat:
|
||||
|
||||
yield e
|
||||
|
||||
|
||||
class MypySettings(pydantic_settings.BaseSettings):
|
||||
model_config = pydantic_settings.SettingsConfigDict(
|
||||
env_prefix='online_fxreader_pr34_mypy_',
|
||||
case_sensitive=False,
|
||||
)
|
||||
|
||||
config_path : pathlib.Path = pathlib.Path.cwd() / '.mypy.ini'
|
||||
max_errors : dict[str, int] = dict()
|
||||
paths : Annotated[list[pathlib.Path], Field(default_factory=lambda : ['.'])]
|
||||
config_path: pathlib.Path = pathlib.Path.cwd() / '.mypy.ini'
|
||||
max_errors: dict[str, int] = dict()
|
||||
paths: Annotated[list[pathlib.Path], Field(default_factory=lambda: ['.'])]
|
||||
|
||||
|
||||
def run(
|
||||
argv: Optional[list[str]] = None,
|
||||
@ -71,7 +88,8 @@ def run(
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'-q', '--quiet',
|
||||
'-q',
|
||||
'--quiet',
|
||||
dest='quiet',
|
||||
action='store_true',
|
||||
help='do not print anything if the program is correct according to max_errors limits',
|
||||
@ -85,14 +103,12 @@ def run(
|
||||
action='append',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-f', '--format',
|
||||
'-f',
|
||||
'--format',
|
||||
dest='_format',
|
||||
help='output format of errors',
|
||||
default=MypyFormat.json.value,
|
||||
choices=[
|
||||
o.value
|
||||
for o in MypyFormat.entries()
|
||||
],
|
||||
choices=[o.value for o in MypyFormat.entries()],
|
||||
)
|
||||
options, args = parser.parse_known_args(argv)
|
||||
|
||||
@ -110,7 +126,8 @@ def run(
|
||||
sys.executable,
|
||||
'-m',
|
||||
'mypy',
|
||||
'--config-file', str(settings.config_path),
|
||||
'--config-file',
|
||||
str(settings.config_path),
|
||||
'--strict',
|
||||
'-O',
|
||||
'json',
|
||||
@ -118,7 +135,6 @@ def run(
|
||||
*options.paths,
|
||||
]
|
||||
|
||||
|
||||
logger.info(dict(cmd=mypy_cmd))
|
||||
|
||||
res = subprocess.run(
|
||||
@ -132,14 +148,13 @@ def run(
|
||||
try:
|
||||
assert not res.returncode is None
|
||||
|
||||
errors = sorted([
|
||||
json.loads(o)
|
||||
for o in res.stdout.decode('utf-8').splitlines()
|
||||
if not o.strip() == ''
|
||||
], key=lambda x: (
|
||||
errors = sorted(
|
||||
[json.loads(o) for o in res.stdout.decode('utf-8').splitlines() if not o.strip() == ''],
|
||||
key=lambda x: (
|
||||
x.get('file', ''),
|
||||
x.get('line', 0),
|
||||
))
|
||||
),
|
||||
)
|
||||
|
||||
if not options.quiet:
|
||||
if (len(res.stderr)) > 0:
|
||||
@ -150,15 +165,14 @@ def run(
|
||||
logger.error(res.stderr.decode('utf-8'))
|
||||
sys.exit(res.returncode)
|
||||
|
||||
|
||||
g : dict[str, Any] = dict()
|
||||
g: dict[str, Any] = dict()
|
||||
for o in errors:
|
||||
if not o['file'] in g:
|
||||
g[o['file']] = []
|
||||
g[o['file']].append(o)
|
||||
|
||||
h = {
|
||||
k : len(v)
|
||||
k: len(v)
|
||||
for k, v in sorted(
|
||||
list(g.items()),
|
||||
key=lambda x: x[0],
|
||||
@ -167,24 +181,26 @@ def run(
|
||||
|
||||
mentioned_paths = marisa_trie.Trie(list(h))
|
||||
|
||||
violated_limits : dict[str, str] = dict()
|
||||
violated_limits: dict[str, str] = dict()
|
||||
|
||||
for k, v in settings.max_errors.items():
|
||||
matching_paths = mentioned_paths.keys(k)
|
||||
total_errors = sum([
|
||||
h[o]
|
||||
for o in matching_paths
|
||||
], 0)
|
||||
total_errors = sum([h[o] for o in matching_paths], 0)
|
||||
|
||||
if total_errors > v:
|
||||
violated_limits[k] = '%s - [%s]: has %d errors > %d' % (
|
||||
k, ', '.join(matching_paths), total_errors, v,
|
||||
k,
|
||||
', '.join(matching_paths),
|
||||
total_errors,
|
||||
v,
|
||||
)
|
||||
|
||||
if len(violated_limits) > 0 or not options.quiet:
|
||||
if options.format == MypyFormat.vscode:
|
||||
for o in errors:
|
||||
sys.stdout.write('[%s] %s:%d,%d %s - %s - %s\n' % (
|
||||
sys.stdout.write(
|
||||
'[%s] %s:%d,%d %s - %s - %s\n'
|
||||
% (
|
||||
o['severity'],
|
||||
o['file'],
|
||||
o['line'],
|
||||
@ -192,25 +208,33 @@ def run(
|
||||
o['message'],
|
||||
o['hint'],
|
||||
o['code'],
|
||||
))
|
||||
)
|
||||
)
|
||||
sys.stdout.flush()
|
||||
#logger.info(json.dumps(errors, indent=4))
|
||||
# logger.info(json.dumps(errors, indent=4))
|
||||
else:
|
||||
logger.info(json.dumps(errors, indent=4))
|
||||
|
||||
#if len(violated_limits) > 0:
|
||||
# if len(violated_limits) > 0:
|
||||
# logger.info(json.dumps(violated_limits, indent=4))
|
||||
logger.info(json.dumps(dict(
|
||||
logger.info(
|
||||
json.dumps(
|
||||
dict(
|
||||
max_errors=settings.max_errors,
|
||||
violated_limits=violated_limits,
|
||||
histogram=h,
|
||||
elapsed=(done_at - started_at).total_seconds(),
|
||||
), indent=4))
|
||||
),
|
||||
indent=4,
|
||||
)
|
||||
)
|
||||
|
||||
if len(violated_limits) > 0:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
from . import logging as _logging
|
||||
|
||||
_logging.setup()
|
||||
run(sys.argv[1:])
|
||||
@ -11,22 +11,31 @@ import dataclasses
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from typing import (overload, Optional, Literal, Any, Annotated,)
|
||||
from typing import (
|
||||
overload,
|
||||
Optional,
|
||||
Literal,
|
||||
Any,
|
||||
Annotated,
|
||||
)
|
||||
|
||||
from .cli_bootstrap import PyProject
|
||||
|
||||
|
||||
@overload
|
||||
def shutil_which(
|
||||
name: str,
|
||||
raise_on_failure: Literal[True],
|
||||
) -> str: ...
|
||||
|
||||
|
||||
@overload
|
||||
def shutil_which(
|
||||
name: str,
|
||||
raise_on_failure: bool,
|
||||
) -> Optional[str]: ...
|
||||
|
||||
|
||||
def shutil_which(
|
||||
name: str,
|
||||
raise_on_failure: bool,
|
||||
@ -37,63 +46,50 @@ def shutil_which(
|
||||
else:
|
||||
return res
|
||||
|
||||
|
||||
def runtime_libdirs_init(
|
||||
project: PyProject,
|
||||
) -> None:
|
||||
if sys.platform == 'linux':
|
||||
ld_library_path : list[pathlib.Path] = [
|
||||
ld_library_path: list[pathlib.Path] = [
|
||||
o
|
||||
for o in [
|
||||
*[
|
||||
o.absolute()
|
||||
for o in (
|
||||
project.runtime_libdirs
|
||||
if project.runtime_libdirs
|
||||
else []
|
||||
)
|
||||
],
|
||||
*[
|
||||
pathlib.Path(o)
|
||||
for o in os.environ.get(
|
||||
'LD_LIBRARY_PATH',
|
||||
''
|
||||
).split(os.path.pathsep)
|
||||
if o != ''
|
||||
]
|
||||
*[o.absolute() for o in (project.runtime_libdirs if project.runtime_libdirs else [])],
|
||||
*[pathlib.Path(o) for o in os.environ.get('LD_LIBRARY_PATH', '').split(os.path.pathsep) if o != ''],
|
||||
]
|
||||
]
|
||||
|
||||
ld_library_path_present : list[pathlib.Path] = []
|
||||
ld_library_path_present: list[pathlib.Path] = []
|
||||
|
||||
for o in ld_library_path:
|
||||
if not o.exists():
|
||||
logger.warning(dict(
|
||||
logger.warning(
|
||||
dict(
|
||||
ld_library_path=o,
|
||||
msg='not found',
|
||||
))
|
||||
)
|
||||
)
|
||||
|
||||
ld_library_path_present.append(o)
|
||||
|
||||
os.environ.update(
|
||||
LD_LIBRARY_PATH=os.path.pathsep.join([
|
||||
str(o) for o in ld_library_path_present
|
||||
])
|
||||
)
|
||||
os.environ.update(LD_LIBRARY_PATH=os.path.pathsep.join([str(o) for o in ld_library_path_present]))
|
||||
|
||||
for preload_path in (project.runtime_preload or []):
|
||||
for preload_found in glob.glob(str(
|
||||
preload_path.parent / ('lib%s.so' % preload_path.name)
|
||||
)):
|
||||
logger.info(dict(
|
||||
preload_path=preload_path, preload_found=preload_found,
|
||||
for preload_path in project.runtime_preload or []:
|
||||
for preload_found in glob.glob(str(preload_path.parent / ('lib%s.so' % preload_path.name))):
|
||||
logger.info(
|
||||
dict(
|
||||
preload_path=preload_path,
|
||||
preload_found=preload_found,
|
||||
# lib_path=o,
|
||||
msg='load_library',
|
||||
))
|
||||
)
|
||||
)
|
||||
|
||||
ctypes.cdll.LoadLibrary(preload_found)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class interfaces_index_t:
|
||||
@dataclasses.dataclass
|
||||
class Interface:
|
||||
@ -106,17 +102,24 @@ class interfaces_index_t:
|
||||
str,
|
||||
pydantic.Field(
|
||||
alias='ifname',
|
||||
)
|
||||
),
|
||||
]
|
||||
addr_info: list[AddrInfo]
|
||||
|
||||
|
||||
def interfaces_index() -> list[interfaces_index_t.Interface]:
|
||||
res = pydantic.RootModel[
|
||||
list[interfaces_index_t.Interface]
|
||||
].model_validate_json(
|
||||
subprocess.check_output([
|
||||
'ip', '-j', 'addr',
|
||||
]).decode('utf-8')
|
||||
).root
|
||||
res = (
|
||||
pydantic.RootModel[list[interfaces_index_t.Interface]]
|
||||
.model_validate_json(
|
||||
subprocess.check_output(
|
||||
[
|
||||
'ip',
|
||||
'-j',
|
||||
'addr',
|
||||
]
|
||||
).decode('utf-8')
|
||||
)
|
||||
.root
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
@ -23,7 +23,10 @@ if typing.TYPE_CHECKING:
|
||||
import pip._internal.operations.prepare
|
||||
|
||||
from typing import (
|
||||
Literal, Optional, Iterable, Any,
|
||||
Literal,
|
||||
Optional,
|
||||
Iterable,
|
||||
Any,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@ -33,6 +36,7 @@ def pip_show(
|
||||
argv: list[str],
|
||||
) -> list['pip._internal.commands.show._PackageInfo']:
|
||||
import pip._internal.commands.show
|
||||
|
||||
return list(
|
||||
pip._internal.commands.show.search_packages_info(
|
||||
argv,
|
||||
@ -43,10 +47,10 @@ def pip_show(
|
||||
class pip_resolve_t:
|
||||
class kwargs_t:
|
||||
class mode_t(enum.StrEnum):
|
||||
copy_paste = "copy_paste"
|
||||
monkey_patch = "monkey_patch"
|
||||
uv_pip_freeze = "uv_pip_freeze"
|
||||
uv_pip_compile = "uv_pip_compile"
|
||||
copy_paste = 'copy_paste'
|
||||
monkey_patch = 'monkey_patch'
|
||||
uv_pip_freeze = 'uv_pip_freeze'
|
||||
uv_pip_compile = 'uv_pip_compile'
|
||||
|
||||
@dataclasses.dataclass
|
||||
class res_t:
|
||||
@ -60,20 +64,9 @@ class pip_resolve_t:
|
||||
entries: Optional[list[download_info_t]] = None
|
||||
|
||||
|
||||
def pip_resolve_entries_to_txt(
|
||||
entries: list[pip_resolve_t.res_t.download_info_t]
|
||||
) -> str:
|
||||
return '\n'.join([
|
||||
'#%s\n%s %s' % (
|
||||
o.url,
|
||||
o.constraint,
|
||||
' '.join([
|
||||
'--hash=sha256:%s' % o2
|
||||
for o2 in o.sha256
|
||||
])
|
||||
)
|
||||
for o in entries
|
||||
])
|
||||
def pip_resolve_entries_to_txt(entries: list[pip_resolve_t.res_t.download_info_t]) -> str:
|
||||
return '\n'.join(['#%s\n%s %s' % (o.url, o.constraint, ' '.join(['--hash=sha256:%s' % o2 for o2 in o.sha256])) for o in entries])
|
||||
|
||||
|
||||
def pip_resolve(
|
||||
argv: list[str],
|
||||
@ -102,8 +95,8 @@ def pip_resolve(
|
||||
|
||||
t2 = pip._internal.cli.main_parser.create_main_parser()
|
||||
|
||||
t3 = t2.parse_args(["download"])
|
||||
t1 = pip._internal.commands.download.DownloadCommand("blah", "shit")
|
||||
t3 = t2.parse_args(['download'])
|
||||
t1 = pip._internal.commands.download.DownloadCommand('blah', 'shit')
|
||||
|
||||
stack.enter_context(t1.main_context())
|
||||
|
||||
@ -143,9 +136,7 @@ def pip_resolve(
|
||||
target_python=target_python,
|
||||
ignore_requires_python=options.ignore_requires_python,
|
||||
)
|
||||
build_tracker = t1.enter_context(
|
||||
pip._internal.operations.build.build_tracker.get_build_tracker()
|
||||
)
|
||||
build_tracker = t1.enter_context(pip._internal.operations.build.build_tracker.get_build_tracker())
|
||||
reqs = t1.get_requirements(
|
||||
[
|
||||
#'pip', 'uv', 'ipython',
|
||||
@ -156,9 +147,7 @@ def pip_resolve(
|
||||
session,
|
||||
)
|
||||
pip._internal.req.req_install.check_legacy_setup_py_options(options, reqs)
|
||||
directory = pip._internal.utils.temp_dir.TempDirectory(
|
||||
delete=True, kind="download", globally_managed=True
|
||||
)
|
||||
directory = pip._internal.utils.temp_dir.TempDirectory(delete=True, kind='download', globally_managed=True)
|
||||
preparer = t1.make_requirement_preparer(
|
||||
temp_build_dir=directory,
|
||||
options=options,
|
||||
@ -195,14 +184,12 @@ def pip_resolve(
|
||||
res.entries.append(
|
||||
pip_resolve_t.res_t.download_info_t(
|
||||
constraint=k,
|
||||
sha256=v.download_info.info.hashes["sha256"],
|
||||
sha256=v.download_info.info.hashes['sha256'],
|
||||
url=v.download_info.url,
|
||||
)
|
||||
)
|
||||
|
||||
res.txt = pip_resolve_entries_to_txt(
|
||||
res.entries
|
||||
)
|
||||
res.txt = pip_resolve_entries_to_txt(res.entries)
|
||||
|
||||
return res
|
||||
elif mode is pip_resolve_t.kwargs_t.mode_t.monkey_patch:
|
||||
@ -248,23 +235,17 @@ def pip_resolve(
|
||||
|
||||
return downloader_call_def(
|
||||
_self,
|
||||
link, location,
|
||||
link,
|
||||
location,
|
||||
)
|
||||
|
||||
batch_downloader_call_def = (
|
||||
pip._internal.network.download.BatchDownloader.__call__
|
||||
)
|
||||
batch_downloader_call_def = pip._internal.network.download.BatchDownloader.__call__
|
||||
|
||||
def batch_downloader_call(
|
||||
_self: pip._internal.network.download.BatchDownloader,
|
||||
links: Iterable[pip._internal.models.link.Link],
|
||||
location: str,
|
||||
) -> Iterable[
|
||||
tuple[
|
||||
pip._internal.models.link.Link,
|
||||
tuple[str, str]
|
||||
]
|
||||
]:
|
||||
) -> Iterable[tuple[pip._internal.models.link.Link, tuple[str, str]]]:
|
||||
# print(args)
|
||||
|
||||
logger.info(
|
||||
@ -274,34 +255,21 @@ def pip_resolve(
|
||||
)
|
||||
)
|
||||
|
||||
return [
|
||||
(o, ("/dev/null", ''))
|
||||
for o in links
|
||||
]
|
||||
return [(o, ('/dev/null', '')) for o in links]
|
||||
|
||||
# base_resolver_resolve_def = pip._internal.resolution.base.BaseResolver.resolve
|
||||
base_resolver_resolve_def = (
|
||||
pip._internal.resolution.resolvelib.resolver.Resolver.resolve
|
||||
)
|
||||
base_resolver_resolve_def = pip._internal.resolution.resolvelib.resolver.Resolver.resolve
|
||||
|
||||
result_requirements : list[
|
||||
RequirementSet | InstallRequirement
|
||||
] = []
|
||||
result_requirements: list[RequirementSet | InstallRequirement] = []
|
||||
|
||||
def base_resolver_resolve(
|
||||
_self: pip._internal.resolution.resolvelib.resolver.Resolver,
|
||||
root_reqs: list[
|
||||
InstallRequirement,
|
||||
],
|
||||
root_reqs: list[InstallRequirement,],
|
||||
check_supported_wheels: bool,
|
||||
) -> RequirementSet:
|
||||
# print(args, kwargs)
|
||||
|
||||
res = base_resolver_resolve_def(
|
||||
_self,
|
||||
root_reqs,
|
||||
check_supported_wheels
|
||||
)
|
||||
res = base_resolver_resolve_def(_self, root_reqs, check_supported_wheels)
|
||||
|
||||
result_requirements.append(res)
|
||||
raise NotImplementedError
|
||||
@ -322,21 +290,16 @@ def pip_resolve(
|
||||
)
|
||||
)
|
||||
|
||||
if link.url.endswith(".whl"):
|
||||
print("blah")
|
||||
if link.url.endswith('.whl'):
|
||||
print('blah')
|
||||
hashes = None
|
||||
|
||||
return File(
|
||||
"/dev/null",
|
||||
'/dev/null',
|
||||
'',
|
||||
)
|
||||
else:
|
||||
return get_http_url_def(
|
||||
link,
|
||||
download,
|
||||
download_dir,
|
||||
hashes
|
||||
)
|
||||
return get_http_url_def(link, download, download_dir, hashes)
|
||||
|
||||
prepare_linked_requirements_more_def = pip._internal.operations.prepare.RequirementPreparer.prepare_linked_requirements_more
|
||||
|
||||
@ -345,9 +308,7 @@ def pip_resolve(
|
||||
reqs: Iterable[InstallRequirement],
|
||||
parallel_builds: bool = False,
|
||||
) -> None:
|
||||
result_requirements.extend(
|
||||
reqs
|
||||
)
|
||||
result_requirements.extend(reqs)
|
||||
raise NotImplementedError
|
||||
|
||||
_complete_partial_requirements_def = pip._internal.operations.prepare.RequirementPreparer._complete_partial_requirements
|
||||
@ -357,18 +318,12 @@ def pip_resolve(
|
||||
partially_downloaded_reqs: Iterable[InstallRequirement],
|
||||
parallel_builds: bool = False,
|
||||
) -> None:
|
||||
result_requirements.extend(
|
||||
partially_downloaded_reqs
|
||||
)
|
||||
result_requirements.extend(partially_downloaded_reqs)
|
||||
raise NotImplementedError
|
||||
|
||||
patches : list[Any] = []
|
||||
patches: list[Any] = []
|
||||
|
||||
patches.append(
|
||||
unittest.mock.patch.object(
|
||||
pip._internal.network.download.Downloader, "__call__", downloader_call
|
||||
)
|
||||
)
|
||||
patches.append(unittest.mock.patch.object(pip._internal.network.download.Downloader, '__call__', downloader_call))
|
||||
# patches.append(
|
||||
# unittest.mock.patch.object(
|
||||
# pip._internal.network.download.BatchDownloader,
|
||||
@ -383,21 +338,21 @@ def pip_resolve(
|
||||
patches.append(
|
||||
unittest.mock.patch.object(
|
||||
pip._internal.resolution.resolvelib.resolver.Resolver,
|
||||
"resolve",
|
||||
'resolve',
|
||||
base_resolver_resolve,
|
||||
)
|
||||
)
|
||||
patches.append(
|
||||
unittest.mock.patch.object(
|
||||
pip._internal.operations.prepare,
|
||||
"get_http_url",
|
||||
'get_http_url',
|
||||
get_http_url,
|
||||
)
|
||||
)
|
||||
patches.append(
|
||||
unittest.mock.patch.object(
|
||||
pip._internal.operations.prepare.RequirementPreparer,
|
||||
"prepare_linked_requirements_more",
|
||||
'prepare_linked_requirements_more',
|
||||
prepare_linked_requirements_more,
|
||||
)
|
||||
)
|
||||
@ -415,11 +370,11 @@ def pip_resolve(
|
||||
|
||||
pip._internal.cli.main.main(
|
||||
[
|
||||
"download",
|
||||
"-q",
|
||||
"--no-cache",
|
||||
"-d",
|
||||
"/dev/null",
|
||||
'download',
|
||||
'-q',
|
||||
'--no-cache',
|
||||
'-d',
|
||||
'/dev/null',
|
||||
*argv,
|
||||
# 'numpy',
|
||||
]
|
||||
@ -445,7 +400,7 @@ def pip_resolve(
|
||||
for o in result_requirements:
|
||||
assert isinstance(o, InstallRequirement)
|
||||
|
||||
sha256_hashes = o.hashes()._allowed["sha256"]
|
||||
sha256_hashes = o.hashes()._allowed['sha256']
|
||||
assert len(sha256_hashes) == 1
|
||||
assert not o.link is None
|
||||
|
||||
@ -457,9 +412,7 @@ def pip_resolve(
|
||||
)
|
||||
)
|
||||
|
||||
res.txt = pip_resolve_entries_to_txt(
|
||||
res.entries
|
||||
)
|
||||
res.txt = pip_resolve_entries_to_txt(res.entries)
|
||||
|
||||
return res
|
||||
elif mode is pip_resolve_t.kwargs_t.mode_t.uv_pip_freeze:
|
||||
@ -468,21 +421,23 @@ def pip_resolve(
|
||||
pip_freeze = subprocess.check_output(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"uv",
|
||||
"pip",
|
||||
"freeze",
|
||||
'-m',
|
||||
'uv',
|
||||
'pip',
|
||||
'freeze',
|
||||
],
|
||||
).decode('utf-8')
|
||||
pip_compile = subprocess.check_output(
|
||||
[
|
||||
sys.executable, '-m',
|
||||
'uv', 'pip', 'compile',
|
||||
sys.executable,
|
||||
'-m',
|
||||
'uv',
|
||||
'pip',
|
||||
'compile',
|
||||
'--generate-hashes',
|
||||
'-',
|
||||
|
||||
],
|
||||
input=pip_freeze.encode('utf-8')
|
||||
input=pip_freeze.encode('utf-8'),
|
||||
).decode('utf-8')
|
||||
|
||||
return pip_resolve_t.res_t(
|
||||
@ -498,9 +453,7 @@ def pip_resolve(
|
||||
suffix='.txt',
|
||||
)
|
||||
)
|
||||
f.write(
|
||||
('\n'.join(requirements)).encode('utf-8')
|
||||
)
|
||||
f.write(('\n'.join(requirements)).encode('utf-8'))
|
||||
f.flush()
|
||||
|
||||
argv.append(f.name)
|
||||
@ -510,8 +463,11 @@ def pip_resolve(
|
||||
|
||||
pip_compile = subprocess.check_output(
|
||||
[
|
||||
sys.executable, '-m',
|
||||
'uv', 'pip', 'compile',
|
||||
sys.executable,
|
||||
'-m',
|
||||
'uv',
|
||||
'pip',
|
||||
'compile',
|
||||
'--generate-hashes',
|
||||
*argv,
|
||||
],
|
||||
|
||||
@ -6,7 +6,8 @@ from typing import Any
|
||||
from typing_extensions import Protocol
|
||||
from abc import abstractmethod
|
||||
|
||||
C = typing.TypeVar("C", bound="Comparable")
|
||||
C = typing.TypeVar('C', bound='Comparable')
|
||||
|
||||
|
||||
class Comparable(Protocol):
|
||||
@abstractmethod
|
||||
@ -24,4 +25,4 @@ class Comparable(Protocol):
|
||||
return self < other or self == other
|
||||
|
||||
def __ge__(self: C, other: C) -> bool:
|
||||
return (not self < other)
|
||||
return not self < other
|
||||
|
||||
@ -8,6 +8,7 @@ async def f1():
|
||||
devices = await bleak.BleakScanner.discover()
|
||||
return devices
|
||||
|
||||
|
||||
async def f2(device, timeout=None):
|
||||
if timeout is None:
|
||||
timeout = 1.0
|
||||
@ -20,19 +21,12 @@ async def f2(device, timeout=None):
|
||||
).__aenter__()
|
||||
return p
|
||||
|
||||
|
||||
async def f3(client):
|
||||
t1 = [
|
||||
dict(
|
||||
service=o.__dict__,
|
||||
characteristics=[
|
||||
o2.__dict__
|
||||
for o2 in o.characteristics
|
||||
]
|
||||
)
|
||||
for o in client.services
|
||||
]
|
||||
t1 = [dict(service=o.__dict__, characteristics=[o2.__dict__ for o2 in o.characteristics]) for o in client.services]
|
||||
return t1
|
||||
|
||||
|
||||
async def f5(
|
||||
name_check=None,
|
||||
):
|
||||
@ -47,18 +41,9 @@ async def f5(
|
||||
if not name_check is None:
|
||||
assert inspect.isfunction(name_check)
|
||||
|
||||
t5 = {
|
||||
i : o.details[0].name()
|
||||
for i, o in enumerate(t1)
|
||||
}
|
||||
t5 = {i: o.details[0].name() for i, o in enumerate(t1)}
|
||||
|
||||
t2.extend(
|
||||
[
|
||||
t1[k]
|
||||
for k, v in t5.items()
|
||||
if isinstance(v, str) and name_check(v)
|
||||
]
|
||||
)
|
||||
t2.extend([t1[k] for k, v in t5.items() if isinstance(v, str) and name_check(v)])
|
||||
else:
|
||||
t2.extend(t1)
|
||||
|
||||
@ -70,6 +55,7 @@ async def f5(
|
||||
|
||||
return t2
|
||||
|
||||
|
||||
async def f4(
|
||||
timeout=None,
|
||||
characteristics=None,
|
||||
|
||||
@ -10,30 +10,35 @@ import threading
|
||||
import cython
|
||||
import datetime
|
||||
|
||||
from typing import (Any, Optional, TypeVar, Type, cast)
|
||||
from typing import Any, Optional, TypeVar, Type, cast
|
||||
# from scoping import scoping as s
|
||||
|
||||
|
||||
def test(
|
||||
_id: int,
|
||||
T: float,
|
||||
a: numpy.ndarray[Any, numpy.dtype[numpy.int32]],
|
||||
) -> None:
|
||||
with cython.nogil:
|
||||
#if True:
|
||||
# if True:
|
||||
started_at = datetime.datetime.now()
|
||||
print('started')
|
||||
|
||||
def elapsed() -> float:
|
||||
return (datetime.datetime.now() - started_at).total_seconds()
|
||||
#a = 0
|
||||
|
||||
# a = 0
|
||||
while elapsed() < T:
|
||||
#a += 1
|
||||
# a += 1
|
||||
for k in range(1024 * 1024):
|
||||
a[_id] += 1
|
||||
|
||||
print(['done', started_at, elapsed(), a[_id]])
|
||||
|
||||
|
||||
M = TypeVar('M', bound=Type[Any])
|
||||
|
||||
|
||||
def build(content: str, module: M) -> M:
|
||||
import pathlib
|
||||
import tempfile
|
||||
@ -44,7 +49,6 @@ def build(content: str, module: M) -> M:
|
||||
|
||||
output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute()
|
||||
|
||||
|
||||
if not output_dir.exists() or True:
|
||||
os.makedirs(str(output_dir), exist_ok=True)
|
||||
|
||||
@ -57,23 +61,16 @@ def build(content: str, module: M) -> M:
|
||||
t1.extensions = Cython.Build.cythonize(str(source_path))
|
||||
t1.build_temp = str(pathlib.Path('/'))
|
||||
t1.build_lib = str(output_dir)
|
||||
#t2 = Cython.Build.Inline.Extension(
|
||||
# t2 = Cython.Build.Inline.Extension(
|
||||
# name=sha256sum,
|
||||
#)
|
||||
# )
|
||||
t1.run()
|
||||
|
||||
return cast(
|
||||
M,
|
||||
Cython.Build.Inline.load_dynamic(
|
||||
'_%s' % sha256sum,
|
||||
glob.glob(
|
||||
str(output_dir / ('_%s*.so' % sha256sum))
|
||||
)[0]
|
||||
)
|
||||
)
|
||||
return cast(M, Cython.Build.Inline.load_dynamic('_%s' % sha256sum, glob.glob(str(output_dir / ('_%s*.so' % sha256sum)))[0]))
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def mypyc_build(file_path: pathlib.Path) -> Any:
|
||||
import pathlib
|
||||
import tempfile
|
||||
@ -83,26 +80,25 @@ def mypyc_build(file_path: pathlib.Path) -> Any:
|
||||
|
||||
assert isinstance(file_path, pathlib.Path)
|
||||
|
||||
#sha256sum = hashlib.sha256(content.encode('utf-8')).digest().hex()
|
||||
# sha256sum = hashlib.sha256(content.encode('utf-8')).digest().hex()
|
||||
|
||||
#output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute()
|
||||
# output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute()
|
||||
output_dir = pathlib.Path('.') / 'tmp' / 'mypyc'
|
||||
sha256sum = file_path.stem
|
||||
lib_pattern = file_path.parent / ('%s.cpython*.so' % sha256sum)
|
||||
lib_dir = pathlib.Path('.')
|
||||
|
||||
|
||||
def lib_path_glob(path: str | pathlib.Path) -> Optional[pathlib.Path]:
|
||||
res : list[str] = glob.glob(str(path))
|
||||
res: list[str] = glob.glob(str(path))
|
||||
|
||||
if len(res) == 0:
|
||||
return None
|
||||
else:
|
||||
return pathlib.Path(res[0])
|
||||
|
||||
need_build : bool = False
|
||||
need_build: bool = False
|
||||
|
||||
lib_path : Optional[pathlib.Path] = None
|
||||
lib_path: Optional[pathlib.Path] = None
|
||||
|
||||
lib_path = lib_path_glob(lib_pattern)
|
||||
|
||||
@ -117,55 +113,46 @@ def mypyc_build(file_path: pathlib.Path) -> Any:
|
||||
else:
|
||||
need_build = True
|
||||
|
||||
|
||||
if need_build:
|
||||
for o in [
|
||||
output_dir,
|
||||
output_dir / 'build' / file_path.parent,
|
||||
]:
|
||||
os.makedirs(
|
||||
str(o),
|
||||
exist_ok=True
|
||||
)
|
||||
#source_path = output_dir / ('_%s.py' % sha256sum)
|
||||
os.makedirs(str(o), exist_ok=True)
|
||||
# source_path = output_dir / ('_%s.py' % sha256sum)
|
||||
source_path = file_path
|
||||
#with io.open(str(source_path), 'w') as f:
|
||||
# with io.open(str(source_path), 'w') as f:
|
||||
# f.write(content)
|
||||
|
||||
t1 = Cython.Build.Inline._get_build_extension()
|
||||
t1.extensions = mypyc.build.mypycify(
|
||||
[str(source_path)],
|
||||
target_dir=str(output_dir / 'build')
|
||||
)
|
||||
t1.extensions = mypyc.build.mypycify([str(source_path)], target_dir=str(output_dir / 'build'))
|
||||
t1.build_temp = str(output_dir)
|
||||
t1.build_lib = str(lib_dir)
|
||||
#t2 = Cython.Build.Inline.Extension(
|
||||
# t2 = Cython.Build.Inline.Extension(
|
||||
# name=sha256sum,
|
||||
#)
|
||||
# )
|
||||
t1.run()
|
||||
|
||||
lib_path = lib_path_glob(lib_pattern)
|
||||
|
||||
return Cython.Build.Inline.load_dynamic(
|
||||
#'_%s' % sha256sum,
|
||||
#t1.extensions[0].name,
|
||||
# t1.extensions[0].name,
|
||||
file_path.stem,
|
||||
str(lib_path),
|
||||
)
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Source:
|
||||
@staticmethod
|
||||
def test2(
|
||||
_a : numpy.ndarray[Any, numpy.dtype[numpy.int64]],
|
||||
_id : numpy.dtype[numpy.int32] | int,
|
||||
T : float=16
|
||||
) -> int:
|
||||
def test2(_a: numpy.ndarray[Any, numpy.dtype[numpy.int64]], _id: numpy.dtype[numpy.int32] | int, T: float = 16) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
source = build(r'''
|
||||
source = build(
|
||||
r"""
|
||||
cimport cython
|
||||
|
||||
@cython.boundscheck(False)
|
||||
@ -226,10 +213,13 @@ def test2(long long [:] _a, int _id, double T=16) -> int:
|
||||
|
||||
return _a[_id]
|
||||
|
||||
''', Source)
|
||||
""",
|
||||
Source,
|
||||
)
|
||||
|
||||
def test_cython(N: int=4, T:int=16) -> None:
|
||||
#a = [0] * N
|
||||
|
||||
def test_cython(N: int = 4, T: int = 16) -> None:
|
||||
# a = [0] * N
|
||||
a = numpy.zeros((N,), dtype=numpy.int64)
|
||||
|
||||
t = [
|
||||
@ -249,18 +239,15 @@ def test_cython(N: int=4, T:int=16) -> None:
|
||||
for o in t:
|
||||
o.join()
|
||||
|
||||
#cython_module['test2'](a, 0)
|
||||
# cython_module['test2'](a, 0)
|
||||
|
||||
def test_mypyc(N: int=4, W:int=35) -> None:
|
||||
cython2 = mypyc_build(
|
||||
(pathlib.Path(__file__).parent / 'cython2.py').relative_to(
|
||||
pathlib.Path.cwd()
|
||||
)
|
||||
)
|
||||
|
||||
def test_mypyc(N: int = 4, W: int = 35) -> None:
|
||||
cython2 = mypyc_build((pathlib.Path(__file__).parent / 'cython2.py').relative_to(pathlib.Path.cwd()))
|
||||
|
||||
# from .cython2 import fib
|
||||
|
||||
#a = [0] * N
|
||||
# a = [0] * N
|
||||
t = [
|
||||
threading.Thread(
|
||||
target=functools.partial(
|
||||
|
||||
@ -1,11 +1,13 @@
|
||||
import time
|
||||
|
||||
|
||||
def fib(n: int) -> int:
|
||||
if n <= 1:
|
||||
return n
|
||||
else:
|
||||
return fib(n - 2) + fib(n - 1)
|
||||
|
||||
|
||||
t0 = time.time()
|
||||
fib(32)
|
||||
print(time.time() - t0)
|
||||
|
||||
@ -16,15 +16,9 @@ def kernel_1_sample_scrap(
|
||||
t10 = p.content.decode('utf-8')
|
||||
t11 = pyquery.PyQuery(t10)
|
||||
t13 = t11('.crayons-story__title > a')
|
||||
t12 = [
|
||||
pyquery.PyQuery(o).attr('href')
|
||||
for o in t13
|
||||
]
|
||||
t12 = [pyquery.PyQuery(o).attr('href') for o in t13]
|
||||
pprint.pprint(t12)
|
||||
t14 = [
|
||||
'https://dev.to/%s' % o
|
||||
for o in t12
|
||||
]
|
||||
t14 = ['https://dev.to/%s' % o for o in t12]
|
||||
|
||||
t8 = []
|
||||
for t7 in t14[:max_articles]:
|
||||
@ -46,7 +40,7 @@ def kernel_1_sample_scrap(
|
||||
)
|
||||
)
|
||||
|
||||
#pprint.pprint(t3)
|
||||
# pprint.pprint(t3)
|
||||
pprint.pprint(t6)
|
||||
t8.append(
|
||||
dict(
|
||||
@ -66,6 +60,7 @@ def kernel_1_sample_scrap(
|
||||
t12=t12,
|
||||
)
|
||||
|
||||
|
||||
def kernel_2():
|
||||
import numpy as np # linear algebra
|
||||
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
|
||||
@ -73,7 +68,7 @@ def kernel_2():
|
||||
from sklearn.model_selection import train_test_split
|
||||
import tensorflow as tf
|
||||
from keras.models import Sequential
|
||||
from keras.layers.recurrent import LSTM, GRU,SimpleRNN
|
||||
from keras.layers.recurrent import LSTM, GRU, SimpleRNN
|
||||
from keras.layers.core import Dense, Activation, Dropout
|
||||
from keras.layers.embeddings import Embedding
|
||||
from keras.layers.normalization import BatchNormalization
|
||||
@ -83,10 +78,10 @@ def kernel_2():
|
||||
from keras.preprocessing import sequence, text
|
||||
from keras.callbacks import EarlyStopping
|
||||
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import seaborn as sns
|
||||
#%matplotlib inline
|
||||
|
||||
# %matplotlib inline
|
||||
from plotly import graph_objs as go
|
||||
import plotly.express as px
|
||||
import plotly.figure_factory as ff
|
||||
@ -114,7 +109,7 @@ def kernel_2():
|
||||
# Default distribution strategy in Tensorflow. Works on CPU and single GPU.
|
||||
strategy = tf.distribute.get_strategy()
|
||||
|
||||
print("REPLICAS: ", strategy.num_replicas_in_sync)
|
||||
print('REPLICAS: ', strategy.num_replicas_in_sync)
|
||||
|
||||
# %% [code]
|
||||
train = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')
|
||||
@ -125,27 +120,25 @@ def kernel_2():
|
||||
# We will drop the other columns and approach this problem as a Binary Classification Problem and also we will have our exercise done on a smaller subsection of the dataset(only 12000 data points) to make it easier to train the models
|
||||
|
||||
# %% [code]
|
||||
train.drop(['severe_toxic','obscene','threat','insult','identity_hate'],axis=1,inplace=True)
|
||||
train.drop(['severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'], axis=1, inplace=True)
|
||||
|
||||
# %% [code]
|
||||
train = train.loc[:12000,:]
|
||||
train = train.loc[:12000, :]
|
||||
train.shape
|
||||
|
||||
# %% [markdown]
|
||||
# We will check the maximum number of words that can be present in a comment , this will help us in padding later
|
||||
|
||||
# %% [code]
|
||||
train['comment_text'].apply(lambda x:len(str(x).split())).max()
|
||||
|
||||
train['comment_text'].apply(lambda x: len(str(x).split())).max()
|
||||
|
||||
# %% [markdown]
|
||||
# ### Data Preparation
|
||||
|
||||
# %% [code]
|
||||
xtrain, xvalid, ytrain, yvalid = train_test_split(train.comment_text.values, train.toxic.values,
|
||||
stratify=train.toxic.values,
|
||||
random_state=42,
|
||||
test_size=0.2, shuffle=True)
|
||||
xtrain, xvalid, ytrain, yvalid = train_test_split(
|
||||
train.comment_text.values, train.toxic.values, stratify=train.toxic.values, random_state=42, test_size=0.2, shuffle=True
|
||||
)
|
||||
|
||||
# %% [markdown]
|
||||
# # Before We Begin
|
||||
@ -199,20 +192,18 @@ def kernel_2():
|
||||
xtrain_seq = token.texts_to_sequences(xtrain)
|
||||
xvalid_seq = token.texts_to_sequences(xvalid)
|
||||
|
||||
#zero pad the sequences
|
||||
# zero pad the sequences
|
||||
xtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len)
|
||||
xvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len)
|
||||
|
||||
word_index = token.word_index
|
||||
|
||||
# %% [code]
|
||||
#%%time
|
||||
# %%time
|
||||
with strategy.scope():
|
||||
# A simpleRNN without any pretrained embeddings and one dense layer
|
||||
model = Sequential()
|
||||
model.add(Embedding(len(word_index) + 1,
|
||||
300,
|
||||
input_length=max_len))
|
||||
model.add(Embedding(len(word_index) + 1, 300, input_length=max_len))
|
||||
model.add(SimpleRNN(100))
|
||||
model.add(Dense(1, activation='sigmoid'))
|
||||
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
|
||||
@ -245,12 +236,13 @@ def kernel_3(
|
||||
# Writing a function for getting auc score for validation
|
||||
|
||||
# %% [code]
|
||||
def roc_auc(predictions,target):
|
||||
def roc_auc(predictions, target):
|
||||
import sklearn.metrics
|
||||
'''
|
||||
|
||||
"""
|
||||
This methods returns the AUC Score when given the Predictions
|
||||
and Labels
|
||||
'''
|
||||
"""
|
||||
|
||||
fpr, tpr, thresholds = sklearn.metrics.roc_curve(target, predictions)
|
||||
roc_auc = sklearn.metrics.auc(fpr, tpr)
|
||||
@ -261,35 +253,17 @@ def kernel_3(
|
||||
o_2['model'].load_weights('model.h5')
|
||||
else:
|
||||
o_2['model'].fit(
|
||||
o_2['xtrain_pad'],
|
||||
o_2['ytrain'],
|
||||
nb_epoch=nb_epochs,
|
||||
batch_size=64*o_2['strategy'].num_replicas_in_sync
|
||||
) #Multiplying by Strategy to run on TPU's
|
||||
o_2['xtrain_pad'], o_2['ytrain'], nb_epoch=nb_epochs, batch_size=64 * o_2['strategy'].num_replicas_in_sync
|
||||
) # Multiplying by Strategy to run on TPU's
|
||||
o_2['model'].save_weights('model.h5')
|
||||
|
||||
# %% [code]
|
||||
scores = o_2['model'].predict(o_2['xvalid_pad'])
|
||||
print(
|
||||
"Auc: %.2f%%" % (
|
||||
roc_auc(
|
||||
scores,
|
||||
o_2['yvalid']
|
||||
)
|
||||
)
|
||||
)
|
||||
print('Auc: %.2f%%' % (roc_auc(scores, o_2['yvalid'])))
|
||||
|
||||
# %% [code]
|
||||
scores_model = []
|
||||
scores_model.append(
|
||||
{
|
||||
'Model': 'SimpleRNN',
|
||||
'AUC_Score': roc_auc(
|
||||
scores,
|
||||
o_2['yvalid']
|
||||
)
|
||||
}
|
||||
)
|
||||
scores_model.append({'Model': 'SimpleRNN', 'AUC_Score': roc_auc(scores, o_2['yvalid'])})
|
||||
|
||||
# %% [markdown]
|
||||
# ## Code Explanantion
|
||||
@ -301,6 +275,7 @@ def kernel_3(
|
||||
# %% [code]
|
||||
o_2['xtrain_seq'][:1]
|
||||
|
||||
|
||||
def kernel_4(
|
||||
o_2,
|
||||
input_texts=None,
|
||||
@ -308,12 +283,7 @@ def kernel_4(
|
||||
import keras.preprocessing.sequence
|
||||
|
||||
if input_texts is None:
|
||||
input_texts = [
|
||||
'blahb blahb blah',
|
||||
'Hello World!',
|
||||
'This is very good!',
|
||||
'A very non toxic comment! This is so polite and polished one!'
|
||||
]
|
||||
input_texts = ['blahb blahb blah', 'Hello World!', 'This is very good!', 'A very non toxic comment! This is so polite and polished one!']
|
||||
|
||||
t6 = []
|
||||
for o in input_texts:
|
||||
@ -321,10 +291,7 @@ def kernel_4(
|
||||
t2 = o_2['token'].texts_to_sequences(
|
||||
[t1],
|
||||
)
|
||||
t3 = keras.preprocessing.sequence.pad_sequences(
|
||||
t2,
|
||||
maxlen=o_2['max_len']
|
||||
)
|
||||
t3 = keras.preprocessing.sequence.pad_sequences(t2, maxlen=o_2['max_len'])
|
||||
t4 = o_2['model'].predict(
|
||||
t3,
|
||||
)
|
||||
@ -348,6 +315,7 @@ def kernel_4(
|
||||
t6=t6,
|
||||
)
|
||||
|
||||
|
||||
def kernel_5(
|
||||
o_1=None,
|
||||
o_2=None,
|
||||
@ -357,23 +325,11 @@ def kernel_5(
|
||||
|
||||
if o_2 is None:
|
||||
o_2 = kernel_2()
|
||||
o_3 = kernel_3(
|
||||
o_2=o_2,
|
||||
nb_epochs=1
|
||||
)
|
||||
o_3 = kernel_3(o_2=o_2, nb_epochs=1)
|
||||
|
||||
t1 = sum(
|
||||
[
|
||||
[
|
||||
o['text'] for o in o2['comments']
|
||||
] for o2 in o_1['t8']
|
||||
], []
|
||||
)
|
||||
t1 = sum([[o['text'] for o in o2['comments']] for o2 in o_1['t8']], [])
|
||||
|
||||
t2 = kernel_4(
|
||||
o_2=o_2,
|
||||
input_texts=t1
|
||||
)
|
||||
t2 = kernel_4(o_2=o_2, input_texts=t1)
|
||||
|
||||
t3 = sorted(
|
||||
t2['t6'],
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user