diff --git a/.gitignore b/.gitignore index 202f0e1..fee01a8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ d2/book1/books .mypy_cache .ruff_cache .tmuxp +*.egg-info +*.whl +*.tar.gz diff --git a/.mypy.ini b/.mypy.ini index d1fcec5..bc4a8f4 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -1,8 +1,13 @@ [mypy] mypy_path = mypy-stubs, - deps/com.github.aiortc.aiortc/src - + deps/com.github.aiortc.aiortc/src, + mypy-stubs/marisa-trie-types, + python plugins = - numpy.typing.mypy_plugin + numpy.typing.mypy_plugin, + pydantic.mypy + +explicit_package_bases = true +namespace_packages = true \ No newline at end of file diff --git a/Makefile b/Makefile index 913c34d..746eccb 100644 --- a/Makefile +++ b/Makefile @@ -4,36 +4,50 @@ host_deps: ./m host_deps python_lint: - ./m mypy -- -f vscode -i deps/com.github.aiortc.aiortc/src/ 2>&1 | less + ./m mypy -- -f vscode 2>&1 | less -python_clean_online_fxreader_vpn: +#python_clean_online_fxreader_vpn: +# rm -fr \ +# deps/com.github.aiortc.aiortc/src/online_fxreader/vpn/dist; + +PYTHON_PROJECTS := \ + deps/com.github.aiortc.aiortc/ \ + deps/com.github.aiortc.aiortc/src/online_fxreader/vpn/ \ + python + +INSTALL_ROOT ?= ~/.local/bin + +#python_clean: python_clean_online_fxreader_vpn +python_clean_env: rm -fr \ - deps/com.github.aiortc.aiortc/src/online_fxreader/vpn/dist; + $(INSTALL_ROOT)/env3; -python_clean: python_clean_online_fxreader_vpn - rm -fr \ - ~/.local/bin/env3 \ - deps/com.github.aiortc.aiortc/dist \ - deps/com.github.aiortc.aiortc/src/online_fxreader/vpn/dist; - -python_put: - [[ -d ~/.local/bin/env3 ]] || (\ - uv venv --system-site-packages --seed ~/.local/bin/env3 && \ - ~/.local/bin/env3/bin/python3 -m pip install uv \ - ); - for f in \ - deps/com.github.aiortc.aiortc/ \ - deps/com.github.aiortc.aiortc/src/online_fxreader/vpn; do \ - echo $$f; \ - [[ -d $$f/dist ]] && continue; \ - python3 -m build --installer uv $$f; \ - ~/.local/bin/env3/bin/python3 -m uv pip install $$f/dist/*.whl; \ +python_clean_dist: + for o in $(PYTHON_PROJECTS); do \ + [[ -d $$o/dist ]] || continue; \ + echo $$o/dist; \ + rm -fr $$o/dist; \ done +python_clean: python_clean_dist python_clean_env + +python_put: + [[ -d $(INSTALL_ROOT)/env3 ]] || (\ + uv venv --system-site-packages --seed $(INSTALL_ROOT)/env3 && \ + $(INSTALL_ROOT)/env3/bin/python3 -m pip install uv \ + ); + for f in \ + $(PYTHON_PROJECTS); do \ + [[ -d $$f/dist ]] && continue; \ + echo $$f; \ + python3 -m build --installer uv $$f; \ + $(INSTALL_ROOT)/env3/bin/python3 -m uv pip install $$f/dist/*.whl; \ + done + ln -sf $(INSTALL_ROOT)/env3/bin/online-fxreader-pr34-commands $(INSTALL_ROOT)/commands + dotfiles_put: - mkdir -p ~/.local/bin - cp dotfiles/.local/bin/commands ~/.local/bin/commands - cp dotfiles/.local/bin/gnome-shortcuts-macbook-air ~/.local/bin/ + mkdir -p $(INSTALL_ROOT) + cp dotfiles/.local/bin/gnome-shortcuts-macbook-air $(INSTALL_ROOT)/ mkdir -p ~/.sway cp dotfiles/.sway/config ~/.sway/config cp dotfiles/.zshenv ~/.zshenv diff --git a/deps/com.github.aiortc.aiortc b/deps/com.github.aiortc.aiortc index 69fe05c..f7c8dfa 160000 --- a/deps/com.github.aiortc.aiortc +++ b/deps/com.github.aiortc.aiortc @@ -1 +1 @@ -Subproject commit 69fe05c5e17c1396a513df5cdc5cbbaba1dbac50 +Subproject commit f7c8dfa4126f1602f0dc0dc315113be28aaf36b3 diff --git a/m b/m index 2fd8e2e..dfd589e 120000 --- a/m +++ b/m @@ -1 +1 @@ -m.py \ No newline at end of file +python/m.py \ No newline at end of file diff --git a/m.py b/m.py deleted file mode 100755 index dc74180..0000000 --- a/m.py +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env python3 -import _m -_m.run() diff --git a/mypy-stubs/marisa-trie-types/marisa_trie/__init__.pyi b/mypy-stubs/marisa-trie-types/marisa_trie/__init__.pyi new file mode 100644 index 0000000..19543cf --- /dev/null +++ b/mypy-stubs/marisa-trie-types/marisa_trie/__init__.pyi @@ -0,0 +1,8 @@ +from typing import (Iterable,) + +class Trie: + def __init__(self, entries: Iterable[str]) -> None: ... + + def keys(self, entry: str) -> list[str]: ... + + def __contains__(self, entry: str) -> bool: ... \ No newline at end of file diff --git a/_m.py b/python/_m.py similarity index 54% rename from _m.py rename to python/_m.py index 4e1c01e..f4b9c08 100644 --- a/_m.py +++ b/python/_m.py @@ -13,37 +13,46 @@ import subprocess import os + from typing import ( Optional, Any, TypeAlias, Literal, cast, BinaryIO, Generator, - ClassVar, + ClassVar, Self, ) logger = logging.getLogger() +@dataclasses.dataclass +class Settings: + project_root : pathlib.Path = pathlib.Path.cwd() + + env_path : pathlib.Path = project_root / 'tmp' / 'env3' + + _settings : ClassVar[Optional['Settings']] = None + + @classmethod + def settings(cls) -> Self: + if cls._settings is None: + cls._settings = cls() + + return cls._settings + def js(argv: list[str]) -> int: return subprocess.check_call([ 'sudo', 'docker-compose', '--project-directory', - os.path.abspath( - os.path.dirname(__file__), - ), + Settings.settings().project_root, '-f', - os.path.abspath( - os.path.join( - os.path.dirname(__file__), - 'docker', 'js', - 'docker-compose.yml', - ) - ), + Settings.settings().project_root / 'docker' / 'js' / 'docker-compose.yml', *argv, ]) def env( argv: Optional[list[str]] = None, + mode: Literal['exec', 'subprocess'] = 'subprocess', **kwargs: Any, ) -> Optional[subprocess.CompletedProcess[bytes]]: - env_path = pathlib.Path(__file__).parent / 'tmp' / 'env3' + env_path = Settings.settings().env_path if not env_path.exists(): subprocess.check_call([ @@ -59,10 +68,24 @@ def env( ]) if not argv is None: - return subprocess.run([ - str(env_path / 'bin' / 'python3'), - *argv, - ], **kwargs) + python_path = str(env_path / 'bin' / 'python3') + + if mode == 'exec': + os.execv( + python_path, + [ + python_path, + *argv, + ], + ) + return None + elif mode == 'subprocess': + return subprocess.run([ + python_path, + *argv, + ], **kwargs) + else: + raise NotImplementedError return None @@ -127,129 +150,6 @@ def ruff(argv: list[str]) -> None: logger.info(json.dumps(errors, indent=4)) logger.info(json.dumps(h, indent=4)) -@dataclasses.dataclass -class MypyFormatEntry: - name : str - value : str - - def __eq__(self, other: object) -> bool: - if not isinstance(other, type(self)): - raise NotImplementedError - - return self.value == other.value - -class MypyFormat: - vscode : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='vscode', value='vscode') - json : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='json', value='json') - - - @classmethod - def from_value(cls, value: str) -> MypyFormatEntry: - for e in cls.entries(): - if value == e.value: - return e - - raise NotImplementedError - - @classmethod - def entries(cls) -> Generator[MypyFormatEntry, None, None,]: - for o in dir(cls): - e = getattr(cls, o) - if not isinstance(e, MypyFormatEntry): - continue - - yield e - -def mypy(argv: list[str]) -> None: - - - parser = argparse.ArgumentParser() - parser.add_argument( - '-i', - dest='paths', - help='specify paths to check', - default=[], - action='append', - ) - parser.add_argument( - '-f', '--format', - dest='_format', - help='output format of errors', - default=MypyFormat.json.value, - choices=[ - o.value - for o in MypyFormat.entries() - ], - ) - options, args = parser.parse_known_args(argv) - - options.format = MypyFormat.from_value(options._format) - - if len(options.paths) == 0: - options.paths.extend([ - 'dotfiles/.local/bin/commands', - 'python', - 'm.py', - ]) - - res = env([ - '-m', - 'mypy', - '--strict', - '-O', - 'json', - *args, - *options.paths, - ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - assert not res is None - - try: - errors = sorted([ - json.loads(o) - for o in res.stdout.decode('utf-8').splitlines() - if not o.strip() == '' - ], key=lambda x: ( - x.get('file', ''), - x.get('line', 0), - )) - except: - logger.exception('') - logger.error(res.stdout.decode('utf-8')) - logger.error(res.stderr.decode('utf-8')) - sys.exit(res.returncode) - - g : dict[str, Any] = dict() - for o in errors: - if not o['file'] in g: - g[o['file']] = [] - g[o['file']].append(o) - - h = { - k : len(v) - for k, v in sorted( - list(g.items()), - key=lambda x: x[0], - ) - } - - if options.format == MypyFormat.vscode: - for o in errors: - sys.stdout.write('[%s] %s:%d,%d %s - %s - %s\n' % ( - o['severity'], - o['file'], - o['line'], - o['column'], - o['message'], - o['hint'], - o['code'], - )) - sys.stdout.flush() - #logger.info(json.dumps(errors, indent=4)) - logger.info(json.dumps(h, indent=4)) - else: - logger.info(json.dumps(errors, indent=4)) - logger.info(json.dumps(h, indent=4)) def inside_env() -> bool: try: @@ -265,6 +165,26 @@ def inside_env() -> bool: # ruff = 'ruff' # m2 = 'm2' +def mypy(argv: list[str]) -> None: + import online.fxreader.pr34.commands_typed.mypy as _mypy + + _mypy.run( + argv, + settings=_mypy.MypySettings( + paths=[ + #Settings.settings().project_root / 'dotfiles/.local/bin/commands', + Settings.settings().project_root / 'python', + Settings.settings().project_root / 'deps/com.github.aiortc.aiortc/src', + #Settings.settings().project_root / 'm.py', + ], + max_errors={ + 'python/online/fxreader/pr34/commands_typed': 0, + 'deps/com.github.aiortc.aiortc/src/online_fxreader': 0, + 'deps/com.github.aiortc.aiortc/src/aiortc/contrib/signaling': 0 + } + ), + ) + def host_deps(argv: list[str]) -> None: if sys.platform in ['linux']: subprocess.check_call(r''' @@ -289,7 +209,7 @@ def run(argv: Optional[list[str]] = None) -> None: ) if argv is None: - argv = sys.argv[1:] + argv = sys.argv[:] parser = argparse.ArgumentParser() @@ -303,10 +223,13 @@ def run(argv: Optional[list[str]] = None) -> None: #required=True, ) - options, args = parser.parse_known_args(argv) + options, args = parser.parse_known_args(argv[1:]) assert options.command in Command_args + if len(args) > 0 and args[0] == '--': + del args[0] + #options.command = Commands(options._command) if options.command == 'js': @@ -314,9 +237,18 @@ def run(argv: Optional[list[str]] = None) -> None: elif options.command == 'host_deps': host_deps(args) elif options.command == 'env': - env(args) + env(args, mode='exec',) elif options.command == 'mypy': - mypy(args) + if not inside_env(): + env( + [ + pathlib.Path(__file__).parent / 'm.py', + *argv[1:], + ], + mode='exec' + ) + else: + mypy(args) elif options.command == 'ruff': ruff(args) elif options.command == 'm2': @@ -332,4 +264,4 @@ def run(argv: Optional[list[str]] = None) -> None: raise NotImplementedError if __name__ == '__main__': - run(sys.argv[1:]) \ No newline at end of file + run() \ No newline at end of file diff --git a/python/m.py b/python/m.py new file mode 100755 index 0000000..e92a601 --- /dev/null +++ b/python/m.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +import sys +import pathlib + +sys.path.append( + str(pathlib.Path.cwd()) +) + +import _m +_m.run() diff --git a/python/__init__.py b/python/online/fxreader/pr34/__init__.py similarity index 100% rename from python/__init__.py rename to python/online/fxreader/pr34/__init__.py diff --git a/dotfiles/.local/bin/commands b/python/online/fxreader/pr34/commands.py old mode 100755 new mode 100644 similarity index 99% rename from dotfiles/.local/bin/commands rename to python/online/fxreader/pr34/commands.py index 315d470..8b50497 --- a/dotfiles/.local/bin/commands +++ b/python/online/fxreader/pr34/commands.py @@ -1619,7 +1619,7 @@ def vpn(argv: list[str]) -> None: '--', ] else: - raise NotImplementedError + python_path = [sys.executable] subprocess.check_call([ 'sudo', @@ -3858,7 +3858,9 @@ def commands_cli( if len(argv) > 0 and argv[0].startswith('media'): msg = media_keys(argv).get('msg') else: - parser = argparse.ArgumentParser('online_fxreader.commands') + parser = argparse.ArgumentParser( + #'online_fxreader.commands' + ) parser.add_argument( '_command', choices=[ @@ -3940,4 +3942,4 @@ def commands_cli( if __name__ == '__main__': - commands_cli() + commands_cli() \ No newline at end of file diff --git a/python/tasks/__init__.py b/python/online/fxreader/pr34/commands_typed/__init__.py similarity index 100% rename from python/tasks/__init__.py rename to python/online/fxreader/pr34/commands_typed/__init__.py diff --git a/python/online/fxreader/pr34/commands_typed/logging.py b/python/online/fxreader/pr34/commands_typed/logging.py new file mode 100644 index 0000000..3c9ef79 --- /dev/null +++ b/python/online/fxreader/pr34/commands_typed/logging.py @@ -0,0 +1,12 @@ +import logging + +def setup() -> None: + logging.basicConfig( + level=logging.INFO, + format=( + '%(levelname)s:%(name)s:%(message)s' + ':%(process)d' + ':%(asctime)s' + ':%(pathname)s:%(funcName)s:%(lineno)s' + ), + ) \ No newline at end of file diff --git a/python/online/fxreader/pr34/commands_typed/mypy.py b/python/online/fxreader/pr34/commands_typed/mypy.py new file mode 100644 index 0000000..9abd9eb --- /dev/null +++ b/python/online/fxreader/pr34/commands_typed/mypy.py @@ -0,0 +1,216 @@ +import pydantic.dataclasses +import datetime +import pydantic_settings +import marisa_trie +import json +import pathlib +import subprocess +import logging +import sys +import argparse + +from pydantic import (Field,) + +from typing import (ClassVar, Generator, Annotated, Optional, Any,) + + +logger = logging.getLogger(__name__) + +@pydantic.dataclasses.dataclass +class MypyFormatEntry: + name : str + value : str + + def __eq__(self, other: object) -> bool: + if not isinstance(other, type(self)): + raise NotImplementedError + + return self.value == other.value + +class MypyFormat: + vscode : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='vscode', value='vscode') + json : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='json', value='json') + + + @classmethod + def from_value(cls, value: str) -> MypyFormatEntry: + for e in cls.entries(): + if value == e.value: + return e + + raise NotImplementedError + + @classmethod + def entries(cls) -> Generator[MypyFormatEntry, None, None,]: + for o in dir(cls): + e = getattr(cls, o) + if not isinstance(e, MypyFormatEntry): + continue + + yield e + +class MypySettings(pydantic_settings.BaseSettings): + model_config = pydantic_settings.SettingsConfigDict( + env_prefix='online_fxreader_pr34_mypy_', + case_sensitive=False, + ) + + config_path : pathlib.Path = pathlib.Path.cwd() / '.mypy.ini' + max_errors : dict[str, int] = dict() + paths : Annotated[list[pathlib.Path], Field(default_factory=lambda : ['.'])] + +def run( + argv: Optional[list[str]] = None, + settings: Optional[MypySettings] = None, +) -> None: + if argv is None: + argv = [] + + if settings is None: + settings = MypySettings() + + parser = argparse.ArgumentParser() + parser.add_argument( + '-q', '--quiet', + dest='quiet', + action='store_true', + help='do not print anything if the program is correct according to max_errors limits', + default=False, + ) + parser.add_argument( + '-i', + dest='paths', + help='specify paths to check', + default=[], + action='append', + ) + parser.add_argument( + '-f', '--format', + dest='_format', + help='output format of errors', + default=MypyFormat.json.value, + choices=[ + o.value + for o in MypyFormat.entries() + ], + ) + options, args = parser.parse_known_args(argv) + + if len(args) > 0 and args[0] == '--': + del args[0] + + options.format = MypyFormat.from_value(options._format) + + if len(options.paths) == 0: + options.paths.extend(settings.paths) + + started_at = datetime.datetime.now() + + mypy_cmd = [ + sys.executable, + '-m', + 'mypy', + '--config-file', str(settings.config_path), + '--strict', + '-O', + 'json', + *args, + *options.paths, + ] + + + logger.info(dict(cmd=mypy_cmd)) + + res = subprocess.run( + mypy_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + done_at = datetime.datetime.now() + + try: + assert not res.returncode is None + + errors = sorted([ + json.loads(o) + for o in res.stdout.decode('utf-8').splitlines() + if not o.strip() == '' + ], key=lambda x: ( + x.get('file', ''), + x.get('line', 0), + )) + + if not options.quiet: + if (len(res.stderr)) > 0: + logger.error(res.stderr.decode('utf-8')) + except: + logger.exception('') + logger.error(res.stdout.decode('utf-8')) + logger.error(res.stderr.decode('utf-8')) + sys.exit(res.returncode) + + + g : dict[str, Any] = dict() + for o in errors: + if not o['file'] in g: + g[o['file']] = [] + g[o['file']].append(o) + + h = { + k : len(v) + for k, v in sorted( + list(g.items()), + key=lambda x: x[0], + ) + } + + mentioned_paths = marisa_trie.Trie(list(h)) + + violated_limits : dict[str, str] = dict() + + for k, v in settings.max_errors.items(): + matching_paths = mentioned_paths.keys(k) + total_errors = sum([ + h[o] + for o in matching_paths + ], 0) + + if total_errors > v: + violated_limits[k] = '%s - [%s]: has %d errors > %d' % ( + k, ', '.join(matching_paths), total_errors, v, + ) + + if len(violated_limits) > 0 or not options.quiet: + if options.format == MypyFormat.vscode: + for o in errors: + sys.stdout.write('[%s] %s:%d,%d %s - %s - %s\n' % ( + o['severity'], + o['file'], + o['line'], + o['column'], + o['message'], + o['hint'], + o['code'], + )) + sys.stdout.flush() + #logger.info(json.dumps(errors, indent=4)) + else: + logger.info(json.dumps(errors, indent=4)) + + #if len(violated_limits) > 0: + # logger.info(json.dumps(violated_limits, indent=4)) + logger.info(json.dumps(dict( + max_errors=settings.max_errors, + violated_limits=violated_limits, + histogram=h, + elapsed=(done_at - started_at).total_seconds(), + ), indent=4)) + + if len(violated_limits) > 0: + sys.exit(1) + +if __name__ == '__main__': + from . import logging as _logging + _logging.setup() + run(sys.argv[1:]) \ No newline at end of file diff --git a/python/online/fxreader/pr34/tasks/__init__.py b/python/online/fxreader/pr34/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/tasks/ble.py b/python/online/fxreader/pr34/tasks/ble.py similarity index 100% rename from python/tasks/ble.py rename to python/online/fxreader/pr34/tasks/ble.py diff --git a/python/tasks/cython.py b/python/online/fxreader/pr34/tasks/cython.py similarity index 100% rename from python/tasks/cython.py rename to python/online/fxreader/pr34/tasks/cython.py diff --git a/python/tasks/cython2.py b/python/online/fxreader/pr34/tasks/cython2.py similarity index 100% rename from python/tasks/cython2.py rename to python/online/fxreader/pr34/tasks/cython2.py diff --git a/python/tasks/jigsaw_toxic.py b/python/online/fxreader/pr34/tasks/jigsaw_toxic.py similarity index 100% rename from python/tasks/jigsaw_toxic.py rename to python/online/fxreader/pr34/tasks/jigsaw_toxic.py diff --git a/python/tasks/mlb_player.py b/python/online/fxreader/pr34/tasks/mlb_player.py similarity index 100% rename from python/tasks/mlb_player.py rename to python/online/fxreader/pr34/tasks/mlb_player.py diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..b6b7dc0 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,40 @@ +[project] +name = 'online.fxreader.pr34' +version = '0.1' + +dependencies = [ + #"-r requirements.txt", + 'mypy', + 'marisa-trie', + 'pydantic', + 'pydantic-settings', +] + +[build-system] +requires = ['setuptools'] +build-backend = 'setuptools.build_meta' + +[tool.setuptools] +include-package-data = false + +[tool.setuptools.package-dir] +'online.fxreader.pr34' = 'online/fxreader/pr34' +#package_dir = '..' +#packages = ['online_fxreader'] +#[tool.setuptools.packages.find] +#where = ['../..'] +#include = ['../../online_fxreader/vpn'] +#exclude =['../../aiortc/*', '../../_cffi_src/*'] + +#[tool.setuptools.packages.find] +#exclude = ['*'] +#include = ['*.py'] + +#[tool.setuptools.exclude-package-data] +#'online_fxreader.vpn' = ['README.rst'] + +#[tool.setuptools.package-data] +#'online_fxreader.vpn' = ['requirements.txt'] + +[project.scripts] +online-fxreader-pr34-commands = 'online.fxreader.pr34.commands:commands_cli' \ No newline at end of file