[~] Refactor

.gitignore

@@ -7,3 +7,6 @@ d2/book1/books
 .mypy_cache
 .ruff_cache
 .tmuxp
+*.egg-info
+*.whl
+*.tar.gz

.mypy.ini

@@ -1,8 +1,13 @@
 [mypy]
 mypy_path =
     mypy-stubs,
-    deps/com.github.aiortc.aiortc/src
+    deps/com.github.aiortc.aiortc/src,
-
+    mypy-stubs/marisa-trie-types,
+    python
 
 plugins =
-    numpy.typing.mypy_plugin
+    numpy.typing.mypy_plugin,
+    pydantic.mypy
+
+explicit_package_bases = true
+namespace_packages = true

Makefile

@@ -4,36 +4,50 @@ host_deps:
 	./m host_deps
 
 python_lint:
-	./m mypy -- -f vscode -i deps/com.github.aiortc.aiortc/src/ 2>&1 | less
+	./m mypy -- -f vscode 2>&1 | less
 
-python_clean_online_fxreader_vpn:
+#python_clean_online_fxreader_vpn:
+#	rm -fr \
+#		deps/com.github.aiortc.aiortc/src/online_fxreader/vpn/dist;
+
+PYTHON_PROJECTS := \
+	deps/com.github.aiortc.aiortc/ \
+	deps/com.github.aiortc.aiortc/src/online_fxreader/vpn/ \
+	python
+
+INSTALL_ROOT ?= ~/.local/bin
+
+#python_clean: python_clean_online_fxreader_vpn
+python_clean_env:
 	rm -fr \
-		deps/com.github.aiortc.aiortc/src/online_fxreader/vpn/dist;
+		$(INSTALL_ROOT)/env3;
 
-python_clean: python_clean_online_fxreader_vpn
+python_clean_dist:
-	rm -fr \
+	for o in $(PYTHON_PROJECTS); do \
-		~/.local/bin/env3 \
+		[[ -d $$o/dist ]] || continue; \
-		deps/com.github.aiortc.aiortc/dist \
+		echo $$o/dist; \
-		deps/com.github.aiortc.aiortc/src/online_fxreader/vpn/dist;
+		rm -fr $$o/dist; \
-
-python_put:
-	[[ -d ~/.local/bin/env3 ]] || (\
-		uv venv --system-site-packages --seed ~/.local/bin/env3 && \
-		~/.local/bin/env3/bin/python3 -m pip install uv \
-	);
-	for f in \
-		deps/com.github.aiortc.aiortc/ \
-		deps/com.github.aiortc.aiortc/src/online_fxreader/vpn; do \
-		echo $$f; \
-		[[ -d $$f/dist ]] && continue; \
-		python3 -m build --installer uv $$f; \
-		~/.local/bin/env3/bin/python3 -m uv pip install $$f/dist/*.whl; \
 	done
 
+python_clean: python_clean_dist python_clean_env
+
+python_put:
+	[[ -d $(INSTALL_ROOT)/env3 ]] || (\
+		uv venv --system-site-packages --seed $(INSTALL_ROOT)/env3 && \
+		$(INSTALL_ROOT)/env3/bin/python3 -m pip install uv \
+	);
+	for f in \
+		$(PYTHON_PROJECTS); do \
+		[[ -d $$f/dist ]] && continue; \
+		echo $$f; \
+		python3 -m build --installer uv $$f; \
+		$(INSTALL_ROOT)/env3/bin/python3 -m uv pip install $$f/dist/*.whl; \
+	done
+	ln -sf $(INSTALL_ROOT)/env3/bin/online-fxreader-pr34-commands $(INSTALL_ROOT)/commands
+
 dotfiles_put:
-	mkdir -p ~/.local/bin
+	mkdir -p $(INSTALL_ROOT)
-	cp dotfiles/.local/bin/commands ~/.local/bin/commands
+	cp dotfiles/.local/bin/gnome-shortcuts-macbook-air $(INSTALL_ROOT)/
-	cp dotfiles/.local/bin/gnome-shortcuts-macbook-air ~/.local/bin/
 	mkdir -p ~/.sway
 	cp dotfiles/.sway/config ~/.sway/config
 	cp dotfiles/.zshenv ~/.zshenv

deps/com.github.aiortc.aiortc

@@ -1 +1 @@
-Subproject commit 69fe05c5e17c1396a513df5cdc5cbbaba1dbac50
+Subproject commit f7c8dfa4126f1602f0dc0dc315113be28aaf36b3

m

@@ -1 +1 @@
-m.py
+python/m.py

m.py

@@ -1,3 +0,0 @@
-#!/usr/bin/env python3
-import _m
-_m.run()

mypy-stubs/marisa-trie-types/marisa_trie/__init__.pyi

@@ -0,0 +1,8 @@
+from typing import (Iterable,)
+
+class Trie:
+    def __init__(self, entries: Iterable[str]) -> None: ...
+
+    def keys(self, entry: str) -> list[str]: ...
+
+    def __contains__(self, entry: str) -> bool: ...

python/_m.py

@@ -13,37 +13,46 @@ import subprocess
 import os
 
 
+
 from typing import (
     Optional, Any, TypeAlias, Literal, cast, BinaryIO, Generator,
-    ClassVar,
+    ClassVar, Self,
 )
 
 logger = logging.getLogger()
 
+@dataclasses.dataclass
+class Settings:
+    project_root : pathlib.Path = pathlib.Path.cwd()
+
+    env_path : pathlib.Path = project_root / 'tmp' / 'env3'
+
+    _settings : ClassVar[Optional['Settings']] = None
+
+    @classmethod
+    def settings(cls) -> Self:
+        if cls._settings is None:
+            cls._settings = cls()
+
+        return cls._settings
+
 def js(argv: list[str]) -> int:
     return subprocess.check_call([
         'sudo',
         'docker-compose',
         '--project-directory',
-        os.path.abspath(
+        Settings.settings().project_root,
-            os.path.dirname(__file__),
-        ),
         '-f',
-        os.path.abspath(
+        Settings.settings().project_root / 'docker' / 'js' / 'docker-compose.yml',
-            os.path.join(
-                os.path.dirname(__file__),
-                'docker', 'js',
-                'docker-compose.yml',
-            )
-        ),
         *argv,
     ])
 
 def env(
     argv: Optional[list[str]] = None,
+    mode: Literal['exec', 'subprocess'] = 'subprocess',
     **kwargs: Any,
 ) -> Optional[subprocess.CompletedProcess[bytes]]:
-    env_path = pathlib.Path(__file__).parent / 'tmp' / 'env3'
+    env_path = Settings.settings().env_path
 
     if not env_path.exists():
         subprocess.check_call([
@@ -59,10 +68,24 @@ def env(
         ])
 
     if not argv is None:
-        return subprocess.run([
+        python_path = str(env_path / 'bin' / 'python3')
-            str(env_path / 'bin' / 'python3'),
+
-            *argv,
+        if mode == 'exec':
-        ], **kwargs)
+            os.execv(
+                python_path,
+                [
+                    python_path,
+                    *argv,
+                ],
+            )
+            return None
+        elif mode == 'subprocess':
+            return subprocess.run([
+                python_path,
+                *argv,
+            ], **kwargs)
+        else:
+            raise NotImplementedError
 
     return None
 
@@ -127,129 +150,6 @@ def ruff(argv: list[str]) -> None:
     logger.info(json.dumps(errors, indent=4))
     logger.info(json.dumps(h, indent=4))
 
-@dataclasses.dataclass
-class MypyFormatEntry:
-    name : str
-    value : str
-
-    def __eq__(self, other: object) -> bool:
-        if not isinstance(other, type(self)):
-            raise NotImplementedError
-
-        return self.value == other.value
-
-class MypyFormat:
-    vscode : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='vscode', value='vscode')
-    json : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='json', value='json')
-
-
-    @classmethod
-    def from_value(cls, value: str) -> MypyFormatEntry:
-        for e in cls.entries():
-            if value == e.value:
-                return e
-
-        raise NotImplementedError
-
-    @classmethod
-    def entries(cls) -> Generator[MypyFormatEntry, None, None,]:
-        for o in dir(cls):
-            e = getattr(cls, o)
-            if not isinstance(e, MypyFormatEntry):
-                continue
-
-            yield e
-
-def mypy(argv: list[str]) -> None:
-
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        '-i',
-        dest='paths',
-        help='specify paths to check',
-        default=[],
-        action='append',
-    )
-    parser.add_argument(
-        '-f', '--format',
-        dest='_format',
-        help='output format of errors',
-        default=MypyFormat.json.value,
-        choices=[
-            o.value
-            for o in MypyFormat.entries()
-        ],
-    )
-    options, args = parser.parse_known_args(argv)
-
-    options.format = MypyFormat.from_value(options._format)
-
-    if len(options.paths) == 0:
-        options.paths.extend([
-            'dotfiles/.local/bin/commands',
-            'python',
-            'm.py',
-        ])
-
-    res = env([
-        '-m',
-        'mypy',
-        '--strict',
-        '-O',
-        'json',
-        *args,
-        *options.paths,
-    ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-    assert not res is None
-
-    try:
-        errors = sorted([
-            json.loads(o)
-            for o in res.stdout.decode('utf-8').splitlines()
-            if not o.strip() == ''
-        ], key=lambda x: (
-            x.get('file', ''),
-            x.get('line', 0),
-        ))
-    except:
-        logger.exception('')
-        logger.error(res.stdout.decode('utf-8'))
-        logger.error(res.stderr.decode('utf-8'))
-        sys.exit(res.returncode)
-
-    g : dict[str, Any] = dict()
-    for o in errors:
-        if not o['file'] in g:
-            g[o['file']] = []
-        g[o['file']].append(o)
-
-    h = {
-        k : len(v)
-        for k, v in sorted(
-            list(g.items()),
-            key=lambda x: x[0],
-        )
-    }
-
-    if options.format == MypyFormat.vscode:
-        for o in errors:
-            sys.stdout.write('[%s] %s:%d,%d %s - %s - %s\n' % (
-                o['severity'],
-                o['file'],
-                o['line'],
-                o['column'],
-                o['message'],
-                o['hint'],
-                o['code'],
-            ))
-            sys.stdout.flush()
-        #logger.info(json.dumps(errors, indent=4))
-        logger.info(json.dumps(h, indent=4))
-    else:
-        logger.info(json.dumps(errors, indent=4))
-        logger.info(json.dumps(h, indent=4))
 
 def inside_env() -> bool:
     try:
@@ -265,6 +165,26 @@ def inside_env() -> bool:
 #    ruff = 'ruff'
 #    m2 = 'm2'
 
+def mypy(argv: list[str]) -> None:
+    import online.fxreader.pr34.commands_typed.mypy as _mypy
+
+    _mypy.run(
+        argv,
+        settings=_mypy.MypySettings(
+            paths=[
+                #Settings.settings().project_root / 'dotfiles/.local/bin/commands',
+                Settings.settings().project_root / 'python',
+                Settings.settings().project_root / 'deps/com.github.aiortc.aiortc/src',
+                #Settings.settings().project_root / 'm.py',
+            ],
+            max_errors={
+                'python/online/fxreader/pr34/commands_typed': 0,
+                'deps/com.github.aiortc.aiortc/src/online_fxreader': 0,
+                'deps/com.github.aiortc.aiortc/src/aiortc/contrib/signaling': 0
+            }
+        ),
+    )
+
 def host_deps(argv: list[str]) -> None:
     if sys.platform in ['linux']:
         subprocess.check_call(r'''
@@ -289,7 +209,7 @@ def run(argv: Optional[list[str]] = None) -> None:
     )
 
     if argv is None:
-        argv = sys.argv[1:]
+        argv = sys.argv[:]
 
 
     parser = argparse.ArgumentParser()
@@ -303,10 +223,13 @@ def run(argv: Optional[list[str]] = None) -> None:
         #required=True,
     )
 
-    options, args = parser.parse_known_args(argv)
+    options, args = parser.parse_known_args(argv[1:])
 
     assert options.command in Command_args
 
+    if len(args) > 0 and args[0] == '--':
+        del args[0]
+
     #options.command = Commands(options._command)
 
     if options.command == 'js':
@@ -314,9 +237,18 @@ def run(argv: Optional[list[str]] = None) -> None:
     elif options.command == 'host_deps':
         host_deps(args)
     elif options.command == 'env':
-        env(args)
+        env(args, mode='exec',)
     elif options.command == 'mypy':
-        mypy(args)
+        if not inside_env():
+            env(
+                [
+                    pathlib.Path(__file__).parent / 'm.py',
+                    *argv[1:],
+                ],
+                mode='exec'
+            )
+        else:
+            mypy(args)
     elif options.command == 'ruff':
         ruff(args)
     elif options.command == 'm2':
@@ -332,4 +264,4 @@ def run(argv: Optional[list[str]] = None) -> None:
         raise NotImplementedError
 
 if __name__ == '__main__':
-    run(sys.argv[1:])
+    run()

python/m.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+import sys
+import pathlib
+
+sys.path.append(
+    str(pathlib.Path.cwd())
+)
+
+import _m
+_m.run()

python/online/fxreader/pr34/commands.py

@@ -1619,7 +1619,7 @@ def vpn(argv: list[str]) -> None:
             '--',
         ]
     else:
-        raise NotImplementedError
+        python_path = [sys.executable]
 
     subprocess.check_call([
         'sudo',
@@ -3858,7 +3858,9 @@ def commands_cli(
         if len(argv) > 0 and argv[0].startswith('media'):
             msg = media_keys(argv).get('msg')
         else:
-            parser = argparse.ArgumentParser('online_fxreader.commands')
+            parser = argparse.ArgumentParser(
+                #'online_fxreader.commands'
+            )
             parser.add_argument(
                 '_command',
                 choices=[

python/online/fxreader/pr34/commands_typed/logging.py

@@ -0,0 +1,12 @@
+import logging
+
+def setup() -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format=(
+            '%(levelname)s:%(name)s:%(message)s'
+            ':%(process)d'
+            ':%(asctime)s'
+            ':%(pathname)s:%(funcName)s:%(lineno)s'
+        ),
+    )

python/online/fxreader/pr34/commands_typed/mypy.py

@@ -0,0 +1,216 @@
+import pydantic.dataclasses
+import datetime
+import pydantic_settings
+import marisa_trie
+import json
+import pathlib
+import subprocess
+import logging
+import sys
+import argparse
+
+from pydantic import (Field,)
+
+from typing import (ClassVar, Generator, Annotated, Optional, Any,)
+
+
+logger = logging.getLogger(__name__)
+
+@pydantic.dataclasses.dataclass
+class MypyFormatEntry:
+    name : str
+    value : str
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, type(self)):
+            raise NotImplementedError
+
+        return self.value == other.value
+
+class MypyFormat:
+    vscode : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='vscode', value='vscode')
+    json : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='json', value='json')
+
+
+    @classmethod
+    def from_value(cls, value: str) -> MypyFormatEntry:
+        for e in cls.entries():
+            if value == e.value:
+                return e
+
+        raise NotImplementedError
+
+    @classmethod
+    def entries(cls) -> Generator[MypyFormatEntry, None, None,]:
+        for o in dir(cls):
+            e = getattr(cls, o)
+            if not isinstance(e, MypyFormatEntry):
+                continue
+
+            yield e
+
+class MypySettings(pydantic_settings.BaseSettings):
+    model_config = pydantic_settings.SettingsConfigDict(
+        env_prefix='online_fxreader_pr34_mypy_',
+        case_sensitive=False,
+    )
+
+    config_path : pathlib.Path = pathlib.Path.cwd() / '.mypy.ini'
+    max_errors : dict[str, int] = dict()
+    paths : Annotated[list[pathlib.Path], Field(default_factory=lambda : ['.'])]
+
+def run(
+    argv: Optional[list[str]] = None,
+    settings: Optional[MypySettings] = None,
+) -> None:
+    if argv is None:
+        argv = []
+
+    if settings is None:
+        settings = MypySettings()
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-q', '--quiet',
+        dest='quiet',
+        action='store_true',
+        help='do not print anything if the program is correct according to max_errors limits',
+        default=False,
+    )
+    parser.add_argument(
+        '-i',
+        dest='paths',
+        help='specify paths to check',
+        default=[],
+        action='append',
+    )
+    parser.add_argument(
+        '-f', '--format',
+        dest='_format',
+        help='output format of errors',
+        default=MypyFormat.json.value,
+        choices=[
+            o.value
+            for o in MypyFormat.entries()
+        ],
+    )
+    options, args = parser.parse_known_args(argv)
+
+    if len(args) > 0 and args[0] == '--':
+        del args[0]
+
+    options.format = MypyFormat.from_value(options._format)
+
+    if len(options.paths) == 0:
+        options.paths.extend(settings.paths)
+
+    started_at = datetime.datetime.now()
+
+    mypy_cmd = [
+        sys.executable,
+        '-m',
+        'mypy',
+        '--config-file', str(settings.config_path),
+        '--strict',
+        '-O',
+        'json',
+        *args,
+        *options.paths,
+    ]
+
+
+    logger.info(dict(cmd=mypy_cmd))
+
+    res = subprocess.run(
+        mypy_cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    done_at = datetime.datetime.now()
+
+    try:
+        assert not res.returncode is None
+
+        errors = sorted([
+            json.loads(o)
+            for o in res.stdout.decode('utf-8').splitlines()
+            if not o.strip() == ''
+        ], key=lambda x: (
+            x.get('file', ''),
+            x.get('line', 0),
+        ))
+
+        if not options.quiet:
+            if (len(res.stderr)) > 0:
+                logger.error(res.stderr.decode('utf-8'))
+    except:
+        logger.exception('')
+        logger.error(res.stdout.decode('utf-8'))
+        logger.error(res.stderr.decode('utf-8'))
+        sys.exit(res.returncode)
+
+
+    g : dict[str, Any] = dict()
+    for o in errors:
+        if not o['file'] in g:
+            g[o['file']] = []
+        g[o['file']].append(o)
+
+    h = {
+        k : len(v)
+        for k, v in sorted(
+            list(g.items()),
+            key=lambda x: x[0],
+        )
+    }
+
+    mentioned_paths = marisa_trie.Trie(list(h))
+
+    violated_limits : dict[str, str] = dict()
+
+    for k, v in settings.max_errors.items():
+        matching_paths = mentioned_paths.keys(k)
+        total_errors = sum([
+            h[o]
+            for o in matching_paths
+        ], 0)
+
+        if total_errors > v:
+            violated_limits[k] = '%s - [%s]: has %d errors > %d' % (
+                k, ', '.join(matching_paths), total_errors, v,
+            )
+
+    if len(violated_limits) > 0 or not options.quiet:
+        if options.format == MypyFormat.vscode:
+            for o in errors:
+                sys.stdout.write('[%s] %s:%d,%d %s - %s - %s\n' % (
+                    o['severity'],
+                    o['file'],
+                    o['line'],
+                    o['column'],
+                    o['message'],
+                    o['hint'],
+                    o['code'],
+                ))
+                sys.stdout.flush()
+            #logger.info(json.dumps(errors, indent=4))
+        else:
+            logger.info(json.dumps(errors, indent=4))
+
+        #if len(violated_limits) > 0:
+        #    logger.info(json.dumps(violated_limits, indent=4))
+        logger.info(json.dumps(dict(
+            max_errors=settings.max_errors,
+            violated_limits=violated_limits,
+            histogram=h,
+            elapsed=(done_at - started_at).total_seconds(),
+        ), indent=4))
+
+    if len(violated_limits) > 0:
+        sys.exit(1)
+
+if __name__ == '__main__':
+    from . import logging as _logging
+    _logging.setup()
+    run(sys.argv[1:])

python/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = 'online.fxreader.pr34'
+version = '0.1'
+
+dependencies = [
+    #"-r requirements.txt",
+    'mypy',
+    'marisa-trie',
+    'pydantic',
+    'pydantic-settings',
+]
+
+[build-system]
+requires = ['setuptools']
+build-backend = 'setuptools.build_meta'
+
+[tool.setuptools]
+include-package-data = false
+
+[tool.setuptools.package-dir]
+'online.fxreader.pr34' = 'online/fxreader/pr34'
+#package_dir = '..'
+#packages = ['online_fxreader']
+#[tool.setuptools.packages.find]
+#where = ['../..']
+#include = ['../../online_fxreader/vpn']
+#exclude =['../../aiortc/*', '../../_cffi_src/*']
+
+#[tool.setuptools.packages.find]
+#exclude = ['*']
+#include = ['*.py']
+
+#[tool.setuptools.exclude-package-data]
+#'online_fxreader.vpn' = ['README.rst']
+
+#[tool.setuptools.package-data]
+#'online_fxreader.vpn' = ['requirements.txt']
+
+[project.scripts]
+online-fxreader-pr34-commands = 'online.fxreader.pr34.commands:commands_cli'