[+] register archlinux as separate project, update cli tooling

1. add archlinux project to cli.py _projects dict;
  2. add archive command delegation in archlinux cli.py;
  3. add --archive-cache flag to compile for multi-date sqlite pool;
  4. add deploy:wheel overwrite protection with --force-whl-overwrite;
  5. add pyrefly command to commands_typed/cli.py;
  6. fix pip.py compatibility with pinned pip version;
  7. fix toml.py merging for list[str] values;

python/cli.py

@@ -37,6 +37,8 @@ class Command(enum.Enum):
 	deploy_wheel = 'deploy:wheel'
 	tests = 'tests'
 	meson_setup = 'meson:setup'
+	module_switch = 'module:switch'
+	pyrefly = 'pyrefly'
 
 
 @dataclasses.dataclass
@@ -59,7 +61,13 @@ class CLI(_cli.CLI):
 				build_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'build',
 				dest_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'install',
 				meson_path=self.settings.base_dir / 'python' / 'meson.build',
-			)
+			),
+			'online.fxreader.pr34.commands_typed.archlinux': _cli.Project(
+				source_dir=self.settings.base_dir / 'meson' / 'online' / 'fxreader' / 'pr34' / 'commands_typed' / 'archlinux',
+				build_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'commands_typed' / 'archlinux' / 'build',
+				dest_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'commands_typed' / 'archlinux' / 'install',
+				meson_path=self.settings.base_dir / 'meson' / 'online' / 'fxreader' / 'pr34' / 'commands_typed' / 'archlinux' / 'meson.build',
+			),
 		}
 
 		self._dependencies: dict[str, _cli.Dependency] = dict()
@@ -162,7 +170,8 @@ class CLI(_cli.CLI):
 				output_dir=options.output_dir,
 				# mypy=True,
 				ruff=True,
-				pyright=True,
+				pyright=False,
+				pyrefly=True,
 			)
 		elif options.command is Command.pyright:
 			self.pyright(
@@ -187,17 +196,59 @@ class CLI(_cli.CLI):
 				argv=args,
 			)
 		elif options.command is Command.tests:
+			from online.fxreader.pr34.commands_typed import argparse as pr34_argparse
+
+			tests_parser = argparse.ArgumentParser()
+			tests_parser.add_argument(
+				'--timeout',
+				default=16,
+				type=int,
+				help='test timeout in seconds, default = 16',
+			)
+
+			tests_options, tests_args = pr34_argparse.parse_args(tests_parser, args)
+
 			for k, v in self.projects.items():
-				subprocess.check_call(
+				if len(tests_args) > 0:
-					[
+					cmd = [
 						sys.executable,
 						'-m',
 						'unittest',
-						'online.fxreader.pr34.tests.test_crypto',
+						*tests_args,
-						*args,
+					]
-					],
+				else:
+					cmd = [
+						sys.executable,
+						'-m',
+						'unittest',
+						'discover',
+						'-s',
+						str(v.source_dir),
+						'-p',
+						'test_*.py',
+						'-t',
+						str(v.source_dir),
+					]
+
+				subprocess.check_call(
+					cmd,
 					cwd=str(v.source_dir),
+					timeout=tests_options.timeout,
 				)
+		elif options.command is Command.module_switch:
+			assert not options.project is None
+
+			self.module_switch(
+				project_name=options.project,
+				argv=args,
+			)
+		elif options.command is Command.pyrefly:
+			assert not options.project is None
+
+			self.pyrefly(
+				project_name=options.project,
+				argv=args,
+			)
 		else:
 			raise NotImplementedError
 

python/online/fxreader/pr34/commands.py

@@ -633,13 +633,15 @@ def eternal_oom(argv: list[str]) -> None:
 			'\n'.join(
 				[
 					(
-						lambda row: '% 8d\t% 6.3f GiB\t% 5.2f %%\t% 10s\t%s'
+						lambda row: (
-						% (
+							'% 8d\t% 6.3f GiB\t% 5.2f %%\t% 10s\t%s'
-							row['PID_x'],
+							% (
-							row['RSS_x'] / 1024 / 1024,
+								row['PID_x'],
-							row['CPU_x'],
+								row['RSS_x'] / 1024 / 1024,
-							row['USER_x'],
+								row['CPU_x'],
-							row['COMMAND_y'],
+								row['USER_x'],
+								row['COMMAND_y'],
+							)
 						)
 					)(pandas_row(current_dataframe, k))
 					for k in range(
@@ -769,7 +771,7 @@ def eternal_oom(argv: list[str]) -> None:
 		mem_used = mem_stat['mem_used']
 
 		if options.memory_limit < mem_stat['mem_total'] and not oom_mem_high(mem_stat['mem_total'] - (mem_stat['mem_total'] - options.memory_limit) / 2):
-			extra_filters = lambda row: ('chrome' in row['COMMAND_y'] and '--type=renderer' in row['COMMAND_y'] or not 'chrome' in row['COMMAND_y'])
+			extra_filters = lambda row: 'chrome' in row['COMMAND_y'] and '--type=renderer' in row['COMMAND_y'] or not 'chrome' in row['COMMAND_y']
 		else:
 			extra_filters = None
 
@@ -957,8 +959,8 @@ def eternal_firefox(
 				# assert os.system('wmctrl -i -r %s -e %s' % (t2, window_position)) == 0
 				# assert os.system('wmctrl -i -r %s -b add,below' % t2) == 0
 				def reposition():
-					t1 = (
+					t1 = lambda s: (
-						lambda s: s.replace('{{PID}}', str(p.pid))
+						s.replace('{{PID}}', str(p.pid))
 						.replace('{{X}}', str(window_position[1]))
 						.replace('{{Y}}', str(window_position[2]))
 						.replace('{{W}}', str(window_position[3]))
@@ -3675,8 +3677,8 @@ def media_keys(argv):
 	msg = None
 
 	mode = None
-	is_mocp = (
+	is_mocp = lambda: (
-		lambda: subprocess.call(
+		subprocess.call(
 			[
 				'pgrep',
 				'-u',

python/online/fxreader/pr34/commands_typed/cli.py

@@ -103,6 +103,32 @@ class CLI(abc.ABC):
 	def dependencies(self) -> dict[str, Dependency]:
 		raise NotImplementedError
 
+	def make_env(
+		self,
+		env: Optional[dict[str, str]] = None,
+		venv_path: bool = True,
+		extra_paths: Optional[list[str]] = None,
+	) -> dict[str, str]:
+		res = dict(list(os.environ.items()))
+
+		if env is not None:
+			res.update(env)
+
+		if venv_path or extra_paths:
+			path_parts: list[str] = []
+
+			if venv_path:
+				path_parts.append(str(pathlib.Path(self.dist_settings.python_path).parent))
+
+			if extra_paths:
+				path_parts.extend(extra_paths)
+
+			path_parts.append(os.environ.get('PATH', ''))
+
+			res['PATH'] = os.pathsep.join(path_parts)
+
+		return res
+
 	def mypy(self, argv: list[str]) -> None:
 		from . import mypy as _mypy
 
@@ -165,6 +191,28 @@ class CLI(abc.ABC):
 
 		subprocess.check_call(cmd)
 
+	def pyrefly(
+		self,
+		project_name: str,
+		argv: list[str],
+	) -> None:
+		project = self.projects[project_name]
+
+		cmd = [
+			str(self.dist_settings.python_path),
+			'-m',
+			'pyrefly',
+			'check',
+			*argv,
+		]
+
+		logger.info(cmd)
+
+		subprocess.check_call(
+			cmd,
+			cwd=str(project.source_dir),
+		)
+
 	def pip_sync(
 		self,
 		project: str,
@@ -303,7 +351,9 @@ class CLI(abc.ABC):
 		mypy: bool = False,
 		ruff: bool = False,
 		pyright: bool = False,
+		pyrefly: bool = False,
 		tests: bool = False,
+		force_whl_overwrite: bool = False,
 	) -> None:
 		project = self.projects[project_name]
 
@@ -317,9 +367,28 @@ class CLI(abc.ABC):
 		if argv is None:
 			argv = []
 
+		from . import argparse as pr34_argparse
+
+		deploy_parser = argparse.ArgumentParser()
+		deploy_parser.add_argument(
+			'--force-whl-overwrite',
+			default=False,
+			action='store_true',
+			help='overwrite existing .whl files in output dir',
+		)
+
+		deploy_options, argv = pr34_argparse.parse_args(deploy_parser, argv)
+		force_whl_overwrite = deploy_options.force_whl_overwrite
+
 		# assert argv is None or len(argv) == 0
 
 		if not project.meson_path is None:
+			if force or not (project.build_dir / 'meson' / 'build.ninja').exists():
+				self.meson_setup(
+					project_name=project_name,
+					force=force if force is not None else False,
+				)
+
 			if tests:
 				self.meson_test(
 					project_name=project_name,
@@ -353,6 +422,12 @@ class CLI(abc.ABC):
 				argv=[],
 			)
 
+		if pyrefly:
+			self.pyrefly(
+				project_name=project_name,
+				argv=[],
+			)
+
 		if env is None:
 			env = dict()
 
@@ -379,22 +454,47 @@ class CLI(abc.ABC):
 			'-w',
 			'-n',
 			*extra_args,
-			'-Csetup-args=-Dmodes=pyproject',
-			'-Cbuild-dir=%s' % str(pyproject_build_dir),
-			'-Csetup-args=-Dinstall_path=%s' % str(project.dest_dir),
-			# '-Cbuild-dir=%s' % str(project.build_dir),
-			str(project.source_dir),
-			*argv,
 		]
 
+		if not project.meson_path is None:
+			cmd.extend(
+				[
+					'-Csetup-args=-Dmodes=pyproject',
+					'-Cbuild-dir=%s' % str(pyproject_build_dir),
+					'-Csetup-args=-Dinstall_path=%s' % str(project.dest_dir),
+				]
+			)
+
+		cmd.extend(
+			[
+				# '-Cbuild-dir=%s' % str(project.build_dir),
+				str(project.source_dir),
+				*argv,
+			]
+		)
+
 		if not output_dir is None:
 			cmd.extend(['-o', str(output_dir)])
 
+			if not force_whl_overwrite:
+				from . import cli_bootstrap
+
+				pyproject = cli_bootstrap.pyproject_load(project.source_dir / 'pyproject.toml')
+				whl_name_prefix = (pyproject.name or project_name).replace('.', '_').replace('-', '_')
+				whl_name_prefix += '-' + pyproject.version if pyproject.version else ''
+
+				conflicting = [o for o in glob.glob(str(pathlib.Path(output_dir) / '*.whl')) if pathlib.Path(o).name.startswith(whl_name_prefix + '-')]
+
+				if len(conflicting) > 0:
+					raise FileExistsError(
+						'wheel(s) already exist in %s: %s. Use --force-whl-overwrite to overwrite.' % (output_dir, [pathlib.Path(o).name for o in conflicting])
+					)
+
 		logger.info(dict(env=env, cmd=cmd))
 
 		subprocess.check_call(
 			cmd,
-			env=dict(list(os.environ.items())) | env,
+			env=self.make_env(env=env),
 		)
 
 		if not project.meson_path is None:
@@ -460,7 +560,10 @@ class CLI(abc.ABC):
 			)
 		)
 
-		subprocess.check_call(cmd)
+		subprocess.check_call(
+			cmd,
+			env=self.make_env(),
+		)
 
 		for o in glob.glob(
 			str(project.dest_dir / 'lib' / 'pkgconfig' / '*.pc'),
@@ -502,7 +605,7 @@ class CLI(abc.ABC):
 				str(project.build_dir / 'meson'),
 				*argv,
 			],
-			env=dict(list(os.environ.items())) | env,
+			env=self.make_env(env=env),
 		)
 
 	def meson_test(
@@ -704,7 +807,7 @@ class CLI(abc.ABC):
 
 		subprocess.check_call(
 			cmd,
-			env=dict(list(os.environ.items())) | env,
+			env=self.make_env(env=env),
 		)
 
 	def venv_compile(
@@ -892,6 +995,20 @@ class CLI(abc.ABC):
 			# 	assert not k in pyproject_tool
 			# 	pyproject_tool[k] = v
 
+			if len(module.scripts) > 0:
+				if 'scripts' not in p:
+					p['scripts'] = dict()
+
+				scripts = p['scripts']
+				assert isinstance(scripts, MutableMapping)
+
+				for k, v in module.scripts.items():
+					scripts[k] = v
+
+			if len(module.project) > 0:
+				for k, v in module.project.items():
+					p[k] = v
+
 			del p
 			del pyproject_tool
 

python/online/fxreader/pr34/commands_typed/pip.py

@@ -313,7 +313,7 @@ def pip_resolve(
 
 			result_requirements.append(res)
 			raise NotImplementedError
-			return res
+			# return res
 
 		get_http_url_def = pip._internal.operations.prepare.get_http_url
 
@@ -504,10 +504,11 @@ def pip_resolve(
 
 				f = stack.enter_context(
 					tempfile.NamedTemporaryFile(
+						mode='w',
 						suffix='.txt',
 					)
 				)
-				f.write(('\n'.join(requirements)).encode('utf-8'))
+				f.write('\n'.join(requirements))
 				f.flush()
 
 				argv.append(f.name)

python/online/fxreader/pr34/commands_typed/toml.py

@@ -1,4 +1,4 @@
-from .cli_bootstrap import check_dict
+from .cli_bootstrap import check_dict, check_list
 
 from typing import (
 	Any,
@@ -21,10 +21,16 @@ def toml_add_overlay(
 		for k, v in overlay2.items():
 			if not k in toml:
 				toml[k] = v
-			else:
+			elif isinstance(toml[k], MutableMapping) and isinstance(v, Mapping):
 				toml_add_overlay(
 					toml[k],
 					v,
 				)
+			elif isinstance(toml[k], list) and isinstance(v, list):
+				check_list(toml[k], str)
+				check_list(v, str)
+				toml[k] = v
+			else:
+				raise NotImplementedError
 	else:
 		raise NotImplementedError