From 9f895d52ee157469fe0c3509d925a6f869e0507a Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Tue, 6 May 2025 18:33:17 +0300 Subject: [PATCH 01/49] [+] add venv_compile --- .../fxreader/pr34/commands_typed/cli.py | 73 ++++++++ python/pyproject.toml | 1 + python/requirements.txt | 162 +++++++++--------- 3 files changed, 158 insertions(+), 78 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index b9e1b01..ad6b168 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -8,6 +8,7 @@ import sys import subprocess import shutil import abc +import argparse from .os import shutil_which @@ -476,3 +477,75 @@ class CLI(abc.ABC): cmd, env=dict(list(os.environ.items())) | env, ) + + def venv_compile( + self, + project_name: str, + force: bool, + argv: Optional[list[str]] = None, + ) -> None: + from . import cli_bootstrap + from . import argparse as pr34_argparse + + project = self.projects[project_name] + + parser = argparse.ArgumentParser() + parser.add_argument( + '-f', + dest='features', + action='append', + default=[], + # type=pathlib.Path, + type=str, + ) + + options, args = pr34_argparse.parse_args( + parser, + argv, + ) + + pyproject = cli_bootstrap.pyproject_load( + project.source_dir / 'pyproject.toml' + ) + + dependencies = sum([ + pyproject.dependencies[o] + for o in options.features + ], []) + + pip_find_links : list[pathlib.Path] = [] + + if not pyproject.pip_find_links is None: + pip_find_links.extend([ + o + for o in pyproject.pip_find_links + if o.exists() + ]) + + logger.info(dict( + dependencies=dependencies, + )) + + with io.open( + project.source_dir / 'requirements.in', + 'w', + ) as f: + f.write( + '\n'.join(dependencies), + ) + f.flush() + + if len(dependencies) > 0: + subprocess.check_call([ + self.dist_settings.python_path, + '-m', + 'uv', 'pip', 'compile', + *args, + *sum([ + ['-f', str(o),] + for o in pip_find_links + ], []), + '--generate-hashes', + project.source_dir / 'requirements.in', + '-o', project.source_dir / 'requirements.txt', + ]) diff --git a/python/pyproject.toml b/python/pyproject.toml index 59a5fa8..94976cd 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -22,6 +22,7 @@ early = [ lint = [ 'tomli', + 'tomlkit', ] [tool.online-fxreader-pr34] diff --git a/python/requirements.txt b/python/requirements.txt index 766bbbd..73dde05 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes -o /home/nartes/Documents/current/freelance-project-34-marketing-blog/python/requirements.txt /tmp/requirementsmeh8aapn.in +# uv pip compile --generate-hashes -o /home/nartes/Documents/current/freelance-project-34-marketing-blog/python/requirements.txt /tmp/requirements83jeiimi.in annotated-types==0.7.0 \ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 @@ -7,7 +7,7 @@ annotated-types==0.7.0 \ build==1.2.2.post1 \ --hash=sha256:1d61c0887fa860c01971625baae8bdd338e517b836a2f70dd1f7aa3a6b2fc5b5 \ --hash=sha256:b36993e92ca9375a219c99e606a122ff365a760a2d4bba0caa09bd5278b608b7 - # via -r /tmp/requirementsmeh8aapn.in + # via -r /tmp/requirements83jeiimi.in cffi==1.17.1 \ --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8 \ --hash=sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2 \ @@ -77,43 +77,45 @@ cffi==1.17.1 \ --hash=sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87 \ --hash=sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b # via cryptography -cryptography==44.0.2 \ - --hash=sha256:04abd71114848aa25edb28e225ab5f268096f44cf0127f3d36975bdf1bdf3390 \ - --hash=sha256:0529b1d5a0105dd3731fa65680b45ce49da4d8115ea76e9da77a875396727b41 \ - --hash=sha256:1bc312dfb7a6e5d66082c87c34c8a62176e684b6fe3d90fcfe1568de675e6688 \ - --hash=sha256:268e4e9b177c76d569e8a145a6939eca9a5fec658c932348598818acf31ae9a5 \ - --hash=sha256:29ecec49f3ba3f3849362854b7253a9f59799e3763b0c9d0826259a88efa02f1 \ - --hash=sha256:2bf7bf75f7df9715f810d1b038870309342bff3069c5bd8c6b96128cb158668d \ - --hash=sha256:3b721b8b4d948b218c88cb8c45a01793483821e709afe5f622861fc6182b20a7 \ - --hash=sha256:3c00b6b757b32ce0f62c574b78b939afab9eecaf597c4d624caca4f9e71e7843 \ - --hash=sha256:3dc62975e31617badc19a906481deacdeb80b4bb454394b4098e3f2525a488c5 \ - --hash=sha256:4973da6ca3db4405c54cd0b26d328be54c7747e89e284fcff166132eb7bccc9c \ - --hash=sha256:4e389622b6927d8133f314949a9812972711a111d577a5d1f4bee5e58736b80a \ - --hash=sha256:51e4de3af4ec3899d6d178a8c005226491c27c4ba84101bfb59c901e10ca9f79 \ - --hash=sha256:5f6f90b72d8ccadb9c6e311c775c8305381db88374c65fa1a68250aa8a9cb3a6 \ - --hash=sha256:6210c05941994290f3f7f175a4a57dbbb2afd9273657614c506d5976db061181 \ - --hash=sha256:6f101b1f780f7fc613d040ca4bdf835c6ef3b00e9bd7125a4255ec574c7916e4 \ - --hash=sha256:7bdcd82189759aba3816d1f729ce42ffded1ac304c151d0a8e89b9996ab863d5 \ - --hash=sha256:7ca25849404be2f8e4b3c59483d9d3c51298a22c1c61a0e84415104dacaf5562 \ - --hash=sha256:81276f0ea79a208d961c433a947029e1a15948966658cf6710bbabb60fcc2639 \ - --hash=sha256:8cadc6e3b5a1f144a039ea08a0bdb03a2a92e19c46be3285123d32029f40a922 \ - --hash=sha256:8e0ddd63e6bf1161800592c71ac794d3fb8001f2caebe0966e77c5234fa9efc3 \ - --hash=sha256:909c97ab43a9c0c0b0ada7a1281430e4e5ec0458e6d9244c0e821bbf152f061d \ - --hash=sha256:96e7a5e9d6e71f9f4fca8eebfd603f8e86c5225bb18eb621b2c1e50b290a9471 \ - --hash=sha256:9a1e657c0f4ea2a23304ee3f964db058c9e9e635cc7019c4aa21c330755ef6fd \ - --hash=sha256:9eb9d22b0a5d8fd9925a7764a054dca914000607dff201a24c791ff5c799e1fa \ - --hash=sha256:af4ff3e388f2fa7bff9f7f2b31b87d5651c45731d3e8cfa0944be43dff5cfbdb \ - --hash=sha256:b042d2a275c8cee83a4b7ae30c45a15e6a4baa65a179a0ec2d78ebb90e4f6699 \ - --hash=sha256:bc821e161ae88bfe8088d11bb39caf2916562e0a2dc7b6d56714a48b784ef0bb \ - --hash=sha256:c505d61b6176aaf982c5717ce04e87da5abc9a36a5b39ac03905c4aafe8de7aa \ - --hash=sha256:c63454aa261a0cf0c5b4718349629793e9e634993538db841165b3df74f37ec0 \ - --hash=sha256:c7362add18b416b69d58c910caa217f980c5ef39b23a38a0880dfd87bdf8cd23 \ - --hash=sha256:d03806036b4f89e3b13b6218fefea8d5312e450935b1a2d55f0524e2ed7c59d9 \ - --hash=sha256:d1b3031093a366ac767b3feb8bcddb596671b3aaff82d4050f984da0c248b615 \ - --hash=sha256:d1c3572526997b36f245a96a2b1713bf79ce99b271bbcf084beb6b9b075f29ea \ - --hash=sha256:efcfe97d1b3c79e486554efddeb8f6f53a4cdd4cf6086642784fa31fc384e1d7 \ - --hash=sha256:f514ef4cd14bb6fb484b4a60203e912cfcb64f2ab139e88c2274511514bf7308 - # via -r /tmp/requirementsmeh8aapn.in +cryptography==44.0.3 \ + --hash=sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259 \ + --hash=sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43 \ + --hash=sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645 \ + --hash=sha256:21a83f6f35b9cc656d71b5de8d519f566df01e660ac2578805ab245ffd8523f8 \ + --hash=sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44 \ + --hash=sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d \ + --hash=sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f \ + --hash=sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d \ + --hash=sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54 \ + --hash=sha256:479d92908277bed6e1a1c69b277734a7771c2b78633c224445b5c60a9f4bc1d9 \ + --hash=sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137 \ + --hash=sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f \ + --hash=sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c \ + --hash=sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334 \ + --hash=sha256:5d20cc348cca3a8aa7312f42ab953a56e15323800ca3ab0706b8cd452a3a056c \ + --hash=sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b \ + --hash=sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2 \ + --hash=sha256:896530bc9107b226f265effa7ef3f21270f18a2026bc09fed1ebd7b66ddf6375 \ + --hash=sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88 \ + --hash=sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5 \ + --hash=sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647 \ + --hash=sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c \ + --hash=sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359 \ + --hash=sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5 \ + --hash=sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d \ + --hash=sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028 \ + --hash=sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01 \ + --hash=sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904 \ + --hash=sha256:cad399780053fb383dc067475135e41c9fe7d901a97dd5d9c5dfb5611afc0d7d \ + --hash=sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93 \ + --hash=sha256:dad80b45c22e05b259e33ddd458e9e2ba099c86ccf4e88db7bbab4b747b18d06 \ + --hash=sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff \ + --hash=sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76 \ + --hash=sha256:e909df4053064a97f1e6565153ff8bb389af12c5c8d29c343308760890560aff \ + --hash=sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759 \ + --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ + --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 + # via -r /tmp/requirements83jeiimi.in marisa-trie==1.2.1 \ --hash=sha256:06b099dd743676dbcd8abd8465ceac8f6d97d8bfaabe2c83b965495523b4cef2 \ --hash=sha256:0ee6cf6a16d9c3d1c94e21c8e63c93d8b34bede170ca4e937e16e1c0700d399f \ @@ -191,15 +193,15 @@ marisa-trie==1.2.1 \ --hash=sha256:f35c2603a6be168088ed1db6ad1704b078aa8f39974c60888fbbced95dcadad4 \ --hash=sha256:f4cd800704a5fc57e53c39c3a6b0c9b1519ebdbcb644ede3ee67a06eb542697d \ --hash=sha256:f713af9b8aa66a34cd3a78c7d150a560a75734713abe818a69021fd269e927fa - # via -r /tmp/requirementsmeh8aapn.in + # via -r /tmp/requirements83jeiimi.in meson==1.8.0 \ --hash=sha256:0a9b23311271519bd03dca12d7d8b0eab582c3a2c5da433d465b6e519dc88e2f \ --hash=sha256:472b7b25da286447333d32872b82d1c6f1a34024fb8ee017d7308056c25fec1f # via meson-python -meson-python==0.17.1 \ - --hash=sha256:30a75c52578ef14aff8392677b09c39346e0a24d2b2c6204b8ed30583c11269c \ - --hash=sha256:efb91f69f2e19eef7bc9a471ed2a4e730088cc6b39eacaf3e49fc4f930eb5f83 - # via -r /tmp/requirementsmeh8aapn.in +meson-python==0.18.0 \ + --hash=sha256:3b0fe051551cc238f5febb873247c0949cd60ded556efa130aa57021804868e2 \ + --hash=sha256:c56a99ec9df669a40662fe46960321af6e4b14106c14db228709c1628e23848d + # via -r /tmp/requirements83jeiimi.in mypy==1.15.0 \ --hash=sha256:1124a18bc11a6a62887e3e137f37f53fbae476dc36c185d549d4f837a2a6a14e \ --hash=sha256:171a9ca9a40cd1843abeca0e405bc1940cd9b305eaeea2dda769ba096932bb22 \ @@ -233,7 +235,7 @@ mypy==1.15.0 \ --hash=sha256:d10d994b41fb3497719bbf866f227b3489048ea4bbbb5015357db306249f7980 \ --hash=sha256:e601a7fa172c2131bff456bb3ee08a88360760d0d2f8cbd7a75a65497e2df078 \ --hash=sha256:f95579473af29ab73a10bada2f9722856792a36ec5af5399b653aa28360290a5 - # via -r /tmp/requirementsmeh8aapn.in + # via -r /tmp/requirements83jeiimi.in mypy-extensions==1.1.0 \ --hash=sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 \ --hash=sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558 @@ -294,7 +296,7 @@ numpy==2.2.5 \ --hash=sha256:ec31367fd6a255dc8de4772bd1658c3e926d8e860a0b6e922b615e532d320ddc \ --hash=sha256:ee461a4eaab4f165b68780a6a1af95fb23a29932be7569b9fab666c407969051 \ --hash=sha256:f5045039100ed58fa817a6227a356240ea1b9a1bc141018864c306c1a16d4175 - # via -r /tmp/requirementsmeh8aapn.in + # via -r /tmp/requirements83jeiimi.in packaging==25.0 \ --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f @@ -302,14 +304,14 @@ packaging==25.0 \ # build # meson-python # pyproject-metadata -pip==25.1 \ - --hash=sha256:13b4aa0aaad055020a11bec8a1c2a70a2b2d080e12d89b962266029fff0a16ba \ - --hash=sha256:272bdd1289f80165e9070a4f881e8f9e1001bbb50378561d1af20e49bf5a2200 - # via -r /tmp/requirementsmeh8aapn.in +pip==25.1.1 \ + --hash=sha256:2913a38a2abf4ea6b64ab507bd9e967f3b53dc1ede74b01b0931e1ce548751af \ + --hash=sha256:3de45d411d308d5054c2168185d8da7f9a2cd753dbac8acbfa88a8909ecd9077 + # via -r /tmp/requirements83jeiimi.in pybind11==2.13.6 \ --hash=sha256:237c41e29157b962835d356b370ededd57594a26d5894a795960f0047cb5caf5 \ --hash=sha256:ba6af10348c12b24e92fa086b39cfba0eff619b61ac77c406167d813b096d39a - # via -r /tmp/requirementsmeh8aapn.in + # via -r /tmp/requirements83jeiimi.in pycparser==2.22 \ --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc @@ -318,7 +320,7 @@ pydantic==2.11.4 \ --hash=sha256:32738d19d63a226a52eed76645a98ee07c1f410ee41d93b4afbfa85ed8111c2d \ --hash=sha256:d9615eaa9ac5a063471da949c8fc16376a84afb5024688b3ff885693506764eb # via - # -r /tmp/requirementsmeh8aapn.in + # -r /tmp/requirements83jeiimi.in # pydantic-settings pydantic-core==2.33.2 \ --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ @@ -424,7 +426,7 @@ pydantic-core==2.33.2 \ pydantic-settings==2.9.1 \ --hash=sha256:59b4f431b1defb26fe620c71a7d3968a710d719f5f4cdbbdb7926edeb770f6ef \ --hash=sha256:c509bf79d27563add44e8446233359004ed85066cd096d8b510f715e6ef5d268 - # via -r /tmp/requirementsmeh8aapn.in + # via -r /tmp/requirements83jeiimi.in pyproject-hooks==1.2.0 \ --hash=sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8 \ --hash=sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913 @@ -437,11 +439,11 @@ python-dotenv==1.1.0 \ --hash=sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5 \ --hash=sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d # via pydantic-settings -setuptools==80.0.1 \ - --hash=sha256:20fe373a22ef9f3925512650d1db90b1b8de01cdb6df91ab1788263139cbf9a2 \ - --hash=sha256:f4b49d457765b3aae7cbbeb1c71f6633a61b729408c2d1a837dae064cca82ef2 +setuptools==80.3.1 \ + --hash=sha256:31e2c58dbb67c99c289f51c16d899afedae292b978f8051efaf6262d8212f927 \ + --hash=sha256:ea8e00d7992054c4c592aeb892f6ad51fe1b4d90cc6947cc45c45717c40ec537 # via - # -r /tmp/requirementsmeh8aapn.in + # -r /tmp/requirements83jeiimi.in # marisa-trie tomli==2.2.1 \ --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \ @@ -476,7 +478,11 @@ tomli==2.2.1 \ --hash=sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272 \ --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \ --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7 - # via -r /tmp/requirementsmeh8aapn.in + # via -r /tmp/requirements83jeiimi.in +tomlkit==0.13.2 \ + --hash=sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde \ + --hash=sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79 + # via -r /tmp/requirements83jeiimi.in typing-extensions==4.13.2 \ --hash=sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c \ --hash=sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef @@ -491,23 +497,23 @@ typing-inspection==0.4.0 \ # via # pydantic # pydantic-settings -uv==0.7.1 \ - --hash=sha256:1d6f914601b769ad0f9a090573e2dc4365e0eaeb377d09cd74c5d47c97002c20 \ - --hash=sha256:2220b942b2eb8a0c5cc91af5d57c2eef7a25053037f9f311e85a2d5dd9154f88 \ - --hash=sha256:40a15f1fc73df852d7655530e5768e29dc7227ab25d9baeb711a8dde9e7f8234 \ - --hash=sha256:425064544f1e20b014447cf523e04e891bf6962e60dd25f495724b271f8911e0 \ - --hash=sha256:53eabd3aabc774d01da7836c58675c3e5cafd4285540e846debddfd056345d2c \ - --hash=sha256:5526f68ce9a5ba35ef13a14d144dc834b4940bd460fedc55f8313f9b7534b63c \ - --hash=sha256:57690b6e3b946dcf8b7b5836806d632f1a0d7667eae7af1302da812dbb7be7e5 \ - --hash=sha256:6bbf096970de17be0c2a1e28f24ebddaad9ad4d0f8d8f75364149cdde75d7462 \ - --hash=sha256:7025c9ba6f6f3d842a2b2915a579ff87eda901736105ee0379653bb4ff6b50d2 \ - --hash=sha256:7239a0ffd4695300a3b6d2004ab664e80be7ef2c46b677b0f47d6409affe2212 \ - --hash=sha256:877145523c348344c6fa2651559e9555dc4210730ad246afb4dd3414424afb3d \ - --hash=sha256:9b503d808310a978453bb91a448ffaf61542b192127c30be136443debac9cdaa \ - --hash=sha256:bf54fab715d6eb2332ff3276f80fddc6ee9e7faf29669d4bfb1918dd53ffc408 \ - --hash=sha256:c5572a2b1d6dbf1cbff315e55931f891d8706ef5ed76e94a7d5e6e6dae075b3a \ - --hash=sha256:c94cb14377c0efa65eb0267cfebfb5212729dc73fd61e4897e38839e3e72d763 \ - --hash=sha256:d9c0c70bd3734cdae20cf22889a0394307a86451bb7c9126f0542eb998dd1472 \ - --hash=sha256:ea2024e6a9daeea3ff6cab8ad4afe3b2aa0be9e07bad57646a749896e58648ad \ - --hash=sha256:ef8765771785a56b2e5485f3c6f9ec04cbd2c077be2fe1f2786ded5710e33c0d - # via -r /tmp/requirementsmeh8aapn.in +uv==0.7.2 \ + --hash=sha256:0445e56d3f9651ad84d5a7f16efabba83bf305b73594f1c1bc0659aeab952040 \ + --hash=sha256:19a64c38657c4fbe7c945055755500116fdaac8e121381a5245ea66823f8c500 \ + --hash=sha256:1fa315366ee36ad1f734734f3153e2f334342900061fc0ed18b06f3b9bb2dfe2 \ + --hash=sha256:28fd5d689ae4f8f16533f091a6dd63e1ddf3b7c782003ac8a18584ddb8823cbe \ + --hash=sha256:45e619bb076916b79df8c5ecc28d1be04d1ccd0b63b080c44ae973b8deb33b25 \ + --hash=sha256:48c115a3c13c3b29748e325093ee04fd48eaf91145bedc68727f78e6a1c34ab8 \ + --hash=sha256:63c97cc5e8029a8dc0e1fc39f15f746be931345bc0aeae85feceaa1828f0de87 \ + --hash=sha256:7236ec776c559fbc3ae4389b7cd506a2428ad9dd0402ac3d9446200ea3dc45f6 \ + --hash=sha256:78ec372b2f5c7ff8a034e16dd04bc579a62561a5eac4b6dfc96af60298a97d31 \ + --hash=sha256:81b86fff996c302be6aa1c1ac6eb72b97a7277c319e52c0def50d40b1ffaa617 \ + --hash=sha256:9aaacb143622cd437a446a4b316a546c02403b438cd7fd7556d62f47a9fd0a99 \ + --hash=sha256:a314a94b42bc6014f18c877f723292306b76c10b455c2b385728e1470e661ced \ + --hash=sha256:be2e8d033936ba8ed9ccf85eb2d15c7a8db3bb3e9c4960bdf7c3c98034a6dbda \ + --hash=sha256:c0edb194c35f1f12c75bec4fe2d7d4d09f0c2cec3a16102217a772620ce1d6e6 \ + --hash=sha256:c388172209ca5a47706666d570a45fef3dd39db9258682e10b2f62ca521f0e91 \ + --hash=sha256:dc1ee6114c824f5880c584a96b2947a35817fdd3a0b752d1adbd926ae6872d1c \ + --hash=sha256:e1e4394b54bc387f227ca1b2aa0348d35f6455b6168ca1826c1dc5f4fc3e8d20 \ + --hash=sha256:e4d1652fe3608fa564dbeaeb2465208f691ac04b57f655ebef62e9ec6d37103d + # via -r /tmp/requirements83jeiimi.in From 24e116882dc67ba36c18003d6dd2c08f6059f123 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Tue, 6 May 2025 18:43:53 +0300 Subject: [PATCH 02/49] [+] improve modules support --- .../pr34/commands_typed/cli_bootstrap.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index aa32a40..ccbe0cb 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -34,6 +34,12 @@ def toml_load(f: BinaryIO) -> Any: @dataclasses.dataclass class PyProject: + @dataclasses.dataclass + class Module: + name: str + meson: Optional[pathlib.Path] = None + + path: pathlib.Path dependencies: dict[str, list[str]] early_features: Optional[list[str]] = None @@ -42,6 +48,10 @@ class PyProject: runtime_preload: Optional[list[pathlib.Path]] = None requirements: dict[str, pathlib.Path] = dataclasses.field(default_factory=lambda : dict()) + modules: list[Module] = dataclasses.field( + default_factory=lambda : [], + ) + def pyproject_load( d: pathlib.Path, ) -> PyProject: @@ -119,6 +129,30 @@ def pyproject_load( for k, v in content['tool'][tool_name]['requirements'].items() } + if 'modules' in content['tool'][tool_name]: + modules = content['tool'][tool_name] + assert isinstance(modules, list) + + # res.modules = [] + + for o in modules: + assert isinstance(o, dict) + assert 'name' in o and isinstance(o['name'], str) + + module = PyProject.Module( + name=o['name'], + ) + + if 'meson' in o: + assert 'meson' in o and isinstance(o['meson'], str) + + module.meson = pathlib.Path(o['meson']) + + res.modules.append(module) + + res.early_features = content['tool'][tool_name]['early_features'] + + return res @dataclasses.dataclass From 3459ee237e9bfbcbeb0827e206eae76922370395 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Tue, 6 May 2025 18:53:13 +0300 Subject: [PATCH 03/49] [+] fix modules from .toml --- python/online/fxreader/pr34/commands_typed/cli_bootstrap.py | 2 +- python/pyproject.toml | 2 +- releases/whl/online_fxreader_pr34-0.1.5.16+27-py3-none-any.whl | 3 +++ releases/whl/online_fxreader_pr34-0.1.5.16+28-py3-none-any.whl | 3 +++ 4 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27-py3-none-any.whl create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+28-py3-none-any.whl diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index ccbe0cb..efeee7a 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -130,7 +130,7 @@ def pyproject_load( } if 'modules' in content['tool'][tool_name]: - modules = content['tool'][tool_name] + modules = content['tool'][tool_name]['modules'] assert isinstance(modules, list) # res.modules = [] diff --git a/python/pyproject.toml b/python/pyproject.toml index 94976cd..fdb78a9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = 'online.fxreader.pr34' -version = '0.1.5.16' +version = '0.1.5.16+28' dependencies = [ #"-r requirements.txt", diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27-py3-none-any.whl new file mode 100644 index 0000000..9230a36 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55b31297a8722395286f647d948ed5a729f03d0989fc7a68dd3324f8faa3ce52 +size 70698 diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+28-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+28-py3-none-any.whl new file mode 100644 index 0000000..033e7c8 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+28-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:558711c29edacb851e2aa9957f85b2b6f4996e06afa80682df6510d212113450 +size 70698 From 3d023ceba3edced5776a207a9aba8261857869c6 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Tue, 6 May 2025 19:01:53 +0300 Subject: [PATCH 04/49] [+] improve modules support --- .../fxreader/pr34/commands_typed/cli.py | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index ad6b168..51a2c02 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -481,7 +481,7 @@ class CLI(abc.ABC): def venv_compile( self, project_name: str, - force: bool, + # force: bool, argv: Optional[list[str]] = None, ) -> None: from . import cli_bootstrap @@ -549,3 +549,46 @@ class CLI(abc.ABC): project.source_dir / 'requirements.in', '-o', project.source_dir / 'requirements.txt', ]) + + def module_switch( + self, + project_name: str, + # force: bool, + argv: Optional[list[str]] = None, + ) -> None: + from . import cli_bootstrap + from . import argparse as pr34_argparse + + project = self.projects[project_name] + + parser = argparse.ArgumentParser() + parser.add_argument( + '-m', + dest='module', + choices=[ + o.name + for o in project.modules + ], + required=True, + # type=pathlib.Path, + type=str, + ) + + parser.add_argument( + '-f', + dest='file', + default='pyproject.common.toml', + default=[], + # type=pathlib.Path, + type=pathlib.Path, + ) + + options, args = pr34_argparse.parse_args( + parser, + argv, + ) + + if not options.file.is_abs(): + options.file = project.source_dir / options.file + + raise NotImplementedError From ffdd17270efeb04036f6e29fed2fa6d0ee480de0 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Wed, 7 May 2025 11:01:40 +0300 Subject: [PATCH 05/49] [+] improve module_switch feature 1. remove a release *+28; 2. add new .whl; --- .../fxreader/pr34/commands_typed/cli.py | 22 +++++++++++++------ python/pyproject.toml | 2 +- ...reader_pr34-0.1.5.16+27.1-py3-none-any.whl | 3 +++ ...reader_pr34-0.1.5.16+27.2-py3-none-any.whl | 3 +++ ...fxreader_pr34-0.1.5.16+28-py3-none-any.whl | 3 --- 5 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.1-py3-none-any.whl create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.2-py3-none-any.whl delete mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+28-py3-none-any.whl diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index 51a2c02..fee6e21 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -565,10 +565,10 @@ class CLI(abc.ABC): parser.add_argument( '-m', dest='module', - choices=[ - o.name - for o in project.modules - ], + # choices=[ + # o.name + # for o in project.modules + # ], required=True, # type=pathlib.Path, type=str, @@ -577,8 +577,7 @@ class CLI(abc.ABC): parser.add_argument( '-f', dest='file', - default='pyproject.common.toml', - default=[], + default=pathlib.Path('pyproject.common.toml'), # type=pathlib.Path, type=pathlib.Path, ) @@ -588,7 +587,16 @@ class CLI(abc.ABC): argv, ) - if not options.file.is_abs(): + if not options.file.is_absolute(): options.file = project.source_dir / options.file + pyproject = cli_bootstrap.pyproject_load( + options.file, + ) + + assert options.module in [ + o.name + for o in pyproject.modules + ] + raise NotImplementedError diff --git a/python/pyproject.toml b/python/pyproject.toml index fdb78a9..6254bbb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = 'online.fxreader.pr34' -version = '0.1.5.16+28' +version = '0.1.5.16+27.2' dependencies = [ #"-r requirements.txt", diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.1-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.1-py3-none-any.whl new file mode 100644 index 0000000..fb75307 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.1-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bae2399f7d1d3f27dabb8a66713f4c440d2ef98df52acb8d9c151932924f324 +size 70869 diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.2-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.2-py3-none-any.whl new file mode 100644 index 0000000..fb8b54e --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.2-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0b831baa4fe596a3442a774748625ffa9a8d5b21d897d424d41933dc390b220 +size 70869 diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+28-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+28-py3-none-any.whl deleted file mode 100644 index 033e7c8..0000000 --- a/releases/whl/online_fxreader_pr34-0.1.5.16+28-py3-none-any.whl +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:558711c29edacb851e2aa9957f85b2b6f4996e06afa80682df6510d212113450 -size 70698 From d6f921eab65fa0d5f5cbd1aa5d018721ed43c786 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Wed, 7 May 2025 17:44:01 +0300 Subject: [PATCH 06/49] [+] partially add pyproject.toml generation --- .../fxreader/pr34/commands_typed/cli.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index fee6e21..a4095a2 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -556,6 +556,9 @@ class CLI(abc.ABC): # force: bool, argv: Optional[list[str]] = None, ) -> None: + import tomlkit + import tomlkit.items + from . import cli_bootstrap from . import argparse as pr34_argparse @@ -599,4 +602,36 @@ class CLI(abc.ABC): for o in pyproject.modules ] + modules: dict[ + str, + cli_bootstrap.PyProject.Module + ] = { + o.name : o + for o in pyproject.modules + } + + module = modules[options.module] + + with io.open(options.file, 'rb') as f: + pyproject2 = tomlkit.load(options.file) + + with io.open( + project.source_dir / 'pyproject.toml', + 'w', + ) as f: + p = pyproject2['project'] + assert isinstance(p, tomlkit.items.Item) + n = p['name'] + assert isinstance(n, tomlkit.items.Item) + n = module.name + + del n + del p + + tomlkit.dump( + pyproject2, + f + ) + del pyproject2 + raise NotImplementedError From 45836381e04b5de325d5f9933e4d0ea5b90a3c0b Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Wed, 7 May 2025 18:03:44 +0300 Subject: [PATCH 07/49] [+] finish pyproject.toml generation 1. release .whl 27.3; --- python/online/fxreader/pr34/commands_typed/cli.py | 12 +++++------- python/pyproject.toml | 2 +- ...line_fxreader_pr34-0.1.5.16+27.3-py3-none-any.whl | 3 +++ 3 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.3-py3-none-any.whl diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index a4095a2..267f861 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -557,7 +557,7 @@ class CLI(abc.ABC): argv: Optional[list[str]] = None, ) -> None: import tomlkit - import tomlkit.items + import tomlkit.container from . import cli_bootstrap from . import argparse as pr34_argparse @@ -620,12 +620,9 @@ class CLI(abc.ABC): 'w', ) as f: p = pyproject2['project'] - assert isinstance(p, tomlkit.items.Item) - n = p['name'] - assert isinstance(n, tomlkit.items.Item) - n = module.name + assert isinstance(p, tomlkit.container.Container) + p['name'] = module.name - del n del p tomlkit.dump( @@ -633,5 +630,6 @@ class CLI(abc.ABC): f ) del pyproject2 + del module - raise NotImplementedError + # raise NotImplementedError diff --git a/python/pyproject.toml b/python/pyproject.toml index 6254bbb..e4bd2dc 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = 'online.fxreader.pr34' -version = '0.1.5.16+27.2' +version = '0.1.5.16+27.3' dependencies = [ #"-r requirements.txt", diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.3-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.3-py3-none-any.whl new file mode 100644 index 0000000..00f1562 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.3-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72a5995097afe73aea5a2ece3c325bc4c5b8b9e26427dda9ce1f23f50d9e080f +size 71034 From 4d4f8d17ac0cd7e6a6fac990e877c02d6597f534 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Wed, 7 May 2025 18:06:17 +0300 Subject: [PATCH 08/49] [+] make tomlkit a default dependency --- python/pyproject.toml | 6 ++++-- .../whl/online_fxreader_pr34-0.1.5.16+27.4-py3-none-any.whl | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.4-py3-none-any.whl diff --git a/python/pyproject.toml b/python/pyproject.toml index e4bd2dc..9b3419c 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = 'online.fxreader.pr34' -version = '0.1.5.16+27.3' +version = '0.1.5.16+27.4' dependencies = [ #"-r requirements.txt", @@ -8,6 +8,7 @@ dependencies = [ 'marisa-trie', 'pydantic', 'pydantic-settings', + 'tomlkit', ] [project.optional-dependencies] @@ -18,11 +19,12 @@ crypto = [ early = [ 'numpy', 'cryptography', + # 'tomlkit', ] lint = [ 'tomli', - 'tomlkit', + # 'tomlkit', ] [tool.online-fxreader-pr34] diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.4-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.4-py3-none-any.whl new file mode 100644 index 0000000..dc2889a --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.4-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4202d8f2b16a9ea030cab0035dad9f35c0c3e471e50d053a4d999b3e74581dcc +size 71034 From db50a5c09e57cd4f0eeb8910beb1cc2c6b3492a6 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Wed, 7 May 2025 18:08:16 +0300 Subject: [PATCH 09/49] [+] fix a typo --- python/online/fxreader/pr34/commands_typed/cli.py | 2 +- python/pyproject.toml | 2 +- .../whl/online_fxreader_pr34-0.1.5.16+27.5-py3-none-any.whl | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.5-py3-none-any.whl diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index 267f861..ba011b3 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -613,7 +613,7 @@ class CLI(abc.ABC): module = modules[options.module] with io.open(options.file, 'rb') as f: - pyproject2 = tomlkit.load(options.file) + pyproject2 = tomlkit.load(f) with io.open( project.source_dir / 'pyproject.toml', diff --git a/python/pyproject.toml b/python/pyproject.toml index 9b3419c..07db22a 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = 'online.fxreader.pr34' -version = '0.1.5.16+27.4' +version = '0.1.5.16+27.5' dependencies = [ #"-r requirements.txt", diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.5-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.5-py3-none-any.whl new file mode 100644 index 0000000..088307a --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.5-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e111101646bfc13ba7cd79b54912b9b414ed7c0297fa1b616df6cfd728ca7867 +size 71030 From a5818c70d9820c2b12963143db7b83ce850a0dbb Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Wed, 7 May 2025 18:16:14 +0300 Subject: [PATCH 10/49] [+] improve module_switch --- python/online/fxreader/pr34/commands_typed/cli.py | 3 ++- python/pyproject.toml | 2 +- .../whl/online_fxreader_pr34-0.1.5.16+27.6-py3-none-any.whl | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.6-py3-none-any.whl diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index ba011b3..b7935c0 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -558,6 +558,7 @@ class CLI(abc.ABC): ) -> None: import tomlkit import tomlkit.container + import tomlkit.items from . import cli_bootstrap from . import argparse as pr34_argparse @@ -620,7 +621,7 @@ class CLI(abc.ABC): 'w', ) as f: p = pyproject2['project'] - assert isinstance(p, tomlkit.container.Container) + assert isinstance(p, tomlkit.items.Table) p['name'] = module.name del p diff --git a/python/pyproject.toml b/python/pyproject.toml index 07db22a..6668525 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = 'online.fxreader.pr34' -version = '0.1.5.16+27.5' +version = '0.1.5.16+27.6' dependencies = [ #"-r requirements.txt", diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.6-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.6-py3-none-any.whl new file mode 100644 index 0000000..d82e714 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.6-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e58c34ec58432632cdf4dbc67e468a69b2947814d3fecbd6ba64c0df07e3af80 +size 71037 From 075b1b61493c80514963ee13b51b5273a384101e Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Thu, 8 May 2025 16:01:37 +0300 Subject: [PATCH 11/49] [+] add pip_check_conflicts --- python/online/fxreader/pr34/commands.py | 24 +++++++++++ .../fxreader/pr34/commands_typed/pip.py | 40 +++++++++++++++++++ python/pyproject.toml | 2 +- ...reader_pr34-0.1.5.16+27.7-py3-none-any.whl | 3 ++ 4 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.7-py3-none-any.whl diff --git a/python/online/fxreader/pr34/commands.py b/python/online/fxreader/pr34/commands.py index c991081..9871644 100644 --- a/python/online/fxreader/pr34/commands.py +++ b/python/online/fxreader/pr34/commands.py @@ -4012,6 +4012,28 @@ class Command(enum.StrEnum): vpn = 'vpn' backup = 'backup' pip_resolve = 'pip_resolve' + pip_check_conflicts = 'pip_check_conflicts' + +def pip_check_conflicts( + args: list[str], +) -> None: + from .commands_typed.pip import pip_check_conflicts + from .commands_typed.argparse import parse_args as pr34_parse_args + parser = argparse.ArgumentParser() + parser.add_argument( + '-p', + dest='venv_path', + type=pathlib.Path, + help='venv path', + default=None, + ) + + options, argv = pr34_parse_args(parser, args) + + res = pip_check_conflicts(options.venv_path) + logger.info(dict(res=res)) + + assert res.status == 'ok' def pip_resolve( args: list[str], @@ -4154,6 +4176,8 @@ def commands_cli( desktop_services(args) elif options.command is Command.pip_resolve: pip_resolve(args) + elif options.command is Command.pip_check_conflicts: + pip_check_conflicts(args) elif options.command is Command.pm_service: pm_service(args) elif options.command is Command.backup: diff --git a/python/online/fxreader/pr34/commands_typed/pip.py b/python/online/fxreader/pr34/commands_typed/pip.py index 53419dd..23739bf 100644 --- a/python/online/fxreader/pr34/commands_typed/pip.py +++ b/python/online/fxreader/pr34/commands_typed/pip.py @@ -1,4 +1,5 @@ import contextlib +import glob import pathlib import sys import enum @@ -522,3 +523,42 @@ def pip_resolve( ) else: raise NotImplementedError + +class pip_check_conflicts_t: + @dataclasses.dataclass + class res_t: + status : Literal['ok', 'error'] + duplicates: list[str] + +def pip_check_conflicts( + venv_path: Optional[pathlib.Path] = None, +) -> pip_check_conflicts_t.res_t: + assert sys.platform == 'linux' + if venv_path is None: + venv_path = pathlib.Path( + sys.executable, + ).parent / '..' + + # records = glob.glob( + # str(venv_path / '*' / 'site-packages' / '*.dist-info' / 'RECORD'), + # recursive=True, + # ) + + duplicates = [ + line + for line in subprocess.check_output(r''' + cat $(find $VENV_PATH/lib/*/*/*.dist-info/RECORD) | sort | uniq -c | (grep -v -P '^\s+1\s'; true;) + ''', shell=True, env=dict( + VENV_PATH=str(venv_path), + )).decode('utf-8').splitlines() + if line.strip() != '' + ] + + return pip_check_conflicts_t.res_t( + status=( + 'error' + if len(duplicates) > 0 + else 'ok' + ), + duplicates=duplicates + ) diff --git a/python/pyproject.toml b/python/pyproject.toml index 6668525..9b3457f 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = 'online.fxreader.pr34' -version = '0.1.5.16+27.6' +version = '0.1.5.16+27.7' dependencies = [ #"-r requirements.txt", diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.7-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.7-py3-none-any.whl new file mode 100644 index 0000000..85091be --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.7-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57974bc5af7c567f214cd720c2e231ab8435f677fba3d997163989f6d254f712 +size 71536 From 7a25882d5733bee50cfe307d875437d98e58d442 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Thu, 8 May 2025 16:54:31 +0300 Subject: [PATCH 12/49] [+] add pip check conflicts into cli_bootstrap --- .../fxreader/pr34/commands_typed/cli_bootstrap.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index efeee7a..12c7863 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import glob +import json import io import tempfile import dataclasses @@ -169,6 +170,12 @@ class BootstrapSettings: ), ).strip() ) + pip_check_conflicts: Optional[bool] = dataclasses.field( + default_factory=lambda : os.environ.get( + 'PIP_CHECK_CONFLICTS', + json.dumps(True) + ) in [json.dumps(True)], + ) uv_args: list[str] = dataclasses.field( default_factory=lambda : os.environ.get( 'UV_ARGS', @@ -309,6 +316,13 @@ def env_bootstrap( '-r', str(requirements_path), ]) + if bootstrap_settings.pip_check_conflicts: + subprocess.check_call([ + bootstrap_settings.python_path, + '-m', + 'online.fxreader.pr34.commands', + 'pip_check_conflicts', + ]) def paths_equal( a: pathlib.Path | str, From 0a6e82b6165b24be9fad1cb224f97171612ddea3 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Thu, 8 May 2025 16:57:01 +0300 Subject: [PATCH 13/49] [+] remove a duplicate line --- python/online/fxreader/pr34/commands_typed/cli_bootstrap.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index 12c7863..0ff3d67 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -151,9 +151,6 @@ def pyproject_load( res.modules.append(module) - res.early_features = content['tool'][tool_name]['early_features'] - - return res @dataclasses.dataclass From 0166dc4756a64d763010575118dd8341757d79a3 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Fri, 9 May 2025 13:21:23 +0300 Subject: [PATCH 14/49] [+] partially migrate pr34 to meson.build 1. add wasm toolchain for meson; --- meson/toolchains/wasm-cross-meson.ini | 13 ++++++ python/cli.py | 1 + python/meson.build | 60 +++++++++++++++++++++++++++ python/pyproject.toml | 31 +++----------- 4 files changed, 80 insertions(+), 25 deletions(-) create mode 100644 meson/toolchains/wasm-cross-meson.ini create mode 100644 python/meson.build diff --git a/meson/toolchains/wasm-cross-meson.ini b/meson/toolchains/wasm-cross-meson.ini new file mode 100644 index 0000000..da0c2e1 --- /dev/null +++ b/meson/toolchains/wasm-cross-meson.ini @@ -0,0 +1,13 @@ +[binaries] +cpp = 'em++' +c = 'emcc' +ar = 'emar' +windres = '/usr/bin/false' +; exe_wrapper = '/usr/bin/false' +exe_wrapper = 'node' + +[host_machine] +system = 'linux' +cpu_family = 'x86_64' +cpu = 'x86_64' +endian = 'little' diff --git a/python/cli.py b/python/cli.py index 3e9e031..b190db9 100644 --- a/python/cli.py +++ b/python/cli.py @@ -47,6 +47,7 @@ class CLI(_cli.CLI): source_dir=self.settings.base_dir / 'python', build_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'build', dest_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'install', + meson_path=self.settings.base_dir / 'python' / 'meson.build', ) } diff --git a/python/meson.build b/python/meson.build new file mode 100644 index 0000000..7f612ef --- /dev/null +++ b/python/meson.build @@ -0,0 +1,60 @@ +project( + run_command( + 'tomlq', '-r', '.project.name', 'pyproject.toml', + check: true + ).stdout().strip('\n'), +# 'online.fxreader.uv', +# ['c', 'cpp'], + version: '0.1.5.16+27.7', +# default_options: [ +# 'cpp_std=c++23', +# # 'prefer_static=true', +# ], +) + +install_path = get_option('install_path') +message('install_path = ' + install_path) + +modes = get_option('modes') + +fs = import('fs') + +assert(modes.length() == 1, 'only one mode allowed') + +mode = modes[0] + + +project_root = '.' +source_dir = project_root +include_dir = project_root + + +if mode == 'meson' +# error() +endif + +if mode == 'pyproject' + py = import('python').find_installation(pure: false) + + namespace_path = meson.project_name().replace('.', '/') + + module_root = py.get_install_dir() / namespace_path + + py.install_sources( + [ + namespace_path, + ], + subdir: namespace_path, + ) + + install_data( + files( +# 'py.typed', + # '__init__.py', +# 'pyproject.toml', + 'py.typed', + ), + install_dir : namespace_path, + install_tag: 'python-runtime', + ) +endif diff --git a/python/pyproject.toml b/python/pyproject.toml index 9b3457f..73e8122 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,9 @@ [project] name = 'online.fxreader.pr34' -version = '0.1.5.16+27.7' +# version = '0.1.5.16+27.7' +dynamic = [ + 'version', +] dependencies = [ #"-r requirements.txt", @@ -31,30 +34,8 @@ lint = [ early_features = ['default', 'early', 'lint',] [build-system] -requires = ['setuptools'] -build-backend = 'setuptools.build_meta' - -[tool.setuptools] -include-package-data = false - -[tool.setuptools.package-dir] -'online.fxreader.pr34' = 'online/fxreader/pr34' -#package_dir = '..' -#packages = ['online_fxreader'] -#[tool.setuptools.packages.find] -#where = ['../..'] -#include = ['../../online_fxreader/vpn'] -#exclude =['../../aiortc/*', '../../_cffi_src/*'] - -#[tool.setuptools.packages.find] -#exclude = ['*'] -#include = ['*.py'] - -# [tool.setuptools.exclude-package-data] -# 'online.fxreader.pr34' = ['online/fxreader/pr34/py.typed'] - -#[tool.setuptools.package-data] -#'online_fxreader.vpn' = ['requirements.txt'] +requires = ["meson-python", "pybind11"] +build-backend = "mesonpy" [project.scripts] online-fxreader-pr34-commands = 'online.fxreader.pr34.commands:commands_cli' From 9a12e714935efd92c9bfaf1970e85a6b5b231ee9 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Fri, 9 May 2025 14:35:27 +0300 Subject: [PATCH 15/49] [+] update module_switch 1. allow per module tool array; 2. partially fix deploy:wheel; --- python/cli.py | 9 +++++++++ python/meson.build | 2 +- python/meson_options.txt | 2 ++ python/online/fxreader/pr34/commands_typed/cli.py | 9 +++++++++ .../online/fxreader/pr34/commands_typed/cli_bootstrap.py | 7 ++++++- 5 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 python/meson_options.txt diff --git a/python/cli.py b/python/cli.py index b190db9..840d0ff 100644 --- a/python/cli.py +++ b/python/cli.py @@ -27,6 +27,7 @@ class Command(enum.StrEnum): mypy = 'mypy' deploy_wheel = 'deploy:wheel' tests = 'tests' + meson_setup = 'meson:setup' @dataclasses.dataclass class Settings( @@ -143,6 +144,14 @@ class CLI(_cli.CLI): output_dir=options.output_dir, mypy=True, ) + elif options.command is Command.meson_setup: + assert not options.project is None + + self.meson_setup( + project_name=options.project, + argv=args, + force=options.force, + ) elif options.command is Command.mypy: self.mypy( argv=args, diff --git a/python/meson.build b/python/meson.build index 7f612ef..1494182 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.7', + version: '0.1.5.16+27.8', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', diff --git a/python/meson_options.txt b/python/meson_options.txt new file mode 100644 index 0000000..bad457a --- /dev/null +++ b/python/meson_options.txt @@ -0,0 +1,2 @@ +option('modes', type: 'array', choices: ['meson', 'pyproject'], value: ['pyproject']) +option('install_path', type : 'string', value: '') diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index b7935c0..db7293d 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -624,7 +624,16 @@ class CLI(abc.ABC): assert isinstance(p, tomlkit.items.Table) p['name'] = module.name + if not pyproject2['tool']: + pyproject2['tool'] = [] + + pyproject_tool = pyproject2['tool'] + assert isinstance(pyproject_tool, tomlkit.items.Array) + + pyproject_tool.extend(module.tool) + del p + del pyproject_tool tomlkit.dump( pyproject2, diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index 0ff3d67..1987543 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -39,7 +39,7 @@ class PyProject: class Module: name: str meson: Optional[pathlib.Path] = None - + tool: list[Any] = dataclasses.field(default_factory=lambda : []) path: pathlib.Path dependencies: dict[str, list[str]] @@ -149,6 +149,11 @@ def pyproject_load( module.meson = pathlib.Path(o['meson']) + if 'tool' in o: + assert isinstance(o['tool'], list) + + module.tool.extend(o['tool']) + res.modules.append(module) return res From e7ec1d1e70efdb6570b951b5696a8f69dc3f4391 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Fri, 9 May 2025 18:20:16 +0300 Subject: [PATCH 16/49] [+] partially fix build for pr34 with meson.build --- python/meson.build | 25 ++++++++++++++----- ...0.1.5.16+27.8-cp313-cp313-linux_x86_64.whl | 3 +++ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.8-cp313-cp313-linux_x86_64.whl diff --git a/python/meson.build b/python/meson.build index 1494182..05f86e8 100644 --- a/python/meson.build +++ b/python/meson.build @@ -40,21 +40,34 @@ if mode == 'pyproject' module_root = py.get_install_dir() / namespace_path + python_sources = run_command( + 'find', namespace_path, '-iname', '*.py', + check: true + ).stdout().strip().split('\n') + py.install_sources( - [ - namespace_path, - ], - subdir: namespace_path, + python_sources, + preserve_path: true +# subdir: namespace_path, ) +# install_subdir( +# namespace_path, +# install_dir: py.get_install_dir(), +# install_tag: 'python-runtime', +# # python_sources, +# # subdir: namespace_path, +# ) + install_data( files( # 'py.typed', # '__init__.py', # 'pyproject.toml', - 'py.typed', + namespace_path / 'py.typed', ), - install_dir : namespace_path, + install_dir : py.get_install_dir(), install_tag: 'python-runtime', + preserve_path: true, ) endif diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.8-cp313-cp313-linux_x86_64.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.8-cp313-cp313-linux_x86_64.whl new file mode 100644 index 0000000..5b177db --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.8-cp313-cp313-linux_x86_64.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00151fd4927f99c427715743cc87611a1c37dda17aa291db9efd3a38f55f766b +size 71488 From 68901a4ec5991b585b3c2b414c008c9ff3872327 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Sat, 10 May 2025 13:18:38 +0300 Subject: [PATCH 17/49] [+] fix build, to be platform independent --- python/meson.build | 24 +++++++++++++++---- ...reader_pr34-0.1.5.16+27.9-py3-none-any.whl | 3 +++ 2 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.9-py3-none-any.whl diff --git a/python/meson.build b/python/meson.build index 05f86e8..71809df 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.8', + version: '0.1.5.16+27.9', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', @@ -38,7 +38,10 @@ if mode == 'pyproject' namespace_path = meson.project_name().replace('.', '/') - module_root = py.get_install_dir() / namespace_path + install_root = py.get_install_dir(pure: true) + install_root_binary = py.get_install_dir(pure: false) + + module_root = install_root / namespace_path python_sources = run_command( 'find', namespace_path, '-iname', '*.py', @@ -47,7 +50,8 @@ if mode == 'pyproject' py.install_sources( python_sources, - preserve_path: true + preserve_path: true, + pure: true, # subdir: namespace_path, ) @@ -61,13 +65,23 @@ if mode == 'pyproject' install_data( files( + [ + namespace_path / 'py.typed', + ], # 'py.typed', # '__init__.py', # 'pyproject.toml', - namespace_path / 'py.typed', ), - install_dir : py.get_install_dir(), + install_dir : install_root, install_tag: 'python-runtime', preserve_path: true, ) +# + install_subdir( + project_root / '..' / 'meson', + install_dir : module_root, + strip_directory: false, +# install_tag: 'devel', + install_tag: 'devel', + ) endif diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.9-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.9-py3-none-any.whl new file mode 100644 index 0000000..a2d5772 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.9-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15d28a44ddb66da99516a1f45a3662200cbcb68118a66f7e19fe55c73f6d5bde +size 71876 From a8f57337ad2bbe6b8be0ab5c34bda61d701560b8 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Sat, 10 May 2025 18:42:02 +0300 Subject: [PATCH 18/49] [+] fix module:switch for tool key --- python/meson.build | 2 +- python/online/fxreader/pr34/commands_typed/cli.py | 7 +++++-- .../fxreader/pr34/commands_typed/cli_bootstrap.py | 11 ++++++++--- ...line_fxreader_pr34-0.1.5.16+27.10-py3-none-any.whl | 3 +++ ...line_fxreader_pr34-0.1.5.16+27.11-py3-none-any.whl | 3 +++ 5 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.10-py3-none-any.whl create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.11-py3-none-any.whl diff --git a/python/meson.build b/python/meson.build index 71809df..80fb864 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.9', + version: '0.1.5.16+27.11', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index db7293d..a76caa6 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -628,9 +628,12 @@ class CLI(abc.ABC): pyproject2['tool'] = [] pyproject_tool = pyproject2['tool'] - assert isinstance(pyproject_tool, tomlkit.items.Array) + # assert isinstance(pyproject_tool, tomlkit.items.Array) + assert isinstance(pyproject_tool, tomlkit.items.Table) - pyproject_tool.extend(module.tool) + for k, v in module.tool.items(): + assert not k in pyproject_tool + pyproject_tool[k] = v del p del pyproject_tool diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index 1987543..259392c 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -39,7 +39,7 @@ class PyProject: class Module: name: str meson: Optional[pathlib.Path] = None - tool: list[Any] = dataclasses.field(default_factory=lambda : []) + tool: dict[str, Any] = dataclasses.field(default_factory=lambda : dict()) path: pathlib.Path dependencies: dict[str, list[str]] @@ -150,9 +150,14 @@ def pyproject_load( module.meson = pathlib.Path(o['meson']) if 'tool' in o: - assert isinstance(o['tool'], list) + assert isinstance(o['tool'], dict) - module.tool.extend(o['tool']) + assert all([ + isinstance(o, str) + for o in o['tool'] + ]) + + module.tool.update(o['tool']) res.modules.append(module) diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.10-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.10-py3-none-any.whl new file mode 100644 index 0000000..9d36afb --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.10-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be90dd9d5f80df201c8de47b63a6d7250545f5421a1e14f2803b597132c96aff +size 71901 diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.11-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.11-py3-none-any.whl new file mode 100644 index 0000000..cc77152 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.11-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04ecbb7f7ef8a50172df7db15a9768d2d4129e44a194f47f6df314a701646c6c +size 71939 From 43c85ca162fec128fa710a59f178940d61c9ca12 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Mon, 12 May 2025 10:30:35 +0300 Subject: [PATCH 19/49] [+] partially update meson_install --- .../fxreader/pr34/commands_typed/cli.py | 49 ++++++++++++++++++- .../pr34/commands_typed/cli_bootstrap.py | 1 + 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index a76caa6..9c969b7 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -5,6 +5,7 @@ import os import pathlib import logging import sys +import pydantic import subprocess import shutil import abc @@ -27,6 +28,29 @@ class Project: dest_dir : pathlib.Path meson_path: Optional[pathlib.Path] = None +class PyProject: + class Tool: + class Meson: + class Args: + install: list[str] + + args: Args + + class MesonPython: + class Args: + install: list[str] + + args: Args + + meson: Optional[Meson] = None + + meson_python: Optional[MesonPython] = pydantic.Field( + alias='meson-python', + default=None, + ) + + tool: Optional[Tool] = None + @dataclasses.dataclass class Dependency: name: str @@ -336,14 +360,35 @@ class CLI(abc.ABC): if force and project.dest_dir.exists(): shutil.rmtree(project.dest_dir) - subprocess.check_call([ + pyproject = cli_bootstrap.pyproject_load( + project.source_dir / 'pyproject.toml' + ) + + pyproject_tool = pydantic.RootModel[ + PyProject + ].model_validate(pyproject.module) + + if ( + pyproject_tool.meson and + pyproject_tool.meson.args and + pyproject_tool.meson.args.install + ): + argv = pyproject_tool.meson.args.install + argv + + cmd = [ shutil_which('meson', True,), 'install', '-C', project.build_dir / 'meson', '--destdir', project.dest_dir, *argv, - ]) + ] + + logger.info(dict( + cmd=cmd, + )) + + subprocess.check_call(cmd) for o in glob.glob( str(project.dest_dir / 'lib' / 'pkgconfig' / '*.pc'), diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index 259392c..5f137b2 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -41,6 +41,7 @@ class PyProject: meson: Optional[pathlib.Path] = None tool: dict[str, Any] = dataclasses.field(default_factory=lambda : dict()) + path: pathlib.Path dependencies: dict[str, list[str]] early_features: Optional[list[str]] = None From 51f08a0ea67a750d5fa540f68f3375f87058649a Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Mon, 12 May 2025 15:51:42 +0300 Subject: [PATCH 20/49] [+] partially update meson_install 1. add parsing of tool section from pyproject.toml; to add install arguments for meson_install; --- python/meson.build | 2 +- .../online/fxreader/pr34/commands_typed/cli.py | 16 ++++++++++++---- .../pr34/commands_typed/cli_bootstrap.py | 4 ++++ ...fxreader_pr34-0.1.5.16+27.12-py3-none-any.whl | 3 +++ 4 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.12-py3-none-any.whl diff --git a/python/meson.build b/python/meson.build index 80fb864..2933a60 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.11', + version: '0.1.5.16+27.12', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index 9c969b7..268b031 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -28,15 +28,21 @@ class Project: dest_dir : pathlib.Path meson_path: Optional[pathlib.Path] = None +@dataclasses.dataclass class PyProject: + @dataclasses.dataclass class Tool: + @dataclasses.dataclass class Meson: + @dataclasses.dataclass class Args: install: list[str] args: Args + @dataclasses.dataclass class MesonPython: + @dataclasses.dataclass class Args: install: list[str] @@ -349,6 +355,8 @@ class CLI(abc.ABC): force: Optional[bool] = None, argv: Optional[list[str]] = None, ) -> None: + from . import cli_bootstrap + project = self.projects[project_name] if force is None: @@ -365,8 +373,8 @@ class CLI(abc.ABC): ) pyproject_tool = pydantic.RootModel[ - PyProject - ].model_validate(pyproject.module) + PyProject.Tool + ].model_validate(pyproject.tool).root if ( pyproject_tool.meson and @@ -379,8 +387,8 @@ class CLI(abc.ABC): shutil_which('meson', True,), 'install', '-C', - project.build_dir / 'meson', - '--destdir', project.dest_dir, + str(project.build_dir / 'meson'), + '--destdir', str(project.dest_dir), *argv, ] diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index 5f137b2..44d089c 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -54,6 +54,10 @@ class PyProject: default_factory=lambda : [], ) + tool: dict[str, Any] = dataclasses.field( + default_factory=dict, + ) + def pyproject_load( d: pathlib.Path, ) -> PyProject: diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.12-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.12-py3-none-any.whl new file mode 100644 index 0000000..25cac92 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.12-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0182a9b408673bfc6dac8706058eaf255d43c791863cff8699ce0d74880ea61c +size 72153 From 540d4ca602c535e955ca446e178fec5da077ae74 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Mon, 12 May 2025 15:59:51 +0300 Subject: [PATCH 21/49] [+] fix pyproject_load 1. store `tool` key; --- python/meson.build | 2 +- .../pr34/commands_typed/cli_bootstrap.py | 17 ++++++++++++++++- ...xreader_pr34-0.1.5.16+27.13-py3-none-any.whl | 3 +++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.13-py3-none-any.whl diff --git a/python/meson.build b/python/meson.build index 2933a60..fb0568a 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.12', + version: '0.1.5.16+27.13', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index 44d089c..f0859e5 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -11,7 +11,7 @@ import os import logging -from typing import (Optional, Any,) +from typing import (Optional, Any, cast,) from typing_extensions import ( Self, BinaryIO, ) @@ -92,6 +92,21 @@ def pyproject_load( tool_name = 'online.fxreader.pr34'.replace('.', '-') + if ( + 'tool' in content and + isinstance( + content['tool'], dict + ) and + all([ + isinstance(o, str) + for o in content['tool'] + ]) + ): + res.tool = cast( + dict[str, Any], + content['tool'] + ) + if ( 'tool' in content and isinstance( diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.13-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.13-py3-none-any.whl new file mode 100644 index 0000000..90d248a --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.13-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d84b500c9ca908db0043a9c346b5165380a370afc287f8a989c1693beb214945 +size 72193 From 52787a89c209ef162bf452004af26a75c19c0aa5 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Wed, 14 May 2025 13:57:10 +0300 Subject: [PATCH 22/49] [+] partially fetch changes for toolchains --- .../online/fxreader/pr34/commands_typed/cli_bootstrap.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index f0859e5..d1b957f 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -185,7 +185,7 @@ def pyproject_load( @dataclasses.dataclass class BootstrapSettings: - env_path: pathlib.Path + env_path: Optional[pathlib.Path] python_path: pathlib.Path base_dir: pathlib.Path python_version: Optional[str] = dataclasses.field( @@ -218,7 +218,12 @@ class BootstrapSettings: if base_dir is None: base_dir = pathlib.Path.cwd() - env_path = base_dir / '.venv' + env_path: Optional[pathlib.Path] = None + if 'ENV_PATH' in os.environ: + env_path = pathlib.Path(os.environ['ENV_PATH']) + else: + env_path = base_dir / '.venv' + python_path = env_path / 'bin' / 'python3' return cls( From f4891526a70f9982735c80d1281075f24d76f9bc Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Wed, 14 May 2025 17:22:51 +0300 Subject: [PATCH 23/49] [m] merge a change from master --- python/online/fxreader/pr34/commands_typed/cli_bootstrap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index d1b957f..8d849a4 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -281,6 +281,7 @@ def env_bootstrap( ], []) logger.info(dict( + requirements_name=requirements_name, early_dependencies=early_dependencies, )) From 8fecbe3e8af4691b5f5d4c9af96fd90691b3d34f Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Thu, 15 May 2025 11:01:38 +0300 Subject: [PATCH 24/49] [+] isolate requirements_name --- .../pr34/commands_typed/cli_bootstrap.py | 69 ++++++++++++++----- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index 8d849a4..82589e2 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -185,7 +185,7 @@ def pyproject_load( @dataclasses.dataclass class BootstrapSettings: - env_path: Optional[pathlib.Path] + env_path: pathlib.Path python_path: pathlib.Path base_dir: pathlib.Path python_version: Optional[str] = dataclasses.field( @@ -232,6 +232,48 @@ class BootstrapSettings: python_path=python_path, ) +class requirements_name_get_t: + @dataclasses.dataclass + class res_t: + not_compiled : pathlib.Path + compiled: pathlib.Path + name: str + +def requirements_name_get( + source_dir: pathlib.Path, + python_version: Optional[str], + features: list[str], + requirements: dict[str, pathlib.Path], +) -> requirements_name_get_t.res_t: + requirements_python_version: Optional[str] = None + if not python_version is None: + requirements_python_version = \ + python_version.replace('.', '_') + + requirements_name = '_'.join(sorted(features)) + + if requirements_python_version: + requirements_name += '_' + requirements_python_version + + requirements_path : Optional[pathlib.Path] = None + + if requirements_name in requirements: + requirements_path = requirements[requirements_name] + else: + requirements_path = source_dir / 'requirements.txt' + + requirements_path_in = requirements_path.parent / ( + requirements_path.stem + '.in' + ) + + requirements_in : list[str] = [] + + return requirements_name_get_t.res_t( + not_compiled=requirements_path_in, + compiled=requirements_path, + name=requirements_name, + ) + def env_bootstrap( bootstrap_settings: BootstrapSettings, pyproject: PyProject, @@ -251,22 +293,13 @@ def env_bootstrap( if pyproject.early_features: features.extend(pyproject.early_features) - requirements_python_version: Optional[str] = None - if not bootstrap_settings.python_version is None: - requirements_python_version = bootstrap_settings.python_version.replace('.', '_') - - - requirements_name = '_'.join(sorted(features)) - - if requirements_python_version: - requirements_name += '_' + requirements_python_version - - requirements_path : Optional[pathlib.Path] = None - - if requirements_name in pyproject.requirements: - requirements_path = pyproject.requirements[requirements_name] - else: - requirements_path = pyproject.path.parent / 'requirements.txt' + requirements_name_get_res = requirements_name_get( + python_version=bootstrap_settings.python_version, + features=features, + requirements=pyproject.requirements, + source_dir=pyproject.path.parent, + ) + requirements_path = requirements_name_get_res.compiled requirements_in : list[str] = [] @@ -281,7 +314,7 @@ def env_bootstrap( ], []) logger.info(dict( - requirements_name=requirements_name, + requirements_name_get_res=requirements_name_get_res, early_dependencies=early_dependencies, )) From 54134ed0a849e4b45dcf1ff241157754ff5b7320 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Thu, 15 May 2025 13:18:00 +0300 Subject: [PATCH 25/49] [+] improve venv_compile --- .../fxreader/pr34/commands_typed/cli.py | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index 268b031..7260fdc 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -551,6 +551,13 @@ class CLI(abc.ABC): # type=pathlib.Path, type=str, ) + parser.add_argument( + '-p', + dest='python_version', + default=None, + # type=pathlib.Path, + type=str, + ) options, args = pr34_argparse.parse_args( parser, @@ -575,12 +582,20 @@ class CLI(abc.ABC): if o.exists() ]) + requirements_name_get_res = cli_bootstrap.requirements_name_get( + source_dir=project.source_dir, + features=options.features, + python_version=options.python_version, + requirements=pyproject.requirements, + ) + logger.info(dict( dependencies=dependencies, + requirements_name_get_res=requirements_name_get_res, )) with io.open( - project.source_dir / 'requirements.in', + requirements_name_get_res.not_compiled, 'w', ) as f: f.write( @@ -599,8 +614,9 @@ class CLI(abc.ABC): for o in pip_find_links ], []), '--generate-hashes', - project.source_dir / 'requirements.in', - '-o', project.source_dir / 'requirements.txt', + requirements_name_get_res.not_compiled, + '-o', + requirements_name_get_res.compiled, ]) def module_switch( From a103f06be82f130f580f2188842fbd2ca948e5f3 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Thu, 15 May 2025 13:24:08 +0300 Subject: [PATCH 26/49] [+] improve cli_bootstrap --- python/cli.py | 19 +++++++++++++++++++ m.py => python/m.py | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) rename m.py => python/m.py (98%) diff --git a/python/cli.py b/python/cli.py index 840d0ff..67032e5 100644 --- a/python/cli.py +++ b/python/cli.py @@ -133,6 +133,25 @@ class CLI(_cli.CLI): options, args = parser.parse_known_args(argv[1:]) + default_project : Optional[str] = None + + for k, v in self.projects.items(): + if ( + cli_bootstrap.paths_equal( + v.source_dir.resolve(), + # pathlib.Path(__file__).parent.resolve(), + pathlib.Path.cwd(), + ) + ): + default_project = k + + if options.project is None: + if not default_project is None: + options.project = default_project + else: + logger.error(dict(msg='not provided project name')) + raise NotImplementedError + options.command = Command(options.command) if options.command is Command.deploy_wheel: diff --git a/m.py b/python/m.py similarity index 98% rename from m.py rename to python/m.py index 31656d3..7d9e342 100755 --- a/m.py +++ b/python/m.py @@ -333,6 +333,6 @@ def run( if __name__ == '__main__': run( - d=pathlib.Path(__file__).parent / 'python' / 'pyproject.toml', - cli_path=pathlib.Path(__file__).parent / 'python' / 'cli.py', + d=pathlib.Path(__file__).parent / 'pyproject.toml', + cli_path=pathlib.Path(__file__).parent / 'cli.py', ) From 98f2c4b965af71e379c9359afddaa8eb6b309d3f Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Thu, 15 May 2025 13:34:55 +0300 Subject: [PATCH 27/49] [+] add .whl release 1. fix mypy paths for mypy.ini; --- .mypy.ini | 18 ------------------ python/.mypy.ini | 18 ++++++++++++++++++ python/cli.py | 11 ++++++----- python/meson.build | 2 +- ...reader_pr34-0.1.5.16+27.14-py3-none-any.whl | 3 +++ 5 files changed, 28 insertions(+), 24 deletions(-) delete mode 100644 .mypy.ini create mode 100644 python/.mypy.ini create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.14-py3-none-any.whl diff --git a/.mypy.ini b/.mypy.ini deleted file mode 100644 index 9811c75..0000000 --- a/.mypy.ini +++ /dev/null @@ -1,18 +0,0 @@ -[mypy] -mypy_path = - mypy-stubs, - deps/com.github.aiortc.aiortc/src, - mypy-stubs/marisa-trie-types, - mypy-stubs/types-debugpy, - python - -exclude = - python/tmp, - python/build - -plugins = - numpy.typing.mypy_plugin, - pydantic.mypy - -explicit_package_bases = true -namespace_packages = true \ No newline at end of file diff --git a/python/.mypy.ini b/python/.mypy.ini new file mode 100644 index 0000000..7b470c3 --- /dev/null +++ b/python/.mypy.ini @@ -0,0 +1,18 @@ +[mypy] +mypy_path = + ../mypy-stubs, + ../deps/com.github.aiortc.aiortc/src, + ../mypy-stubs/marisa-trie-types, + ../mypy-stubs/types-debugpy, + . + +exclude = + tmp, + build + +plugins = + numpy.typing.mypy_plugin, + pydantic.mypy + +explicit_package_bases = true +namespace_packages = true diff --git a/python/cli.py b/python/cli.py index 67032e5..885014a 100644 --- a/python/cli.py +++ b/python/cli.py @@ -81,16 +81,17 @@ class CLI(_cli.CLI): project.source_dir / '_m.py', project.source_dir / 'online', project.source_dir / 'cli.py', - self.settings.base_dir / 'm.py', + project.source_dir / 'm.py', # Settings.settings().project_root / 'deps/com.github.aiortc.aiortc/src', #Settings.settings().project_root / 'm.py', ], max_errors={ - 'python/online/fxreader/pr34/commands_typed': 0, - 'python/cli.py': 0, + 'online/fxreader/pr34/commands_typed': 0, + # 'online/fxreader/pr34/commands': 0, + 'cli.py': 0, 'm.py': 0, - 'deps/com.github.aiortc.aiortc/src/online_fxreader': 0, - 'deps/com.github.aiortc.aiortc/src/aiortc/contrib/signaling': 0 + '../deps/com.github.aiortc.aiortc/src/online_fxreader': 0, + '../deps/com.github.aiortc.aiortc/src/aiortc/contrib/signaling': 0 } ), ) diff --git a/python/meson.build b/python/meson.build index fb0568a..f807817 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.13', + version: '0.1.5.16+27.14', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.14-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.14-py3-none-any.whl new file mode 100644 index 0000000..55b3c2a --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.14-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:028a358026924f4e4002ca4720570b2fbfe60fe66763720451a25172f3fdfa1a +size 72486 From ac03137032ce83c4ff1eb9c6e3ce656c90bf1652 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Fri, 16 May 2025 10:59:35 +0300 Subject: [PATCH 28/49] [+] add ruff, pyright into lint extras --- python/meson.build | 2 +- python/pyproject.toml | 3 +++ .../whl/online_fxreader_pr34-0.1.5.16+27.15-py3-none-any.whl | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.15-py3-none-any.whl diff --git a/python/meson.build b/python/meson.build index f807817..5d4af1f 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.14', + version: '0.1.5.16+27.15', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', diff --git a/python/pyproject.toml b/python/pyproject.toml index 73e8122..0fe33e8 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -27,6 +27,9 @@ early = [ lint = [ 'tomli', + 'mypy', + 'pyright', + 'ruff', # 'tomlkit', ] diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.15-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.15-py3-none-any.whl new file mode 100644 index 0000000..13881c1 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.15-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cc5ecb131bd3e15da75902f5428306eaa0df5ad5a0588fd2d880dbea557ae75 +size 72507 From 1d0f7195314ef3d54f1ed2703dbefde6d4260ca9 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Fri, 16 May 2025 14:11:19 +0300 Subject: [PATCH 29/49] [+] fix module:switch --- python/meson.build | 2 +- python/online/fxreader/pr34/commands_typed/cli.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/python/meson.build b/python/meson.build index 5d4af1f..0ba40dc 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.15', + version: '0.1.5.16+27.16', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index 7260fdc..050a511 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -696,6 +696,9 @@ class CLI(abc.ABC): if not pyproject2['tool']: pyproject2['tool'] = [] + if not 'tool' in pyproject2: + pyproject2['tool'] = dict() + pyproject_tool = pyproject2['tool'] # assert isinstance(pyproject_tool, tomlkit.items.Array) assert isinstance(pyproject_tool, tomlkit.items.Table) From a54fc8aadac0f76c03bf3824294be260b4e91c79 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Fri, 16 May 2025 14:12:44 +0300 Subject: [PATCH 30/49] [+] add .whl --- .../whl/online_fxreader_pr34-0.1.5.16+27.16-py3-none-any.whl | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.16-py3-none-any.whl diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.16-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.16-py3-none-any.whl new file mode 100644 index 0000000..e946f41 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.16-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f6d2e5319a2870456f132bff8b0952b55608cc218e12bf8e61c03f3d1a55e32 +size 72524 From 8c07cea8b738a6c179b305cf55c88051b5569268 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Fri, 16 May 2025 14:18:12 +0300 Subject: [PATCH 31/49] [+] improve module:switch --- python/meson.build | 2 +- python/online/fxreader/pr34/commands_typed/cli.py | 3 ++- .../whl/online_fxreader_pr34-0.1.5.16+27.17-py3-none-any.whl | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.17-py3-none-any.whl diff --git a/python/meson.build b/python/meson.build index 0ba40dc..fa1a713 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.16', + version: '0.1.5.16+27.17', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index 050a511..e10b8d6 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -17,6 +17,7 @@ from typing import ( Optional, Literal, Any, + MutableMapping, ) logger = logging.getLogger(__name__) @@ -701,7 +702,7 @@ class CLI(abc.ABC): pyproject_tool = pyproject2['tool'] # assert isinstance(pyproject_tool, tomlkit.items.Array) - assert isinstance(pyproject_tool, tomlkit.items.Table) + assert isinstance(pyproject_tool, MutableMapping) for k, v in module.tool.items(): assert not k in pyproject_tool diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.17-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.17-py3-none-any.whl new file mode 100644 index 0000000..8e60c29 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.17-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81a87abcabe1b11f58a609cd0923d736c77008e7eeb4f10676cc29a3c7d06836 +size 72530 From c6b2b62ba462084529c0fed4a56eeb6af5fa99b1 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Fri, 16 May 2025 16:58:23 +0300 Subject: [PATCH 32/49] [+] improve venv:compile --- python/meson.build | 2 +- python/online/fxreader/pr34/commands_typed/cli.py | 15 ++++++++++----- ..._fxreader_pr34-0.1.5.16+27.18-py3-none-any.whl | 3 +++ 3 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.18-py3-none-any.whl diff --git a/python/meson.build b/python/meson.build index fa1a713..38da911 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.17', + version: '0.1.5.16+27.18', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index e10b8d6..a140781 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -605,8 +605,8 @@ class CLI(abc.ABC): f.flush() if len(dependencies) > 0: - subprocess.check_call([ - self.dist_settings.python_path, + cmd = [ + str(self.dist_settings.python_path), '-m', 'uv', 'pip', 'compile', *args, @@ -615,10 +615,15 @@ class CLI(abc.ABC): for o in pip_find_links ], []), '--generate-hashes', - requirements_name_get_res.not_compiled, + str(requirements_name_get_res.not_compiled), '-o', - requirements_name_get_res.compiled, - ]) + str(requirements_name_get_res.compiled), + ] + + logger.info(dict( + cmd=cmd, + )) + subprocess.check_call(cmd) def module_switch( self, diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.18-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.18-py3-none-any.whl new file mode 100644 index 0000000..2f61f91 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.18-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0343401686b765a8e83c4d3f3b4d6100b497c2160cbeb14fdc67a8ccd998f499 +size 72556 From cf840258c299cb5963c9140d0bacdca4ad46b78e Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Fri, 16 May 2025 17:05:51 +0300 Subject: [PATCH 33/49] [+] pass python_version to uv pip compile --- python/meson.build | 2 +- python/online/fxreader/pr34/commands_typed/cli.py | 1 + .../whl/online_fxreader_pr34-0.1.5.16+27.19-py3-none-any.whl | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.19-py3-none-any.whl diff --git a/python/meson.build b/python/meson.build index 38da911..702ea32 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.18', + version: '0.1.5.16+27.19', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index a140781..23db06c 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -610,6 +610,7 @@ class CLI(abc.ABC): '-m', 'uv', 'pip', 'compile', *args, + '--python-version', options.python_version, *sum([ ['-f', str(o),] for o in pip_find_links diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.19-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.19-py3-none-any.whl new file mode 100644 index 0000000..09abdc1 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.19-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e6f5f4e32aaace54ac1d1602500cc9ea7f9a369de0441bcef133e2cd1f11290 +size 72567 From 3ef16b70de82911e7e3d0de2adf3573148f58773 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Sat, 17 May 2025 11:10:37 +0300 Subject: [PATCH 34/49] [+] improve logging --- .../online/fxreader/pr34/commands_typed/cli.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index 23db06c..88643ce 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -107,13 +107,19 @@ class CLI(abc.ABC): if len(argv) == 0: argv = ['check', '.',] - subprocess.check_call([ - self.dist_settings.python_path, + cmd = [ + str(self.dist_settings.python_path), '-m', 'ruff', '--config', str(project.source_dir / 'pyproject.toml'), *argv, - ]) + ] + + logger.info(dict( + cmd=cmd, + )) + + subprocess.check_call(cmd) def pyright( self, @@ -543,6 +549,8 @@ class CLI(abc.ABC): project = self.projects[project_name] + bootstrap_settings = cli_bootstrap.BootstrapSettings.get() + parser = argparse.ArgumentParser() parser.add_argument( '-f', @@ -555,7 +563,7 @@ class CLI(abc.ABC): parser.add_argument( '-p', dest='python_version', - default=None, + default=bootstrap_settings.python_version, # type=pathlib.Path, type=str, ) From 81138aa12eee573c4012ccc6d541ec42476bc167 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Sat, 17 May 2025 11:11:40 +0300 Subject: [PATCH 35/49] [+] add .whl --- python/meson.build | 2 +- .../whl/online_fxreader_pr34-0.1.5.16+27.20-py3-none-any.whl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.20-py3-none-any.whl diff --git a/python/meson.build b/python/meson.build index 702ea32..3613f6b 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.19', + version: '0.1.5.16+27.20', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.20-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.20-py3-none-any.whl new file mode 100644 index 0000000..d966e72 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.20-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cb01468f91874ee860999ce072dd4c7a552f56ca325650882eae353b153f07d +size 72581 From 218af2d173c53e7306895aa3f0faede344de5cba Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Sat, 17 May 2025 11:24:26 +0300 Subject: [PATCH 36/49] [+] integrate ruff, pyright into deploy_wheel --- python/meson.build | 2 +- .../fxreader/pr34/commands_typed/cli.py | 19 +++++++++++++++++++ ...eader_pr34-0.1.5.16+27.21-py3-none-any.whl | 3 +++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 releases/whl/online_fxreader_pr34-0.1.5.16+27.21-py3-none-any.whl diff --git a/python/meson.build b/python/meson.build index 3613f6b..23cd8ba 100644 --- a/python/meson.build +++ b/python/meson.build @@ -5,7 +5,7 @@ project( ).stdout().strip('\n'), # 'online.fxreader.uv', # ['c', 'cpp'], - version: '0.1.5.16+27.20', + version: '0.1.5.16+27.21', # default_options: [ # 'cpp_std=c++23', # # 'prefer_static=true', diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index 88643ce..ad62dce 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -278,6 +278,8 @@ class CLI(abc.ABC): force: Optional[bool] = None, env: Optional[dict[str, str]] = None, mypy: bool = False, + ruff: bool = False, + pyright: bool = False, tests: bool = False, ) -> None: project = self.projects[project_name] @@ -305,9 +307,26 @@ class CLI(abc.ABC): force=force, ) + if ruff: + self.ruff( + project_name=project_name, + argv=['format', '--check',], + ) + + self.ruff( + project_name=project_name, + argv=[], + ) + if mypy: self.mypy([]) + if pyright: + self.pyright( + project_name=project_name, + argv=[], + ) + if env is None: env = dict() diff --git a/releases/whl/online_fxreader_pr34-0.1.5.16+27.21-py3-none-any.whl b/releases/whl/online_fxreader_pr34-0.1.5.16+27.21-py3-none-any.whl new file mode 100644 index 0000000..3e62256 --- /dev/null +++ b/releases/whl/online_fxreader_pr34-0.1.5.16+27.21-py3-none-any.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7da8412a74439f76efc90cb147f91ad1ef7f6af49bd472eee351500541ac9476 +size 72636 From 6a2b14d8b94336d276999313ab3b91d896df6249 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Sat, 17 May 2025 13:36:08 +0300 Subject: [PATCH 37/49] [+] partially add ruff, pyright --- python/cli.py | 18 +++- python/pyproject.toml | 176 ++++++++++++++++++++++++++++++++++++++++ python/requirements.txt | 103 ++++++++++++++--------- 3 files changed, 258 insertions(+), 39 deletions(-) diff --git a/python/cli.py b/python/cli.py index 885014a..5d0becd 100644 --- a/python/cli.py +++ b/python/cli.py @@ -25,6 +25,8 @@ logger = logging.getLogger(__name__) class Command(enum.StrEnum): mypy = 'mypy' + pyright = 'pyright' + ruff = 'ruff' deploy_wheel = 'deploy:wheel' tests = 'tests' meson_setup = 'meson:setup' @@ -36,8 +38,8 @@ class Settings( base_dir: pathlib.Path = pathlib.Path(__file__).parent.parent build_dir: pathlib.Path = base_dir / 'tmp' / 'build' wheel_dir: pathlib.Path = base_dir / 'deps' / 'dist' - env_path: pathlib.Path = cli_bootstrap.BootstrapSettings.get(base_dir).env_path - python_path: pathlib.Path = cli_bootstrap.BootstrapSettings.get(base_dir).python_path + env_path: pathlib.Path = cli_bootstrap.BootstrapSettings.get().env_path + python_path: pathlib.Path = pathlib.Path(sys.executable) class CLI(_cli.CLI): @@ -163,6 +165,18 @@ class CLI(_cli.CLI): argv=args, output_dir=options.output_dir, mypy=True, + ruff=True, + pyright=True, + ) + elif options.command is Command.pyright: + self.pyright( + project_name=options.project, + argv=args, + ) + elif options.command is Command.ruff: + self.ruff( + project_name=options.project, + argv=args, ) elif options.command is Command.meson_setup: assert not options.project is None diff --git a/python/pyproject.toml b/python/pyproject.toml index 0fe33e8..6cb5a70 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,4 +1,13 @@ [project] +description = 'set of tools for software development' +requires-python = '>= 3.10' +maintainers = [ + { name = 'Siarhei Siniak', email = 'siarheisiniak@gmail.com' }, +] +classifiers = [ + 'Programming Language :: Python', +] + name = 'online.fxreader.pr34' # version = '0.1.5.16+27.7' dynamic = [ @@ -42,3 +51,170 @@ build-backend = "mesonpy" [project.scripts] online-fxreader-pr34-commands = 'online.fxreader.pr34.commands:commands_cli' + + +[tool.ruff] +line-length = 160 +target-version = 'py310' +# builtins = ['_', 'I', 'P'] +include = [ + # 'follow_the_leader/**/*.py', + #'*.py', + # '*.recipe', + '*.py', + 'online/**/*.py', + 'online/**/*.pyi', +] +exclude = [ + '.venv', +] + +[tool.ruff.format] +quote-style = 'single' +indent-style = 'tab' +skip-magic-trailing-comma = false + + +[tool.ruff.lint] +ignore = [ + 'E402', 'E722', 'E741', 'W191', 'E101', 'E501', 'I001', 'F401', 'E714', + 'E713', + # remove lambdas later on + 'E731', + # fix this too + 'E712', + 'E703', + # remove unused variables, or fix a bug + 'F841', + # fix * imports + 'F403', + +] +select = ['E', 'F', 'I', 'W', 'INT'] + + +[tool.ruff.lint.isort] +detect-same-package = true +# extra-standard-library = ["aes", "elementmaker", "encodings"] +# known-first-party = ["calibre_extensions", "calibre_plugins", "polyglot"] +# known-third-party = ["odf", "qt", "templite", "tinycss", "css_selectors"] +relative-imports-order = "closest-to-furthest" +split-on-trailing-comma = true +section-order = ['__python__', "future", "standard-library", "third-party", "first-party", "local-folder"] +force-wrap-aliases = true + +# [tool.ruff.lint.isort.sections] +# '__python__' = ['__python__'] + +[tool.pylsp-mypy] +enabled = false + +[tool.pyright] +include = [ + #'../../../../../follow_the_leader/views2/payments.py', + #'../../../../../follow_the_leader/logic/payments.py', + #'../../../../../follow_the_leader/logic/paypal.py', + 'python', +] +extraPaths = [ + 'python', + # '../../../../../', +] +#strict = ["src"] + +analyzeUnannotatedFunctions = true +disableBytesTypePromotions = true +strictParameterNoneValue = true +enableTypeIgnoreComments = true +enableReachabilityAnalysis = true +strictListInference = true +strictDictionaryInference = true +strictSetInference = true +deprecateTypingAliases = false +enableExperimentalFeatures = false +reportMissingTypeStubs ="error" +reportMissingModuleSource = "warning" +reportInvalidTypeForm = "error" +reportMissingImports = "error" +reportUndefinedVariable = "error" +reportAssertAlwaysTrue = "error" +reportInvalidStringEscapeSequence = "error" +reportInvalidTypeVarUse = "error" +reportSelfClsParameterName = "error" +reportUnsupportedDunderAll = "error" +reportUnusedExpression = "error" +reportWildcardImportFromLibrary = "error" +reportAbstractUsage = "error" +reportArgumentType = "error" +reportAssertTypeFailure = "error" +reportAssignmentType = "error" +reportAttributeAccessIssue = "error" +reportCallIssue = "error" +reportGeneralTypeIssues = "error" +reportInconsistentOverload = "error" +reportIndexIssue = "error" +reportInvalidTypeArguments = "error" +reportNoOverloadImplementation = "error" +reportOperatorIssue = "error" +reportOptionalSubscript = "error" +reportOptionalMemberAccess = "error" +reportOptionalCall = "error" +reportOptionalIterable = "error" +reportOptionalContextManager = "error" +reportOptionalOperand = "error" +reportRedeclaration = "error" +reportReturnType = "error" +reportTypedDictNotRequiredAccess = "error" +reportPrivateImportUsage = "error" +reportUnboundVariable = "error" +reportUnhashable = "error" +reportUnusedCoroutine = "error" +reportUnusedExcept = "error" +reportFunctionMemberAccess = "error" +reportIncompatibleMethodOverride = "error" +reportIncompatibleVariableOverride = "error" +reportOverlappingOverload = "error" +reportPossiblyUnboundVariable = "error" +reportConstantRedefinition = "error" +#reportDeprecated = "error" +reportDeprecated = "warning" +reportDuplicateImport = "error" +reportIncompleteStub = "error" +reportInconsistentConstructor = "error" +reportInvalidStubStatement = "error" +reportMatchNotExhaustive = "error" +reportMissingParameterType = "error" +reportMissingTypeArgument = "error" +reportPrivateUsage = "error" +reportTypeCommentUsage = "error" +reportUnknownArgumentType = "error" +reportUnknownLambdaType = "error" +reportUnknownMemberType = "error" +reportUnknownParameterType = "error" +reportUnknownVariableType = "error" +#reportUnknownVariableType = "warning" +reportUnnecessaryCast = "error" +reportUnnecessaryComparison = "error" +reportUnnecessaryContains = "error" +#reportUnnecessaryIsInstance = "error" +reportUnnecessaryIsInstance = "warning" +reportUnusedClass = "error" +#reportUnusedImport = "error" +reportUnusedImport = "none" +reportUnusedFunction = "error" +#reportUnusedVariable = "error" +reportUnusedVariable = "warning" +reportUntypedBaseClass = "error" +reportUntypedClassDecorator = "error" +reportUntypedFunctionDecorator = "error" +reportUntypedNamedTuple = "error" +reportCallInDefaultInitializer = "none" +reportImplicitOverride = "none" +reportImplicitStringConcatenation = "none" +reportImportCycles = "none" +reportMissingSuperCall = "none" +reportPropertyTypeMismatch = "none" +reportShadowedImports = "none" +reportUninitializedInstanceVariable = "none" +reportUnnecessaryTypeIgnoreComment = "none" +reportUnusedCallResult = "none" diff --git a/python/requirements.txt b/python/requirements.txt index 73dde05..8bc513e 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes -o /home/nartes/Documents/current/freelance-project-34-marketing-blog/python/requirements.txt /tmp/requirements83jeiimi.in +# uv pip compile --generate-hashes --offline -o /home/nartes/Documents/current/freelance-project-34-marketing-blog/python/requirements.txt /tmp/requirementsguod07w5.in annotated-types==0.7.0 \ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 @@ -7,7 +7,7 @@ annotated-types==0.7.0 \ build==1.2.2.post1 \ --hash=sha256:1d61c0887fa860c01971625baae8bdd338e517b836a2f70dd1f7aa3a6b2fc5b5 \ --hash=sha256:b36993e92ca9375a219c99e606a122ff365a760a2d4bba0caa09bd5278b608b7 - # via -r /tmp/requirements83jeiimi.in + # via -r /tmp/requirementsguod07w5.in cffi==1.17.1 \ --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8 \ --hash=sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2 \ @@ -115,7 +115,7 @@ cryptography==44.0.3 \ --hash=sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759 \ --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 - # via -r /tmp/requirements83jeiimi.in + # via -r /tmp/requirementsguod07w5.in marisa-trie==1.2.1 \ --hash=sha256:06b099dd743676dbcd8abd8465ceac8f6d97d8bfaabe2c83b965495523b4cef2 \ --hash=sha256:0ee6cf6a16d9c3d1c94e21c8e63c93d8b34bede170ca4e937e16e1c0700d399f \ @@ -193,7 +193,7 @@ marisa-trie==1.2.1 \ --hash=sha256:f35c2603a6be168088ed1db6ad1704b078aa8f39974c60888fbbced95dcadad4 \ --hash=sha256:f4cd800704a5fc57e53c39c3a6b0c9b1519ebdbcb644ede3ee67a06eb542697d \ --hash=sha256:f713af9b8aa66a34cd3a78c7d150a560a75734713abe818a69021fd269e927fa - # via -r /tmp/requirements83jeiimi.in + # via -r /tmp/requirementsguod07w5.in meson==1.8.0 \ --hash=sha256:0a9b23311271519bd03dca12d7d8b0eab582c3a2c5da433d465b6e519dc88e2f \ --hash=sha256:472b7b25da286447333d32872b82d1c6f1a34024fb8ee017d7308056c25fec1f @@ -201,7 +201,7 @@ meson==1.8.0 \ meson-python==0.18.0 \ --hash=sha256:3b0fe051551cc238f5febb873247c0949cd60ded556efa130aa57021804868e2 \ --hash=sha256:c56a99ec9df669a40662fe46960321af6e4b14106c14db228709c1628e23848d - # via -r /tmp/requirements83jeiimi.in + # via -r /tmp/requirementsguod07w5.in mypy==1.15.0 \ --hash=sha256:1124a18bc11a6a62887e3e137f37f53fbae476dc36c185d549d4f837a2a6a14e \ --hash=sha256:171a9ca9a40cd1843abeca0e405bc1940cd9b305eaeea2dda769ba096932bb22 \ @@ -235,11 +235,15 @@ mypy==1.15.0 \ --hash=sha256:d10d994b41fb3497719bbf866f227b3489048ea4bbbb5015357db306249f7980 \ --hash=sha256:e601a7fa172c2131bff456bb3ee08a88360760d0d2f8cbd7a75a65497e2df078 \ --hash=sha256:f95579473af29ab73a10bada2f9722856792a36ec5af5399b653aa28360290a5 - # via -r /tmp/requirements83jeiimi.in + # via -r /tmp/requirementsguod07w5.in mypy-extensions==1.1.0 \ --hash=sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 \ --hash=sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558 # via mypy +nodeenv==1.9.1 \ + --hash=sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f \ + --hash=sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9 + # via pyright numpy==2.2.5 \ --hash=sha256:0255732338c4fdd00996c0421884ea8a3651eea555c3a56b84892b66f696eb70 \ --hash=sha256:02f226baeefa68f7d579e213d0f3493496397d8f1cff5e2b222af274c86a552a \ @@ -296,7 +300,7 @@ numpy==2.2.5 \ --hash=sha256:ec31367fd6a255dc8de4772bd1658c3e926d8e860a0b6e922b615e532d320ddc \ --hash=sha256:ee461a4eaab4f165b68780a6a1af95fb23a29932be7569b9fab666c407969051 \ --hash=sha256:f5045039100ed58fa817a6227a356240ea1b9a1bc141018864c306c1a16d4175 - # via -r /tmp/requirements83jeiimi.in + # via -r /tmp/requirementsguod07w5.in packaging==25.0 \ --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f @@ -307,11 +311,11 @@ packaging==25.0 \ pip==25.1.1 \ --hash=sha256:2913a38a2abf4ea6b64ab507bd9e967f3b53dc1ede74b01b0931e1ce548751af \ --hash=sha256:3de45d411d308d5054c2168185d8da7f9a2cd753dbac8acbfa88a8909ecd9077 - # via -r /tmp/requirements83jeiimi.in + # via -r /tmp/requirementsguod07w5.in pybind11==2.13.6 \ --hash=sha256:237c41e29157b962835d356b370ededd57594a26d5894a795960f0047cb5caf5 \ --hash=sha256:ba6af10348c12b24e92fa086b39cfba0eff619b61ac77c406167d813b096d39a - # via -r /tmp/requirements83jeiimi.in + # via -r /tmp/requirementsguod07w5.in pycparser==2.22 \ --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc @@ -320,7 +324,7 @@ pydantic==2.11.4 \ --hash=sha256:32738d19d63a226a52eed76645a98ee07c1f410ee41d93b4afbfa85ed8111c2d \ --hash=sha256:d9615eaa9ac5a063471da949c8fc16376a84afb5024688b3ff885693506764eb # via - # -r /tmp/requirements83jeiimi.in + # -r /tmp/requirementsguod07w5.in # pydantic-settings pydantic-core==2.33.2 \ --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ @@ -426,7 +430,7 @@ pydantic-core==2.33.2 \ pydantic-settings==2.9.1 \ --hash=sha256:59b4f431b1defb26fe620c71a7d3968a710d719f5f4cdbbdb7926edeb770f6ef \ --hash=sha256:c509bf79d27563add44e8446233359004ed85066cd096d8b510f715e6ef5d268 - # via -r /tmp/requirements83jeiimi.in + # via -r /tmp/requirementsguod07w5.in pyproject-hooks==1.2.0 \ --hash=sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8 \ --hash=sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913 @@ -435,15 +439,39 @@ pyproject-metadata==0.9.1 \ --hash=sha256:b8b2253dd1b7062b78cf949a115f02ba7fa4114aabe63fa10528e9e1a954a816 \ --hash=sha256:ee5efde548c3ed9b75a354fc319d5afd25e9585fa918a34f62f904cc731973ad # via meson-python +pyright==1.1.400 \ + --hash=sha256:b8a3ba40481aa47ba08ffb3228e821d22f7d391f83609211335858bf05686bdb \ + --hash=sha256:c80d04f98b5a4358ad3a35e241dbf2a408eee33a40779df365644f8054d2517e + # via -r /tmp/requirementsguod07w5.in python-dotenv==1.1.0 \ --hash=sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5 \ --hash=sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d # via pydantic-settings -setuptools==80.3.1 \ - --hash=sha256:31e2c58dbb67c99c289f51c16d899afedae292b978f8051efaf6262d8212f927 \ - --hash=sha256:ea8e00d7992054c4c592aeb892f6ad51fe1b4d90cc6947cc45c45717c40ec537 +ruff==0.11.10 \ + --hash=sha256:1067245bad978e7aa7b22f67113ecc6eb241dca0d9b696144256c3a879663bca \ + --hash=sha256:2f071b0deed7e9245d5820dac235cbdd4ef99d7b12ff04c330a241ad3534319f \ + --hash=sha256:3afead355f1d16d95630df28d4ba17fb2cb9c8dfac8d21ced14984121f639bad \ + --hash=sha256:4a60e3a0a617eafba1f2e4186d827759d65348fa53708ca547e384db28406a0b \ + --hash=sha256:5a94acf798a82db188f6f36575d80609072b032105d114b0f98661e1679c9125 \ + --hash=sha256:5b6a9cc5b62c03cc1fea0044ed8576379dbaf751d5503d718c973d5418483641 \ + --hash=sha256:5cc725fbb4d25b0f185cb42df07ab6b76c4489b4bfb740a175f3a59c70e8a224 \ + --hash=sha256:607ecbb6f03e44c9e0a93aedacb17b4eb4f3563d00e8b474298a201622677947 \ + --hash=sha256:7b3a522fa389402cd2137df9ddefe848f727250535c70dafa840badffb56b7a4 \ + --hash=sha256:859a7bfa7bc8888abbea31ef8a2b411714e6a80f0d173c2a82f9041ed6b50f58 \ + --hash=sha256:8b4564e9f99168c0f9195a0fd5fa5928004b33b377137f978055e40008a082c5 \ + --hash=sha256:968220a57e09ea5e4fd48ed1c646419961a0570727c7e069842edd018ee8afed \ + --hash=sha256:d522fb204b4959909ecac47da02830daec102eeb100fb50ea9554818d47a5fa6 \ + --hash=sha256:da8ec977eaa4b7bf75470fb575bea2cb41a0e07c7ea9d5a0a97d13dbca697bf2 \ + --hash=sha256:dc061a98d32a97211af7e7f3fa1d4ca2fcf919fb96c28f39551f35fc55bdbc19 \ + --hash=sha256:ddf8967e08227d1bd95cc0851ef80d2ad9c7c0c5aab1eba31db49cf0a7b99523 \ + --hash=sha256:ef69637b35fb8b210743926778d0e45e1bffa850a7c61e428c6b971549b5f5d1 \ + --hash=sha256:f4854fd09c7aed5b1590e996a81aeff0c9ff51378b084eb5a0b9cd9518e6cff2 + # via -r /tmp/requirementsguod07w5.in +setuptools==80.7.1 \ + --hash=sha256:ca5cc1069b85dc23070a6628e6bcecb3292acac802399c7f8edc0100619f9009 \ + --hash=sha256:f6ffc5f0142b1bd8d0ca94ee91b30c0ca862ffd50826da1ea85258a06fd94552 # via - # -r /tmp/requirements83jeiimi.in + # -r /tmp/requirementsguod07w5.in # marisa-trie tomli==2.2.1 \ --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \ @@ -478,11 +506,11 @@ tomli==2.2.1 \ --hash=sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272 \ --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \ --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7 - # via -r /tmp/requirements83jeiimi.in + # via -r /tmp/requirementsguod07w5.in tomlkit==0.13.2 \ --hash=sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde \ --hash=sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79 - # via -r /tmp/requirements83jeiimi.in + # via -r /tmp/requirementsguod07w5.in typing-extensions==4.13.2 \ --hash=sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c \ --hash=sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef @@ -490,6 +518,7 @@ typing-extensions==4.13.2 \ # mypy # pydantic # pydantic-core + # pyright # typing-inspection typing-inspection==0.4.0 \ --hash=sha256:50e72559fcd2a6367a19f7a7e610e6afcb9fac940c650290eed893d61386832f \ @@ -497,23 +526,23 @@ typing-inspection==0.4.0 \ # via # pydantic # pydantic-settings -uv==0.7.2 \ - --hash=sha256:0445e56d3f9651ad84d5a7f16efabba83bf305b73594f1c1bc0659aeab952040 \ - --hash=sha256:19a64c38657c4fbe7c945055755500116fdaac8e121381a5245ea66823f8c500 \ - --hash=sha256:1fa315366ee36ad1f734734f3153e2f334342900061fc0ed18b06f3b9bb2dfe2 \ - --hash=sha256:28fd5d689ae4f8f16533f091a6dd63e1ddf3b7c782003ac8a18584ddb8823cbe \ - --hash=sha256:45e619bb076916b79df8c5ecc28d1be04d1ccd0b63b080c44ae973b8deb33b25 \ - --hash=sha256:48c115a3c13c3b29748e325093ee04fd48eaf91145bedc68727f78e6a1c34ab8 \ - --hash=sha256:63c97cc5e8029a8dc0e1fc39f15f746be931345bc0aeae85feceaa1828f0de87 \ - --hash=sha256:7236ec776c559fbc3ae4389b7cd506a2428ad9dd0402ac3d9446200ea3dc45f6 \ - --hash=sha256:78ec372b2f5c7ff8a034e16dd04bc579a62561a5eac4b6dfc96af60298a97d31 \ - --hash=sha256:81b86fff996c302be6aa1c1ac6eb72b97a7277c319e52c0def50d40b1ffaa617 \ - --hash=sha256:9aaacb143622cd437a446a4b316a546c02403b438cd7fd7556d62f47a9fd0a99 \ - --hash=sha256:a314a94b42bc6014f18c877f723292306b76c10b455c2b385728e1470e661ced \ - --hash=sha256:be2e8d033936ba8ed9ccf85eb2d15c7a8db3bb3e9c4960bdf7c3c98034a6dbda \ - --hash=sha256:c0edb194c35f1f12c75bec4fe2d7d4d09f0c2cec3a16102217a772620ce1d6e6 \ - --hash=sha256:c388172209ca5a47706666d570a45fef3dd39db9258682e10b2f62ca521f0e91 \ - --hash=sha256:dc1ee6114c824f5880c584a96b2947a35817fdd3a0b752d1adbd926ae6872d1c \ - --hash=sha256:e1e4394b54bc387f227ca1b2aa0348d35f6455b6168ca1826c1dc5f4fc3e8d20 \ - --hash=sha256:e4d1652fe3608fa564dbeaeb2465208f691ac04b57f655ebef62e9ec6d37103d - # via -r /tmp/requirements83jeiimi.in +uv==0.7.3 \ + --hash=sha256:0646e463365e7277f22200ce2d43b7a44e5a3192320500b4983b4fe34d69a5fb \ + --hash=sha256:0a446d4e5b10ce8a793156a276727bb7affa96a85e80dc5ad34e0c2de7e71cc8 \ + --hash=sha256:3e6e1fd5755d4ef4c6e1ce55bd2c6d9dec278a8bef5752703d702ce03704fe29 \ + --hash=sha256:44e2f3fcbd1ab519bdb68986449b2e3103d2261be95f985cadcf7ec7c510b595 \ + --hash=sha256:4809e5f7f5b2d6423d6573fda5655389c955ca649499fe9750b61af95daf9b7d \ + --hash=sha256:5eb4872888a9fb10b62cc00be8e84822d63d3e622a5f340248e53ecf321dba96 \ + --hash=sha256:863ceb63aefc7c2db9918313a1cb3c8bf3fc3d59b656b617db9e4abad90373f3 \ + --hash=sha256:90990e4c289feee24164c8e463fc0ebc9a336960119cd256acca7c1439f0f536 \ + --hash=sha256:acef117a0c52299e60c6f7a3e60849050cd233704c561f688fac1100d113da2e \ + --hash=sha256:acff7fba5ff40dcb5a42de496db92a3965edac7a3d687d9b013ba6e0336995df \ + --hash=sha256:b1414a026c153ae0731daed0812b17bf77d34eafedaeb3a5c72e08181aea116b \ + --hash=sha256:c976fce3d1068a1d007f50127cc7873d67643c1a60439564970f092d9be41877 \ + --hash=sha256:cb2547fd1466698e9b4f11de5eef7055b8cbcc3c693d79f6d747e3f8e6be2ab7 \ + --hash=sha256:cc27207c35c959d2e0e873e86a80a2470a77b7a34a4512a831e8d4f7c87f4404 \ + --hash=sha256:d246243f348796730e8ea9736ddd48702d4448d98af5e61693063ed616e30378 \ + --hash=sha256:db8a5d5995b160158405379deadf0ffccf849a5e7ce048900b73517daf109e2c \ + --hash=sha256:f37c8a6b172776fb5305afe0699907aff44a778669de7a8fbe5a9c09c1a88a97 \ + --hash=sha256:fbb2d322d453e498e1431c51421cee597962ecd3f93fcef853b258e9c7e7636c + # via -r /tmp/requirementsguod07w5.in From 9f4b426229d38f98f674b410155103a7eeec60b6 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Mon, 19 May 2025 11:38:25 +0300 Subject: [PATCH 38/49] [+] improve typing --- .../pr34/commands_typed/cli_bootstrap.py | 23 +++++++++++++++++-- python/pyproject.toml | 7 ++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index 82589e2..c114649 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -11,7 +11,7 @@ import os import logging -from typing import (Optional, Any, cast,) +from typing import (Optional, Any, cast, Type, TypeVar,) from typing_extensions import ( Self, BinaryIO, ) @@ -55,7 +55,26 @@ class PyProject: ) tool: dict[str, Any] = dataclasses.field( - default_factory=dict, + default_factory=lambda : dict(), + ) + +Key = TypeVar('Key') +Value = TypeVar('Value') + +def check_dict( + value: Any, + KT: Type[Key], + VT: Type[Value], +) -> dict[Key, Value]: + assert isinstance(value, dict) + value2 = cast(dict[Any, Any], value) + assert all([ + isinstance(k, KT) and isinstance(v, VT) + for k, v in value2.items() + ]) + return cast( + dict[Key, Value], + value, ) def pyproject_load( diff --git a/python/pyproject.toml b/python/pyproject.toml index 6cb5a70..c7fb665 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -114,10 +114,13 @@ include = [ #'../../../../../follow_the_leader/views2/payments.py', #'../../../../../follow_the_leader/logic/payments.py', #'../../../../../follow_the_leader/logic/paypal.py', - 'python', + 'online/fxreader/pr34/commands_typed/**/*.py', ] +# stubPath = '../mypy-stubs' extraPaths = [ - 'python', + '.', + '../mypy-stubs', + '../mypy-stubs/types-debugpy', # '../../../../../', ] #strict = ["src"] From 591c893909164da7324a364936e5bea54b498773 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Mon, 19 May 2025 12:24:51 +0300 Subject: [PATCH 39/49] [+] improve typing --- .../pr34/commands_typed/cli_bootstrap.py | 54 +++++++++++++++---- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index c114649..44304ef 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -13,7 +13,7 @@ import logging from typing import (Optional, Any, cast, Type, TypeVar,) from typing_extensions import ( - Self, BinaryIO, + Self, BinaryIO, overload, ) logger = logging.getLogger(__name__) @@ -61,21 +61,45 @@ class PyProject: Key = TypeVar('Key') Value = TypeVar('Value') +@overload def check_dict( value: Any, KT: Type[Key], VT: Type[Value], +) -> dict[Key, Value]: ... + +@overload +def check_dict( + value: Any, + KT: Type[Key], +) -> dict[Key, Any]: ... + +def check_dict( + value: Any, + KT: Type[Key], + VT: Optional[Type[Value]] = None, ) -> dict[Key, Value]: assert isinstance(value, dict) value2 = cast(dict[Any, Any], value) + assert all([ - isinstance(k, KT) and isinstance(v, VT) + isinstance(k, KT) and ( + VT is None or + isinstance(v, VT) + ) for k, v in value2.items() ]) - return cast( - dict[Key, Value], - value, - ) + + if VT is None: + return cast( + dict[Key, Any], + value, + ) + else: + return cast( + dict[Key, Value], + value, + ) def pyproject_load( d: pathlib.Path, @@ -97,9 +121,21 @@ def pyproject_load( dict ) - for k, v in content['project']['optional-dependencies'].items(): - assert isinstance(v, list) - assert isinstance(k, str) + for k, v in check_dict( + check_dict( + check_dict( + content, + str, + # Any, + )['project'], + str, + # Any, + )['optional-dependencies'], + str, + list[Any], + ).items(): + # assert isinstance(v, list) + # assert isinstance(k, str) dependencies[k] = v From 1dbba925f160ff52df31cd3c2ab2f584dbd34904 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Mon, 19 May 2025 12:33:28 +0300 Subject: [PATCH 40/49] [+] improve typing --- .../pr34/commands_typed/cli_bootstrap.py | 94 +++++++++++++------ 1 file changed, 67 insertions(+), 27 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index 44304ef..0fd3b20 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -101,6 +101,43 @@ def check_dict( value, ) +@overload +def check_list( + value: Any, + VT: Type[Value], +) -> list[Value]: ... + +@overload +def check_list( + value: Any, +) -> list[Any]: ... + +def check_list( + value: Any, + VT: Optional[Type[Value]] = None, +) -> list[Value] | list[Any]: + assert isinstance(value, list) + value2 = cast(list[Any], value) + + assert all([ + ( + VT is None or + isinstance(o, VT) + ) + for o in value2 + ]) + + if VT is None: + return cast( + list[Any], + value, + ) + else: + return cast( + list[Value], + value, + ) + def pyproject_load( d: pathlib.Path, ) -> PyProject: @@ -148,18 +185,11 @@ def pyproject_load( tool_name = 'online.fxreader.pr34'.replace('.', '-') if ( - 'tool' in content and - isinstance( - content['tool'], dict - ) and - all([ - isinstance(o, str) - for o in content['tool'] - ]) + 'tool' in content ): - res.tool = cast( - dict[str, Any], - content['tool'] + res.tool = check_dict( + content['tool'], + str, ) if ( @@ -173,42 +203,52 @@ def pyproject_load( dict ) ): - if 'early_features' in content['tool'][tool_name]: - res.early_features = content['tool'][tool_name]['early_features'] + pr34_tool = check_dict( + check_dict( + content['tool'], + str, + )[tool_name], + str + ) - if 'pip_find_links' in content['tool'][tool_name]: + if 'early_features' in pr34_tool: + res.early_features = pr34_tool['early_features'] + + if 'pip_find_links' in pr34_tool: res.pip_find_links = [ d.parent / pathlib.Path(o) - for o in content['tool'][tool_name]['pip_find_links'] + for o in pr34_tool['pip_find_links'] ] - if 'runtime_libdirs' in content['tool'][tool_name]: + if 'runtime_libdirs' in pr34_tool: res.runtime_libdirs = [ d.parent / pathlib.Path(o) # pathlib.Path(o) - for o in content['tool'][tool_name]['runtime_libdirs'] + for o in pr34_tool['runtime_libdirs'] ] - if 'runtime_preload' in content['tool'][tool_name]: + if 'runtime_preload' in pr34_tool: res.runtime_preload = [ d.parent / pathlib.Path(o) # pathlib.Path(o) - for o in content['tool'][tool_name]['runtime_preload'] + for o in pr34_tool['runtime_preload'] ] - if 'requirements' in content['tool'][tool_name]: - assert isinstance(content['tool'][tool_name]['requirements'], dict) - + if 'requirements' in pr34_tool: res.requirements = { k : d.parent / pathlib.Path(v) # pathlib.Path(o) - for k, v in content['tool'][tool_name]['requirements'].items() + for k, v in check_dict( + pr34_tool['requirements'], + str, + str + ).items() } - if 'modules' in content['tool'][tool_name]: - modules = content['tool'][tool_name]['modules'] - assert isinstance(modules, list) - + if 'modules' in pr34_tool: + modules = check_list( + pr34_tool['modules'] + ) # res.modules = [] for o in modules: From 91af45c582d6ec6403f13fce6ca39e04acb076a6 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Mon, 19 May 2025 13:11:26 +0300 Subject: [PATCH 41/49] [+] improve typing --- .../pr34/commands_typed/cli_bootstrap.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index 0fd3b20..cec9b65 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -265,14 +265,12 @@ def pyproject_load( module.meson = pathlib.Path(o['meson']) if 'tool' in o: - assert isinstance(o['tool'], dict) - - assert all([ - isinstance(o, str) - for o in o['tool'] - ]) - - module.tool.update(o['tool']) + module.tool.update( + check_dict( + o['tool'], + str, + ) + ) res.modules.append(module) @@ -381,7 +379,7 @@ def env_bootstrap( pip_find_links_args = sum([ ['-f', str(o),] for o in pip_find_links - ], []) + ], cast(list[str], [])) features : list[str] = [] @@ -406,7 +404,7 @@ def env_bootstrap( early_dependencies = sum([ pyproject.dependencies[o] for o in pyproject.early_features - ], []) + ], cast(list[str], [])) logger.info(dict( requirements_name_get_res=requirements_name_get_res, From cc506baa9e4ec32ec0748eedff20a582178ff898 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Mon, 19 May 2025 13:16:08 +0300 Subject: [PATCH 42/49] [+] improve typing --- python/online/fxreader/pr34/commands_typed/cli.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index ad62dce..c3119a9 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -20,6 +20,10 @@ from typing import ( MutableMapping, ) +from typing_extensions import ( + cast, +) + logger = logging.getLogger(__name__) @dataclasses.dataclass @@ -158,7 +162,7 @@ class CLI(abc.ABC): dependencies = sum([ pyproject.dependencies[o] for o in features - ], []) + ], cast(list[str], [])) pip_find_links : list[pathlib.Path] = [] @@ -178,7 +182,7 @@ class CLI(abc.ABC): *sum([ ['-f', str(o),] for o in pip_find_links - ], []), + ], cast(list[str], [])), # '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'), '--offline', *dependencies, @@ -599,7 +603,7 @@ class CLI(abc.ABC): dependencies = sum([ pyproject.dependencies[o] for o in options.features - ], []) + ], cast(list[str], [])) pip_find_links : list[pathlib.Path] = [] @@ -641,7 +645,7 @@ class CLI(abc.ABC): *sum([ ['-f', str(o),] for o in pip_find_links - ], []), + ], cast(list[str], [])), '--generate-hashes', str(requirements_name_get_res.not_compiled), '-o', From 21326dd2229bb0ffb11e70f4d0ad20781f1e1f96 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Mon, 19 May 2025 13:23:02 +0300 Subject: [PATCH 43/49] [+] improve typing --- python/online/fxreader/pr34/commands_typed/cli.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index c3119a9..0e8c342 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -18,10 +18,13 @@ from typing import ( Literal, Any, MutableMapping, + Mapping, + IO, ) from typing_extensions import ( cast, + Callable, ) logger = logging.getLogger(__name__) @@ -748,10 +751,14 @@ class CLI(abc.ABC): del p del pyproject_tool - tomlkit.dump( + cast( + Callable[[Mapping[Any, Any], IO[str] | IO[bytes]], None], + getattr(tomlkit, 'dump'), + )( pyproject2, f ) + del pyproject2 del module From 5e6b2b2e6342d1e285920a207976693f9536c524 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Mon, 19 May 2025 16:46:37 +0300 Subject: [PATCH 44/49] [+] improve typing --- .../fxreader/pr34/commands_typed/crypto.py | 1 + .../fxreader/pr34/commands_typed/mypy.py | 4 +-- .../fxreader/pr34/commands_typed/pip.py | 29 +++++++++++++++---- python/pyproject.toml | 1 + 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/crypto.py b/python/online/fxreader/pr34/commands_typed/crypto.py index 827fb0d..d76f8f8 100644 --- a/python/online/fxreader/pr34/commands_typed/crypto.py +++ b/python/online/fxreader/pr34/commands_typed/crypto.py @@ -2,6 +2,7 @@ import base64 import os import cryptography.hazmat.primitives.kdf.scrypt +import cryptography.exceptions from typing import (Literal, overload, Optional,) diff --git a/python/online/fxreader/pr34/commands_typed/mypy.py b/python/online/fxreader/pr34/commands_typed/mypy.py index 9abd9eb..2e5ad9d 100644 --- a/python/online/fxreader/pr34/commands_typed/mypy.py +++ b/python/online/fxreader/pr34/commands_typed/mypy.py @@ -67,7 +67,7 @@ def run( argv = [] if settings is None: - settings = MypySettings() + settings = MypySettings.model_validate(dict()) parser = argparse.ArgumentParser() parser.add_argument( @@ -213,4 +213,4 @@ def run( if __name__ == '__main__': from . import logging as _logging _logging.setup() - run(sys.argv[1:]) \ No newline at end of file + run(sys.argv[1:]) diff --git a/python/online/fxreader/pr34/commands_typed/pip.py b/python/online/fxreader/pr34/commands_typed/pip.py index 23739bf..b8e086a 100644 --- a/python/online/fxreader/pr34/commands_typed/pip.py +++ b/python/online/fxreader/pr34/commands_typed/pip.py @@ -11,6 +11,8 @@ import unittest.mock import logging import typing +from typing_extensions import (cast,) + if typing.TYPE_CHECKING: import pip._internal.commands.show import pip._internal.commands.download @@ -30,13 +32,24 @@ from typing import ( logger = logging.getLogger(__name__) +class pip_show_t: + class res_t: + class package_info_t: + pass + def pip_show( argv: list[str], -) -> list['pip._internal.commands.show._PackageInfo']: +) -> list[ + # 'pip._internal.commands.show._PackageInfo' + pip_show_t.res_t.package_info_t, +]: import pip._internal.commands.show - return list( - pip._internal.commands.show.search_packages_info( - argv, + return cast( + list[pip_show_t.res_t.package_info_t], + list( + pip._internal.commands.show.search_packages_info( + argv, + ) ) ) @@ -138,7 +151,13 @@ def pip_resolve( # t1._in_main_context = True session = t1.get_default_session(options) target_python = pip._internal.cli.cmdoptions.make_target_python(options) - finder = t1._build_package_finder( + finder = cast( + getattr( + t1, + '_build_package_finder' + ) + ) + t1._build_package_finder( options=options, session=session, target_python=target_python, diff --git a/python/pyproject.toml b/python/pyproject.toml index c7fb665..378767e 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -121,6 +121,7 @@ extraPaths = [ '.', '../mypy-stubs', '../mypy-stubs/types-debugpy', + '../mypy-stubs/marisa-trie-types', # '../../../../../', ] #strict = ["src"] From d30cdc7d7066366158adbe6eaceadc1b99982527 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Mon, 19 May 2025 16:58:40 +0300 Subject: [PATCH 45/49] [+] improve typing --- .../online/fxreader/pr34/commands_typed/pip.py | 17 ++++++++++++++--- python/pyproject.toml | 3 ++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/pip.py b/python/online/fxreader/pr34/commands_typed/pip.py index b8e086a..24f4c4e 100644 --- a/python/online/fxreader/pr34/commands_typed/pip.py +++ b/python/online/fxreader/pr34/commands_typed/pip.py @@ -11,7 +11,7 @@ import unittest.mock import logging import typing -from typing_extensions import (cast,) +from typing_extensions import (cast, Protocol,) if typing.TYPE_CHECKING: import pip._internal.commands.show @@ -24,6 +24,7 @@ if typing.TYPE_CHECKING: import pip._internal.resolution.base import pip._internal.resolution.resolvelib.resolver import pip._internal.operations.prepare + import pip._internal.index.package_finder from typing import ( Literal, Optional, Iterable, Any, @@ -55,6 +56,15 @@ def pip_show( class pip_resolve_t: + class build_package_finder_t(Protocol): + def __call__( + self, + options: Any, + session: Any, + target_python: Any, + ignore_requires_python: Any, + ) -> 'pip._internal.index.package_finder.PackageFinder': ... + class kwargs_t: class mode_t(enum.StrEnum): copy_paste = "copy_paste" @@ -152,17 +162,18 @@ def pip_resolve( session = t1.get_default_session(options) target_python = pip._internal.cli.cmdoptions.make_target_python(options) finder = cast( + pip_resolve_t.build_package_finder_t, getattr( t1, '_build_package_finder' ) - ) - t1._build_package_finder( + )( options=options, session=session, target_python=target_python, ignore_requires_python=options.ignore_requires_python, ) + build_tracker = t1.enter_context( pip._internal.operations.build.build_tracker.get_build_tracker() ) diff --git a/python/pyproject.toml b/python/pyproject.toml index 378767e..f6e1df0 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -205,7 +205,8 @@ reportUnnecessaryIsInstance = "warning" reportUnusedClass = "error" #reportUnusedImport = "error" reportUnusedImport = "none" -reportUnusedFunction = "error" +# reportUnusedFunction = "error" +reportUnusedFunction = "warning" #reportUnusedVariable = "error" reportUnusedVariable = "warning" reportUntypedBaseClass = "error" From a7f6bd927cd7ebe3440aafc2374d9781444dbdfb Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Mon, 19 May 2025 17:06:01 +0300 Subject: [PATCH 46/49] [+] improve typing --- .../online/fxreader/pr34/commands_typed/pip.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/python/online/fxreader/pr34/commands_typed/pip.py b/python/online/fxreader/pr34/commands_typed/pip.py index 24f4c4e..bf90c5c 100644 --- a/python/online/fxreader/pr34/commands_typed/pip.py +++ b/python/online/fxreader/pr34/commands_typed/pip.py @@ -25,6 +25,7 @@ if typing.TYPE_CHECKING: import pip._internal.resolution.resolvelib.resolver import pip._internal.operations.prepare import pip._internal.index.package_finder + from pip._internal.req.req_install import InstallRequirement from typing import ( Literal, Optional, Iterable, Any, @@ -65,6 +66,14 @@ class pip_resolve_t: ignore_requires_python: Any, ) -> 'pip._internal.index.package_finder.PackageFinder': ... + class complete_partial_requirements_t(Protocol): + def __call__( + self, + _self: 'pip._internal.resolution.resolvelib.resolver.Resolver', + partially_downloaded_reqs: Iterable['InstallRequirement',], + parallel_builds: bool = False, + ) -> None: ... + class kwargs_t: class mode_t(enum.StrEnum): copy_paste = "copy_paste" @@ -381,7 +390,13 @@ def pip_resolve( ) raise NotImplementedError - _complete_partial_requirements_def = pip._internal.operations.prepare.RequirementPreparer._complete_partial_requirements + _complete_partial_requirements_def = cast( + pip_resolve_t.complete_partial_requirements_t, + getattr( + pip._internal.operations.prepare.RequirementPreparer, + '_complete_partial_requirements', + ) + ) def _complete_partial_requirements( _self: pip._internal.resolution.resolvelib.resolver.Resolver, From 5491f92bbea78c759e7f6486897dd5c591e7c6fc Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Tue, 20 May 2025 10:46:04 +0300 Subject: [PATCH 47/49] [+] improve typing --- python/online/fxreader/pr34/commands_typed/pip.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/pip.py b/python/online/fxreader/pr34/commands_typed/pip.py index bf90c5c..9953aa7 100644 --- a/python/online/fxreader/pr34/commands_typed/pip.py +++ b/python/online/fxreader/pr34/commands_typed/pip.py @@ -273,6 +273,8 @@ def pip_resolve( from pip._internal.utils.hashes import Hashes from pip._internal.req.req_install import InstallRequirement + from . import cli_bootstrap + downloader_call_def = pip._internal.network.download.Downloader.__call__ def downloader_call( @@ -482,7 +484,7 @@ def pip_resolve( # ] # for o in result_requirements # ], []) - logger.warn(result_requirements) + logger.warning(result_requirements) res = pip_resolve_t.res_t() @@ -491,7 +493,15 @@ def pip_resolve( for o in result_requirements: assert isinstance(o, InstallRequirement) - sha256_hashes = o.hashes()._allowed["sha256"] + sha256_hashes = cli_bootstrap.check_list( + cli_bootstrap.check_dict( + getattr(o.hashes(), '_allowed'), + str, + list[str], + )['sha256'], + str, + ) + assert len(sha256_hashes) == 1 assert not o.link is None From cf9ede1dde97962fa720db299ee31ee5dcb37096 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Tue, 20 May 2025 10:56:48 +0300 Subject: [PATCH 48/49] [+] fix ruff issues --- .../fxreader/pr34/commands_typed/cli.py | 2 +- .../online/fxreader/pr34/tasks/mlb_player.py | 30 +++++++++---------- python/pyproject.toml | 8 ++++- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index 0e8c342..638134a 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -263,7 +263,7 @@ class CLI(abc.ABC): subprocess.check_call([ self.dist_settings.python_path, '-m', 'pip', - 'install', + 'install', latest_file['path'], ]) diff --git a/python/online/fxreader/pr34/tasks/mlb_player.py b/python/online/fxreader/pr34/tasks/mlb_player.py index ff28e08..38780c9 100644 --- a/python/online/fxreader/pr34/tasks/mlb_player.py +++ b/python/online/fxreader/pr34/tasks/mlb_player.py @@ -5,12 +5,9 @@ import cv2 import re import json import io -import glob import xarray import numpy -import json import glob -import io import os import pandas import pickle @@ -360,7 +357,7 @@ def kernel_7( import torch.optim as optim import torchvision.transforms as transforms import torchvision.datasets as datasets - import torchvision.models as models + import torchvision.models from torch.autograd import Variable @@ -1731,7 +1728,10 @@ def kernel_28( --outdir %s ''' % (t13, t2) if False: - pprint.pprint([t4, t2, t6]) + pprint.pprint([ + # t4, + t2, t6, + ]) with subprocess.Popen(t6, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: if False: pprint.pprint(p.communicate()) @@ -1794,7 +1794,7 @@ def kernel_29(video_path=None, video_id=None,): o['data'] for o in t1 ], []) - t10 = re.compile('frame-(\d+)\.jpg') + t10 = re.compile(r'frame-(\d+)\.jpg') for i, o in enumerate(t8): o['frame_id'] = int(t10.match(o['image_id'])[1]) @@ -2035,7 +2035,7 @@ def kernel_30( def kernel_31(image_id, image_size, keypoints): def get_angle(a,b): - from math import sqrt, acos, degrees, atan, degrees + from math import sqrt, acos, degrees, atan #print(a) #print(b) del_y = a[1]-b[1] @@ -2054,7 +2054,7 @@ def kernel_31(image_id, image_size, keypoints): def angle_gor(a,b,c,d): import numpy as np - from math import sqrt, acos, degrees, atan, degrees + from math import sqrt, acos, degrees, atan ab=[a[0]-b[0],a[1]-b[1]] ab1=[c[0]-d[0],c[1]-d[1]] cos = \ @@ -2075,14 +2075,14 @@ def kernel_31(image_id, image_size, keypoints): def sit_ang(a,b,c,d): ang=angle_gor(a,b,c,d) s1=0 - if ang != None: + if not ang is None: #print("Angle",ang) if ang < 120 and ang>40: s1=1 return s1 def sit_rec(a,b,c,d): - from math import sqrt, acos, degrees, atan, degrees + from math import sqrt, acos, degrees, atan ab = [a[0] - b[0], a[1] - b[1]] ab1 = [c[0] - d[0], c[1] - d[1]] @@ -2294,7 +2294,7 @@ def kernel_36(): """ - import os + # import os from os.path import exists, join, basename, splitext git_repo_url = 'https://github.com/CMU-Perceptual-Computing-Lab/openpose.git' @@ -2318,8 +2318,8 @@ def kernel_36(): """## From a Google Drive's folder""" - import os - from os.path import exists, join, basename, splitext + # import os + # from os.path import exists, join, basename, splitext folder_path = '/content/drive/My Drive/openpose/' files = os.listdir(folder_path) @@ -2354,8 +2354,8 @@ def kernel_36(): # video_folder = os.path.dirname(colab_video_path) # video_base_name = os.path.basename(colab_video_path) # print(video_base_name) - import os - from os.path import exists, join, basename, splitext + # import os + # from os.path import exists, join, basename, splitext #colab_video_path = '/content/drive/My Drive/bachata.mp4' colab_video_path = '/content/output.mp4' colab_openpose_video_path = colab_video_path.replace('.mp4', '') + '-openpose.mp4' diff --git a/python/pyproject.toml b/python/pyproject.toml index f6e1df0..02f4682 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -88,6 +88,8 @@ ignore = [ 'F841', # fix * imports 'F403', + # don't care about trailing new lines + 'W292', ] select = ['E', 'F', 'I', 'W', 'INT'] @@ -100,7 +102,11 @@ detect-same-package = true # known-third-party = ["odf", "qt", "templite", "tinycss", "css_selectors"] relative-imports-order = "closest-to-furthest" split-on-trailing-comma = true -section-order = ['__python__', "future", "standard-library", "third-party", "first-party", "local-folder"] +section-order = [ + # '__python__', + "future", + "standard-library", "third-party", "first-party", "local-folder" +] force-wrap-aliases = true # [tool.ruff.lint.isort.sections] From 64a898ce44b7bdf274f39b7685e5001dad203050 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Tue, 20 May 2025 11:03:00 +0300 Subject: [PATCH 49/49] [+] reformat with ruff --- d1/jigsaw_toxic.py | 694 -- d1/mlb_player.py | 757 -- d1/mlb_player_v2.py | 1399 ---- d1/mlb_player_v3.py | 168 - d1/mlb_player_v4.py | 576 -- d1/mlb_player_v5.py | 56 - d1/mlb_player_v6.py | 172 - python/_m.py | 390 +- python/cli.py | 306 +- python/m.py | 529 +- python/online/fxreader/pr34/commands.py | 6965 ++++++++--------- .../fxreader/pr34/commands_typed/argparse.py | 31 +- .../fxreader/pr34/commands_typed/asyncio.py | 23 +- .../fxreader/pr34/commands_typed/cli.py | 1457 ++-- .../pr34/commands_typed/cli_bootstrap.py | 797 +- .../fxreader/pr34/commands_typed/crypto.py | 151 +- .../fxreader/pr34/commands_typed/debug.py | 52 +- .../fxreader/pr34/commands_typed/logging.py | 22 +- .../fxreader/pr34/commands_typed/mypy.py | 340 +- .../online/fxreader/pr34/commands_typed/os.py | 165 +- .../fxreader/pr34/commands_typed/pip.py | 990 ++- .../fxreader/pr34/commands_typed/typing.py | 27 +- python/online/fxreader/pr34/tasks/ble.py | 170 +- python/online/fxreader/pr34/tasks/cython.py | 307 +- python/online/fxreader/pr34/tasks/cython2.py | 10 +- .../fxreader/pr34/tasks/jigsaw_toxic.py | 590 +- .../online/fxreader/pr34/tasks/mlb_player.py | 4087 +++++----- .../online/fxreader/pr34/tests/test_crypto.py | 52 +- 28 files changed, 8683 insertions(+), 12600 deletions(-) delete mode 100644 d1/jigsaw_toxic.py delete mode 100644 d1/mlb_player.py delete mode 100644 d1/mlb_player_v2.py delete mode 100644 d1/mlb_player_v3.py delete mode 100644 d1/mlb_player_v4.py delete mode 100644 d1/mlb_player_v5.py delete mode 100644 d1/mlb_player_v6.py diff --git a/d1/jigsaw_toxic.py b/d1/jigsaw_toxic.py deleted file mode 100644 index 41a0287..0000000 --- a/d1/jigsaw_toxic.py +++ /dev/null @@ -1,694 +0,0 @@ -# %% [markdown] -# # About this Notebook -# -# NLP is a very hot topic right now and as belived by many experts '2020 is going to be NLP's Year' ,with its ever changing dynamics it is experiencing a boom , same as computer vision once did. Owing to its popularity Kaggle launched two NLP competitions recently and me being a lover of this Hot topic prepared myself to join in my first Kaggle Competition.

-# As I joined the competitions and since I was a complete beginner with Deep Learning Techniques for NLP, all my enthusiasm took a beating when I saw everyone Using all kinds of BERT , everything just went over my head,I thought to quit but there is a special thing about Kaggle ,it just hooks you. I thought I have to learn someday , why not now , so I braced myself and sat on the learning curve. I wrote a kernel on the Tweet Sentiment Extraction competition that has now got a gold medal , it can be viewed here : https://www.kaggle.com/tanulsingh077/twitter-sentiment-extaction-analysis-eda-and-model

-# After 10 days of extensive learning(finishing all the latest NLP approaches) , I am back here to share my leaning , by writing a kernel that starts from the very Basic RNN's to built over , all the way to BERT . I invite you all to come and learn alongside with me and take a step closer towards becoming an NLP expert - -# %% [markdown] -# # Contents -# -# In this Notebook I will start with the very Basics of RNN's and Build all the way to latest deep learning architectures to solve NLP problems. It will cover the Following: -# * Simple RNN's -# * Word Embeddings : Definition and How to get them -# * LSTM's -# * GRU's -# * BI-Directional RNN's -# * Encoder-Decoder Models (Seq2Seq Models) -# * Attention Models -# * Transformers - Attention is all you need -# * BERT -# -# I will divide every Topic into four subsections: -# * Basic Overview -# * In-Depth Understanding : In this I will attach links of articles and videos to learn about the topic in depth -# * Code-Implementation -# * Code Explanation -# -# This is a comprehensive kernel and if you follow along till the end , I promise you would learn all the techniques completely -# -# Note that the aim of this notebook is not to have a High LB score but to present a beginner guide to understand Deep Learning techniques used for NLP. Also after discussing all of these ideas , I will present a starter solution for this competiton - -# %% [markdown] -# **This kernel has been a work of more than 10 days If you find my kernel useful and my efforts appreciable, Please Upvote it , it motivates me to write more Quality content** - -# %% [code] -import numpy as np # linear algebra -import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) -from tqdm import tqdm -from sklearn.model_selection import train_test_split -import tensorflow as tf -from keras.models import Sequential -from keras.layers.recurrent import LSTM, GRU,SimpleRNN -from keras.layers.core import Dense, Activation, Dropout -from keras.layers.embeddings import Embedding -from keras.layers.normalization import BatchNormalization -from keras.utils import np_utils -from sklearn import preprocessing, decomposition, model_selection, metrics, pipeline -from keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D -from keras.preprocessing import sequence, text -from keras.callbacks import EarlyStopping - - -import matplotlib.pyplot as plt -import seaborn as sns -#%matplotlib inline -from plotly import graph_objs as go -import plotly.express as px -import plotly.figure_factory as ff - -# %% [markdown] -# # Configuring TPU's -# -# For this version of Notebook we will be using TPU's as we have to built a BERT Model - -# %% [code] -# Detect hardware, return appropriate distribution strategy -try: - # TPU detection. No parameters necessary if TPU_NAME environment variable is - # set: this is always the case on Kaggle. - tpu = tf.distribute.cluster_resolver.TPUClusterResolver() - print('Running on TPU ', tpu.master()) -except ValueError: - tpu = None - -if tpu: - tf.config.experimental_connect_to_cluster(tpu) - tf.tpu.experimental.initialize_tpu_system(tpu) - strategy = tf.distribute.experimental.TPUStrategy(tpu) -else: - # Default distribution strategy in Tensorflow. Works on CPU and single GPU. - strategy = tf.distribute.get_strategy() - -print("REPLICAS: ", strategy.num_replicas_in_sync) - -# %% [code] -train = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv') -validation = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv') -test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv') - -# %% [markdown] -# We will drop the other columns and approach this problem as a Binary Classification Problem and also we will have our exercise done on a smaller subsection of the dataset(only 12000 data points) to make it easier to train the models - -# %% [code] -train.drop(['severe_toxic','obscene','threat','insult','identity_hate'],axis=1,inplace=True) - -# %% [code] -train = train.loc[:12000,:] -train.shape - -# %% [markdown] -# We will check the maximum number of words that can be present in a comment , this will help us in padding later - -# %% [code] -train['comment_text'].apply(lambda x:len(str(x).split())).max() - -# %% [markdown] -# Writing a function for getting auc score for validation - -# %% [code] -def roc_auc(predictions,target): - ''' - This methods returns the AUC Score when given the Predictions - and Labels - ''' - - fpr, tpr, thresholds = metrics.roc_curve(target, predictions) - roc_auc = metrics.auc(fpr, tpr) - return roc_auc - -# %% [markdown] -# ### Data Preparation - -# %% [code] -xtrain, xvalid, ytrain, yvalid = train_test_split(train.comment_text.values, train.toxic.values, - stratify=train.toxic.values, - random_state=42, - test_size=0.2, shuffle=True) - -# %% [markdown] -# # Before We Begin -# -# Before we Begin If you are a complete starter with NLP and never worked with text data, I am attaching a few kernels that will serve as a starting point of your journey -# * https://www.kaggle.com/arthurtok/spooky-nlp-and-topic-modelling-tutorial -# * https://www.kaggle.com/abhishek/approaching-almost-any-nlp-problem-on-kaggle -# -# If you want a more basic dataset to practice with here is another kernel which I wrote: -# * https://www.kaggle.com/tanulsingh077/what-s-cooking -# -# Below are some Resources to get started with basic level Neural Networks, It will help us to easily understand the upcoming parts -# * https://www.youtube.com/watch?v=aircAruvnKk&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv -# * https://www.youtube.com/watch?v=IHZwWFHWa-w&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=2 -# * https://www.youtube.com/watch?v=Ilg3gGewQ5U&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=3 -# * https://www.youtube.com/watch?v=tIeHLnjs5U8&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=4 -# -# For Learning how to visualize test data and what to use view: -# * https://www.kaggle.com/tanulsingh077/twitter-sentiment-extaction-analysis-eda-and-model -# * https://www.kaggle.com/jagangupta/stop-the-s-toxic-comments-eda - -# %% [markdown] -# # Simple RNN -# -# ## Basic Overview -# -# What is a RNN? -# -# Recurrent Neural Network(RNN) are a type of Neural Network where the output from previous step are fed as input to the current step. In traditional neural networks, all the inputs and outputs are independent of each other, but in cases like when it is required to predict the next word of a sentence, the previous words are required and hence there is a need to remember the previous words. Thus RNN came into existence, which solved this issue with the help of a Hidden Layer. -# -# Why RNN's? -# -# https://www.quora.com/Why-do-we-use-an-RNN-instead-of-a-simple-neural-network -# -# ## In-Depth Understanding -# -# * https://medium.com/mindorks/understanding-the-recurrent-neural-network-44d593f112a2 -# * https://www.youtube.com/watch?v=2E65LDnM2cA&list=PL1F3ABbhcqa3BBWo170U4Ev2wfsF7FN8l -# * https://www.d2l.ai/chapter_recurrent-neural-networks/rnn.html -# -# ## Code Implementation -# -# So first I will implement the and then I will explain the code step by step - -# %% [code] -# using keras tokenizer here -token = text.Tokenizer(num_words=None) -max_len = 1500 - -token.fit_on_texts(list(xtrain) + list(xvalid)) -xtrain_seq = token.texts_to_sequences(xtrain) -xvalid_seq = token.texts_to_sequences(xvalid) - -#zero pad the sequences -xtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len) -xvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len) - -word_index = token.word_index - -# %% [code] -#%%time -with strategy.scope(): - # A simpleRNN without any pretrained embeddings and one dense layer - model = Sequential() - model.add(Embedding(len(word_index) + 1, - 300, - input_length=max_len)) - model.add(SimpleRNN(100)) - model.add(Dense(1, activation='sigmoid')) - model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) - -model.summary() - -# %% [code] -model.fit(xtrain_pad, ytrain, nb_epoch=5, batch_size=64*strategy.num_replicas_in_sync) #Multiplying by Strategy to run on TPU's - -# %% [code] -scores = model.predict(xvalid_pad) -print("Auc: %.2f%%" % (roc_auc(scores,yvalid))) - -# %% [code] -scores_model = [] -scores_model.append({'Model': 'SimpleRNN','AUC_Score': roc_auc(scores,yvalid)}) - -# %% [markdown] -# ## Code Explanantion -# * Tokenization

-# So if you have watched the videos and referred to the links, you would know that in an RNN we input a sentence word by word. We represent every word as one hot vectors of dimensions : Numbers of words in Vocab +1.
-# What keras Tokenizer does is , it takes all the unique words in the corpus,forms a dictionary with words as keys and their number of occurences as values,it then sorts the dictionary in descending order of counts. It then assigns the first value 1 , second value 2 and so on. So let's suppose word 'the' occured the most in the corpus then it will assigned index 1 and vector representing 'the' would be a one-hot vector with value 1 at position 1 and rest zereos.
-# Try printing first 2 elements of xtrain_seq you will see every word is represented as a digit now - -# %% [code] -xtrain_seq[:1] - -# %% [markdown] -# Now you might be wondering What is padding? Why its done

-# -# Here is the answer : -# * https://www.quora.com/Which-effect-does-sequence-padding-have-on-the-training-of-a-neural-network -# * https://machinelearningmastery.com/data-preparation-variable-length-input-sequences-sequence-prediction/ -# * https://www.coursera.org/lecture/natural-language-processing-tensorflow/padding-2Cyzs -# -# Also sometimes people might use special tokens while tokenizing like EOS(end of string) and BOS(Begining of string). Here is the reason why it's done -# * https://stackoverflow.com/questions/44579161/why-do-we-do-padding-in-nlp-tasks -# -# -# The code token.word_index simply gives the dictionary of vocab that keras created for us - -# %% [markdown] -# * Building the Neural Network -# -# To understand the Dimensions of input and output given to RNN in keras her is a beautiful article : https://medium.com/@shivajbd/understanding-input-and-output-shape-in-lstm-keras-c501ee95c65e -# -# The first line model.Sequential() tells keras that we will be building our network sequentially . Then we first add the Embedding layer. -# Embedding layer is also a layer of neurons which takes in as input the nth dimensional one hot vector of every word and converts it into 300 dimensional vector , it gives us word embeddings similar to word2vec. We could have used word2vec but the embeddings layer learns during training to enhance the embeddings. -# Next we add an 100 LSTM units without any dropout or regularization -# At last we add a single neuron with sigmoid function which takes output from 100 LSTM cells (Please note we have 100 LSTM cells not layers) to predict the results and then we compile the model using adam optimizer -# -# * Comments on the model

-# We can see our model achieves an accuracy of 1 which is just insane , we are clearly overfitting I know , but this was the simplest model of all ,we can tune a lot of hyperparameters like RNN units, we can do batch normalization , dropouts etc to get better result. The point is we got an AUC score of 0.82 without much efforts and we know have learnt about RNN's .Deep learning is really revolutionary - -# %% [markdown] -# # Word Embeddings -# -# While building our simple RNN models we talked about using word-embeddings , So what is word-embeddings and how do we get word-embeddings? -# Here is the answer : -# * https://www.coursera.org/learn/nlp-sequence-models/lecture/6Oq70/word-representation -# * https://machinelearningmastery.com/what-are-word-embeddings/ -#

-# The latest approach to getting word Embeddings is using pretained GLoVe or using Fasttext. Without going into too much details, I would explain how to create sentence vectors and how can we use them to create a machine learning model on top of it and since I am a fan of GloVe vectors, word2vec and fasttext. In this Notebook, I'll be using the GloVe vectors. You can download the GloVe vectors from here http://www-nlp.stanford.edu/data/glove.840B.300d.zip or you can search for GloVe in datasets on Kaggle and add the file - -# %% [code] -# load the GloVe vectors in a dictionary: - -embeddings_index = {} -f = open('/kaggle/input/glove840b300dtxt/glove.840B.300d.txt','r',encoding='utf-8') -for line in tqdm(f): - values = line.split(' ') - word = values[0] - coefs = np.asarray([float(val) for val in values[1:]]) - embeddings_index[word] = coefs -f.close() - -print('Found %s word vectors.' % len(embeddings_index)) - -# %% [markdown] -# # LSTM's -# -# ## Basic Overview -# -# Simple RNN's were certainly better than classical ML algorithms and gave state of the art results, but it failed to capture long term dependencies that is present in sentences . So in 1998-99 LSTM's were introduced to counter to these drawbacks. -# -# ## In Depth Understanding -# -# Why LSTM's? -# * https://www.coursera.org/learn/nlp-sequence-models/lecture/PKMRR/vanishing-gradients-with-rnns -# * https://www.analyticsvidhya.com/blog/2017/12/fundamentals-of-deep-learning-introduction-to-lstm/ -# -# What are LSTM's? -# * https://www.coursera.org/learn/nlp-sequence-models/lecture/KXoay/long-short-term-memory-lstm -# * https://distill.pub/2019/memorization-in-rnns/ -# * https://towardsdatascience.com/illustrated-guide-to-lstms-and-gru-s-a-step-by-step-explanation-44e9eb85bf21 -# -# # Code Implementation -# -# We have already tokenized and paded our text for input to LSTM's - -# %% [code] -# create an embedding matrix for the words we have in the dataset -embedding_matrix = np.zeros((len(word_index) + 1, 300)) -for word, i in tqdm(word_index.items()): - embedding_vector = embeddings_index.get(word) - if embedding_vector is not None: - embedding_matrix[i] = embedding_vector - -# %% [code] -#%%time -with strategy.scope(): - - # A simple LSTM with glove embeddings and one dense layer - model = Sequential() - model.add(Embedding(len(word_index) + 1, - 300, - weights=[embedding_matrix], - input_length=max_len, - trainable=False)) - - model.add(LSTM(100, dropout=0.3, recurrent_dropout=0.3)) - model.add(Dense(1, activation='sigmoid')) - model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy']) - -model.summary() - -# %% [code] -model.fit(xtrain_pad, ytrain, nb_epoch=5, batch_size=64*strategy.num_replicas_in_sync) - -# %% [code] -scores = model.predict(xvalid_pad) -print("Auc: %.2f%%" % (roc_auc(scores,yvalid))) - -# %% [code] -scores_model.append({'Model': 'LSTM','AUC_Score': roc_auc(scores,yvalid)}) - -# %% [markdown] -# ## Code Explanation -# -# As a first step we calculate embedding matrix for our vocabulary from the pretrained GLoVe vectors . Then while building the embedding layer we pass Embedding Matrix as weights to the layer instead of training it over Vocabulary and thus we pass trainable = False. -# Rest of the model is same as before except we have replaced the SimpleRNN By LSTM Units -# -# * Comments on the Model -# -# We now see that the model is not overfitting and achieves an auc score of 0.96 which is quite commendable , also we close in on the gap between accuracy and auc . -# We see that in this case we used dropout and prevented overfitting the data - -# %% [markdown] -# # GRU's -# -# ## Basic Overview -# -# Introduced by Cho, et al. in 2014, GRU (Gated Recurrent Unit) aims to solve the vanishing gradient problem which comes with a standard recurrent neural network. GRU's are a variation on the LSTM because both are designed similarly and, in some cases, produce equally excellent results . GRU's were designed to be simpler and faster than LSTM's and in most cases produce equally good results and thus there is no clear winner. -# -# ## In Depth Explanation -# -# * https://towardsdatascience.com/understanding-gru-networks-2ef37df6c9be -# * https://www.coursera.org/learn/nlp-sequence-models/lecture/agZiL/gated-recurrent-unit-gru -# * https://www.geeksforgeeks.org/gated-recurrent-unit-networks/ -# -# ## Code Implementation - -# %% [code] -#%%time -with strategy.scope(): - # GRU with glove embeddings and two dense layers - model = Sequential() - model.add(Embedding(len(word_index) + 1, - 300, - weights=[embedding_matrix], - input_length=max_len, - trainable=False)) - model.add(SpatialDropout1D(0.3)) - model.add(GRU(300)) - model.add(Dense(1, activation='sigmoid')) - - model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy']) - -model.summary() - -# %% [code] -model.fit(xtrain_pad, ytrain, nb_epoch=5, batch_size=64*strategy.num_replicas_in_sync) - -# %% [code] -scores = model.predict(xvalid_pad) -print("Auc: %.2f%%" % (roc_auc(scores,yvalid))) - -# %% [code] -scores_model.append({'Model': 'GRU','AUC_Score': roc_auc(scores,yvalid)}) - -# %% [code] -scores_model - -# %% [markdown] -# # Bi-Directional RNN's -# -# ## In Depth Explanation -# -# * https://www.coursera.org/learn/nlp-sequence-models/lecture/fyXnn/bidirectional-rnn -# * https://towardsdatascience.com/understanding-bidirectional-rnn-in-pytorch-5bd25a5dd66 -# * https://d2l.ai/chapter_recurrent-modern/bi-rnn.html -# -# ## Code Implementation - -# %% [code] -#%%time -with strategy.scope(): - # A simple bidirectional LSTM with glove embeddings and one dense layer - model = Sequential() - model.add(Embedding(len(word_index) + 1, - 300, - weights=[embedding_matrix], - input_length=max_len, - trainable=False)) - model.add(Bidirectional(LSTM(300, dropout=0.3, recurrent_dropout=0.3))) - - model.add(Dense(1,activation='sigmoid')) - model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy']) - - -model.summary() - -# %% [code] -model.fit(xtrain_pad, ytrain, nb_epoch=5, batch_size=64*strategy.num_replicas_in_sync) - -# %% [code] -scores = model.predict(xvalid_pad) -print("Auc: %.2f%%" % (roc_auc(scores,yvalid))) - -# %% [code] -scores_model.append({'Model': 'Bi-directional LSTM','AUC_Score': roc_auc(scores,yvalid)}) - -# %% [markdown] -# ## Code Explanation -# -# Code is same as before,only we have added bidirectional nature to the LSTM cells we used before and is self explanatory. We have achieve similar accuracy and auc score as before and now we have learned all the types of typical RNN architectures - -# %% [markdown] -# **We are now at the end of part 1 of this notebook and things are about to go wild now as we Enter more complex and State of the art models .If you have followed along from the starting and read all the articles and understood everything , these complex models would be fairly easy to understand.I recommend Finishing Part 1 before continuing as the upcoming techniques can be quite overwhelming** - -# %% [markdown] -# # Seq2Seq Model Architecture -# -# ## Overview -# -# RNN's are of many types and different architectures are used for different purposes. Here is a nice video explanining different types of model architectures : https://www.coursera.org/learn/nlp-sequence-models/lecture/BO8PS/different-types-of-rnns. -# Seq2Seq is a many to many RNN architecture where the input is a sequence and the output is also a sequence (where input and output sequences can be or cannot be of different lengths). This architecture is used in a lot of applications like Machine Translation, text summarization, question answering etc -# -# ## In Depth Understanding -# -# I will not write the code implementation for this,but rather I will provide the resources where code has already been implemented and explained in a much better way than I could have ever explained. -# -# * https://www.coursera.org/learn/nlp-sequence-models/lecture/HyEui/basic-models ---> A basic idea of different Seq2Seq Models -# -# * https://blog.keras.io/a-ten-minute-introduction-to-sequence-to-sequence-learning-in-keras.html , https://machinelearningmastery.com/define-encoder-decoder-sequence-sequence-model-neural-machine-translation-keras/ ---> Basic Encoder-Decoder Model and its explanation respectively -# -# * https://towardsdatascience.com/how-to-implement-seq2seq-lstm-model-in-keras-shortcutnlp-6f355f3e5639 ---> A More advanced Seq2seq Model and its explanation -# -# * https://d2l.ai/chapter_recurrent-modern/machine-translation-and-dataset.html , https://d2l.ai/chapter_recurrent-modern/encoder-decoder.html ---> Implementation of Encoder-Decoder Model from scratch -# -# * https://www.youtube.com/watch?v=IfsjMg4fLWQ&list=PLtmWHNX-gukKocXQOkQjuVxglSDYWsSh9&index=8&t=0s ---> Introduction to Seq2seq By fast.ai - -# %% [code] -# Visualization of Results obtained from various Deep learning models -results = pd.DataFrame(scores_model).sort_values(by='AUC_Score',ascending=False) -results.style.background_gradient(cmap='Blues') - -# %% [code] -fig = go.Figure(go.Funnelarea( - text =results.Model, - values = results.AUC_Score, - title = {"position": "top center", "text": "Funnel-Chart of Sentiment Distribution"} - )) -fig.show() - -# %% [markdown] -# # Attention Models -# -# This is the toughest and most tricky part. If you are able to understand the intiuition and working of attention block , understanding transformers and transformer based architectures like BERT will be a piece of cake. This is the part where I spent the most time on and I suggest you do the same . Please read and view the following resources in the order I am providing to ignore getting confused, also at the end of this try to write and draw an attention block in your own way :- -# -# * https://www.coursera.org/learn/nlp-sequence-models/lecture/RDXpX/attention-model-intuition --> Only watch this video and not the next one -# * https://towardsdatascience.com/sequence-2-sequence-model-with-attention-mechanism-9e9ca2a613a -# * https://towardsdatascience.com/attention-and-its-different-forms-7fc3674d14dc -# * https://distill.pub/2016/augmented-rnns/ -# -# ## Code Implementation -# -# * https://www.analyticsvidhya.com/blog/2019/11/comprehensive-guide-attention-mechanism-deep-learning/ --> Basic Level -# * https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html ---> Implementation from Scratch in Pytorch - -# %% [markdown] -# # Transformers : Attention is all you need -# -# So finally we have reached the end of the learning curve and are about to start learning the technology that changed NLP completely and are the reasons for the state of the art NLP techniques .Transformers were introduced in the paper Attention is all you need by Google. If you have understood the Attention models,this will be very easy , Here is transformers fully explained: -# -# * http://jalammar.github.io/illustrated-transformer/ -# -# ## Code Implementation -# -# * http://nlp.seas.harvard.edu/2018/04/03/attention.html ---> This presents the code implementation of the architecture presented in the paper by Google - -# %% [markdown] -# # BERT and Its Implementation on this Competition -# -# As Promised I am back with Resiurces , to understand about BERT architecture , please follow the contents in the given order :- -# -# * http://jalammar.github.io/illustrated-bert/ ---> In Depth Understanding of BERT -# -# After going through the post Above , I guess you must have understood how transformer architecture have been utilized by the current SOTA models . Now these architectures can be used in two ways :

-# 1) We can use the model for prediction on our problems using the pretrained weights without fine-tuning or training the model for our sepcific tasks -# * EG: http://jalammar.github.io/a-visual-guide-to-using-bert-for-the-first-time/ ---> Using Pre-trained BERT without Tuning -# -# 2) We can fine-tune or train these transformer models for our task by tweaking the already pre-trained weights and training on a much smaller dataset -# * EG:* https://www.youtube.com/watch?v=hinZO--TEk4&t=2933s ---> Tuning BERT For your TASK -# -# We will be using the first example as a base for our implementation of BERT model using Hugging Face and KERAS , but contrary to first example we will also Fine-Tune our model for our task -# -# Acknowledgements : https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras -# -# -# Steps Involved : -# * Data Preparation : Tokenization and encoding of data -# * Configuring TPU's -# * Building a Function for Model Training and adding an output layer for classification -# * Train the model and get the results - -# %% [code] -# Loading Dependencies -import os -import tensorflow as tf -from tensorflow.keras.layers import Dense, Input -from tensorflow.keras.optimizers import Adam -from tensorflow.keras.models import Model -from tensorflow.keras.callbacks import ModelCheckpoint -from kaggle_datasets import KaggleDatasets -import transformers - -from tokenizers import BertWordPieceTokenizer - -# %% [code] -# LOADING THE DATA - -train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv") -valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv') -test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv') -sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv') - -# %% [markdown] -# Encoder FOr DATA for understanding waht encode batch does read documentation of hugging face tokenizer : -# https://huggingface.co/transformers/main_classes/tokenizer.html here - -# %% [code] -def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512): - """ - Encoder for encoding the text into sequence of integers for BERT Input - """ - tokenizer.enable_truncation(max_length=maxlen) - tokenizer.enable_padding(max_length=maxlen) - all_ids = [] - - for i in tqdm(range(0, len(texts), chunk_size)): - text_chunk = texts[i:i+chunk_size].tolist() - encs = tokenizer.encode_batch(text_chunk) - all_ids.extend([enc.ids for enc in encs]) - - return np.array(all_ids) - -# %% [code] -#IMP DATA FOR CONFIG - -AUTO = tf.data.experimental.AUTOTUNE - - -# Configuration -EPOCHS = 3 -BATCH_SIZE = 16 * strategy.num_replicas_in_sync -MAX_LEN = 192 - -# %% [markdown] -# ## Tokenization -# -# For understanding please refer to hugging face documentation again - -# %% [code] -# First load the real tokenizer -tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased') -# Save the loaded tokenizer locally -tokenizer.save_pretrained('.') -# Reload it with the huggingface tokenizers library -fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False) -fast_tokenizer - -# %% [code] -x_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN) -x_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN) -x_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN) - -y_train = train1.toxic.values -y_valid = valid.toxic.values - -# %% [code] -train_dataset = ( - tf.data.Dataset - .from_tensor_slices((x_train, y_train)) - .repeat() - .shuffle(2048) - .batch(BATCH_SIZE) - .prefetch(AUTO) -) - -valid_dataset = ( - tf.data.Dataset - .from_tensor_slices((x_valid, y_valid)) - .batch(BATCH_SIZE) - .cache() - .prefetch(AUTO) -) - -test_dataset = ( - tf.data.Dataset - .from_tensor_slices(x_test) - .batch(BATCH_SIZE) -) - -# %% [code] -def build_model(transformer, max_len=512): - """ - function for training the BERT model - """ - input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids") - sequence_output = transformer(input_word_ids)[0] - cls_token = sequence_output[:, 0, :] - out = Dense(1, activation='sigmoid')(cls_token) - - model = Model(inputs=input_word_ids, outputs=out) - model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy']) - - return model - -# %% [markdown] -# ## Starting Training -# -# If you want to use any another model just replace the model name in transformers._____ and use accordingly - -# %% [code] -#%%time -with strategy.scope(): - transformer_layer = ( - transformers.TFDistilBertModel - .from_pretrained('distilbert-base-multilingual-cased') - ) - model = build_model(transformer_layer, max_len=MAX_LEN) -model.summary() - -# %% [code] -n_steps = x_train.shape[0] // BATCH_SIZE -train_history = model.fit( - train_dataset, - steps_per_epoch=n_steps, - validation_data=valid_dataset, - epochs=EPOCHS -) - -# %% [code] -n_steps = x_valid.shape[0] // BATCH_SIZE -train_history_2 = model.fit( - valid_dataset.repeat(), - steps_per_epoch=n_steps, - epochs=EPOCHS*2 -) - -# %% [code] -sub['toxic'] = model.predict(test_dataset, verbose=1) -sub.to_csv('submission.csv', index=False) - -# %% [markdown] -# # End Notes -# -# This was my effort to share my learnings so that everyone can benifit from it.As this community has been very kind to me and helped me in learning all of this , I want to take this forward. I have shared all the resources I used to learn all the stuff .Join me and make these NLP competitions your first ,without being overwhelmed by the shear number of techniques used . It took me 10 days to learn all of this , you can learn it at your pace and dont give in , at the end of all this you will be a different person and it will all be worth it. -# -# -# ### I am attaching more resources if you want NLP end to end: -# -# 1) Books -# -# * https://d2l.ai/ -# * Jason Brownlee's Books -# -# 2) Courses -# -# * https://www.coursera.org/learn/nlp-sequence-models/home/welcome -# * Fast.ai NLP Course -# -# 3) Blogs and websites -# -# * Machine Learning Mastery -# * https://distill.pub/ -# * http://jalammar.github.io/ -# -# **This is subtle effort of contributing towards the community, if it helped you in any way please show a token of love by upvoting** diff --git a/d1/mlb_player.py b/d1/mlb_player.py deleted file mode 100644 index 0d8d28b..0000000 --- a/d1/mlb_player.py +++ /dev/null @@ -1,757 +0,0 @@ -# %% [markdown] -#
-#

MLB Player Digital Engagement Forecasting

-#

LightGBM + CatBoost + ANN 2505f2

-#
- -# %% [markdown] -#
-#
- -# %% [markdown] -#
-#

If you find this work useful, please don't forget upvoting :)

-#
- -# %% [markdown] -# #### Thanks to: @lhagiimn https://www.kaggle.com/lhagiimn/lightgbm-catboost-ann-2505f2 -# -# #### https://www.kaggle.com/columbia2131/mlb-lightgbm-starter-dataset-code-en-ja -# -# #### https://www.kaggle.com/mlconsult/1-3816-lb-lbgm-descriptive-stats-param-tune -# -# #### https://www.kaggle.com/batprem/lightgbm-ann-weight-with-love -# -# #### https://www.kaggle.com/mlconsult/1-3816-lb-lbgm-descriptive-stats-param-tune -# -# #### https://www.kaggle.com/ulrich07/mlb-ann-with-lags-tf-keras -# - -# %% [markdown] -#
-#
- -# %% [markdown] -# ## About Dataset - -# %% [markdown] -# Train.csv is stored as a csv file with each column as follows. -# -# train.csvを以下のようにして各カラムをcsvファイルとして保管しています。 - -# %% [code] {"execution":{"iopub.status.busy":"2021-06-26T07:16:47.242749Z","iopub.execute_input":"2021-06-26T07:16:47.243324Z","iopub.status.idle":"2021-06-26T07:16:48.030215Z","shell.execute_reply.started":"2021-06-26T07:16:47.243266Z","shell.execute_reply":"2021-06-26T07:16:48.029Z"}} -import os - -assert os.system(r'''cp ../input/fork-of-1-35-lightgbm-ann-2505f2-c4e96a/* .''') == 0 - -# %% [code] {"execution":{"iopub.status.busy":"2021-06-26T07:16:48.031858Z","iopub.execute_input":"2021-06-26T07:16:48.032396Z","iopub.status.idle":"2021-06-26T07:16:48.799514Z","shell.execute_reply.started":"2021-06-26T07:16:48.032357Z","shell.execute_reply":"2021-06-26T07:16:48.798628Z"}} -assert os.system(r'''ls''') == 0 - -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:16:48.801992Z","iopub.execute_input":"2021-06-26T07:16:48.802645Z","iopub.status.idle":"2021-06-26T07:16:48.813801Z","shell.execute_reply.started":"2021-06-26T07:16:48.802592Z","shell.execute_reply":"2021-06-26T07:16:48.812863Z"}} -#%%capture - -""" -!pip install pandarallel - -import gc - -import numpy as np -import pandas as pd -from pathlib import Path - -from pandarallel import pandarallel -pandarallel.initialize() - -BASE_DIR = Path('../input/mlb-player-digital-engagement-forecasting') -train = pd.read_csv(BASE_DIR / 'train.csv') - -null = np.nan -true = True -false = False - -for col in train.columns: - - if col == 'date': continue - - _index = train[col].notnull() - train.loc[_index, col] = train.loc[_index, col].parallel_apply(lambda x: eval(x)) - - outputs = [] - for index, date, record in train.loc[_index, ['date', col]].itertuples(): - _df = pd.DataFrame(record) - _df['index'] = index - _df['date'] = date - outputs.append(_df) - - outputs = pd.concat(outputs).reset_index(drop=True) - - outputs.to_csv(f'{col}_train.csv', index=False) - outputs.to_pickle(f'{col}_train.pkl') - - del outputs - del train[col] - gc.collect() -""" - -# %% [markdown] {"execution":{"iopub.status.busy":"2021-06-16T09:14:33.869464Z","iopub.execute_input":"2021-06-16T09:14:33.869905Z","iopub.status.idle":"2021-06-16T09:14:33.874766Z","shell.execute_reply.started":"2021-06-16T09:14:33.869879Z","shell.execute_reply":"2021-06-16T09:14:33.873097Z"}} -# ## Training - -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:16:48.81564Z","iopub.execute_input":"2021-06-26T07:16:48.816326Z","iopub.status.idle":"2021-06-26T07:16:50.081995Z","shell.execute_reply.started":"2021-06-26T07:16:48.816246Z","shell.execute_reply":"2021-06-26T07:16:50.080828Z"}} -import numpy as np -import pandas as pd -from pathlib import Path -from sklearn.metrics import mean_absolute_error -from datetime import timedelta -from functools import reduce -from tqdm import tqdm -import lightgbm as lgbm -import mlb -import os - -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:16:50.083534Z","iopub.execute_input":"2021-06-26T07:16:50.083899Z","iopub.status.idle":"2021-06-26T07:16:50.088159Z","shell.execute_reply.started":"2021-06-26T07:16:50.083863Z","shell.execute_reply":"2021-06-26T07:16:50.087357Z"}} -BASE_DIR = Path('../input/mlb-player-digital-engagement-forecasting') -TRAIN_DIR = Path('../input/mlb-pdef-train-dataset') - -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:16:50.08951Z","iopub.execute_input":"2021-06-26T07:16:50.090053Z","iopub.status.idle":"2021-06-26T07:16:54.221868Z","shell.execute_reply.started":"2021-06-26T07:16:50.090018Z","shell.execute_reply":"2021-06-26T07:16:54.220656Z"}} -players = pd.read_csv(BASE_DIR / 'players.csv') - -rosters = pd.read_pickle(TRAIN_DIR / 'rosters_train.pkl') -targets = pd.read_pickle(TRAIN_DIR / 'nextDayPlayerEngagement_train.pkl') -scores = pd.read_pickle(TRAIN_DIR / 'playerBoxScores_train.pkl') -scores = scores.groupby(['playerId', 'date']).sum().reset_index() - -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:16:54.223547Z","iopub.execute_input":"2021-06-26T07:16:54.224Z","iopub.status.idle":"2021-06-26T07:16:54.243132Z","shell.execute_reply.started":"2021-06-26T07:16:54.22395Z","shell.execute_reply":"2021-06-26T07:16:54.242076Z"}} -targets_cols = ['playerId', 'target1', 'target2', 'target3', 'target4', 'date'] -players_cols = ['playerId', 'primaryPositionName'] -rosters_cols = ['playerId', 'teamId', 'status', 'date'] -scores_cols = ['playerId', 'battingOrder', 'gamesPlayedBatting', 'flyOuts', - 'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns', - 'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch', - 'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay', - 'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi', - 'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference', - 'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching', - 'completeGamesPitching', 'shutoutsPitching', 'winsPitching', - 'lossesPitching', 'flyOutsPitching', 'airOutsPitching', - 'groundOutsPitching', 'runsPitching', 'doublesPitching', - 'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching', - 'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching', - 'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching', - 'stolenBasesPitching', 'inningsPitched', 'saveOpportunities', - 'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls', - 'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching', - 'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners', - 'inheritedRunnersScored', 'catchersInterferencePitching', - 'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves', - 'assists', 'putOuts', 'errors', 'chances', 'date'] - -feature_cols = ['label_playerId', 'label_primaryPositionName', 'label_teamId', - 'label_status', 'battingOrder', 'gamesPlayedBatting', 'flyOuts', - 'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns', - 'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch', - 'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay', - 'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi', - 'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference', - 'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching', - 'completeGamesPitching', 'shutoutsPitching', 'winsPitching', - 'lossesPitching', 'flyOutsPitching', 'airOutsPitching', - 'groundOutsPitching', 'runsPitching', 'doublesPitching', - 'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching', - 'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching', - 'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching', - 'stolenBasesPitching', 'inningsPitched', 'saveOpportunities', - 'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls', - 'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching', - 'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners', - 'inheritedRunnersScored', 'catchersInterferencePitching', - 'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves', - 'assists', 'putOuts', 'errors', 'chances','target1_mean', - 'target1_median', - 'target1_std', - 'target1_min', - 'target1_max', - 'target1_prob', - 'target2_mean', - 'target2_median', - 'target2_std', - 'target2_min', - 'target2_max', - 'target2_prob', - 'target3_mean', - 'target3_median', - 'target3_std', - 'target3_min', - 'target3_max', - 'target3_prob', - 'target4_mean', - 'target4_median', - 'target4_std', - 'target4_min', - 'target4_max', - 'target4_prob'] -feature_cols2 = ['label_playerId', 'label_primaryPositionName', 'label_teamId', - 'label_status', 'battingOrder', 'gamesPlayedBatting', 'flyOuts', - 'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns', - 'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch', - 'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay', - 'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi', - 'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference', - 'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching', - 'completeGamesPitching', 'shutoutsPitching', 'winsPitching', - 'lossesPitching', 'flyOutsPitching', 'airOutsPitching', - 'groundOutsPitching', 'runsPitching', 'doublesPitching', - 'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching', - 'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching', - 'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching', - 'stolenBasesPitching', 'inningsPitched', 'saveOpportunities', - 'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls', - 'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching', - 'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners', - 'inheritedRunnersScored', 'catchersInterferencePitching', - 'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves', - 'assists', 'putOuts', 'errors', 'chances','target1_mean', - 'target1_median', - 'target1_std', - 'target1_min', - 'target1_max', - 'target1_prob', - 'target2_mean', - 'target2_median', - 'target2_std', - 'target2_min', - 'target2_max', - 'target2_prob', - 'target3_mean', - 'target3_median', - 'target3_std', - 'target3_min', - 'target3_max', - 'target3_prob', - 'target4_mean', - 'target4_median', - 'target4_std', - 'target4_min', - 'target4_max', - 'target4_prob', - 'target1'] - -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:16:54.244866Z","iopub.execute_input":"2021-06-26T07:16:54.24532Z","iopub.status.idle":"2021-06-26T07:16:54.296844Z","shell.execute_reply.started":"2021-06-26T07:16:54.245257Z","shell.execute_reply":"2021-06-26T07:16:54.295689Z"}} -player_target_stats = pd.read_csv("../input/player-target-stats/player_target_stats.csv") -data_names=player_target_stats.columns.values.tolist() -data_names - -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:16:54.300157Z","iopub.execute_input":"2021-06-26T07:16:54.300622Z","iopub.status.idle":"2021-06-26T07:17:02.252208Z","shell.execute_reply.started":"2021-06-26T07:16:54.300578Z","shell.execute_reply":"2021-06-26T07:17:02.250423Z"}} -# creat dataset -train = targets[targets_cols].merge(players[players_cols], on=['playerId'], how='left') -train = train.merge(rosters[rosters_cols], on=['playerId', 'date'], how='left') -train = train.merge(scores[scores_cols], on=['playerId', 'date'], how='left') -train = train.merge(player_target_stats, how='inner', left_on=["playerId"],right_on=["playerId"]) - - -# label encoding -player2num = {c: i for i, c in enumerate(train['playerId'].unique())} -position2num = {c: i for i, c in enumerate(train['primaryPositionName'].unique())} -teamid2num = {c: i for i, c in enumerate(train['teamId'].unique())} -status2num = {c: i for i, c in enumerate(train['status'].unique())} -train['label_playerId'] = train['playerId'].map(player2num) -train['label_primaryPositionName'] = train['primaryPositionName'].map(position2num) -train['label_teamId'] = train['teamId'].map(teamid2num) -train['label_status'] = train['status'].map(status2num) - -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:17:02.253453Z","iopub.status.idle":"2021-06-26T07:17:02.254076Z"}} -train_X = train[feature_cols] -train_y = train[['target1', 'target2', 'target3', 'target4']] - -_index = (train['date'] < 20210401) -x_train1 = train_X.loc[_index].reset_index(drop=True) -y_train1 = train_y.loc[_index].reset_index(drop=True) -x_valid1 = train_X.loc[~_index].reset_index(drop=True) -y_valid1 = train_y.loc[~_index].reset_index(drop=True) - -# %% [code] {"execution":{"iopub.status.busy":"2021-06-26T07:17:02.255068Z","iopub.status.idle":"2021-06-26T07:17:02.255685Z"}} -train_X = train[feature_cols2] -train_y = train[['target1', 'target2', 'target3', 'target4']] - -_index = (train['date'] < 20210401) -x_train2 = train_X.loc[_index].reset_index(drop=True) -y_train2 = train_y.loc[_index].reset_index(drop=True) -x_valid2 = train_X.loc[~_index].reset_index(drop=True) -y_valid2 = train_y.loc[~_index].reset_index(drop=True) - -# %% [code] {"execution":{"iopub.status.busy":"2021-06-26T07:17:02.256629Z","iopub.status.idle":"2021-06-26T07:17:02.257215Z"}} -train_X - -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:17:02.258224Z","iopub.status.idle":"2021-06-26T07:17:02.258854Z"}} -def fit_lgbm(x_train, y_train, x_valid, y_valid, params: dict=None, verbose=100): - oof_pred = np.zeros(len(y_valid), dtype=np.float32) - model = lgbm.LGBMRegressor(**params) - model.fit(x_train, y_train, - eval_set=[(x_valid, y_valid)], - early_stopping_rounds=verbose, - verbose=verbose) - oof_pred = model.predict(x_valid) - score = mean_absolute_error(oof_pred, y_valid) - print('mae:', score) - return oof_pred, model, score - - -# training lightgbm - -params1 = {'objective':'mae', - 'reg_alpha': 0.14947461820098767, - 'reg_lambda': 0.10185644384043743, - 'n_estimators': 3633, - 'learning_rate': 0.08046301304430488, - 'num_leaves': 674, - 'feature_fraction': 0.9101240539122566, - 'bagging_fraction': 0.9884451442950513, - 'bagging_freq': 8, - 'min_child_samples': 51} - -params2 = { - 'objective':'mae', - 'reg_alpha': 0.1, - 'reg_lambda': 0.1, - 'n_estimators': 80, - 'learning_rate': 0.1, - 'random_state': 42, - "num_leaves": 22 -} - -params4 = {'objective':'mae', - 'reg_alpha': 0.016468100279441976, - 'reg_lambda': 0.09128335764019105, - 'n_estimators': 9868, - 'learning_rate': 0.10528150510326864, - 'num_leaves': 157, - 'feature_fraction': 0.5419185713426886, - 'bagging_fraction': 0.2637405128936662, - 'bagging_freq': 19, - 'min_child_samples': 71} - - -params = { - 'objective':'mae', - 'reg_alpha': 0.1, - 'reg_lambda': 0.1, - 'n_estimators': 10000, - 'learning_rate': 0.1, - 'random_state': 42, - "num_leaves": 100 -} - - -# Slow from this point !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -oof1, model1, score1 = fit_lgbm( - x_train1, y_train1['target1'], - x_valid1, y_valid1['target1'], - params1 - ) - -oof2, model2, score2 = fit_lgbm( - x_train2, y_train2['target2'], - x_valid2, y_valid2['target2'], - params2 -) - -oof3, model3, score3 = fit_lgbm( - x_train2, y_train2['target3'], - x_valid2, y_valid2['target3'], - params -) - -oof4, model4, score4 = fit_lgbm( - x_train2, y_train2['target4'], - x_valid2, y_valid2['target4'], - params4 -) - -score = (score1+score2+score3+score4) / 4 -print(f'score: {score}') - -# %% [code] -import pickle -from catboost import CatBoostRegressor - -def fit_lgbm(x_train, y_train, x_valid, y_valid, target, params: dict=None, verbose=100): - oof_pred_lgb = np.zeros(len(y_valid), dtype=np.float32) - oof_pred_cat = np.zeros(len(y_valid), dtype=np.float32) - - if os.path.isfile(f'../input/mlb-lgbm-and-catboost-models/model_lgb_{target}.pkl'): - with open(f'../input/mlb-lgbm-and-catboost-models/model_lgb_{target}.pkl', 'rb') as fin: - model = pickle.load(fin) - else: - - model = lgbm.LGBMRegressor(**params) - model.fit(x_train, y_train, - eval_set=[(x_valid, y_valid)], - early_stopping_rounds=verbose, - verbose=verbose) - - with open(f'model_lgb_{target}.pkl', 'wb') as handle: - pickle.dump(model, handle, protocol=pickle.HIGHEST_PROTOCOL) - - oof_pred_lgb = model.predict(x_valid) - score_lgb = mean_absolute_error(oof_pred_lgb, y_valid) - print('mae:', score_lgb) - - if os.path.isfile(f'../input/mlb-lgbm-and-catboost-models/model_cb_{target}.pkl'): - with open(f'../input/mlb-lgbm-and-catboost-models/model_cb_{target}.pkl', 'rb') as fin: - model_cb = pickle.load(fin) - else: - - model_cb = CatBoostRegressor( - n_estimators=2000, - learning_rate=0.05, - loss_function='MAE', - eval_metric='MAE', - max_bin=50, - subsample=0.9, - colsample_bylevel=0.5, - verbose=100) - - model_cb.fit(x_train, y_train, use_best_model=True, - eval_set=(x_valid, y_valid), - early_stopping_rounds=25) - - with open(f'model_cb_{target}.pkl', 'wb') as handle: - pickle.dump(model_cb, handle, protocol=pickle.HIGHEST_PROTOCOL) - - oof_pred_cat = model_cb.predict(x_valid) - score_cat = mean_absolute_error(oof_pred_cat, y_valid) - print('mae:', score_cat) - - return oof_pred_lgb, model, oof_pred_cat, model_cb, score_lgb, score_cat - - -# training lightgbm -params = { -'boosting_type': 'gbdt', -'objective':'mae', -'subsample': 0.5, -'subsample_freq': 1, -'learning_rate': 0.03, -'num_leaves': 2**11-1, -'min_data_in_leaf': 2**12-1, -'feature_fraction': 0.5, -'max_bin': 100, -'n_estimators': 2500, -'boost_from_average': False, -"random_seed":42, -} - -oof_pred_lgb2, model_lgb2, oof_pred_cat2, model_cb2, score_lgb2, score_cat2 = fit_lgbm( - x_train1, y_train1['target2'], - x_valid1, y_valid1['target2'], - 2, params -) - -oof_pred_lgb1, model_lgb1, oof_pred_cat1, model_cb1, score_lgb1, score_cat1 = fit_lgbm( - x_train1, y_train1['target1'], - x_valid1, y_valid1['target1'], - 1, params -) - -oof_pred_lgb3, model_lgb3, oof_pred_cat3, model_cb3, score_lgb3, score_cat3 = fit_lgbm( - x_train1, y_train1['target3'], - x_valid1, y_valid1['target3'], - 3, params -) -oof_pred_lgb4, model_lgb4, oof_pred_cat4, model_cb4, score_lgb4, score_cat4= fit_lgbm( - x_train1, y_train1['target4'], - x_valid1, y_valid1['target4'], - 4, params -) - -score = (score_lgb1+score_lgb2+score_lgb3+score_lgb4) / 4 -print(f'LightGBM score: {score}') - -score = (score_cat1+score_cat2+score_cat3+score_cat4) / 4 -print(f'Catboost score: {score}') - -# %% [markdown] -# ## Inference - -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:17:02.259872Z","iopub.status.idle":"2021-06-26T07:17:02.260506Z"}} -players_cols = ['playerId', 'primaryPositionName'] -rosters_cols = ['playerId', 'teamId', 'status'] -scores_cols = ['playerId', 'battingOrder', 'gamesPlayedBatting', 'flyOuts', - 'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns', - 'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch', - 'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay', - 'groundIntoTriplePlay', 'plateAppearances', 'totalBases', 'rbi', - 'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference', - 'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching', - 'completeGamesPitching', 'shutoutsPitching', 'winsPitching', - 'lossesPitching', 'flyOutsPitching', 'airOutsPitching', - 'groundOutsPitching', 'runsPitching', 'doublesPitching', - 'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching', - 'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching', - 'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching', - 'stolenBasesPitching', 'inningsPitched', 'saveOpportunities', - 'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls', - 'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching', - 'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners', - 'inheritedRunnersScored', 'catchersInterferencePitching', - 'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves', - 'assists', 'putOuts', 'errors', 'chances'] - -null = np.nan -true = True -false = False - -# %% [code] {"execution":{"iopub.status.busy":"2021-06-26T07:17:02.26162Z","iopub.status.idle":"2021-06-26T07:17:02.262287Z"}} -import pandas as pd -import numpy as np -from datetime import timedelta -from tqdm import tqdm -import gc -from functools import reduce -from sklearn.model_selection import StratifiedKFold - -ROOT_DIR = "../input/mlb-player-digital-engagement-forecasting" - -#=======================# -def flatten(df, col): - du = (df.pivot(index="playerId", columns="EvalDate", - values=col).add_prefix(f"{col}_"). - rename_axis(None, axis=1).reset_index()) - return du -#============================# -def reducer(left, right): - return left.merge(right, on="playerId") -#======================== - -TGTCOLS = ["target1","target2","target3","target4"] -def train_lag(df, lag=1): - dp = df[["playerId","EvalDate"]+TGTCOLS].copy() - dp["EvalDate"] =dp["EvalDate"] + timedelta(days=lag) - df = df.merge(dp, on=["playerId", "EvalDate"], suffixes=["",f"_{lag}"], how="left") - return df -#================================= -def test_lag(sub): - sub["playerId"] = sub["date_playerId"].apply(lambda s: int( s.split("_")[1] ) ) - assert sub.date.nunique() == 1 - dte = sub["date"].unique()[0] - - eval_dt = pd.to_datetime(dte, format="%Y%m%d") - dtes = [eval_dt + timedelta(days=-k) for k in LAGS] - mp_dtes = {eval_dt + timedelta(days=-k):k for k in LAGS} - - sl = LAST.loc[LAST.EvalDate.between(dtes[-1], dtes[0]), ["EvalDate","playerId"]+TGTCOLS].copy() - sl["EvalDate"] = sl["EvalDate"].map(mp_dtes) - du = [flatten(sl, col) for col in TGTCOLS] - du = reduce(reducer, du) - return du, eval_dt - # -#=============== - -tr = pd.read_csv("../input/mlb-data/target.csv") -print(tr.shape) -gc.collect() - -tr["EvalDate"] = pd.to_datetime(tr["EvalDate"]) -tr["EvalDate"] = tr["EvalDate"] + timedelta(days=-1) -tr["EvalYear"] = tr["EvalDate"].dt.year - -MED_DF = tr.groupby(["playerId","EvalYear"])[TGTCOLS].median().reset_index() -MEDCOLS = ["tgt1_med","tgt2_med", "tgt3_med", "tgt4_med"] -MED_DF.columns = ["playerId","EvalYear"] + MEDCOLS - -LAGS = list(range(1,21)) -FECOLS = [f"{col}_{lag}" for lag in reversed(LAGS) for col in TGTCOLS] - -for lag in tqdm(LAGS): - tr = train_lag(tr, lag=lag) - gc.collect() -#=========== -tr = tr.sort_values(by=["playerId", "EvalDate"]) -print(tr.shape) -tr = tr.dropna() -print(tr.shape) -tr = tr.merge(MED_DF, on=["playerId","EvalYear"]) -gc.collect() - -X = tr[FECOLS+MEDCOLS].values -y = tr[TGTCOLS].values -cl = tr["playerId"].values - -NFOLDS = 6 -skf = StratifiedKFold(n_splits=NFOLDS) -folds = skf.split(X, cl) -folds = list(folds) - -import tensorflow as tf -import tensorflow.keras.layers as L -import tensorflow.keras.models as M -from sklearn.metrics import mean_absolute_error, mean_squared_error -from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping - -tf.random.set_seed(777) - -def make_model(n_in): - inp = L.Input(name="inputs", shape=(n_in,)) - x = L.Dense(50, activation="relu", name="d1")(inp) - x = L.Dense(50, activation="relu", name="d2")(x) - preds = L.Dense(4, activation="linear", name="preds")(x) - - model = M.Model(inp, preds, name="ANN") - model.compile(loss="mean_absolute_error", optimizer="adam") - return model - -net = make_model(X.shape[1]) -print(net.summary()) - -oof = np.zeros(y.shape) -nets = [] -for idx in range(NFOLDS): - print("FOLD:", idx) - tr_idx, val_idx = folds[idx] - ckpt = ModelCheckpoint(f"w{idx}.h5", monitor='val_loss', verbose=1, save_best_only=True,mode='min') - reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,patience=3, min_lr=0.0005) - es = EarlyStopping(monitor='val_loss', patience=6) - reg = make_model(X.shape[1]) -# reg.fit(X[tr_idx], y[tr_idx], epochs=10, batch_size=35_000, validation_data=(X[val_idx], y[val_idx]), -# verbose=1, callbacks=[ckpt, reduce_lr, es]) - reg.load_weights(f"w{idx}.h5") - oof[val_idx] = reg.predict(X[val_idx], batch_size=50_000, verbose=1) - nets.append(reg) - gc.collect() - # -# - -mae = mean_absolute_error(y, oof) -mse = mean_squared_error(y, oof, squared=False) -print("mae:", mae) -print("mse:", mse) - -# Historical information to use in prediction time -bound_dt = pd.to_datetime("2021-01-01") -LAST = tr.loc[tr.EvalDate>bound_dt].copy() - -LAST_MED_DF = MED_DF.loc[MED_DF.EvalYear==2021].copy() -LAST_MED_DF.drop("EvalYear", axis=1, inplace=True) -del tr - -#""" -import mlb -FE = []; SUB = []; - -# %% [markdown] -#
-#
- -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:17:02.263332Z","iopub.status.idle":"2021-06-26T07:17:02.263974Z"}} -import copy - -env = mlb.make_env() # initialize the environment -iter_test = env.iter_test() # iterator which loops over each date in test set - -for (test_df, sample_prediction_df) in iter_test: # make predictions here - - sub = copy.deepcopy(sample_prediction_df.reset_index()) - sample_prediction_df = copy.deepcopy(sample_prediction_df.reset_index(drop=True)) - - # LGBM summit - # creat dataset - sample_prediction_df['playerId'] = sample_prediction_df['date_playerId']\ - .map(lambda x: int(x.split('_')[1])) - # Dealing with missing values - if test_df['rosters'].iloc[0] == test_df['rosters'].iloc[0]: - test_rosters = pd.DataFrame(eval(test_df['rosters'].iloc[0])) - else: - test_rosters = pd.DataFrame({'playerId': sample_prediction_df['playerId']}) - for col in rosters.columns: - if col == 'playerId': continue - test_rosters[col] = np.nan - - if test_df['playerBoxScores'].iloc[0] == test_df['playerBoxScores'].iloc[0]: - test_scores = pd.DataFrame(eval(test_df['playerBoxScores'].iloc[0])) - else: - test_scores = pd.DataFrame({'playerId': sample_prediction_df['playerId']}) - for col in scores.columns: - if col == 'playerId': continue - test_scores[col] = np.nan - test_scores = test_scores.groupby('playerId').sum().reset_index() - test = sample_prediction_df[['playerId']].copy() - test = test.merge(players[players_cols], on='playerId', how='left') - test = test.merge(test_rosters[rosters_cols], on='playerId', how='left') - test = test.merge(test_scores[scores_cols], on='playerId', how='left') - test = test.merge(player_target_stats, how='inner', left_on=["playerId"],right_on=["playerId"]) - - - test['label_playerId'] = test['playerId'].map(player2num) - test['label_primaryPositionName'] = test['primaryPositionName'].map(position2num) - test['label_teamId'] = test['teamId'].map(teamid2num) - test['label_status'] = test['status'].map(status2num) - - test_X = test[feature_cols] - # predict - pred1 = model1.predict(test_X) - - # predict - pred_lgd1 = model_lgb1.predict(test_X) - pred_lgd2 = model_lgb2.predict(test_X) - pred_lgd3 = model_lgb3.predict(test_X) - pred_lgd4 = model_lgb4.predict(test_X) - - pred_cat1 = model_cb1.predict(test_X) - pred_cat2 = model_cb2.predict(test_X) - pred_cat3 = model_cb3.predict(test_X) - pred_cat4 = model_cb4.predict(test_X) - - test['target1'] = np.clip(pred1,0,100) - test_X = test[feature_cols2] - - pred2 = model2.predict(test_X) - pred3 = model3.predict(test_X) - pred4 = model4.predict(test_X) - - # merge submission - sample_prediction_df['target1'] = 0.65*np.clip(pred1, 0, 100)+0.25*np.clip(pred_lgd1, 0, 100)+0.10*np.clip(pred_cat1, 0, 100) - sample_prediction_df['target2'] = 0.65*np.clip(pred2, 0, 100)+0.25*np.clip(pred_lgd2, 0, 100)+0.10*np.clip(pred_cat2, 0, 100) - sample_prediction_df['target3'] = 0.65*np.clip(pred3, 0, 100)+0.25*np.clip(pred_lgd3, 0, 100)+0.10*np.clip(pred_cat3, 0, 100) - sample_prediction_df['target4'] = 0.65*np.clip(pred4, 0, 100)+0.25*np.clip(pred_lgd4, 0, 100)+0.10*np.clip(pred_cat4, 0, 100) - sample_prediction_df = sample_prediction_df.fillna(0.) - del sample_prediction_df['playerId'] - # TF summit - # Features computation at Evaluation Date - sub_fe, eval_dt = test_lag(sub) - sub_fe = sub_fe.merge(LAST_MED_DF, on="playerId", how="left") - sub_fe = sub_fe.fillna(0.) - - _preds = 0. - for reg in nets: - _preds += reg.predict(sub_fe[FECOLS + MEDCOLS]) / NFOLDS - sub_fe[TGTCOLS] = np.clip(_preds, 0, 100) - sub.drop(["date"]+TGTCOLS, axis=1, inplace=True) - sub = sub.merge(sub_fe[["playerId"]+TGTCOLS], on="playerId", how="left") - sub.drop("playerId", axis=1, inplace=True) - sub = sub.fillna(0.) - # Blending - blend = pd.concat( - [sub[['date_playerId']], - (0.35*sub.drop('date_playerId', axis=1) + 0.65*sample_prediction_df.drop('date_playerId', axis=1))], - axis=1 - ) - env.predict(blend) - # Update Available information - sub_fe["EvalDate"] = eval_dt - #sub_fe.drop(MEDCOLS, axis=1, inplace=True) - LAST = LAST.append(sub_fe) - LAST = LAST.drop_duplicates(subset=["EvalDate","playerId"], keep="last") - -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:17:02.264951Z","iopub.status.idle":"2021-06-26T07:17:02.265581Z"}} -pd.concat( - [sub[['date_playerId']], - (sub.drop('date_playerId', axis=1) + sample_prediction_df.drop('date_playerId', axis=1)) / 2], - axis=1 -) - -# %% [code] {"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-26T07:17:02.26657Z","iopub.status.idle":"2021-06-26T07:17:02.267169Z"}} -sample_prediction_df - -# %% [markdown] -#
-#
diff --git a/d1/mlb_player_v2.py b/d1/mlb_player_v2.py deleted file mode 100644 index 37052bc..0000000 --- a/d1/mlb_player_v2.py +++ /dev/null @@ -1,1399 +0,0 @@ -# %% [markdown] {"papermill":{"duration":0.099672,"end_time":"2021-06-11T18:42:32.853471","exception":false,"start_time":"2021-06-11T18:42:32.753799","status":"completed"},"tags":[]} -# # **MLB Player Digital Engagementコンペ概略**😀 -# -# ## ※ English page is here : https://www.kaggle.com/chumajin/eda-of-mlb-for-starter-english-ver -# -# ## このコンペは、MLBのplayer idごとに、次の日(将来)にファンがデジタルコンテンツへのエンゲージメント（「反応」「行動」みたいなもの)をどれくらい起こすかというのを数値化したもの(target)を予測するコンペだと思います。targetは1～4で、それぞれ異なる指標で4つあって、0-100のスケールで数値化したものだそうです。 -# (コメントいただきました。ありがとうございます!!　たしかにサポーターなどのtwitterの書き込みとか、どこかのサイトへのアクセスなどそういうのを想像するとイメージしやすいですね。) -# -# - -# %% [markdown] {"papermill":{"duration":0.10241,"end_time":"2021-06-11T18:42:33.052395","exception":false,"start_time":"2021-06-11T18:42:32.949985","status":"completed"},"tags":[]} -# ## もし、少しでもお役に立てば、**upvote**いただけたら嬉しいです！　他notebookでもupvoteいただけた方いつもありがとうございます。 -# -# ## また、基本的には、この事務局のスターターを見て、EDAを理解していきました(一部抜粋)。ありがとうございます。 -# -# ## こちらもupvoteお願いいたします。 -# -# https://www.kaggle.com/ryanholbrook/getting-started-with-mlb-player-digital-engagement - -# %% [code] {"papermill":{"duration":1.060051,"end_time":"2021-06-11T18:42:34.209063","exception":false,"start_time":"2021-06-11T18:42:33.149012","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:06:08.741917Z","iopub.execute_input":"2021-06-14T09:06:08.742307Z","iopub.status.idle":"2021-06-14T09:06:09.566612Z","shell.execute_reply.started":"2021-06-14T09:06:08.742273Z","shell.execute_reply":"2021-06-14T09:06:09.565664Z"}} -import pprint -def display(*args, **kwargs): - pprint.pprint( - dict( - args=args, - kwargs=kwargs, - ), - ) - -import gc -import sys -import warnings -from pathlib import Path - -import os - -import ipywidgets as widgets -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -import seaborn as sns -warnings.simplefilter("ignore") - -# %% [markdown] {"papermill":{"duration":0.099753,"end_time":"2021-06-11T18:42:34.425178","exception":false,"start_time":"2021-06-11T18:42:34.325425","status":"completed"},"tags":[]} -# # 0. 何を予測するか (submissionファイルから見ちゃいます) - -# %% [code] {"papermill":{"duration":0.148821,"end_time":"2021-06-11T18:42:34.687273","exception":false,"start_time":"2021-06-11T18:42:34.538452","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:06:13.75186Z","iopub.execute_input":"2021-06-14T09:06:13.752423Z","iopub.status.idle":"2021-06-14T09:06:13.804719Z","shell.execute_reply.started":"2021-06-14T09:06:13.752383Z","shell.execute_reply":"2021-06-14T09:06:13.803356Z"}} -example_sample_submission = pd.read_csv("../input/mlb-player-digital-engagement-forecasting/example_sample_submission.csv") -example_sample_submission - -# %% [markdown] {"papermill":{"duration":0.092961,"end_time":"2021-06-11T18:42:34.873875","exception":false,"start_time":"2021-06-11T18:42:34.780914","status":"completed"},"tags":[]} -# playeridごとに、次の日(将来)にファンがデジタルコンテンツへのエンゲージメント（「反応」「行動」みたいなもの)をどれくらい起こすかというのを数値化したもの(target)を予測するコンペ。 -# -# targetは1～4で、それぞれ異なる指標で4つあって、0-100のスケールで数値化したものだそうです。 -# - -# %% [markdown] {"papermill":{"duration":0.09515,"end_time":"2021-06-11T18:42:35.062356","exception":false,"start_time":"2021-06-11T18:42:34.967206","status":"completed"},"tags":[]} -# ## 0.1 どの情報から推測 ? (先にテストデータを見ちゃいます) - -# %% [code] {"papermill":{"duration":0.895014,"end_time":"2021-06-11T18:42:36.051171","exception":false,"start_time":"2021-06-11T18:42:35.156157","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:07:01.149956Z","iopub.execute_input":"2021-06-14T09:07:01.150331Z","iopub.status.idle":"2021-06-14T09:07:02.04348Z","shell.execute_reply.started":"2021-06-14T09:07:01.1503Z","shell.execute_reply":"2021-06-14T09:07:02.042485Z"}} -example_test = pd.read_csv("../input/mlb-player-digital-engagement-forecasting/example_test.csv") -example_test - -# %% [markdown] {"papermill":{"duration":0.093244,"end_time":"2021-06-11T18:42:36.239027","exception":false,"start_time":"2021-06-11T18:42:36.145783","status":"completed"},"tags":[]} -# パッと見て、submissionに出てくるplayer IDとかがすぐわかる感じではなさそう。json形式でいろいろな情報が入っていそう。 -# -# -# テストデータは1日に1行のデータからなっている。 -# -# -# 例えば、starterコードからの関数を使用すると、以下のように展開できる。 - -# %% [code] {"papermill":{"duration":0.105661,"end_time":"2021-06-11T18:42:36.437921","exception":false,"start_time":"2021-06-11T18:42:36.33226","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:07:46.275371Z","iopub.execute_input":"2021-06-14T09:07:46.275752Z","iopub.status.idle":"2021-06-14T09:07:46.280719Z","shell.execute_reply.started":"2021-06-14T09:07:46.275719Z","shell.execute_reply":"2021-06-14T09:07:46.279482Z"}} -# Helper function to unpack json found in daily data -def unpack_json(json_str): - return np.nan if pd.isna(json_str) else pd.read_json(json_str) - -# %% [code] {"papermill":{"duration":0.211983,"end_time":"2021-06-11T18:42:36.743198","exception":false,"start_time":"2021-06-11T18:42:36.531215","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:07:52.045349Z","iopub.execute_input":"2021-06-14T09:07:52.045719Z","iopub.status.idle":"2021-06-14T09:07:52.170803Z","shell.execute_reply.started":"2021-06-14T09:07:52.045686Z","shell.execute_reply":"2021-06-14T09:07:52.169701Z"}} -example_test.head(3) - -# %% [markdown] {"papermill":{"duration":0.093974,"end_time":"2021-06-11T18:42:36.931685","exception":false,"start_time":"2021-06-11T18:42:36.837711","status":"completed"},"tags":[]} -# example_test["games"].iloc[0] の中身を見てみる - -# %% [code] {"papermill":{"duration":0.165624,"end_time":"2021-06-11T18:42:37.192619","exception":false,"start_time":"2021-06-11T18:42:37.026995","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:08:00.433132Z","iopub.execute_input":"2021-06-14T09:08:00.43354Z","iopub.status.idle":"2021-06-14T09:08:00.50561Z","shell.execute_reply.started":"2021-06-14T09:08:00.433495Z","shell.execute_reply":"2021-06-14T09:08:00.50448Z"}} -unpack_json(example_test["games"].iloc[0]) - -# %% [markdown] {"papermill":{"duration":0.094383,"end_time":"2021-06-11T18:42:37.381562","exception":false,"start_time":"2021-06-11T18:42:37.287179","status":"completed"},"tags":[]} -# example_test["rosters"].iloc[0] の中身を見てみる - -# %% [code] {"papermill":{"duration":0.126356,"end_time":"2021-06-11T18:42:37.603497","exception":false,"start_time":"2021-06-11T18:42:37.477141","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:09:08.225885Z","iopub.execute_input":"2021-06-14T09:09:08.226269Z","iopub.status.idle":"2021-06-14T09:09:08.255073Z","shell.execute_reply.started":"2021-06-14T09:09:08.226235Z","shell.execute_reply":"2021-06-14T09:09:08.254404Z"}} -unpack_json(example_test["rosters"].iloc[0]) - -# %% [markdown] {"papermill":{"duration":0.094364,"end_time":"2021-06-11T18:42:37.794102","exception":false,"start_time":"2021-06-11T18:42:37.699738","status":"completed"},"tags":[]} -# この辺の情報から、player idごとに次の日のtarget1～4という評価項目の期待値を推測するコンペだと思います。 - -# %% [code] {"papermill":{"duration":0.097753,"end_time":"2021-06-11T18:42:37.988036","exception":false,"start_time":"2021-06-11T18:42:37.890283","status":"completed"},"tags":[]} - - -# %% [markdown] {"papermill":{"duration":0.095565,"end_time":"2021-06-11T18:42:38.18146","exception":false,"start_time":"2021-06-11T18:42:38.085895","status":"completed"},"tags":[]} -# ---------以上を踏まえて、trainデータなど他のデータを見ていきます--------- - -# %% [markdown] {"papermill":{"duration":0.094539,"end_time":"2021-06-11T18:42:38.370792","exception":false,"start_time":"2021-06-11T18:42:38.276253","status":"completed"},"tags":[]} -# # 1. train.csv - -# %% [code] {"papermill":{"duration":73.740436,"end_time":"2021-06-11T18:43:52.206935","exception":false,"start_time":"2021-06-11T18:42:38.466499","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:09:47.628029Z","iopub.execute_input":"2021-06-14T09:09:47.628644Z","iopub.status.idle":"2021-06-14T09:11:03.816877Z","shell.execute_reply.started":"2021-06-14T09:09:47.6286Z","shell.execute_reply":"2021-06-14T09:11:03.815822Z"}} -# 読み込みに少し時間かかります。 -training = pd.read_csv("../input/mlb-player-digital-engagement-forecasting/train.csv") -training - -# %% [code] {"papermill":{"duration":0.108401,"end_time":"2021-06-11T18:43:52.411026","exception":false,"start_time":"2021-06-11T18:43:52.302625","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:12:42.628153Z","iopub.execute_input":"2021-06-14T09:12:42.628536Z","iopub.status.idle":"2021-06-14T09:12:42.64072Z","shell.execute_reply.started":"2021-06-14T09:12:42.628496Z","shell.execute_reply":"2021-06-14T09:12:42.639545Z"}} -# dateはdatetimeに変換 -training['date'] = pd.to_datetime(training['date'], format="%Y%m%d") - -# %% [code] {"papermill":{"duration":0.124469,"end_time":"2021-06-11T18:43:52.63163","exception":false,"start_time":"2021-06-11T18:43:52.507161","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:12:46.690978Z","iopub.execute_input":"2021-06-14T09:12:46.691391Z","iopub.status.idle":"2021-06-14T09:12:46.716353Z","shell.execute_reply.started":"2021-06-14T09:12:46.691356Z","shell.execute_reply":"2021-06-14T09:12:46.715114Z"}} -training.info() - -# %% [markdown] {"papermill":{"duration":0.096956,"end_time":"2021-06-11T18:43:52.824788","exception":false,"start_time":"2021-06-11T18:43:52.727832","status":"completed"},"tags":[]} -# 1216日分のデータ。nullデータは無し。nanデータがところどころにある。 - -# %% [markdown] {"papermill":{"duration":0.09861,"end_time":"2021-06-11T18:43:53.01957","exception":false,"start_time":"2021-06-11T18:43:52.92096","status":"completed"},"tags":[]} -# --------------------------------------------------------------------- - -# %% [markdown] {"papermill":{"duration":0.095137,"end_time":"2021-06-11T18:43:53.213022","exception":false,"start_time":"2021-06-11T18:43:53.117885","status":"completed"},"tags":[]} -# ## ここから**カラムごとにデータがあるところのjsonを事例として1つ見てみます**。 -# -# 上述したように、train.csvの中身も1つのセルの中にjsonファイル形式で、dataframeがさらにそれぞれ入っているような複雑な形をしています。 -# -# (結果から言うと、全部で1216日分のデータの1日に対して、約11個(nanもあるのでも少し少ないですが)のDataFrameが情報量としてぶら下がっているイメージで、かなりの情報量です。 -# -# ので、ここから少し長いです。イメージだけつかんで、読み流しても良いかもです。) -# - -# %% [code] {"papermill":{"duration":0.105297,"end_time":"2021-06-11T18:43:53.414068","exception":false,"start_time":"2021-06-11T18:43:53.308771","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:13:37.94206Z","iopub.execute_input":"2021-06-14T09:13:37.942469Z","iopub.status.idle":"2021-06-14T09:13:37.94928Z","shell.execute_reply.started":"2021-06-14T09:13:37.942422Z","shell.execute_reply":"2021-06-14T09:13:37.948339Z"}} -training.columns - -# %% [markdown] {"papermill":{"duration":0.098188,"end_time":"2021-06-11T18:43:53.612128","exception":false,"start_time":"2021-06-11T18:43:53.51394","status":"completed"},"tags":[]} -# 1つ1つ入力するのが、めんどくさいので、naを抜いて、n番目(0だと一番上)のサンプルをdataframeにしてcolumn名と中身を見る関数を作っちゃいます。 - -# %% [code] {"papermill":{"duration":0.1263,"end_time":"2021-06-11T18:43:53.851813","exception":false,"start_time":"2021-06-11T18:43:53.725513","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:14:01.593291Z","iopub.execute_input":"2021-06-14T09:14:01.593682Z","iopub.status.idle":"2021-06-14T09:14:01.598603Z","shell.execute_reply.started":"2021-06-14T09:14:01.593649Z","shell.execute_reply":"2021-06-14T09:14:01.597731Z"}} -def exshow(col,n): - tmp = training[col] - tmp = tmp.dropna() - tmpdf = unpack_json(tmp.iloc[n]) - print(tmpdf.columns) - return tmpdf - -# %% [markdown] {"papermill":{"duration":0.112367,"end_time":"2021-06-11T18:43:54.094326","exception":false,"start_time":"2021-06-11T18:43:53.981959","status":"completed"},"tags":[]} -# ## 1.1 nextDayPlayerEngagement (train.csvのcolumn1番目) -# 翌日以降のすべてのモデリング ターゲットを含むネストされた JSON。 - -# %% [code] {"papermill":{"duration":0.13531,"end_time":"2021-06-11T18:43:54.391578","exception":false,"start_time":"2021-06-11T18:43:54.256268","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:14:10.526008Z","iopub.execute_input":"2021-06-14T09:14:10.526394Z","iopub.status.idle":"2021-06-14T09:14:10.552638Z","shell.execute_reply.started":"2021-06-14T09:14:10.526362Z","shell.execute_reply":"2021-06-14T09:14:10.551683Z"}} -training.head(3) - -# %% [code] {"papermill":{"duration":0.15348,"end_time":"2021-06-11T18:43:54.656058","exception":false,"start_time":"2021-06-11T18:43:54.502578","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:14:19.888531Z","iopub.execute_input":"2021-06-14T09:14:19.888888Z","iopub.status.idle":"2021-06-14T09:14:19.928418Z","shell.execute_reply.started":"2021-06-14T09:14:19.888857Z","shell.execute_reply":"2021-06-14T09:14:19.927415Z"}} -tmpdf = exshow("nextDayPlayerEngagement",0) -tmpdf - -# %% [markdown] {"papermill":{"duration":0.101315,"end_time":"2021-06-11T18:43:54.857283","exception":false,"start_time":"2021-06-11T18:43:54.755968","status":"completed"},"tags":[]} -# * engagementMetricsDate - 米国太平洋時間に基づくプレーヤーエンゲージメント指標の日付（前日のゲーム、名簿、フィールド統計、トランザクション、賞などと一致します）。 -# * playerId -# * target1 -# * target2 -# * target3 -# * target4 -# -# -# target1-target4は、0から100のスケールでのデジタルエンゲージメントの毎日のインデックスです。 - -# %% [markdown] {"papermill":{"duration":0.099862,"end_time":"2021-06-11T18:43:55.056252","exception":false,"start_time":"2021-06-11T18:43:54.95639","status":"completed"},"tags":[]} -# ここから、plyaerIdと次の日以降のtarget1～4を抜くんですね。 - -# %% [markdown] {"papermill":{"duration":0.108076,"end_time":"2021-06-11T18:43:55.265858","exception":false,"start_time":"2021-06-11T18:43:55.157782","status":"completed"},"tags":[]} -# ## 1.2 games(train.csvのcolumn2番目) -# 特定の日のすべてのゲーム情報を含むネストされた JSON。レギュラー シーズン、ポストシーズン、オールスター ゲームに加えて、スプリング トレーニングとエキシビション ゲームが含まれています。 - -# %% [code] {"papermill":{"duration":0.138443,"end_time":"2021-06-11T18:43:55.51297","exception":false,"start_time":"2021-06-11T18:43:55.374527","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:14:48.471833Z","iopub.execute_input":"2021-06-14T09:14:48.472245Z","iopub.status.idle":"2021-06-14T09:14:48.498208Z","shell.execute_reply.started":"2021-06-14T09:14:48.472214Z","shell.execute_reply":"2021-06-14T09:14:48.497415Z"}} -training.head(3) - -# %% [code] {"papermill":{"duration":0.153552,"end_time":"2021-06-11T18:43:55.769473","exception":false,"start_time":"2021-06-11T18:43:55.615921","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:14:56.122131Z","iopub.execute_input":"2021-06-14T09:14:56.122758Z","iopub.status.idle":"2021-06-14T09:14:56.169907Z","shell.execute_reply.started":"2021-06-14T09:14:56.122703Z","shell.execute_reply":"2021-06-14T09:14:56.168899Z"}} -tmpdf = exshow("games",1) # 0番目（1番上はデータが一行しかなかったので、1にしました。) -tmpdf - -# %% [markdown] {"papermill":{"duration":0.105256,"end_time":"2021-06-11T18:43:55.985414","exception":false,"start_time":"2021-06-11T18:43:55.880158","status":"completed"},"tags":[]} -# カラムの意味の翻訳は↓を開いてください。(長いので、hideしています。) - -# %% [markdown] {"_kg_hide-input":true,"papermill":{"duration":0.109485,"end_time":"2021-06-11T18:43:56.199254","exception":false,"start_time":"2021-06-11T18:43:56.089769","status":"completed"},"tags":[]} -# * gamePk : ゲームの一意の識別子。 -# * gameType : ゲームの種類、さまざまな種類がここにあります。 -# * season : -# * gameDate : -# * gameTimeUTC : UTCでの始球式。 -# * resumeDate : タイムゲームが再開されました（放棄された場合、それ以外の場合はnull）。 -# * resumedFrom : タイムゲームは元々放棄されていました（放棄された場合、それ以外の場合はnull）。 -# * codedGameState : ゲームのステータスコード、さまざまなタイプがここにあります。 -# * detailedGameState : ゲームのステータス、さまざまな種類がここにあります。 -# * isTie : ブール値。ゲームが引き分けで終了した場合はtrue。 -# * gameNumber : ダブルヘッダーを区別するためのゲーム番号フラグ -# * doubleHeader : YはDH、Nはシングルゲーム、Sはスプリット -# * dayNight : スケジュールされた開始時間の昼または夜のフラグ。 -# * scheduledInnings : 予定イニング数。 -# * gamesInSeries : 現在のシリーズのゲーム数。 -# * seriesDescription : 現在のシリーズのテキスト説明。 -# * homeId : ホームチームの一意の識別子。 -# * homeName : ホームチーム名。 -# * homeAbbrev : ホームチームの略語。 -# * homeWins : ホームチームのシーズンの現在の勝利数。 -# * homeLosses : ホームチームのシーズンでの現在の損失数。 -# * homeWinPct : ホームチームの現在の勝率。 -# * homeWinner : ブール値。ホームチームが勝った場合はtrue。 -# * homeScore : ホームチームが得点するラン。 -# * awayId : アウェイチームの一意の識別子。 -# * awayName : アウェイチームの一意の識別子。 -# * awayAbbrev : アウェイチームの略。 -# * awayWins : アウェイチームのシーズンの現在の勝利数。 -# * awayLosses : アウェイチームのシーズン中の現在の敗北数。 -# * awayWinPct : アウェイチームの現在の勝率。 -# * awayWinner : ブール値。離れたチームが勝った場合はtrue。 -# * awayScore : アウェイチームが得点したラン。 - -# %% [markdown] {"papermill":{"duration":0.099984,"end_time":"2021-06-11T18:43:56.401399","exception":false,"start_time":"2021-06-11T18:43:56.301415","status":"completed"},"tags":[]} -# ## 1.3 rosters(train.csvのcolumn3番目) -# 特定の日のすべての名簿情報を含むネストされた JSON。インシーズンとオフシーズンのチーム名簿が含まれます。 - -# %% [code] {"papermill":{"duration":0.133667,"end_time":"2021-06-11T18:43:56.635608","exception":false,"start_time":"2021-06-11T18:43:56.501941","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:15:42.681691Z","iopub.execute_input":"2021-06-14T09:15:42.682297Z","iopub.status.idle":"2021-06-14T09:15:42.710025Z","shell.execute_reply.started":"2021-06-14T09:15:42.682226Z","shell.execute_reply":"2021-06-14T09:15:42.708629Z"}} -training.head(3) - -# %% [code] {"papermill":{"duration":0.136156,"end_time":"2021-06-11T18:43:56.876362","exception":false,"start_time":"2021-06-11T18:43:56.740206","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:15:47.818485Z","iopub.execute_input":"2021-06-14T09:15:47.818837Z","iopub.status.idle":"2021-06-14T09:15:47.849949Z","shell.execute_reply.started":"2021-06-14T09:15:47.818807Z","shell.execute_reply":"2021-06-14T09:15:47.848977Z"}} -tmpdf = exshow("rosters",0) -tmpdf - -# %% [markdown] {"papermill":{"duration":0.110068,"end_time":"2021-06-11T18:43:57.094794","exception":false,"start_time":"2021-06-11T18:43:56.984726","status":"completed"},"tags":[]} -# * playerId-プレーヤーの一意の識別子。 -# * gameDate -# * teamId-そのプレーヤーがその日にいるteamId。 -# * statusCode-名簿ステータスの略語。 -# * status-説明的な名簿のステータス。 - -# %% [markdown] {"papermill":{"duration":0.10842,"end_time":"2021-06-11T18:43:57.309461","exception":false,"start_time":"2021-06-11T18:43:57.201041","status":"completed"},"tags":[]} -# ## 1.4 playerBoxScores(train.csvのcolumn4番目) -# 特定の日のプレイヤー ゲーム レベルで集計されたゲーム統計を含むネストされた JSON。レギュラーシーズン、ポストシーズン、オールスターゲームが含まれます。 - -# %% [code] {"papermill":{"duration":0.131819,"end_time":"2021-06-11T18:43:57.543572","exception":false,"start_time":"2021-06-11T18:43:57.411753","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:15:51.122412Z","iopub.execute_input":"2021-06-14T09:15:51.122822Z","iopub.status.idle":"2021-06-14T09:15:51.150918Z","shell.execute_reply.started":"2021-06-14T09:15:51.122791Z","shell.execute_reply":"2021-06-14T09:15:51.149868Z"}} -training.head(3) - -# %% [code] {"papermill":{"duration":0.193827,"end_time":"2021-06-11T18:43:57.843961","exception":false,"start_time":"2021-06-11T18:43:57.650134","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:15:52.259031Z","iopub.execute_input":"2021-06-14T09:15:52.259431Z","iopub.status.idle":"2021-06-14T09:15:52.344774Z","shell.execute_reply.started":"2021-06-14T09:15:52.259394Z","shell.execute_reply":"2021-06-14T09:15:52.343636Z"}} -tmpdf = exshow("playerBoxScores",0) -tmpdf.head(5) - -# %% [markdown] {"papermill":{"duration":0.107718,"end_time":"2021-06-11T18:43:58.056077","exception":false,"start_time":"2021-06-11T18:43:57.948359","status":"completed"},"tags":[]} -# カラムの意味の翻訳は↓を開いてください。(長いので、hideしています。) - -# %% [markdown] {"_kg_hide-input":true,"papermill":{"duration":0.106169,"end_time":"2021-06-11T18:43:58.272551","exception":false,"start_time":"2021-06-11T18:43:58.166382","status":"completed"},"tags":[]} -# * home : バイナリ、ホームチームの場合は1、離れている場合は0。 -# * gamePk : ゲームの一意の識別子。 -# * gameDate : -# * gameTimeUTC : UTCでの始球式。 -# * teamId : チームの一意の識別子。 -# * teamName : -# * playerId : プレーヤーの一意の識別子。 -# * playerName : -# * jerseyNum : -# * positionCode : 番号の位置コード、詳細はこちらです。 -# * positionName : テキスト位置の表示、詳細はこちらです。 -# * positionType : ポジショングループ、詳細はこちらです。 -# * battingOrder : 形式：「###」。最初の桁は打順スポットを示し、次の2桁はそのプレーヤーがその打順スポットを占めた順序を示します。例：「300」は、打順の3番目のスポットのスターターを示します。 4人目（900、901、902以降）が打順9位を占めることを示す「903」。ゲームに登場した場合にのみ入力されます。 -# * gamesPlayedBatting : プレーヤーが打者、ランナー、または野手としてゲームに参加した場合は1。 -# * flyOuts : ゲームの合計フライアウト。 -# * groundOuts : ゲームのトータルグラウンドアウト。 -# * runsScored : ゲームの合計ランが記録されました。 -# * doubles : ゲームの合計は2倍です。 -# * triples : ゲームの合計トリプル。 -# * homeRuns : ゲームの総本塁打。 -# * strikeOuts : ゲームの合計三振。 -# * baseOnBalls : ゲームの合計ウォーク。 -# * intentionalWalks : ゲームの故意四球。 -# * hits : ゲームの総ヒット数。 -# * hitByPitch : ピッチによるゲームの合計ヒット。 -# * atBats : でのゲーム合計 -# * caughtStealing : ゲームの合計が盗塁をキャッチしました。 -# * stolenBases : ゲームの盗塁総数。 -# * groundIntoDoublePlay : ゲームの合計併殺はに基づいています。 -# * groundIntoTriplePlay : ゲームの合計 3 回プレイが基礎になります。 -# * plateAppearances : ゲームの総打席。 -# * totalBases : ゲームの総拠点数。 -# * rbi : ゲームの合計打点。 -# * leftOnBase : ゲームの総ランナーはベースに残った。 -# * sacBunts : ゲームの合計犠牲バント。 -# * sacFlies : ゲームの総犠牲フライ。 -# * catchersInterference : ゲームのトータルキャッチャーの干渉が発生しました。 -# * pickoffs : ゲームの合計回数がベースから外れました。 -# * gamesPlayedPitching : バイナリ、プレーヤーが投手としてゲームに参加した場合は 1。 -# * gamesStartedPitching : バイナリ、プレーヤーがゲームの先発投手だった場合は1。 -# * completeGamesPitching : バイナリ、完投でクレジットされている場合は1。 -# * shutoutsPitching : バイナリ、完封でクレジットされている場合は1。 -# * winsPitching : バイナリ、勝利でクレジットされている場合は 1。 -# * lossesPitching : バイナリ、損失がクレジットされている場合は1。 -# * flyOutsPitching : 許可されたフライアウトのゲーム合計。 -# * airOutsPitching : エアアウト（フライアウト+ポップアウト）のゲーム合計が許可されます。 -# * groundOutsPitching : ゲームの合計グラウンドアウトが許可されます。 -# * runsPitching : ゲームの合計実行が許可されます。 -# * doublesPitching : ゲームの合計は2倍になります。 -# * triplesPitching : ゲームの合計トリプルが許可されます。 -# * homeRunsPitching : ゲームの合計ホームランが許可されます。 -# * strikeOutsPitching : ゲームの合計三振が許可されます。 -# * baseOnBallsPitching : ゲームの合計歩行が許可されます。 -# * intentionalWalksPitching : ゲームの故意四球の合計が許可されます。 -# * hitsPitching : 許可されるゲームの合計ヒット数。 -# * hitByPitchPitching : 許可されたピッチによるゲームの合計ヒット。 -# * atBatsPitching : でのゲーム合計 -# * caughtStealingPitching : ゲームの合計は、盗みをキャッチしました。 -# * stolenBasesPitching : ゲームの盗塁の合計は許可されます。 -# * inningsPitched : ゲームの総投球回。 -# * saveOpportunities : バイナリ、保存の機会がある場合は1。 -# * earnedRuns : ゲームの合計自責点は許可されています。 -# * battersFaced : 直面したゲームの総打者。 -# * outsPitching : ゲームの合計アウトが記録されました。 -# * pitchesThrown : 投げられた投球のゲーム総数。 -# * balls : 投げられたゲームの合計ボール。 -# * strikes : スローされたゲームの合計ストライク。 -# * hitBatsmen : ゲームの総死球打者。 -# * balks : ゲームの合計はボークします。 -# * wildPitches : 投げられた暴投のゲーム総数。 -# * pickoffsPitching : ゲームのピックオフの総数。 -# * rbiPitching : 打点のゲーム総数は許可されています。 -# * inheritedRunners : 継承されたランナーのゲーム合計を想定。 -# * inheritedRunnersScored : 得点した継承されたランナーのゲーム合計。 -# * catchersInterferencePitching : キャッチャーの干渉のゲーム合計はバッテリーによって発生しました。 -# * sacBuntsPitching : ゲームの犠牲バントの合計が許可されます。 -# * sacFliesPitching : ゲームの犠牲フライは許可されています。 -# * saves : バイナリ、保存でクレジットされている場合は1。 -# * holds : バイナリ、保留がクレジットされている場合は1。 -# * blownSaves : バイナリ、ブローセーブでクレジットされている場合は1。 -# * assists : ゲームのアシスト総数。 -# * putOuts : ゲームの刺殺の総数。 -# * errors : ゲームのエラーの総数。 -# * chances : ゲームのトータルフィールディングチャンス。 - -# %% [code] {"papermill":{"duration":0.102554,"end_time":"2021-06-11T18:43:58.478581","exception":false,"start_time":"2021-06-11T18:43:58.376027","status":"completed"},"tags":[]} - - -# %% [markdown] {"papermill":{"duration":0.108051,"end_time":"2021-06-11T18:43:58.692413","exception":false,"start_time":"2021-06-11T18:43:58.584362","status":"completed"},"tags":[]} -# ## 1.5 teamBoxScores(train.csvのcolumn5番目) -# 特定の日のチーム ゲーム レベルで集計されたゲーム統計を含むネストされた JSON。レギュラーシーズン、ポストシーズン、オールスターゲームが含まれます。 - -# %% [code] {"papermill":{"duration":0.131738,"end_time":"2021-06-11T18:43:58.930844","exception":false,"start_time":"2021-06-11T18:43:58.799106","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:16:00.738588Z","iopub.execute_input":"2021-06-14T09:16:00.738972Z","iopub.status.idle":"2021-06-14T09:16:00.765648Z","shell.execute_reply.started":"2021-06-14T09:16:00.738941Z","shell.execute_reply":"2021-06-14T09:16:00.764551Z"}} -training.head(3) - -# %% [code] {"papermill":{"duration":0.15405,"end_time":"2021-06-11T18:43:59.189049","exception":false,"start_time":"2021-06-11T18:43:59.034999","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:16:01.366547Z","iopub.execute_input":"2021-06-14T09:16:01.367082Z","iopub.status.idle":"2021-06-14T09:16:01.4172Z","shell.execute_reply.started":"2021-06-14T09:16:01.36705Z","shell.execute_reply":"2021-06-14T09:16:01.415972Z"}} -tmpdf = exshow("teamBoxScores",0) -tmpdf.head(5) - -# %% [markdown] {"papermill":{"duration":0.121084,"end_time":"2021-06-11T18:43:59.427565","exception":false,"start_time":"2021-06-11T18:43:59.306481","status":"completed"},"tags":[]} -# カラムの意味の翻訳は↓を開いてください。(長いので、hideしています。) - -# %% [markdown] {"_kg_hide-input":true,"papermill":{"duration":0.120566,"end_time":"2021-06-11T18:43:59.660476","exception":false,"start_time":"2021-06-11T18:43:59.53991","status":"completed"},"tags":[]} -# * home : バイナリ、ホームチームの場合は1、離れている場合は0。 -# * teamId : チームの一意の識別子。 -# * gamePk : ゲームの一意の識別子。 -# * gameDate : -# * gameTimeUTC : UTCでの始球式。 -# * flyOuts : ゲームの合計フライアウト。 -# * groundOuts : ゲームのトータルグラウンドアウト。 -# * runsScored : ゲームの合計ランが記録されました。 -# * doubles : ゲームの合計は2倍です。 -# * triples : ゲームの合計トリプル。 -# * homeRuns : ゲームの総本塁打。 -# * strikeOuts : ゲームの合計三振。 -# * baseOnBalls : ゲームの合計ウォーク。 -# * intentionalWalks : ゲームの故意四球。 -# * hits : ゲームの総ヒット数。 -# * hitByPitch : ピッチによるゲームの合計ヒット。 -# * atBats : でのゲーム合計 -# * caughtStealing : ゲームの合計が盗塁をキャッチしました。 -# * stolenBases : ゲームの盗塁総数。 -# * groundIntoDoublePlay : ゲームの合計併殺はに基づいています。 -# * groundIntoTriplePlay : ゲームの合計 3 回プレイが基礎になります。 -# * plateAppearances : ゲームの総打席。 -# * totalBases : ゲームの総拠点数。 -# * rbi : ゲームの合計打点。 -# * leftOnBase : ゲームの総ランナーはベースに残った。 -# * sacBunts : ゲームの合計犠牲バント。 -# * sacFlies : ゲームの総犠牲フライ。 -# * catchersInterference : ゲームのトータルキャッチャーの干渉が発生しました。 -# * pickoffs : ゲームの合計回数がベースから外れました。 -# * airOutsPitching : エアアウト（フライアウト+ポップアウト）のゲーム合計が許可されます。 -# * groundOutsPitching : ゲームの合計グラウンドアウトが許可されます。 -# * runsPitching : ゲームの合計実行が許可されます。 -# * doublesPitching : ゲームの合計は2倍になります。 -# * triplesPitching : ゲームの合計トリプルが許可されます。 -# * homeRunsPitching : ゲームの合計ホームランが許可されます。 -# * strikeOutsPitching : ゲームの合計三振が許可されます。 -# * baseOnBallsPitching : ゲームの合計歩行が許可されます。 -# * intentionalWalksPitching : ゲームの故意四球の合計が許可されます。 -# * hitsPitching : 許可されるゲームの合計ヒット数。 -# * hitByPitchPitching : 許可されたピッチによるゲームの合計ヒット。 -# * atBatsPitching : でのゲーム合計 -# * caughtStealingPitching : ゲームの合計は、盗みをキャッチしました。 -# * stolenBasesPitching : ゲームの盗塁の合計は許可されます。 -# * inningsPitched : ゲームの総投球回。 -# * earnedRuns : ゲームの合計自責点は許可されています。 -# * battersFaced : 直面したゲームの総打者。 -# * outsPitching : ゲームの合計アウトが記録されました。 -# * hitBatsmen : ゲームの総死球打者。 -# * balks : ゲームの合計はボークします。 -# * wildPitches : 投げられた暴投のゲーム総数。 -# * pickoffsPitching : ゲームのピックオフの総数。 -# * rbiPitching : 打点のゲーム総数は許可されています。 -# * inheritedRunners : 継承されたランナーのゲーム合計を想定。 -# * inheritedRunnersScored : 得点した継承されたランナーのゲーム合計。 -# * catchersInterferencePitching : キャッチャーの干渉のゲーム合計はバッテリーによって発生しました。 -# * sacBuntsPitching : ゲームの犠牲バントの合計が許可されます。 -# * sacFliesPitching : ゲームの犠牲フライは許可されています。 - -# %% [code] {"papermill":{"duration":0.120236,"end_time":"2021-06-11T18:43:59.90294","exception":false,"start_time":"2021-06-11T18:43:59.782704","status":"completed"},"tags":[]} - - -# %% [markdown] {"papermill":{"duration":0.124656,"end_time":"2021-06-11T18:44:00.150824","exception":false,"start_time":"2021-06-11T18:44:00.026168","status":"completed"},"tags":[]} -# ## 1.6 transactions(train.csvのcolumn6番目) -# 特定の日の MLB チームに関係するすべてのトランザクション情報を含むネストされた JSON。 - -# %% [code] {"papermill":{"duration":0.271551,"end_time":"2021-06-11T18:44:00.569742","exception":false,"start_time":"2021-06-11T18:44:00.298191","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:16:04.232745Z","iopub.execute_input":"2021-06-14T09:16:04.233092Z","iopub.status.idle":"2021-06-14T09:16:04.26048Z","shell.execute_reply.started":"2021-06-14T09:16:04.233063Z","shell.execute_reply":"2021-06-14T09:16:04.25898Z"}} -training.head(3) - -# %% [code] {"papermill":{"duration":0.164462,"end_time":"2021-06-11T18:44:00.857988","exception":false,"start_time":"2021-06-11T18:44:00.693526","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:16:04.801025Z","iopub.execute_input":"2021-06-14T09:16:04.801379Z","iopub.status.idle":"2021-06-14T09:16:04.834846Z","shell.execute_reply.started":"2021-06-14T09:16:04.801351Z","shell.execute_reply":"2021-06-14T09:16:04.833668Z"}} -tmpdf = exshow("transactions",1) -tmpdf - -# %% [markdown] {"papermill":{"duration":0.108288,"end_time":"2021-06-11T18:44:01.074241","exception":false,"start_time":"2021-06-11T18:44:00.965953","status":"completed"},"tags":[]} -# * transactionId : トランザクションの一意の識別子。 -# * playerId : プレーヤーの一意の識別子。 -# * playerName : -# * date : -# * fromTeamId : プレーヤーの出身チームの一意の識別子。 -# * fromTeamName : -# * toTeamId : プレーヤーが行くチームの一意の識別子。 -# * toTeamName : -# * effectiveDate : -# * resolutionDate : -# * typeCode : トランザクションステータスの略語。 -# * typeDesc : トランザクションステータスの説明。 -# * description : トランザクションのテキスト説明。 - -# %% [markdown] {"papermill":{"duration":0.112168,"end_time":"2021-06-11T18:44:01.294478","exception":false,"start_time":"2021-06-11T18:44:01.18231","status":"completed"},"tags":[]} -# ## 1.7 standings(train.csvのcolumn7番目) -# 特定の日の MLB チームに関するすべての順位情報を含むネストされた JSON。 - -# %% [code] {"papermill":{"duration":0.140989,"end_time":"2021-06-11T18:44:01.541726","exception":false,"start_time":"2021-06-11T18:44:01.400737","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:16:06.386009Z","iopub.execute_input":"2021-06-14T09:16:06.386417Z","iopub.status.idle":"2021-06-14T09:16:06.415615Z","shell.execute_reply.started":"2021-06-14T09:16:06.386382Z","shell.execute_reply":"2021-06-14T09:16:06.414552Z"}} -training.head(3) - -# %% [code] {"papermill":{"duration":0.164115,"end_time":"2021-06-11T18:44:01.812397","exception":false,"start_time":"2021-06-11T18:44:01.648282","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:16:07.341281Z","iopub.execute_input":"2021-06-14T09:16:07.341726Z","iopub.status.idle":"2021-06-14T09:16:07.391635Z","shell.execute_reply.started":"2021-06-14T09:16:07.34169Z","shell.execute_reply":"2021-06-14T09:16:07.390416Z"}} -tmpdf = exshow("standings",0) -tmpdf.head(5) - -# %% [markdown] {"papermill":{"duration":0.112066,"end_time":"2021-06-11T18:44:02.037524","exception":false,"start_time":"2021-06-11T18:44:01.925458","status":"completed"},"tags":[]} -# カラムの意味の翻訳は↓を開いてください。(長いので、hideしています。) - -# %% [markdown] {"_kg_hide-input":true,"papermill":{"duration":0.106878,"end_time":"2021-06-11T18:44:02.255318","exception":false,"start_time":"2021-06-11T18:44:02.14844","status":"completed"},"tags":[]} -# * season : -# * gameDate : -# * divisionId : このチームが所属する部門を表す一意識別子。 -# * teamId : チームの一意の識別子。 -# * teamName : -# * streakCode : チームの現在の勝ち負けの連続の略語。最初の文字は勝ち負けを示し、数字はゲームの数です。 -# * divisionRank : チームの部門における現在のランク。 -# * leagueRank : リーグでのチームの現在のランク。 -# * wildCardRank : ワイルドカードバースのチームの現在のランク。 -# * leagueGamesBack : ゲームはチームのリーグに戻ります。 -# * sportGamesBack : MLBのすべてに戻ってゲーム。 -# * divisionGamesBack : チームの部門にゲームが戻ってきました。 -# * wins : 現在の勝利。 -# * losses : 現在の損失。 -# * pct : 現在の勝率。 -# * runsAllowed : シーズン中に許可された実行。 -# * runsScored : シーズンに得点したラン。 -# * divisionChamp : チームが部門タイトルを獲得した場合はtrue。 -# * divisionLeader : チームがディビジョンレースをリードしている場合はtrue。 -# * wildCardLeader : チームがワイルドカードリーダーの場合はtrue。 -# * eliminationNumber : ディビジョンレースから排除されるまでのゲーム数（チームの敗北+対戦相手の勝利）。 -# * wildCardEliminationNumber : ワイルドカードレースから排除されるまでのゲーム数（チームの敗北+対戦相手の勝利）。 -# * homeWins : ホームはシーズンに勝ちます。 -# * homeLosses : シーズン中のホームロス。 -# * awayWins : アウェイはシーズンに勝ちます。 -# * awayLosses : シーズンのアウェイロス。 -# * lastTenWins : 過去10試合で勝ちました。 -# * lastTenLosses : 過去10試合で負けました。 -# * extraInningWins : シーズンの追加イニングで勝ちます。 -# * extraInningLosses : シーズンの追加イニングでの損失。 -# * oneRunWins : シーズン中に1ランで勝ちます。 -# * oneRunLosses : シーズン中に1ランで負けます。 -# * dayWins : デイゲームはシーズンに勝ちます。 -# * dayLosses Day game losses on the season. : -# * nightWins : ナイトゲームはシーズンに勝ちます。 -# * nightLosses : シーズン中のナイトゲームの敗北。 -# * grassWins : 芝生のフィールドがシーズンに勝ちます。 -# * grassLosses : 季節の草地の損失。 -# * turfWins : 芝フィールドはシーズンに勝ちます。 -# * turfLosses : シーズン中の芝フィールドの損失。 -# * divWins : シーズン中にディビジョンの対戦相手に勝ちます。 -# * divLosses : シーズン中のディビジョンの対戦相手に対する敗北。 -# * alWins : シーズン中にALチームに勝ちます。 -# * alLosses : シーズン中のALチームに対する敗北。 -# * nlWins : シーズン中にNLチームに勝ちます。 -# * nlLosses : シーズン中のNLチームに対する敗北。 -# * xWinLossPct : スコアリングおよび許可されたランに基づく予想勝率. - -# %% [code] {"papermill":{"duration":0.106775,"end_time":"2021-06-11T18:44:02.471542","exception":false,"start_time":"2021-06-11T18:44:02.364767","status":"completed"},"tags":[]} - - -# %% [markdown] {"papermill":{"duration":0.107466,"end_time":"2021-06-11T18:44:02.687027","exception":false,"start_time":"2021-06-11T18:44:02.579561","status":"completed"},"tags":[]} -# ## 1.8 awards(train.csvのcolumn8番目) -# 特定の日に配られたすべての賞または栄誉を含むネストされた JSON。 - -# %% [code] {"papermill":{"duration":0.134686,"end_time":"2021-06-11T18:44:02.93004","exception":false,"start_time":"2021-06-11T18:44:02.795354","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:16:11.266915Z","iopub.execute_input":"2021-06-14T09:16:11.267266Z","iopub.status.idle":"2021-06-14T09:16:11.294052Z","shell.execute_reply.started":"2021-06-14T09:16:11.267232Z","shell.execute_reply":"2021-06-14T09:16:11.29274Z"}} -training.head(3) - -# %% [code] {"papermill":{"duration":0.135331,"end_time":"2021-06-11T18:44:03.177168","exception":false,"start_time":"2021-06-11T18:44:03.041837","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:16:12.039564Z","iopub.execute_input":"2021-06-14T09:16:12.039924Z","iopub.status.idle":"2021-06-14T09:16:12.061189Z","shell.execute_reply.started":"2021-06-14T09:16:12.039892Z","shell.execute_reply":"2021-06-14T09:16:12.060285Z"}} -tmpdf = exshow("awards",0) -tmpdf - -# %% [markdown] {"papermill":{"duration":0.109992,"end_time":"2021-06-11T18:44:03.399011","exception":false,"start_time":"2021-06-11T18:44:03.289019","status":"completed"},"tags":[]} -# * awardId : -# * awardName : -# * awardDate : 日付賞が与えられました。 -# * awardSeason : シーズンアワードはからでした。 -# * playerId : プレーヤーの一意の識別子。 -# * playerName : -# * awardPlayerTeamId : - -# %% [markdown] {"papermill":{"duration":0.1095,"end_time":"2021-06-11T18:44:03.617125","exception":false,"start_time":"2021-06-11T18:44:03.507625","status":"completed"},"tags":[]} -# ## 1.9 events(train.csvのcolumn9番目) -# 特定の日のすべてのオンフィールド ゲーム イベントを含むネストされた JSON。レギュラーシーズンとポストシーズンの試合が含まれます。 - -# %% [code] {"papermill":{"duration":0.137444,"end_time":"2021-06-11T18:44:03.868309","exception":false,"start_time":"2021-06-11T18:44:03.730865","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:16:15.023498Z","iopub.execute_input":"2021-06-14T09:16:15.023833Z","iopub.status.idle":"2021-06-14T09:16:15.049979Z","shell.execute_reply.started":"2021-06-14T09:16:15.023804Z","shell.execute_reply":"2021-06-14T09:16:15.048934Z"}} -training.head(3) - -# %% [code] {"papermill":{"duration":0.464152,"end_time":"2021-06-11T18:44:04.44451","exception":false,"start_time":"2021-06-11T18:44:03.980358","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:16:15.920155Z","iopub.execute_input":"2021-06-14T09:16:15.920493Z","iopub.status.idle":"2021-06-14T09:16:16.20824Z","shell.execute_reply.started":"2021-06-14T09:16:15.920465Z","shell.execute_reply":"2021-06-14T09:16:16.206929Z"}} -tmpdf = exshow("events",0) -tmpdf.head(5) - -# %% [markdown] {"papermill":{"duration":0.110823,"end_time":"2021-06-11T18:44:04.666523","exception":false,"start_time":"2021-06-11T18:44:04.5557","status":"completed"},"tags":[]} -# カラムの意味の翻訳は↓を開いてください。(長いので、hideしています。) - -# %% [markdown] {"_kg_hide-input":true,"papermill":{"duration":0.110798,"end_time":"2021-06-11T18:44:04.89031","exception":false,"start_time":"2021-06-11T18:44:04.779512","status":"completed"},"tags":[]} -# * gamePk : ゲームの一意の識別子。 -# * gameDate : -# * gameTimeUTC : UTCでの始球式。 -# * season : -# * gameType : ゲームの種類、さまざまな種類がここにあります。 -# * playId : スタットキャストのプレイガイド。 -# * eventId : -# * inning : イニングABが発生しました。 -# * halfInning : 「上」または「下」のイニングインジケーター。 -# * homeScore : イベント開始時のホームスコア。 -# * awayScore : イベント開始時のアウェイスコア。 -# * menOnBase : 走者がベースにいる場合に使用されるスプリット–すなわち（RISP、空）。 -# * atBatIndex : で -# * atBatDesc : 演奏する -# * atBatEvent : atBatのイベントタイプの結果。さまざまなタイプがここにあります。 -# * hasOut : バイナリ、ランナーが場に出ている場合は1。 -# * pitcherTeamId : ピッチングチームの一意の識別子。 -# * isPitcherHome : バイナリ、投手がホームチームの場合は1。 -# * pitcherTeam : ピッチングチームのチーム名。 -# * hitterTeamId : 打撃チームの一意の識別子。 -# * hitterTeam : 打撃チームのチーム名。 -# * pitcherId : -# * pitcherName : -# * isStarter : バイナリ、プレーヤーがゲームの先発投手だった場合は1。 -# * pitcherHand : プレーヤーが手を投げる：「L」、「R」。 -# * hitterId : -# * hitterName : -# * batSide : プレーヤーのバット側：「L」、「R」。 -# * pitchNumber : ABのピッチシーケンス番号。 -# * balls : イベント後のボール数。 -# * strikes : イベント後のストライクカウント。 -# * isGB : バイナリ、打席がグラウンドボールの場合は1。 -# * isLD : バイナリ、打席がラインドライブの場合は1。 -# * isFB : バイナリ、打席が飛球の場合は1。 -# * isPU : バイナリ、打席がポップアップの場合は1。 -# * launchSpeed : 打球の測定速度。 -# * launchAngle : ヒットが開始された地平線に対する垂直角度。 -# * totalDistance : ボールが移動した合計距離。 -# * event : で発生する可能性のあるイベント -# * description : イベントのテキスト説明。 -# * rbi : AB中に打点を打った回数。 -# * pitchType : ピッチタイプ分類コード。さまざまなタイプがここにあります。 -# * call : 投球または投球の結果分類コード。さまざまなタイプがここにあります。 -# * outs : ABの現在/最終アウト。 -# * inPlay : ボールが場に出た場合は真/偽。 -# * isPaOver : バイナリ、このイベントがプレートの外観の終わりである場合は1。 -# * startSpeed : ホームプレートの前50フィートでのボールのMPHでの速度。 -# * endSpeed : ボールがホームプレートの前端（x軸で0,0）を横切るときのボールのMPHでの速度。 -# * nastyFactor : 各ピッチのいくつかのプロパティを評価し、ピッチの「不快感」を0からのスケールで評価します -# * breakAngle : ピッチの平面が垂直から外れる時計回り（打者の視点）の角度。 -# * breakLength : ピッチがピッチ開始とピッチ終了の間の直線から離れる最大距離。 -# * breakY : ブレークが最大のホームプレートからの距離。 -# * spinRate : ピッチャーによってRPMでリリースされた後のボールのスピン率。 -# * spinDirection : スピンがボールの弾道にどのように影響するかを反映する角度として与えられる、リリース時のボールの回転軸。ピュアバック -# * pX : ボールがホームプレートの前軸と交差するときのボールのフィート単位の水平位置。 -# * pZ : ボールがホームプレートの前軸と交差するときの、ボールのホームプレートからのフィート単位の垂直位置。 -# * aX : z軸のボール加速度。 -# * aY : y軸のボール加速度。 -# * aZ : z 軸上のボールの加速度。 -# * pfxX : インチ単位のボールの水平方向の動き。 -# * pfxZ : インチ単位のボールの垂直方向の動き。 -# * vX0 : x軸からのボールの速度。 -# * vY0 : y軸からのボールの速度。 0,0,0 はバッターの後ろにあり、ボールはピッチャー マウンドから 0,0,0 に向かって移動するため、これは負です。 -# * vZ0 : z軸からのボールの速度。 -# * x : ピッチがホームプレートの前を横切ったX座標。 -# * y : ピッチがホームプレートの前面と交差するY座標。 -# * x0 : ピッチャーの手を離したときのボールの x 軸上の座標位置 (時間 = 0)。 -# * y0 : y軸上でピッチャーの手からボールがリリースされたポイントでのボールの座標位置（時間= 0）。 -# * z0 : z軸上でピッチャーの手からボールがリリースされたポイントでのボールの座標位置（時間= 0）。 -# * type : 「ピッチ」または「アクション」のいずれかのイベントのタイプ -# * zone : ゾーンロケーション番号.下を参照 -# -# ![image.png](attachment:1ad951bc-0f08-4424-83c4-6ff88a557d7d.png) -# - -# %% [markdown] {"papermill":{"duration":0.114746,"end_time":"2021-06-11T18:44:05.117226","exception":false,"start_time":"2021-06-11T18:44:05.00248","status":"completed"},"tags":[]} -# ## 1.10 playerTwitterFollowers(train.csvのcolumn10番目) -# その日の一部のプレイヤーの Twitter フォロワー数を含むネストされた JSON。 - -# %% [code] {"papermill":{"duration":0.142856,"end_time":"2021-06-11T18:44:05.374626","exception":false,"start_time":"2021-06-11T18:44:05.23177","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:16:19.415876Z","iopub.execute_input":"2021-06-14T09:16:19.416249Z","iopub.status.idle":"2021-06-14T09:16:19.445478Z","shell.execute_reply.started":"2021-06-14T09:16:19.416219Z","shell.execute_reply":"2021-06-14T09:16:19.444498Z"}} -training.head(3) - -# %% [code] {"papermill":{"duration":0.148668,"end_time":"2021-06-11T18:44:05.64077","exception":false,"start_time":"2021-06-11T18:44:05.492102","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:16:20.69924Z","iopub.execute_input":"2021-06-14T09:16:20.699607Z","iopub.status.idle":"2021-06-14T09:16:20.734173Z","shell.execute_reply.started":"2021-06-14T09:16:20.699576Z","shell.execute_reply":"2021-06-14T09:16:20.732938Z"}} -tmpdf = exshow("playerTwitterFollowers",0) -tmpdf.head(3) - -# %% [markdown] {"papermill":{"duration":0.115258,"end_time":"2021-06-11T18:44:05.87703","exception":false,"start_time":"2021-06-11T18:44:05.761772","status":"completed"},"tags":[]} -# Twitterのフォローデータは、MLBによってメジャーリーグプレーヤーのTwitter APIから毎月1日に収集され、2018年1月1日までさかのぼります。 すべてのプレーヤーがTwitterアカウントを持っている/持っているわけではない、プレーヤーがランダムにアカウントを作成/削除/復元する、または特定の日にフォロワーデータを収集できないその他のシナリオがあるため、このデータセットはすべての月にわたってすべてのプレーヤーを網羅しているわけではありません。 - -# %% [markdown] {"papermill":{"duration":0.1152,"end_time":"2021-06-11T18:44:06.109077","exception":false,"start_time":"2021-06-11T18:44:05.993877","status":"completed"},"tags":[]} -# * date : フォロワー数の日付。 -# * playerId : プレーヤーの一意の識別子。 -# * playerName : -# * accountName : プレイヤーのTwitterアカウントの名前。 -# * twitterHandle : プレイヤーのツイッターハンドル。 -# * numberOfFollowers : フォロワー数 - -# %% [markdown] {"papermill":{"duration":0.117931,"end_time":"2021-06-11T18:44:06.340356","exception":false,"start_time":"2021-06-11T18:44:06.222425","status":"completed"},"tags":[]} -# ## 1.11 teamTwitterFollowers(train.csvのcolumn11番目) -# その日の各チームの Twitter フォロワー数を含むネストされた JSON。 - -# %% [code] {"papermill":{"duration":0.144909,"end_time":"2021-06-11T18:44:06.602116","exception":false,"start_time":"2021-06-11T18:44:06.457207","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:17:20.07115Z","iopub.execute_input":"2021-06-14T09:17:20.071555Z","iopub.status.idle":"2021-06-14T09:17:20.098844Z","shell.execute_reply.started":"2021-06-14T09:17:20.071521Z","shell.execute_reply":"2021-06-14T09:17:20.097876Z"}} -training.head(3) - -# %% [code] {"papermill":{"duration":0.140245,"end_time":"2021-06-11T18:44:06.858481","exception":false,"start_time":"2021-06-11T18:44:06.718236","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:17:21.031613Z","iopub.execute_input":"2021-06-14T09:17:21.03204Z","iopub.status.idle":"2021-06-14T09:17:21.057308Z","shell.execute_reply.started":"2021-06-14T09:17:21.032005Z","shell.execute_reply":"2021-06-14T09:17:21.056119Z"}} -tmpdf = exshow("teamTwitterFollowers",0) -tmpdf.head(3) - -# %% [markdown] {"papermill":{"duration":0.119312,"end_time":"2021-06-11T18:44:07.09097","exception":false,"start_time":"2021-06-11T18:44:06.971658","status":"completed"},"tags":[]} -# Twitterのフォローデータは、2018年1月1日までさかのぼって、毎月1日に、メジャーリーグの30チームすべてのTwitterAPIからMLBによって収集されました。 - -# %% [markdown] {"papermill":{"duration":0.11246,"end_time":"2021-06-11T18:44:07.317214","exception":false,"start_time":"2021-06-11T18:44:07.204754","status":"completed"},"tags":[]} -# * date : フォロワー数の日付。 -# * teamId : チームの一意の識別子。 -# * teamName : -# * accountName : チームのTwitterアカウントの名前。 -# * twitterHandle : チームのツイッターハンドル。 - -# %% [markdown] {"papermill":{"duration":0.119547,"end_time":"2021-06-11T18:44:07.551012","exception":false,"start_time":"2021-06-11T18:44:07.431465","status":"completed"},"tags":[]} -# やっとこ中身確認完了。おつかれさまでした。。。 - -# %% [code] {"papermill":{"duration":0.120035,"end_time":"2021-06-11T18:44:07.783273","exception":false,"start_time":"2021-06-11T18:44:07.663238","status":"completed"},"tags":[]} - - -# %% [markdown] {"papermill":{"duration":0.112525,"end_time":"2021-06-11T18:44:08.009126","exception":false,"start_time":"2021-06-11T18:44:07.896601","status":"completed"},"tags":[]} -# # 2. 他のadditional data ( awards.csv, players.csv, seasons.csv, teams.csv) - -# %% [markdown] {"papermill":{"duration":0.12237,"end_time":"2021-06-11T18:44:08.250799","exception":false,"start_time":"2021-06-11T18:44:08.128429","status":"completed"},"tags":[]} -# ## 2.1 starterにあったwidgetの練習(こんなことできるんだーと思いましたので・・・) - -# %% [code] {"papermill":{"duration":0.120605,"end_time":"2021-06-11T18:44:08.489884","exception":false,"start_time":"2021-06-11T18:44:08.369279","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:17:55.345428Z","iopub.execute_input":"2021-06-14T09:17:55.345843Z","iopub.status.idle":"2021-06-14T09:17:55.35036Z","shell.execute_reply.started":"2021-06-14T09:17:55.345812Z","shell.execute_reply":"2021-06-14T09:17:55.349032Z"}} -df_names = ['seasons', 'teams', 'players', 'awards'] - -path = "../input/mlb-player-digital-engagement-forecasting" - -# %% [code] {"papermill":{"duration":0.155209,"end_time":"2021-06-11T18:44:08.759151","exception":false,"start_time":"2021-06-11T18:44:08.603942","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:17:58.414493Z","iopub.execute_input":"2021-06-14T09:17:58.414844Z","iopub.status.idle":"2021-06-14T09:17:58.448828Z","shell.execute_reply.started":"2021-06-14T09:17:58.414816Z","shell.execute_reply":"2021-06-14T09:17:58.447643Z"}} -kaggle_data_tabs = widgets.Tab() -# widgetsにそれぞれのDataFrameをchildrenの中にタブで表示 -kaggle_data_tabs.children = list([widgets.Output() for df_name in df_names]) - -# %% [code] {"papermill":{"duration":0.295868,"end_time":"2021-06-11T18:44:09.169213","exception":false,"start_time":"2021-06-11T18:44:08.873345","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:17:59.564008Z","iopub.execute_input":"2021-06-14T09:17:59.564414Z","iopub.status.idle":"2021-06-14T09:17:59.734201Z","shell.execute_reply.started":"2021-06-14T09:17:59.564381Z","shell.execute_reply":"2021-06-14T09:17:59.733044Z"}} -for index in range(len(df_names)): - # タブのタイトルを設定 - kaggle_data_tabs.set_title(index, df_names[index]) - - df = pd.read_csv(os.path.join(path,df_names[index]) + ".csv") - - # それぞれのタブにDataFrameを埋め込む - with kaggle_data_tabs.children[index]: - display(df) - -# %% [code] {"papermill":{"duration":0.127693,"end_time":"2021-06-11T18:44:09.419366","exception":false,"start_time":"2021-06-11T18:44:09.291673","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:18:01.219466Z","iopub.execute_input":"2021-06-14T09:18:01.221633Z","iopub.status.idle":"2021-06-14T09:18:01.229718Z","shell.execute_reply.started":"2021-06-14T09:18:01.22159Z","shell.execute_reply":"2021-06-14T09:18:01.228737Z"}} -display(kaggle_data_tabs) - -# %% [markdown] {"papermill":{"duration":0.112987,"end_time":"2021-06-11T18:44:09.648047","exception":false,"start_time":"2021-06-11T18:44:09.53506","status":"completed"},"tags":[]} -# -----------細かく一つ一つ見ていきます----------- - -# %% [markdown] {"papermill":{"duration":0.128132,"end_time":"2021-06-11T18:44:09.894007","exception":false,"start_time":"2021-06-11T18:44:09.765875","status":"completed"},"tags":[]} -# ## 2.2 Seasons.csv - -# %% [code] {"papermill":{"duration":0.150781,"end_time":"2021-06-11T18:44:10.190127","exception":false,"start_time":"2021-06-11T18:44:10.039346","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:19:00.781853Z","iopub.execute_input":"2021-06-14T09:19:00.782327Z","iopub.status.idle":"2021-06-14T09:19:00.807238Z","shell.execute_reply.started":"2021-06-14T09:19:00.782296Z","shell.execute_reply":"2021-06-14T09:19:00.806039Z"}} -seasons = pd.read_csv("../input/mlb-player-digital-engagement-forecasting/seasons.csv") -seasons - -# %% [markdown] {"papermill":{"duration":0.116735,"end_time":"2021-06-11T18:44:10.442337","exception":false,"start_time":"2021-06-11T18:44:10.325602","status":"completed"},"tags":[]} -# * seasonId : シーズンID -# * seasonStartDate : シーズンスタート日 -# * seasonEndDate : シーズン終了日 -# * preSeasonStartDate : 1つ前のシーズンスタート日 -# * preSeasonEndDate : 1つ前のシーズンの終わりの日 -# * regularSeasonStartDate : レギュラーシーズンのスタートの日 -# * regularSeasonEndDate : レギュラーシーズンの終わりの日 -# * lastDate1stHalf : 1st halfの最終日 -# * allStarDate : オールスター戦の日付 -# * firstDate2ndHalf : 2nd halfの始まり日 -# * postSeasonStartDate : 次のシーズンのスタート日 -# * postSeasonEndDate : 次のシーズンの終わり日 - -# %% [code] {"papermill":{"duration":0.118911,"end_time":"2021-06-11T18:44:10.677851","exception":false,"start_time":"2021-06-11T18:44:10.55894","status":"completed"},"tags":[]} - - -# %% [markdown] {"papermill":{"duration":0.11553,"end_time":"2021-06-11T18:44:10.912934","exception":false,"start_time":"2021-06-11T18:44:10.797404","status":"completed"},"tags":[]} -# ## 2.3 teams.csv - -# %% [code] {"papermill":{"duration":0.138966,"end_time":"2021-06-11T18:44:11.167983","exception":false,"start_time":"2021-06-11T18:44:11.029017","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:19:09.180321Z","iopub.execute_input":"2021-06-14T09:19:09.180697Z","iopub.status.idle":"2021-06-14T09:19:09.204548Z","shell.execute_reply.started":"2021-06-14T09:19:09.180668Z","shell.execute_reply":"2021-06-14T09:19:09.203438Z"}} -teams = pd.read_csv("../input/mlb-player-digital-engagement-forecasting/teams.csv") -teams.head(3) - -# %% [markdown] {"papermill":{"duration":0.125126,"end_time":"2021-06-11T18:44:11.428349","exception":false,"start_time":"2021-06-11T18:44:11.303223","status":"completed"},"tags":[]} -# ## teams.csv -# * id - チームID -# * name : 名前 -# * teamName : チームの名前 -# * teamCode : チームのコード -# * shortName : 短い名前 -# * abbreviation : 略語 -# * locationName : 場所の名前 -# * leagueId : リーグのid -# * leagueName : リーグの名前 -# * divisionId : 部門id -# * divisionName : 部門名 -# * venueId : 会場id -# * venueName : 会場名 - -# %% [markdown] {"papermill":{"duration":0.115273,"end_time":"2021-06-11T18:44:11.660746","exception":false,"start_time":"2021-06-11T18:44:11.545473","status":"completed"},"tags":[]} -# ## 2.4 players.csv - -# %% [code] {"papermill":{"duration":0.146866,"end_time":"2021-06-11T18:44:11.923004","exception":false,"start_time":"2021-06-11T18:44:11.776138","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:19:17.018473Z","iopub.execute_input":"2021-06-14T09:19:17.019073Z","iopub.status.idle":"2021-06-14T09:19:17.050961Z","shell.execute_reply.started":"2021-06-14T09:19:17.019024Z","shell.execute_reply":"2021-06-14T09:19:17.049924Z"}} -players = pd.read_csv("../input/mlb-player-digital-engagement-forecasting/players.csv") -players.head(3) - -# %% [markdown] {"papermill":{"duration":0.116271,"end_time":"2021-06-11T18:44:12.158173","exception":false,"start_time":"2021-06-11T18:44:12.041902","status":"completed"},"tags":[]} -# * playerId - Unique identifier for a player. : プレーヤーID-プレーヤーの一意の識別子。 -# * playerName : プレーヤの名前 -# * DOB - Player’s date of birth. : DOB-プレーヤーの生年月日。 -# * mlbDebutDate : MLBデビュー日 -# * birthCity : 生まれた町 -# * birthStateProvince : 出生州 -# * birthCountry : 生まれた国 -# * heightInches : 身長(inch) -# * weight : 体重 -# * primaryPositionCode - Player’s primary position code : 主要ポジションコード -# * primaryPositionName - player’s primary position : 主要ポジション名 -# * playerForTestSetAndFuturePreds - Boolean, true if player is among those for whom predictions are to be made in test data -# -# : **ブール値、プレーヤーがテストデータで予測が行われる対象の1人である場合はtrue** - -# %% [code] {"papermill":{"duration":0.11651,"end_time":"2021-06-11T18:44:12.393967","exception":false,"start_time":"2021-06-11T18:44:12.277457","status":"completed"},"tags":[]} - - -# %% [markdown] {"papermill":{"duration":0.118357,"end_time":"2021-06-11T18:44:12.629827","exception":false,"start_time":"2021-06-11T18:44:12.51147","status":"completed"},"tags":[]} -# ## 2.5 awards.csv - -# %% [code] {"papermill":{"duration":0.148565,"end_time":"2021-06-11T18:44:12.897385","exception":false,"start_time":"2021-06-11T18:44:12.74882","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:19:57.113713Z","iopub.execute_input":"2021-06-14T09:19:57.114402Z","iopub.status.idle":"2021-06-14T09:19:57.152653Z","shell.execute_reply.started":"2021-06-14T09:19:57.114352Z","shell.execute_reply":"2021-06-14T09:19:57.151616Z"}} -awards = pd.read_csv("../input/mlb-player-digital-engagement-forecasting/awards.csv") -awards.head(3) - -# %% [markdown] {"papermill":{"duration":0.118007,"end_time":"2021-06-11T18:44:13.133412","exception":false,"start_time":"2021-06-11T18:44:13.015405","status":"completed"},"tags":[]} -# このファイルには、日次データの開始前（つまり、2018年以前）にトレーニングセットのプレーヤーが獲得した賞が含まれています。 -# -# * awardDate - Date award was given. : 授与日 - 授与された日付。 -# * awardSeason - Season award was from. : アワードシーズン-シーズンアワードはからでした。 -# * awardId : アワードid -# * awardName : アワード名 -# * playerId - Unique identifier for a player. : プレーヤーID-プレーヤーの一意の識別子。 -# * playerName : プレーヤーの名前 -# * awardPlayerTeamId : プレイヤーのチームID - -# %% [code] {"papermill":{"duration":0.116296,"end_time":"2021-06-11T18:44:13.36787","exception":false,"start_time":"2021-06-11T18:44:13.251574","status":"completed"},"tags":[]} - - -# %% [markdown] {"papermill":{"duration":0.115578,"end_time":"2021-06-11T18:44:13.601453","exception":false,"start_time":"2021-06-11T18:44:13.485875","status":"completed"},"tags":[]} -# # 3. Data Merge - -# %% [markdown] {"papermill":{"duration":0.120359,"end_time":"2021-06-11T18:44:13.841012","exception":false,"start_time":"2021-06-11T18:44:13.720653","status":"completed"},"tags":[]} -# とりあえず、スターターhttps://www.kaggle.com/ryanholbrook/getting-started-with-mlb-player-digital-engagement　 -# -# のコピーです。けっこう時間かかります。 - -# %% [code] {"papermill":{"duration":0.164068,"end_time":"2021-06-11T18:44:14.121247","exception":false,"start_time":"2021-06-11T18:44:13.957179","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:20:09.176809Z","iopub.execute_input":"2021-06-14T09:20:09.177183Z","iopub.status.idle":"2021-06-14T09:20:09.220929Z","shell.execute_reply.started":"2021-06-14T09:20:09.177151Z","shell.execute_reply":"2021-06-14T09:20:09.2202Z"}} -for name in df_names: - globals()[name] = pd.read_csv(os.path.join(path,name)+ ".csv") - -# %% [code] {"papermill":{"duration":306.845173,"end_time":"2021-06-11T18:49:21.088167","exception":false,"start_time":"2021-06-11T18:44:14.242994","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:20:10.410562Z","iopub.execute_input":"2021-06-14T09:20:10.411092Z","iopub.status.idle":"2021-06-14T09:25:36.408897Z","shell.execute_reply.started":"2021-06-14T09:20:10.411043Z","shell.execute_reply":"2021-06-14T09:25:36.40774Z"}} -#### Unnest various nested data within training (daily) data #### -daily_data_unnested_dfs = pd.DataFrame(data = { - 'dfName': training.drop('date', axis = 1).columns.values.tolist() - }) - -# Slow from this point !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1 - -daily_data_unnested_dfs['df'] = [pd.DataFrame() for row in - daily_data_unnested_dfs.iterrows()] - -for df_index, df_row in daily_data_unnested_dfs.iterrows(): - nestedTableName = str(df_row['dfName']) - - date_nested_table = training[['date', nestedTableName]] - - date_nested_table = (date_nested_table[ - ~pd.isna(date_nested_table[nestedTableName]) - ]. - reset_index(drop = True) - ) - - daily_dfs_collection = [] - - for date_index, date_row in date_nested_table.iterrows(): - daily_df = unpack_json(date_row[nestedTableName]) - - daily_df['dailyDataDate'] = date_row['date'] - - daily_dfs_collection = daily_dfs_collection + [daily_df] - - unnested_table = pd.concat(daily_dfs_collection, - ignore_index = True).set_index('dailyDataDate').reset_index() - - # Creates 1 pandas df per unnested df from daily data read in, with same name - globals()[df_row['dfName']] = unnested_table - - daily_data_unnested_dfs['df'][df_index] = unnested_table - -del training -gc.collect() - - - -#### Get some information on each date in daily data (using season dates of interest) #### -dates = pd.DataFrame(data = - {'dailyDataDate': nextDayPlayerEngagement['dailyDataDate'].unique()}) - -dates['date'] = pd.to_datetime(dates['dailyDataDate'].astype(str)) - -dates['year'] = dates['date'].dt.year -dates['month'] = dates['date'].dt.month - -dates_with_info = pd.merge( - dates, - seasons, - left_on = 'year', - right_on = 'seasonId' - ) - -dates_with_info['inSeason'] = ( - dates_with_info['date'].between( - dates_with_info['regularSeasonStartDate'], - dates_with_info['postSeasonEndDate'], - inclusive = True - ) - ) - -dates_with_info['seasonPart'] = np.select( - [ - dates_with_info['date'] < dates_with_info['preSeasonStartDate'], - dates_with_info['date'] < dates_with_info['regularSeasonStartDate'], - dates_with_info['date'] <= dates_with_info['lastDate1stHalf'], - dates_with_info['date'] < dates_with_info['firstDate2ndHalf'], - dates_with_info['date'] <= dates_with_info['regularSeasonEndDate'], - dates_with_info['date'] < dates_with_info['postSeasonStartDate'], - dates_with_info['date'] <= dates_with_info['postSeasonEndDate'], - dates_with_info['date'] > dates_with_info['postSeasonEndDate'] - ], - [ - 'Offseason', - 'Preseason', - 'Reg Season 1st Half', - 'All-Star Break', - 'Reg Season 2nd Half', - 'Between Reg and Postseason', - 'Postseason', - 'Offseason' - ], - default = np.nan - ) - -#### Add some pitching stats/pieces of info to player game level stats #### - -player_game_stats = (playerBoxScores.copy(). - # Change team Id/name to reflect these come from player game, not roster - rename(columns = {'teamId': 'gameTeamId', 'teamName': 'gameTeamName'}) - ) - -# Adds in field for innings pitched as fraction (better for aggregation) -player_game_stats['inningsPitchedAsFrac'] = np.where( - pd.isna(player_game_stats['inningsPitched']), - np.nan, - np.floor(player_game_stats['inningsPitched']) + - (player_game_stats['inningsPitched'] - - np.floor(player_game_stats['inningsPitched'])) * 10/3 - ) - -# Add in Tom Tango pitching game score (https://www.mlb.com/glossary/advanced-stats/game-score) -player_game_stats['pitchingGameScore'] = (40 -# + 2 * player_game_stats['outs'] - + 1 * player_game_stats['strikeOutsPitching'] - - 2 * player_game_stats['baseOnBallsPitching'] - - 2 * player_game_stats['hitsPitching'] - - 3 * player_game_stats['runsPitching'] - - 6 * player_game_stats['homeRunsPitching'] - ) - -# Add in criteria for no-hitter by pitcher (individual, not multiple pitchers) -player_game_stats['noHitter'] = np.where( - (player_game_stats['gamesStartedPitching'] == 1) & - (player_game_stats['inningsPitched'] >= 9) & - (player_game_stats['hitsPitching'] == 0), - 1, 0 - ) - -player_date_stats_agg = pd.merge( - (player_game_stats. - groupby(['dailyDataDate', 'playerId'], as_index = False). - # Some aggregations that are not simple sums - agg( - numGames = ('gamePk', 'nunique'), - # Should be 1 team per player per day, but adding here for 1 exception: - # playerId 518617 (Jake Diekman) had 2 games for different teams marked - # as played on 5/19/19, due to resumption of game after he was traded - numTeams = ('gameTeamId', 'nunique'), - # Should be only 1 team for almost all player-dates, taking min to simplify - gameTeamId = ('gameTeamId', 'min') - ) - ), - # Merge with a bunch of player stats that can be summed at date/player level - (player_game_stats. - groupby(['dailyDataDate', 'playerId'], as_index = False) - [['runsScored', 'homeRuns', 'strikeOuts', 'baseOnBalls', 'hits', - 'hitByPitch', 'atBats', 'caughtStealing', 'stolenBases', - 'groundIntoDoublePlay', 'groundIntoTriplePlay', 'plateAppearances', - 'totalBases', 'rbi', 'leftOnBase', 'sacBunts', 'sacFlies', - 'gamesStartedPitching', 'runsPitching', 'homeRunsPitching', - 'strikeOutsPitching', 'baseOnBallsPitching', 'hitsPitching', - 'inningsPitchedAsFrac', 'earnedRuns', - 'battersFaced','saves', 'blownSaves', 'pitchingGameScore', - 'noHitter' - ]]. - sum() - ), - on = ['dailyDataDate', 'playerId'], - how = 'inner' - ) - -#### Turn games table into 1 row per team-game, then merge with team box scores #### -# Filter to regular or Postseason games w/ valid scores for this part -games_for_stats = games[ - np.isin(games['gameType'], ['R', 'F', 'D', 'L', 'W', 'C', 'P']) & - ~pd.isna(games['homeScore']) & - ~pd.isna(games['awayScore']) - ] - -# Get games table from home team perspective -games_home_perspective = games_for_stats.copy() - -# Change column names so that "team" is "home", "opp" is "away" -games_home_perspective.columns = [ - col_value.replace('home', 'team').replace('away', 'opp') for - col_value in games_home_perspective.columns.values] - -games_home_perspective['isHomeTeam'] = 1 - -# Get games table from away team perspective -games_away_perspective = games_for_stats.copy() - -# Change column names so that "opp" is "home", "team" is "away" -games_away_perspective.columns = [ - col_value.replace('home', 'opp').replace('away', 'team') for - col_value in games_away_perspective.columns.values] - -games_away_perspective['isHomeTeam'] = 0 - -# Put together games from home/away perspective to get df w/ 1 row per team game -team_games = (pd.concat([ - games_home_perspective, - games_away_perspective - ], - ignore_index = True) - ) - -# Copy over team box scores data to modify -team_game_stats = teamBoxScores.copy() - -# Change column names to reflect these are all "team" stats - helps -# to differentiate from individual player stats if/when joining later -team_game_stats.columns = [ - (col_value + 'Team') - if (col_value not in ['dailyDataDate', 'home', 'teamId', 'gamePk', - 'gameDate', 'gameTimeUTC']) - else col_value - for col_value in team_game_stats.columns.values - ] - -# Merge games table with team game stats -team_games_with_stats = pd.merge( - team_games, - team_game_stats. - # Drop some fields that are already present in team_games table - drop(['home', 'gameDate', 'gameTimeUTC'], axis = 1), - on = ['dailyDataDate', 'gamePk', 'teamId'], - # Doing this as 'inner' join excludes spring training games, postponed games, - # etc. from original games table, but this may be fine for purposes here - how = 'inner' - ) - -team_date_stats_agg = (team_games_with_stats. - groupby(['dailyDataDate', 'teamId', 'gameType', 'oppId', 'oppName'], - as_index = False). - agg( - numGamesTeam = ('gamePk', 'nunique'), - winsTeam = ('teamWinner', 'sum'), - lossesTeam = ('oppWinner', 'sum'), - runsScoredTeam = ('teamScore', 'sum'), - runsAllowedTeam = ('oppScore', 'sum') - ) - ) - -# Prepare standings table for merge w/ player digital engagement data -# Pick only certain fields of interest from standings for merge -standings_selected_fields = (standings[['dailyDataDate', 'teamId', - 'streakCode', 'divisionRank', 'leagueRank', 'wildCardRank', 'pct' - ]]. - rename(columns = {'pct': 'winPct'}) - ) - -# Change column names to reflect these are all "team" standings - helps -# to differentiate from player-related fields if/when joining later -standings_selected_fields.columns = [ - (col_value + 'Team') - if (col_value not in ['dailyDataDate', 'teamId']) - else col_value - for col_value in standings_selected_fields.columns.values - ] - -standings_selected_fields['streakLengthTeam'] = ( - standings_selected_fields['streakCodeTeam']. - str.replace('W', ''). - str.replace('L', ''). - astype(float) - ) - -# Add fields to separate winning and losing streak from streak code -standings_selected_fields['winStreakTeam'] = np.where( - standings_selected_fields['streakCodeTeam'].str[0] == 'W', - standings_selected_fields['streakLengthTeam'], - np.nan - ) - -standings_selected_fields['lossStreakTeam'] = np.where( - standings_selected_fields['streakCodeTeam'].str[0] == 'L', - standings_selected_fields['streakLengthTeam'], - np.nan - ) - -standings_for_digital_engagement_merge = (pd.merge( - standings_selected_fields, - dates_with_info[['dailyDataDate', 'inSeason']], - on = ['dailyDataDate'], - how = 'left' - ). - # Limit down standings to only in season version - query("inSeason"). - # Drop fields no longer necessary (in derived values, etc.) - drop(['streakCodeTeam', 'streakLengthTeam', 'inSeason'], axis = 1). - reset_index(drop = True) - ) - -#### Merge together various data frames to add date, player, roster, and team info #### -# Copy over player engagement df to add various pieces to it -player_engagement_with_info = nextDayPlayerEngagement.copy() - -# Take "row mean" across targets to add (helps with studying all 4 targets at once) -player_engagement_with_info['targetAvg'] = np.mean( - player_engagement_with_info[['target1', 'target2', 'target3', 'target4']], - axis = 1) - -# Merge in date information -player_engagement_with_info = pd.merge( - player_engagement_with_info, - dates_with_info[['dailyDataDate', 'date', 'year', 'month', 'inSeason', - 'seasonPart']], - on = ['dailyDataDate'], - how = 'left' - ) - -# Merge in some player information -player_engagement_with_info = pd.merge( - player_engagement_with_info, - players[['playerId', 'playerName', 'DOB', 'mlbDebutDate', 'birthCity', - 'birthStateProvince', 'birthCountry', 'primaryPositionName']], - on = ['playerId'], - how = 'left' - ) - -# Merge in some player roster information by date -player_engagement_with_info = pd.merge( - player_engagement_with_info, - (rosters[['dailyDataDate', 'playerId', 'statusCode', 'status', 'teamId']]. - rename(columns = { - 'statusCode': 'rosterStatusCode', - 'status': 'rosterStatus', - 'teamId': 'rosterTeamId' - }) - ), - on = ['dailyDataDate', 'playerId'], - how = 'left' - ) - -# Merge in team name from player's roster team -player_engagement_with_info = pd.merge( - player_engagement_with_info, - (teams[['id', 'teamName']]. - rename(columns = { - 'id': 'rosterTeamId', - 'teamName': 'rosterTeamName' - }) - ), - on = ['rosterTeamId'], - how = 'left' - ) - -# Merge in some player game stats (previously aggregated) from that date -player_engagement_with_info = pd.merge( - player_engagement_with_info, - player_date_stats_agg, - on = ['dailyDataDate', 'playerId'], - how = 'left' - ) - -# Merge in team name from player's game team -player_engagement_with_info = pd.merge( - player_engagement_with_info, - (teams[['id', 'teamName']]. - rename(columns = { - 'id': 'gameTeamId', - 'teamName': 'gameTeamName' - }) - ), - on = ['gameTeamId'], - how = 'left' - ) - -# Merge in some team game stats/results (previously aggregated) from that date -player_engagement_with_info = pd.merge( - player_engagement_with_info, - team_date_stats_agg.rename(columns = {'teamId': 'gameTeamId'}), - on = ['dailyDataDate', 'gameTeamId'], - how = 'left' - ) - -# Merge in player transactions of note on that date - -# Merge in some pieces of team standings (previously filter/processed) from that date -player_engagement_with_info = pd.merge( - player_engagement_with_info, - standings_for_digital_engagement_merge. - rename(columns = {'teamId': 'gameTeamId'}), - on = ['dailyDataDate', 'gameTeamId'], - how = 'left' - ) - -display(player_engagement_with_info) - -# %% [code] {"papermill":{"duration":0.143732,"end_time":"2021-06-11T18:49:21.348227","exception":false,"start_time":"2021-06-11T18:49:21.204495","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:26:55.997662Z","iopub.execute_input":"2021-06-14T09:26:55.998073Z","iopub.status.idle":"2021-06-14T09:26:56.018325Z","shell.execute_reply.started":"2021-06-14T09:26:55.998038Z","shell.execute_reply":"2021-06-14T09:26:56.017469Z"}} -player_engagement_with_info.info() - -# %% [markdown] {"papermill":{"duration":0.116904,"end_time":"2021-06-11T18:49:21.581773","exception":false,"start_time":"2021-06-11T18:49:21.464869","status":"completed"},"tags":[]} -# output結果をreferenceできるように、一応pickleで保存しておきます。 - -# %% [code] {"execution":{"iopub.execute_input":"2021-06-11T18:49:21.819366Z","iopub.status.busy":"2021-06-11T18:49:21.81832Z","iopub.status.idle":"2021-06-11T18:49:29.543521Z","shell.execute_reply":"2021-06-11T18:49:29.542493Z","shell.execute_reply.started":"2021-06-11T18:29:24.484735Z"},"papermill":{"duration":7.84534,"end_time":"2021-06-11T18:49:29.543677","exception":false,"start_time":"2021-06-11T18:49:21.698337","status":"completed"},"tags":[]} -player_engagement_with_info.to_pickle("player_engagement_with_info.pkl") - -# %% [code] {"papermill":{"duration":0.134207,"end_time":"2021-06-11T18:49:29.803106","exception":false,"start_time":"2021-06-11T18:49:29.668899","status":"completed"},"tags":[]} - - -# %% [markdown] {"papermill":{"duration":0.136875,"end_time":"2021-06-11T18:49:30.071632","exception":false,"start_time":"2021-06-11T18:49:29.934757","status":"completed"},"tags":[]} -# #### スターターではここからkerasで簡単なモデル作成をしていますので、興味ある方はそちらをご覧ください。 - -# %% [code] {"papermill":{"duration":0.153007,"end_time":"2021-06-11T18:49:30.347614","exception":false,"start_time":"2021-06-11T18:49:30.194607","status":"completed"},"tags":[]} - - -# %% [markdown] {"papermill":{"duration":0.12281,"end_time":"2021-06-11T18:49:30.601655","exception":false,"start_time":"2021-06-11T18:49:30.478845","status":"completed"},"tags":[]} -# ### 以下、検証用として、target1～4をすべて中間値(スコア上がるため、v8でmeanからmedianに変更しました)でsubmitします。 - -# %% [code] {"papermill":{"duration":0.163708,"end_time":"2021-06-11T18:49:30.885597","exception":false,"start_time":"2021-06-11T18:49:30.721889","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:27:36.099392Z","iopub.execute_input":"2021-06-14T09:27:36.099805Z","iopub.status.idle":"2021-06-14T09:27:36.314269Z","shell.execute_reply.started":"2021-06-14T09:27:36.09976Z","shell.execute_reply":"2021-06-14T09:27:36.313164Z"}} -t1_median = player_engagement_with_info["target1"].median() -t2_median = player_engagement_with_info["target2"].median() -t3_median = player_engagement_with_info["target3"].median() -t4_median = player_engagement_with_info["target4"].median() - -# %% [code] {"papermill":{"duration":0.131632,"end_time":"2021-06-11T18:49:31.142954","exception":false,"start_time":"2021-06-11T18:49:31.011322","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:27:37.123159Z","iopub.execute_input":"2021-06-14T09:27:37.12362Z","iopub.status.idle":"2021-06-14T09:27:37.130077Z","shell.execute_reply.started":"2021-06-14T09:27:37.123585Z","shell.execute_reply":"2021-06-14T09:27:37.128856Z"}} -print(t1_median,t2_median,t3_median,t4_median) - -# %% [markdown] {"papermill":{"duration":0.11749,"end_time":"2021-06-11T18:49:33.163029","exception":false,"start_time":"2021-06-11T18:49:33.045539","status":"completed"},"tags":[]} -# # 4. submitの形式 -# riiidの https://www.kaggle.com/chumajin/eda-for-biginner　で解説したのと同じく、1部のtest dataをget → 1部を予測　→　1部を提出　をどんどん繰り返していく方式です。 -# 今回は1日分のtest data→次の日を予測提出、んで、次の日のtest data→その次の日を予測、提出　の流れです。 -# -# -# ## **↓のmake_envは1回しか実行できません。** -# ## **失敗したら、データをrestart(上の方のFactory resetボタンを押す)して、再度やることになりますので、注意が必要です!** - -# %% [markdown] {"papermill":{"duration":0.118225,"end_time":"2021-06-11T18:49:33.398072","exception":false,"start_time":"2021-06-11T18:49:33.279847","status":"completed"},"tags":[]} -# #### 最終形はこんな感じです(スターターから抜粋。解説用に少し細かくやっていきます - -# %% [code] {"execution":{"iopub.execute_input":"2021-06-11T18:49:33.63826Z","iopub.status.busy":"2021-06-11T18:49:33.637334Z","iopub.status.idle":"2021-06-11T18:49:33.641655Z","shell.execute_reply":"2021-06-11T18:49:33.642107Z","shell.execute_reply.started":"2021-06-11T18:29:31.450494Z"},"papermill":{"duration":0.127157,"end_time":"2021-06-11T18:49:33.642278","exception":false,"start_time":"2021-06-11T18:49:33.515121","status":"completed"},"tags":[]} -""" -if 'kaggle_secrets' in sys.modules: # only run while on Kaggle - import mlb - - env = mlb.make_env() - iter_test = env.iter_test() - - for (test_df, sample_prediction_df) in iter_test: - - # Example: unpack a dataframe from a json column - today_games = unpack_json(test_df['games'].iloc[0]) - - # Make your predictions for the next day's engagement - sample_prediction_df['target1'] = 100.00 - - # Submit your predictions - env.predict(sample_prediction_df) - - -""" - -# %% [markdown] {"papermill":{"duration":0.118402,"end_time":"2021-06-11T18:49:33.878823","exception":false,"start_time":"2021-06-11T18:49:33.760421","status":"completed"},"tags":[]} -# #### ここから↑のサンプルコードを少し解説 - -# %% [markdown] {"papermill":{"duration":0.117862,"end_time":"2021-06-11T18:49:34.114753","exception":false,"start_time":"2021-06-11T18:49:33.996891","status":"completed"},"tags":[]} -# mlbのダウンロード - -# %% [code] {"papermill":{"duration":0.157437,"end_time":"2021-06-11T18:49:34.38967","exception":false,"start_time":"2021-06-11T18:49:34.232233","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:28:19.253637Z","iopub.execute_input":"2021-06-14T09:28:19.254017Z","iopub.status.idle":"2021-06-14T09:28:19.298491Z","shell.execute_reply.started":"2021-06-14T09:28:19.253987Z","shell.execute_reply":"2021-06-14T09:28:19.297258Z"}} -if 'kaggle_secrets' in sys.modules: # only run while on Kaggle - import mlb - -# %% [markdown] {"papermill":{"duration":0.124681,"end_time":"2021-06-11T18:49:34.711578","exception":false,"start_time":"2021-06-11T18:49:34.586897","status":"completed"},"tags":[]} -# envとiter_testの定義 (お決まりの作業と思ってもらえれば) - -# %% [code] {"papermill":{"duration":0.125932,"end_time":"2021-06-11T18:49:34.955809","exception":false,"start_time":"2021-06-11T18:49:34.829877","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:28:24.550083Z","iopub.execute_input":"2021-06-14T09:28:24.550516Z","iopub.status.idle":"2021-06-14T09:28:24.55605Z","shell.execute_reply.started":"2021-06-14T09:28:24.550468Z","shell.execute_reply":"2021-06-14T09:28:24.554671Z"}} -env = mlb.make_env() -iter_test = env.iter_test() - -# %% [markdown] {"papermill":{"duration":0.118437,"end_time":"2021-06-11T18:49:35.193711","exception":false,"start_time":"2021-06-11T18:49:35.075274","status":"completed"},"tags":[]} -# iter_testの中身を見てみる (とりあえずbreakで1個だけ見る。break外すとエラーでます。理由はそのあと解説しています) - -# %% [code] {"papermill":{"duration":0.991035,"end_time":"2021-06-11T18:49:36.301953","exception":false,"start_time":"2021-06-11T18:49:35.310918","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:28:40.080548Z","iopub.execute_input":"2021-06-14T09:28:40.081101Z","iopub.status.idle":"2021-06-14T09:28:40.942286Z","shell.execute_reply.started":"2021-06-14T09:28:40.081052Z","shell.execute_reply":"2021-06-14T09:28:40.941273Z"}} -for (test_df, sample_prediction_df) in iter_test: - display(test_df) - display(sample_prediction_df) - break - -# %% [markdown] {"papermill":{"duration":0.121287,"end_time":"2021-06-11T18:49:36.544242","exception":false,"start_time":"2021-06-11T18:49:36.422955","status":"completed"},"tags":[]} -# 1日分のtest dataと、submissionファイルが出てくるのがわかる -# -# -# ここで、submissionファイルに予測値を記入して、提出しないと、次の日のtest dataを受け取ることができないというエラーが出る(以下のように、もう一度走らせると怒られる) - -# %% [code] {"papermill":{"duration":0.226649,"end_time":"2021-06-11T18:49:36.890061","exception":false,"start_time":"2021-06-11T18:49:36.663412","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:29:31.022394Z","iopub.execute_input":"2021-06-14T09:29:31.022795Z","iopub.status.idle":"2021-06-14T09:29:31.080859Z","shell.execute_reply.started":"2021-06-14T09:29:31.022764Z","shell.execute_reply":"2021-06-14T09:29:31.079283Z"}} -for (test_df, sample_prediction_df) in iter_test: - display(test_df) - display(sample_prediction_df) - break - -# %% [markdown] {"papermill":{"duration":0.121178,"end_time":"2021-06-11T18:49:37.134545","exception":false,"start_time":"2021-06-11T18:49:37.013367","status":"completed"},"tags":[]} -# なので、一度submissionファイルに記入して提出してあげないといけません。ここではお試しに、先ほどのtargetのそれぞれの中間値を全部入れてみます。 - -# %% [code] {"papermill":{"duration":0.145229,"end_time":"2021-06-11T18:49:37.401468","exception":false,"start_time":"2021-06-11T18:49:37.256239","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:29:51.650122Z","iopub.execute_input":"2021-06-14T09:29:51.650494Z","iopub.status.idle":"2021-06-14T09:29:51.671854Z","shell.execute_reply.started":"2021-06-14T09:29:51.650465Z","shell.execute_reply":"2021-06-14T09:29:51.670792Z"}} -sample_prediction_df["target1"] = t1_median -sample_prediction_df["target2"] = t2_median -sample_prediction_df["target3"] = t3_median -sample_prediction_df["target4"] = t4_median - - -sample_prediction_df - -# %% [markdown] {"papermill":{"duration":0.121948,"end_time":"2021-06-11T18:49:37.64384","exception":false,"start_time":"2021-06-11T18:49:37.521892","status":"completed"},"tags":[]} -# 予測値を入れたらこの時点で一度下記のコードでsubmitします - -# %% [code] {"papermill":{"duration":0.129576,"end_time":"2021-06-11T18:49:37.895614","exception":false,"start_time":"2021-06-11T18:49:37.766038","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:29:59.007068Z","iopub.execute_input":"2021-06-14T09:29:59.007417Z","iopub.status.idle":"2021-06-14T09:29:59.012104Z","shell.execute_reply.started":"2021-06-14T09:29:59.00739Z","shell.execute_reply":"2021-06-14T09:29:59.010732Z"}} -env.predict(sample_prediction_df) - -# %% [markdown] {"papermill":{"duration":0.123149,"end_time":"2021-06-11T18:49:38.142959","exception":false,"start_time":"2021-06-11T18:49:38.01981","status":"completed"},"tags":[]} -# そうすると、次の日のデータが受け取れるようになります。(以下のように先ほどと同じコードを流してもエラーで怒られません) - -# %% [code] {"papermill":{"duration":0.383184,"end_time":"2021-06-11T18:49:38.648033","exception":false,"start_time":"2021-06-11T18:49:38.264849","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:30:09.450715Z","iopub.execute_input":"2021-06-14T09:30:09.45107Z","iopub.status.idle":"2021-06-14T09:30:09.707916Z","shell.execute_reply.started":"2021-06-14T09:30:09.451042Z","shell.execute_reply":"2021-06-14T09:30:09.707035Z"}} -for (test_df, sample_prediction_df) in iter_test: - display(test_df) - display(sample_prediction_df) - break - -# %% [markdown] {"papermill":{"duration":0.121984,"end_time":"2021-06-11T18:49:38.89173","exception":false,"start_time":"2021-06-11T18:49:38.769746","status":"completed"},"tags":[]} -# 以下のスターターのコードをもう一度みて見ると、for文の中でこれを繰り返してsubmitしていることがわかります。 -# -# 基本的には以下のfor文の中身を、test dfの前処理と、predictionして、sample_prediction_dfの書き換え、env.predictで提出していく流れですね。 -# -# riiidコンペの場合は、一つ前の情報の正解が流れてきてましたので、それを使って次の予測のためのデータとして使用していました(今回もおそらくそうなのかな??)。 - -# %% [code] {"papermill":{"duration":0.121709,"end_time":"2021-06-11T18:49:39.135494","exception":false,"start_time":"2021-06-11T18:49:39.013785","status":"completed"},"tags":[]} - - -# %% [code] {"execution":{"iopub.execute_input":"2021-06-11T18:49:39.386397Z","iopub.status.busy":"2021-06-11T18:49:39.385345Z","iopub.status.idle":"2021-06-11T18:49:39.390058Z","shell.execute_reply":"2021-06-11T18:49:39.389555Z"},"papermill":{"duration":0.133027,"end_time":"2021-06-11T18:49:39.390209","exception":false,"start_time":"2021-06-11T18:49:39.257182","status":"completed"},"tags":[]} -""" -if 'kaggle_secrets' in sys.modules: # only run while on Kaggle - import mlb - - env = mlb.make_env() - iter_test = env.iter_test() - - for (test_df, sample_prediction_df) in iter_test: - - # Example: unpack a dataframe from a json column - today_games = unpack_json(test_df['games'].iloc[0]) - - # Make your predictions for the next day's engagement - sample_prediction_df['target1'] = 100.00 - - # Submit your predictions - env.predict(sample_prediction_df) - - -""" - -# %% [markdown] {"papermill":{"duration":0.123008,"end_time":"2021-06-11T18:49:39.63599","exception":false,"start_time":"2021-06-11T18:49:39.512982","status":"completed"},"tags":[]} -# ## 今回はこのままsubmitしたいので、2回目の提出後にfor文で最後まで回します。 - -# %% [code] {"papermill":{"duration":0.132458,"end_time":"2021-06-11T18:49:39.894569","exception":false,"start_time":"2021-06-11T18:49:39.762111","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:30:26.736705Z","iopub.execute_input":"2021-06-14T09:30:26.737092Z","iopub.status.idle":"2021-06-14T09:30:26.745284Z","shell.execute_reply.started":"2021-06-14T09:30:26.737059Z","shell.execute_reply":"2021-06-14T09:30:26.743878Z"}} -# 2回目の提出 - -sample_prediction_df["target1"] = t1_median -sample_prediction_df["target2"] = t2_median -sample_prediction_df["target3"] = t3_median -sample_prediction_df["target4"] = t4_median -env.predict(sample_prediction_df) - - -# %% [code] {"papermill":{"duration":0.760212,"end_time":"2021-06-11T18:49:40.779803","exception":false,"start_time":"2021-06-11T18:49:40.019591","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-14T09:31:21.734487Z","iopub.execute_input":"2021-06-14T09:31:21.734853Z","iopub.status.idle":"2021-06-14T09:31:22.380877Z","shell.execute_reply.started":"2021-06-14T09:31:21.734822Z","shell.execute_reply":"2021-06-14T09:31:22.379995Z"}} -# 残り最後まで - -for (test_df, sample_prediction_df) in iter_test: - - # Example: unpack a dataframe from a json column - #today_games = unpack_json(test_df['games'].iloc[0]) - - # Make your predictions for the next day's engagement - sample_prediction_df["target1"] = t1_median - sample_prediction_df["target2"] = t2_median - sample_prediction_df["target3"] = t3_median - sample_prediction_df["target4"] = t4_median - - # Submit your predictions - env.predict(sample_prediction_df) - -# %% [markdown] {"papermill":{"duration":0.121297,"end_time":"2021-06-11T18:49:41.02355","exception":false,"start_time":"2021-06-11T18:49:40.902253","status":"completed"},"tags":[]} -# ご参考> コード要件 -# -# * これはコードコンペティションです -# * このコンテストへの提出は、ノートブックを通じて行う必要があります。コミット後に[送信]ボタンをアクティブにするには、次の条件が満たされている必要があります。 -# * -# * CPUノートブック<= 6時間の実行時間 -# * GPUノートブック<= 6時間の実行時間 -# * **インターネットアクセスが無効** -# * 事前にトレーニングされたモデルを含む、無料で公開されている外部データが許可されます -# * 提出は、mlbPythonモジュールを使用して行う必要があります - -# %% [code] {"papermill":{"duration":0.135355,"end_time":"2021-06-11T18:49:41.280874","exception":false,"start_time":"2021-06-11T18:49:41.145519","status":"completed"},"tags":[]} - - -# %% [code] {"papermill":{"duration":0.122852,"end_time":"2021-06-11T18:49:41.541567","exception":false,"start_time":"2021-06-11T18:49:41.418715","status":"completed"},"tags":[]} - - -# %% [markdown] {"papermill":{"duration":0.122603,"end_time":"2021-06-11T18:49:41.786428","exception":false,"start_time":"2021-06-11T18:49:41.663825","status":"completed"},"tags":[]} -# # ここまで読んでいただいてありがとうございます！ -# # お役にたてば、upvote/followいただけたら嬉しいです！ -# # よろしくお願いいたします !! - -# %% [code] {"papermill":{"duration":0.123659,"end_time":"2021-06-11T18:49:42.033425","exception":false,"start_time":"2021-06-11T18:49:41.909766","status":"completed"},"tags":[]} diff --git a/d1/mlb_player_v3.py b/d1/mlb_player_v3.py deleted file mode 100644 index 8372f21..0000000 --- a/d1/mlb_player_v3.py +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -# # Overview -# The kernel shows how to use the [tf_pose_estimation](https://github.com/ildoonet/tf-pose-estimation) package in Python on a series of running videos. - -# ## Libraries we need -# Install tf_pose and pycocotools - -# In[1]: - - -import os -def get_ipython(): - return os - -get_ipython().system('pip install -qq https://www.github.com/ildoonet/tf-pose-estimation') - - -# In[2]: - - -get_ipython().system('pip install -qq pycocotools') - - -# In[3]: - - -get_ipython().run_line_magic('load_ext', 'autoreload') -get_ipython().run_line_magic('autoreload', '2') -import seaborn as sns -import matplotlib.pyplot as plt -plt.rcParams["figure.figsize"] = (8, 8) -plt.rcParams["figure.dpi"] = 125 -plt.rcParams["font.size"] = 14 -plt.rcParams['font.family'] = ['sans-serif'] -plt.rcParams['font.sans-serif'] = ['DejaVu Sans'] -plt.style.use('ggplot') -sns.set_style("whitegrid", {'axes.grid': False}) - - -# In[4]: - - -get_ipython().run_line_magic('matplotlib', 'inline') -import tf_pose -import cv2 -from glob import glob -from tqdm import tqdm_notebook -from PIL import Image -import numpy as np -import os -def video_gen(in_path): - c_cap = cv2.VideoCapture(in_path) - while c_cap.isOpened(): - ret, frame = c_cap.read() - if not ret: - break - yield c_cap.get(cv2.CAP_PROP_POS_MSEC), frame[:, :, ::-1] - c_cap.release() - - -# In[5]: - - -video_paths = glob('../input/*.mp4') -c_video = video_gen(video_paths[0]) -for _ in range(300): - c_ts, c_frame = next(c_video) -plt.imshow(c_frame) - - -# In[6]: - - -from tf_pose.estimator import TfPoseEstimator -from tf_pose.networks import get_graph_path, model_wh -tfpe = tf_pose.get_estimator() - - -# In[7]: - - -humans = tfpe.inference(npimg=c_frame, upsample_size=4.0) -print(humans) - - -# In[8]: - - -new_image = TfPoseEstimator.draw_humans(c_frame[:, :, ::-1], humans, imgcopy=False) -fig, ax1 = plt.subplots(1, 1, figsize=(10, 10)) -ax1.imshow(new_image[:, :, ::-1]) - - -# In[9]: - - -body_to_dict = lambda c_fig: {'bp_{}_{}'.format(k, vec_name): vec_val - for k, part_vec in c_fig.body_parts.items() - for vec_name, vec_val in zip(['x', 'y', 'score'], - (part_vec.x, 1-part_vec.y, part_vec.score))} -c_fig = humans[0] -body_to_dict(c_fig) - - -# In[10]: - - -MAX_FRAMES = 200 -body_pose_list = [] -for vid_path in tqdm_notebook(video_paths, desc='Files'): - c_video = video_gen(vid_path) - c_ts, c_frame = next(c_video) - out_path = '{}_out.avi'.format(os.path.split(vid_path)[1]) - out = cv2.VideoWriter(out_path, - cv2.VideoWriter_fourcc('M','J','P','G'), - 10, - (c_frame.shape[1], c_frame.shape[0])) - for (c_ts, c_frame), _ in zip(c_video, - tqdm_notebook(range(MAX_FRAMES), desc='Frames')): - bgr_frame = c_frame[:,:,::-1] - humans = tfpe.inference(npimg=bgr_frame, upsample_size=4.0) - for c_body in humans: - body_pose_list += [dict(video=out_path, time=c_ts, **body_to_dict(c_body))] - new_image = TfPoseEstimator.draw_humans(bgr_frame, humans, imgcopy=False) - out.write(new_image) - out.release() - - -# In[11]: - - -import pandas as pd -body_pose_df = pd.DataFrame(body_pose_list) -body_pose_df.describe() - - -# In[12]: - - -fig, m_axs = plt.subplots(1, 2, figsize=(15, 5)) -for c_ax, (c_name, c_rows) in zip(m_axs, body_pose_df.groupby('video')): - for i in range(17): - c_ax.plot(c_rows['time'], c_rows['bp_{}_y'.format(i)], label='x {}'.format(i)) - c_ax.legend() - c_ax.set_title(c_name) - - -# In[13]: - - -fig, m_axs = plt.subplots(1, 2, figsize=(15, 5)) -for c_ax, (c_name, n_rows) in zip(m_axs, body_pose_df.groupby('video')): - for i in range(17): - c_rows = n_rows.query('bp_{}_score>0.6'.format(i)) # only keep confident results - c_ax.plot(c_rows['bp_{}_x'.format(i)], c_rows['bp_{}_y'.format(i)], label='BP {}'.format(i)) - c_ax.legend() - c_ax.set_title(c_name) - - -# In[14]: - - -body_pose_df.to_csv('body_pose.csv', index=False) - - -# In[15]: diff --git a/d1/mlb_player_v4.py b/d1/mlb_player_v4.py deleted file mode 100644 index feaf276..0000000 --- a/d1/mlb_player_v4.py +++ /dev/null @@ -1,576 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -# -# -# NOTE: Turn on Internet and GPU - -# The code hidden below handles all the imports and function definitions (the heavy lifting). If you're a beginner I'd advice you skip this for now. When you are able to understand the rest of the code, come back here and understand each function to get a deeper knowledge. - -# In[1]: - - -# !/usr/bin/env python3 -# coding=utf-8 -# author=dave.fang@outlook.com -# create=20171225 - -import os -import pprint -import cv2 -import sys -import math -import time -import tempfile -import numpy as np -import matplotlib.pyplot as plt - -import torch -import torch.nn as nn -import torch.nn.parallel -import torch.backends.cudnn as cudnn -import torch.optim as optim -import torchvision.transforms as transforms -import torchvision.datasets as datasets -import torchvision.models as models - -from torch.autograd import Variable - -from scipy.ndimage.filters import gaussian_filter - -#get_ipython().run_line_magic('matplotlib', 'inline') -#get_ipython().run_line_magic('config', "InlineBackend.figure_format = 'retina'") - -# find connection in the specified sequence, center 29 is in the position 15 -limb_seq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], - [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], [1, 15], [15, 17], - [1, 16], [16, 18], [3, 17], [6, 18]] - -# the middle joints heatmap correpondence -map_ids = [[31, 32], [39, 40], [33, 34], [35, 36], [41, 42], [43, 44], [19, 20], [21, 22], - [23, 24], [25, 26], [27, 28], [29, 30], [47, 48], [49, 50], [53, 54], [51, 52], - [55, 56], [37, 38], [45, 46]] - -# these are the colours for the 18 body points -colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], - [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], - [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]] - - -class PoseEstimation(nn.Module): - def __init__(self, model_dict): - super(PoseEstimation, self).__init__() - - self.model0 = model_dict['block_0'] - self.model1_1 = model_dict['block1_1'] - self.model2_1 = model_dict['block2_1'] - self.model3_1 = model_dict['block3_1'] - self.model4_1 = model_dict['block4_1'] - self.model5_1 = model_dict['block5_1'] - self.model6_1 = model_dict['block6_1'] - - self.model1_2 = model_dict['block1_2'] - self.model2_2 = model_dict['block2_2'] - self.model3_2 = model_dict['block3_2'] - self.model4_2 = model_dict['block4_2'] - self.model5_2 = model_dict['block5_2'] - self.model6_2 = model_dict['block6_2'] - - def forward(self, x): - out1 = self.model0(x) - - out1_1 = self.model1_1(out1) - out1_2 = self.model1_2(out1) - out2 = torch.cat([out1_1, out1_2, out1], 1) - - out2_1 = self.model2_1(out2) - out2_2 = self.model2_2(out2) - out3 = torch.cat([out2_1, out2_2, out1], 1) - - out3_1 = self.model3_1(out3) - out3_2 = self.model3_2(out3) - out4 = torch.cat([out3_1, out3_2, out1], 1) - - out4_1 = self.model4_1(out4) - out4_2 = self.model4_2(out4) - out5 = torch.cat([out4_1, out4_2, out1], 1) - - out5_1 = self.model5_1(out5) - out5_2 = self.model5_2(out5) - out6 = torch.cat([out5_1, out5_2, out1], 1) - - out6_1 = self.model6_1(out6) - out6_2 = self.model6_2(out6) - - return out6_1, out6_2 - - -def make_layers(layer_dict): - layers = [] - - for i in range(len(layer_dict) - 1): - layer = layer_dict[i] - for k in layer: - v = layer[k] - if 'pool' in k: - layers += [nn.MaxPool2d(kernel_size=v[0], stride=v[1], padding=v[2])] - else: - conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1], kernel_size=v[2], stride=v[3], padding=v[4]) - layers += [conv2d, nn.ReLU(inplace=True)] - layer = list(layer_dict[-1].keys()) - k = layer[0] - v = layer_dict[-1][k] - - conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1], kernel_size=v[2], stride=v[3], padding=v[4]) - layers += [conv2d] - - return nn.Sequential(*layers) - - -def get_pose_model(): - blocks = {} - - block_0 = [{'conv1_1': [3, 64, 3, 1, 1]}, {'conv1_2': [64, 64, 3, 1, 1]}, {'pool1_stage1': [2, 2, 0]}, - {'conv2_1': [64, 128, 3, 1, 1]}, {'conv2_2': [128, 128, 3, 1, 1]}, {'pool2_stage1': [2, 2, 0]}, - {'conv3_1': [128, 256, 3, 1, 1]}, {'conv3_2': [256, 256, 3, 1, 1]}, {'conv3_3': [256, 256, 3, 1, 1]}, - {'conv3_4': [256, 256, 3, 1, 1]}, {'pool3_stage1': [2, 2, 0]}, {'conv4_1': [256, 512, 3, 1, 1]}, - {'conv4_2': [512, 512, 3, 1, 1]}, {'conv4_3_CPM': [512, 256, 3, 1, 1]}, - {'conv4_4_CPM': [256, 128, 3, 1, 1]}] - - blocks['block1_1'] = [{'conv5_1_CPM_L1': [128, 128, 3, 1, 1]}, {'conv5_2_CPM_L1': [128, 128, 3, 1, 1]}, - {'conv5_3_CPM_L1': [128, 128, 3, 1, 1]}, {'conv5_4_CPM_L1': [128, 512, 1, 1, 0]}, - {'conv5_5_CPM_L1': [512, 38, 1, 1, 0]}] - - blocks['block1_2'] = [{'conv5_1_CPM_L2': [128, 128, 3, 1, 1]}, {'conv5_2_CPM_L2': [128, 128, 3, 1, 1]}, - {'conv5_3_CPM_L2': [128, 128, 3, 1, 1]}, {'conv5_4_CPM_L2': [128, 512, 1, 1, 0]}, - {'conv5_5_CPM_L2': [512, 19, 1, 1, 0]}] - - for i in range(2, 7): - blocks['block%d_1' % i] = [{'Mconv1_stage%d_L1' % i: [185, 128, 7, 1, 3]}, - {'Mconv2_stage%d_L1' % i: [128, 128, 7, 1, 3]}, - {'Mconv3_stage%d_L1' % i: [128, 128, 7, 1, 3]}, - {'Mconv4_stage%d_L1' % i: [128, 128, 7, 1, 3]}, - {'Mconv5_stage%d_L1' % i: [128, 128, 7, 1, 3]}, - {'Mconv6_stage%d_L1' % i: [128, 128, 1, 1, 0]}, - {'Mconv7_stage%d_L1' % i: [128, 38, 1, 1, 0]}] - blocks['block%d_2' % i] = [{'Mconv1_stage%d_L2' % i: [185, 128, 7, 1, 3]}, - {'Mconv2_stage%d_L2' % i: [128, 128, 7, 1, 3]}, - {'Mconv3_stage%d_L2' % i: [128, 128, 7, 1, 3]}, - {'Mconv4_stage%d_L2' % i: [128, 128, 7, 1, 3]}, - {'Mconv5_stage%d_L2' % i: [128, 128, 7, 1, 3]}, - {'Mconv6_stage%d_L2' % i: [128, 128, 1, 1, 0]}, - {'Mconv7_stage%d_L2' % i: [128, 19, 1, 1, 0]}] - - layers = [] - for block in block_0: - # print(block) - for key in block: - v = block[key] - if 'pool' in key: - layers += [nn.MaxPool2d(kernel_size=v[0], stride=v[1], padding=v[2])] - else: - conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1], kernel_size=v[2], stride=v[3], padding=v[4]) - layers += [conv2d, nn.ReLU(inplace=True)] - - models = { - 'block_0': nn.Sequential(*layers) - } - - for k in blocks: - v = blocks[k] - models[k] = make_layers(v) - - return PoseEstimation(models) - - -def get_paf_and_heatmap(model, img_raw, scale_search, param_stride=8, box_size=368): - multiplier = [scale * box_size / img_raw.shape[0] for scale in scale_search] - - heatmap_avg = torch.zeros((len(multiplier), 19, img_raw.shape[0], img_raw.shape[1])).cuda() - paf_avg = torch.zeros((len(multiplier), 38, img_raw.shape[0], img_raw.shape[1])).cuda() - - for i, scale in enumerate(multiplier): - img_test = cv2.resize(img_raw, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) - img_test_pad, pad = pad_right_down_corner(img_test, param_stride, param_stride) - img_test_pad = np.transpose(np.float32(img_test_pad[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5 - - feed = Variable(torch.from_numpy(img_test_pad)).cuda() - output1, output2 = model(feed) - - print(output1.size()) - print(output2.size()) - - heatmap = nn.UpsamplingBilinear2d((img_raw.shape[0], img_raw.shape[1])).cuda()(output2) - - paf = nn.UpsamplingBilinear2d((img_raw.shape[0], img_raw.shape[1])).cuda()(output1) - - heatmap_avg[i] = heatmap[0].data - paf_avg[i] = paf[0].data - - heatmap_avg = torch.transpose(torch.transpose(torch.squeeze(torch.mean(heatmap_avg, 0)), 0, 1), 1, 2).cuda() - heatmap_avg = heatmap_avg.cpu().numpy() - - paf_avg = torch.transpose(torch.transpose(torch.squeeze(torch.mean(paf_avg, 0)), 0, 1), 1, 2).cuda() - paf_avg = paf_avg.cpu().numpy() - - return paf_avg, heatmap_avg - - -def extract_heatmap_info(heatmap_avg, param_thre1=0.1): - all_peaks = [] - peak_counter = 0 - - for part in range(18): - map_ori = heatmap_avg[:, :, part] - map_gau = gaussian_filter(map_ori, sigma=3) - - map_left = np.zeros(map_gau.shape) - map_left[1:, :] = map_gau[:-1, :] - map_right = np.zeros(map_gau.shape) - map_right[:-1, :] = map_gau[1:, :] - map_up = np.zeros(map_gau.shape) - map_up[:, 1:] = map_gau[:, :-1] - map_down = np.zeros(map_gau.shape) - map_down[:, :-1] = map_gau[:, 1:] - - peaks_binary = np.logical_and.reduce( - (map_gau >= map_left, map_gau >= map_right, map_gau >= map_up, - map_gau >= map_down, map_gau > param_thre1)) - - peaks = zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0]) # note reverse - peaks = list(peaks) - peaks_with_score = [x + (map_ori[x[1], x[0]],) for x in peaks] - ids = range(peak_counter, peak_counter + len(peaks)) - peaks_with_score_and_id = [peaks_with_score[i] + (ids[i],) for i in range(len(ids))] - - all_peaks.append(peaks_with_score_and_id) - peak_counter += len(peaks) - - return all_peaks - - -def extract_paf_info(img_raw, paf_avg, all_peaks, param_thre2=0.05, param_thre3=0.5): - connection_all = [] - special_k = [] - mid_num = 10 - - for k in range(len(map_ids)): - score_mid = paf_avg[:, :, [x - 19 for x in map_ids[k]]] - candA = all_peaks[limb_seq[k][0] - 1] - candB = all_peaks[limb_seq[k][1] - 1] - nA = len(candA) - nB = len(candB) - if nA != 0 and nB != 0: - connection_candidate = [] - for i in range(nA): - for j in range(nB): - vec = np.subtract(candB[j][:2], candA[i][:2]) - norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1]) - vec = np.divide(vec, norm) - - startend = zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), - np.linspace(candA[i][1], candB[j][1], num=mid_num)) - startend = list(startend) - - vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] - for I in range(len(startend))]) - vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] - for I in range(len(startend))]) - - score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1]) - score_with_dist_prior = sum(score_midpts) / len(score_midpts) - score_with_dist_prior += min(0.5 * img_raw.shape[0] / norm - 1, 0) - - criterion1 = len(np.nonzero(score_midpts > param_thre2)[0]) > 0.8 * len(score_midpts) - criterion2 = score_with_dist_prior > 0 - if criterion1 and criterion2: - connection_candidate.append( - [i, j, score_with_dist_prior, score_with_dist_prior + candA[i][2] + candB[j][2]]) - - connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True) - connection = np.zeros((0, 5)) - for c in range(len(connection_candidate)): - i, j, s = connection_candidate[c][0:3] - if i not in connection[:, 3] and j not in connection[:, 4]: - connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]]) - if len(connection) >= min(nA, nB): - break - - connection_all.append(connection) - else: - special_k.append(k) - connection_all.append([]) - - return special_k, connection_all - - -def get_subsets(connection_all, special_k, all_peaks): - # last number in each row is the total parts number of that person - # the second last number in each row is the score of the overall configuration - subset = -1 * np.ones((0, 20)) - candidate = np.array([item for sublist in all_peaks for item in sublist]) - - for k in range(len(map_ids)): - if k not in special_k: - partAs = connection_all[k][:, 0] - partBs = connection_all[k][:, 1] - indexA, indexB = np.array(limb_seq[k]) - 1 - - for i in range(len(connection_all[k])): # = 1:size(temp,1) - found = 0 - subset_idx = [-1, -1] - for j in range(len(subset)): # 1:size(subset,1): - if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]: - subset_idx[found] = j - found += 1 - - if found == 1: - j = subset_idx[0] - if (subset[j][indexB] != partBs[i]): - subset[j][indexB] = partBs[i] - subset[j][-1] += 1 - subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] - elif found == 2: # if found 2 and disjoint, merge them - j1, j2 = subset_idx - print("found = 2") - membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2] - if len(np.nonzero(membership == 2)[0]) == 0: # merge - subset[j1][:-2] += (subset[j2][:-2] + 1) - subset[j1][-2:] += subset[j2][-2:] - subset[j1][-2] += connection_all[k][i][2] - subset = np.delete(subset, j2, 0) - else: # as like found == 1 - subset[j1][indexB] = partBs[i] - subset[j1][-1] += 1 - subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] - - # if find no partA in the subset, create a new subset - elif not found and k < 17: - row = -1 * np.ones(20) - row[indexA] = partAs[i] - row[indexB] = partBs[i] - row[-1] = 2 - row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2] - subset = np.vstack([subset, row]) - return subset, candidate - - -def draw_key_point(subset, all_peaks, img_raw): - del_ids = [] - for i in range(len(subset)): - if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4: - del_ids.append(i) - subset = np.delete(subset, del_ids, axis=0) - - img_canvas = img_raw.copy() # B,G,R order - - for i in range(18): - for j in range(len(all_peaks[i])): - cv2.circle(img_canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1) - - return subset, img_canvas - - -def link_key_point(img_canvas, candidate, subset, stickwidth=4): - for i in range(17): - for n in range(len(subset)): - index = subset[n][np.array(limb_seq[i]) - 1] - if -1 in index: - continue - cur_canvas = img_canvas.copy() - Y = candidate[index.astype(int), 0] - X = candidate[index.astype(int), 1] - mX = np.mean(X) - mY = np.mean(Y) - length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5 - angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) - polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1) - cv2.fillConvexPoly(cur_canvas, polygon, colors[i]) - img_canvas = cv2.addWeighted(img_canvas, 0.4, cur_canvas, 0.6, 0) - - return img_canvas - -def pad_right_down_corner(img, stride, pad_value): - h = img.shape[0] - w = img.shape[1] - - pad = 4 * [None] - pad[0] = 0 # up - pad[1] = 0 # left - pad[2] = 0 if (h % stride == 0) else stride - (h % stride) # down - pad[3] = 0 if (w % stride == 0) else stride - (w % stride) # right - - img_padded = img - pad_up = np.tile(img_padded[0:1, :, :] * 0 + pad_value, (pad[0], 1, 1)) - img_padded = np.concatenate((pad_up, img_padded), axis=0) - pad_left = np.tile(img_padded[:, 0:1, :] * 0 + pad_value, (1, pad[1], 1)) - img_padded = np.concatenate((pad_left, img_padded), axis=1) - pad_down = np.tile(img_padded[-2:-1, :, :] * 0 + pad_value, (pad[2], 1, 1)) - img_padded = np.concatenate((img_padded, pad_down), axis=0) - pad_right = np.tile(img_padded[:, -2:-1, :] * 0 + pad_value, (1, pad[3], 1)) - img_padded = np.concatenate((img_padded, pad_right), axis=1) - - return img_padded, pad - - -if __name__ == '__main__': - print(get_pose_model()) - - -# First let's download the pre-trained model. - -# In[2]: - - -# Using gdown to download the model directly from Google Drive - -#assert os.system(' conda install -y gdown') == 0 -import gdown - - -# In[3]: - - -model = 'coco_pose_iter_440000.pth.tar' -if not os.path.exists(model): - url = 'https://drive.google.com/u/0/uc?export=download&confirm=f_Ix&id=0B1asvDK18cu_MmY1ZkpaOUhhRHM' - gdown.download( - url, - model, - quiet=False - ) - - -# In[4]: - - -state_dict = torch.load('./coco_pose_iter_440000.pth.tar')['state_dict'] # getting the pre-trained model's parameters -# A state_dict is simply a Python dictionary object that maps each layer to its parameter tensor. - -model_pose = get_pose_model() # building the model (see fn. defn. above). To see the architecture, see below cell. -model_pose.load_state_dict(state_dict) # Loading the parameters (weights, biases) into the model. - -model_pose.float() # I'm not sure why this is used. No difference if you remove it. - - -# In[5]: - - -arch_image = '../input/indonesian-traditional-dance/tgagrakanyar/tga_0000.jpg' -img_ori = cv2.imread(arch_image) -plt.figure(figsize=(15, 8)) -plt.imshow(img_ori[...,::-1]) - - -# Notice, the first 10 layers are from VGG-19. But here instead of downloading the model and loading the layers from there, we simply hardcoaded it in get_pose_model() - -# In[6]: - - -# Run this to view the model's architecture -#model_pose.eval() - - -# In[7]: - - -use_gpu = True - -if use_gpu: - model_pose.cuda() - model_pose = torch.nn.DataParallel(model_pose, device_ids=range(torch.cuda.device_count())) - cudnn.benchmark = True - - -# In[8]: - - -def estimate_pose(img_ori, name=None): - if name is None: - name = tempfile.mktemp( - dir='/kaggle/working', - suffix='.png', - ) - pprint.pprint( - ['estimate_pose', dict(name=name)], - ) - - # People might be at different scales in the image, perform inference at multiple scales to boost results - scale_param = [0.5, 1.0, 1.5, 2.0] - - # Predict Heatmaps for approximate joint position - # Use Part Affinity Fields (PAF's) as guidance to link joints to form skeleton - # PAF's are just unit vectors along the limb encoding the direction of the limb - # A dot product of possible joint connection will be high if actual limb else low - - paf_info, heatmap_info = get_paf_and_heatmap(model_pose, img_ori, scale_param) - peaks = extract_heatmap_info(heatmap_info) - sp_k, con_all = extract_paf_info(img_ori, paf_info, peaks) - - subsets, candidates = get_subsets(con_all, sp_k, peaks) - subsets, img_points = draw_key_point(subsets, peaks, img_ori) - - # After predicting Heatmaps and PAF's, proceeed to link joints correctly - img_canvas = link_key_point(img_points, candidates, subsets) - - - f = plt.figure(figsize=(15, 10)) - - plt.subplot(1, 2, 1) - plt.imshow(img_points[...,::-1]) - - plt.subplot(1, 2, 2) - plt.imshow(img_canvas[...,::-1]) - - f.savefig(name) - - -# In[9]: - - -test_image = '../input/indonesian-traditional-dance/tgagrakanyar/tga_0000.jpg' -img_ori = cv2.imread(test_image) -estimate_pose(img_ori) - - -# In[10]: - - -test_image = '../input/indonesian-traditional-dance/tgagrakanyar/tga_0010.jpg' -img_ori = cv2.imread(test_image) -estimate_pose(img_ori) - - -# In[11]: - - -test_image = '../input/indonesian-traditional-dance/tgagrakanyar/tga_0020.jpg' -img_ori = cv2.imread(test_image) -estimate_pose(img_ori) - - -# In[12]: - - -test_image = '../input/indonesian-traditional-dance/tgagrakanyar/tga_0030.jpg' -img_ori = cv2.imread(test_image) -estimate_pose(img_ori) - - -# In[13]: - - -test_image = '../input/indonesian-traditional-dance/tgagrakanyar/tga_0040.jpg' -img_ori = cv2.imread(test_image) -estimate_pose(img_ori) - - -# In[14]: - - -test_image = '../input/indonesian-traditional-dance/tgagrakanyar/tga_0050.jpg' -img_ori = cv2.imread(test_image) -estimate_pose(img_ori) - - -# In[ ]: diff --git a/d1/mlb_player_v5.py b/d1/mlb_player_v5.py deleted file mode 100644 index b296083..0000000 --- a/d1/mlb_player_v5.py +++ /dev/null @@ -1,56 +0,0 @@ -import os - -if os.system(r''' pip show alphapose''') != 0: - t1 = r''' - pip install pycocotools - rm -fr /kaggle/working/AlphaPose - pip install pyyaml==5.2 - pip install scipy==1.1.0 - git clone https://github.com/WildflowerSchools/AlphaPose - python -m pip install cython gdown - apt-get install libyaml-dev - cd /kaggle/working/AlphaPose && python setup.py build develop - ''' - - for o in t1.splitlines(): - print(o) - assert os.system(o) == 0 - -import os -#!git clone https://github.com/MVIG-SJTU/AlphaPose.git - -import torch -print(torch.__version__) -import yaml, scipy -print(yaml.__version__) -print(scipy.__version__) - -import gdown -import os -for o1, o2 in [ - ( - '1D47msNOOiJKvPOXlnpyzdKA3k6E97NTC', - '/kaggle/working/AlphaPose/detector/yolo/data/yolov3-spp.weights', - ), - ( - '1nlnuYfGNuHWZztQHXwVZSL_FvfE551pA', - '/kaggle/working/AlphaPose/detector/tracker/data/JDE-1088x608-uncertainty', - ), - ( - '1kQhnMRURFiy7NsdS8EFL-8vtqEXOgECn', - '/kaggle/working/AlphaPose/pretrained_models/fast_res50_256x192.pth' - ), -]: - os.makedirs(os.path.split(o2)[0], exist_ok=True) - if not os.path.exists(o2): - gdown.download( - 'https://drive.google.com/u/0/uc?export=download&confirm=f_Ix&id=%s' % o1, - o2, - quiet=False - ) - - -assert os.system(r''' -mkdir -p /kaggle/working/test-input && mkdir -p /kaggle/working/test-output && cp /kaggle/working/AlphaPose/examples/demo/*.jpg /kaggle/working/test-input -cd /kaggle/working/AlphaPose && python3 scripts/demo_inference.py --cfg configs/coco/resnet/256x192_res50_lr1e-3_1x.yaml --checkpoint pretrained_models/fast_res50_256x192.pth --indir /kaggle/working/test-input --outdir /kaggle/working/test-output --save_img -''') == 0 diff --git a/d1/mlb_player_v6.py b/d1/mlb_player_v6.py deleted file mode 100644 index ba47ac8..0000000 --- a/d1/mlb_player_v6.py +++ /dev/null @@ -1,172 +0,0 @@ -# https://raw.githubusercontent.com/hafizas101/Real-time-human-pose-estimation-and-classification/master/main.py -# From Python -# It requires OpenCV installed for Python -import sys -import cv2 -import os -from sys import platform -import argparse -from math import sqrt, acos, degrees, atan, degrees -import numpy as np - -# ----------------------------------------- Arslan Part ---------------------------------------------------------------------------------- -def get_angle(a,b): - #print(a) - #print(b) - del_y = a[1]-b[1] - del_x = b[0]-a[0] - if del_x == 0: - del_x = 0.1 - #print("Del_X : "+str(del_x)+"-----Del_Y: "+str(del_y)) - angle = 0 - - if del_x > 0 and del_y > 0: - angle = degrees(atan(del_y / del_x)) - elif del_x < 0 and del_y > 0: - angle = degrees(atan(del_y / del_x)) + 180 - - return angle - -# ------------------------------------------------------------------------------------------------------------------------------------------ - -# ----------------------------------------- Maksim Part ---------------------------------------------------------------------------------- - -def angle_gor(a,b,c,d): - ab=[a[0]-b[0],a[1]-b[1]] - ab1=[c[0]-d[0],c[1]-d[1]] - cos=abs(ab[0]*ab1[0]+ab[1]*ab1[1])/(sqrt(ab[0]**2+ab[1]**2)*sqrt(ab1[0]**2+ab1[1]**2)) - ang = acos(cos) - return ang*180/np.pi - - -def sit_ang(a,b,c,d): - ang=angle_gor(a,b,c,d) - s1=0 - if ang != None: - #print("Angle",ang) - if ang < 120 and ang>40: - s1=1 - return s1 - -def sit_rec(a,b,c,d): - ab = [a[0] - b[0], a[1] - b[1]] - ab1 = [c[0] - d[0], c[1] - d[1]] - l1=sqrt(ab[0]**2+ab[1]**2) - l2=sqrt(ab1[0]**2+ab1[1]**2) - s=0 - if l1!=0 and l2!=0: - #print(l1,l2, "---------->>>") - if l2/l1>=1.5: - s=1 - return s - -# ------------------------------------------------------------------------------------------------------------------------------------------ - -# ----------------------------------------------------------- OpenPose Example Code ---------------------------------------------------------- - -# Import Openpose (Windows/Ubuntu/OSX) -dir_path = os.path.dirname(os.path.realpath(__file__)) -try: - # Windows Import - if platform == "win32": - # Change these variables to point to the correct folder (Release/x64 etc.) - sys.path.append(dir_path + '/../../python/openpose/Release'); - os.environ['PATH'] = os.environ['PATH'] + ';' + dir_path + '/../../x64/Release;' + dir_path + '/../../bin;' - import pyopenpose as op - else: - # Change these variables to point to the correct folder (Release/x64 etc.) - sys.path.append('../../python'); - # If you run `make install` (default path is `/usr/local/python` for Ubuntu), you can also access the OpenPose/python module from there. This will install OpenPose and the python library at your desired installation path. Ensure that this is in your python path in order to use it. - # sys.path.append('/usr/local/python') - from openpose import pyopenpose as op -except ImportError as e: - print('Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?') - raise e - -# Flags -parser = argparse.ArgumentParser() -parser.add_argument("--image_path", default="../../../examples/media/COCO_val2014_000000000192.jpg", help="Process an image. Read all standard formats (jpg, png, bmp, etc.).") -args = parser.parse_known_args() - -# Custom Params (refer to include/openpose/flags.hpp for more parameters) -params = dict() -params["model_folder"] = "/home/nvidia/openpose/models/" - -# Add others in path? -for i in range(0, len(args[1])): - curr_item = args[1][i] - if i != len(args[1])-1: next_item = args[1][i+1] - else: next_item = "1" - if "--" in curr_item and "--" in next_item: - key = curr_item.replace('-','') - if key not in params: params[key] = "1" - elif "--" in curr_item and "--" not in next_item: - key = curr_item.replace('-','') - if key not in params: params[key] = next_item - -# Construct it from system arguments -# op.init_argv(args[1]) -# oppython = op.OpenposePython() - -c=0 -# Starting OpenPose -opWrapper = op.WrapperPython() -opWrapper.configure(params) -opWrapper.start() - -# ------------------------------------------------------- OUR CONTRIBUTIONS ---------------------------------------------------------------- - -cam = cv2.VideoCapture(1) -for i in range(1000): - # Process Image - datum = op.Datum() - s, im = cam.read() # captures image - #cv2.imshow("Test Picture", im) # displays captured image - #im=cv2.resize(im,(480,270), interpolation = cv2.INTER_AREA) - image1 = im - #imageToProcess = cv2.imread(args[0].image_path) - c+=1 - if c==8: - c=0 - datum.cvInputData = image1 - opWrapper.emplaceAndPop([datum]) # OpenPose being applied to the frame image. - # Display Image - #print("Body keypoints: \n" + str(datum.poseKeypoints)) - #print(datum.poseKeypoints.shape) - if len(datum.poseKeypoints.shape)>=2: - x1=0 - x2=0 - - for j in range(len(datum.poseKeypoints)): - x1=0 - x2=0 - s=0 - s1=0 - ang1 = get_angle(datum.poseKeypoints[j][3], datum.poseKeypoints[j][4]) - ang2 = get_angle(datum.poseKeypoints[j][6], datum.poseKeypoints[j][7]) - if (30 < ang1 < 150): - x1 = 1 - if (30 < ang2 < 150): - x2 = 1 - x3 = x1+x2 - if (x3 == 1): - print("The {} person says: HELLO !".format(j+1)) - #cv2.putText(datum.cvOutputData,'OpenPose using Python-OpenCV',(20,30), cv2.FONT_HERSHEY_SIMPLEX, 1,(255,255,255),1,cv2.LINE_AA) - elif (x3 == 2): - print("The {} person says: STOP PLEASE !".format(j+1)) - s += sit_rec(datum.poseKeypoints[j][9], datum.poseKeypoints[j][10],datum.poseKeypoints[j][10],datum.poseKeypoints[j][11]) - s += sit_rec(datum.poseKeypoints[j][12], datum.poseKeypoints[j][13],datum.poseKeypoints[j][13],datum.poseKeypoints[j][14]) - s1+=sit_ang(datum.poseKeypoints[j][9], datum.poseKeypoints[j][10],datum.poseKeypoints[j][10],datum.poseKeypoints[j][11]) - s1+=sit_ang(datum.poseKeypoints[j][12], datum.poseKeypoints[j][13],datum.poseKeypoints[j][13],datum.poseKeypoints[j][14]) - if s > 0 or s1>0: - print("The {} person is sitting".format(j+1)) - if s == 0 and s1 == 0: - print("The {} person is standing".format(j+1)) - print("___________________________") - print(" ") - im=cv2.resize(datum.cvOutputData,(960,540), interpolation = cv2.INTER_AREA) - cv2.imshow("OpenPose 1.4.0 - Tutorial Python API", im) - cv2.waitKey(1) - - -# ------------------------------------------------------------------------------------------------------------------------------------------ diff --git a/python/_m.py b/python/_m.py index f2a003d..b63426e 100644 --- a/python/_m.py +++ b/python/_m.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -#vim: set filetype=python +# vim: set filetype=python import logging import json @@ -7,158 +7,184 @@ import enum import pathlib import sys import argparse -#import optparse + +# import optparse import dataclasses import subprocess import os - from typing import ( - Optional, Any, TypeAlias, Literal, cast, BinaryIO, Generator, - ClassVar, Self, + Optional, + Any, + TypeAlias, + Literal, + cast, + BinaryIO, + Generator, + ClassVar, + Self, ) logger = logging.getLogger() + @dataclasses.dataclass class Settings: - project_root : pathlib.Path = pathlib.Path.cwd() + project_root: pathlib.Path = pathlib.Path.cwd() - env_path : pathlib.Path = project_root / 'tmp' / 'env3' + env_path: pathlib.Path = project_root / 'tmp' / 'env3' - _settings : ClassVar[Optional['Settings']] = None + _settings: ClassVar[Optional['Settings']] = None - @classmethod - def settings(cls) -> Self: - if cls._settings is None: - cls._settings = cls() + @classmethod + def settings(cls) -> Self: + if cls._settings is None: + cls._settings = cls() + + return cls._settings - return cls._settings def js(argv: list[str]) -> int: - return subprocess.check_call([ - 'sudo', - 'docker-compose', - '--project-directory', - Settings.settings().project_root, - '-f', - Settings.settings().project_root / 'docker' / 'js' / 'docker-compose.yml', - *argv, - ]) + return subprocess.check_call( + [ + 'sudo', + 'docker-compose', + '--project-directory', + Settings.settings().project_root, + '-f', + Settings.settings().project_root / 'docker' / 'js' / 'docker-compose.yml', + *argv, + ] + ) + def env( - argv: Optional[list[str]] = None, - mode: Literal['exec', 'subprocess'] = 'subprocess', - **kwargs: Any, + argv: Optional[list[str]] = None, + mode: Literal['exec', 'subprocess'] = 'subprocess', + **kwargs: Any, ) -> Optional[subprocess.CompletedProcess[bytes]]: - env_path = Settings.settings().env_path + env_path = Settings.settings().env_path - if not env_path.exists(): - subprocess.check_call([ - sys.executable, '-m', 'venv', - '--system-site-packages', - str(env_path) - ]) + if not env_path.exists(): + subprocess.check_call([sys.executable, '-m', 'venv', '--system-site-packages', str(env_path)]) - subprocess.check_call([ - env_path / 'bin' / 'python3', - '-m', 'pip', - 'install', '-r', 'requirements.txt', - ]) + subprocess.check_call( + [ + env_path / 'bin' / 'python3', + '-m', + 'pip', + 'install', + '-r', + 'requirements.txt', + ] + ) - if not argv is None: - python_path = str(env_path / 'bin' / 'python3') + if not argv is None: + python_path = str(env_path / 'bin' / 'python3') - if mode == 'exec': - os.execv( - python_path, - [ - python_path, - *argv, - ], - ) - return None - elif mode == 'subprocess': - return subprocess.run([ - python_path, - *argv, - ], **kwargs) - else: - raise NotImplementedError + if mode == 'exec': + os.execv( + python_path, + [ + python_path, + *argv, + ], + ) + return None + elif mode == 'subprocess': + return subprocess.run( + [ + python_path, + *argv, + ], + **kwargs, + ) + else: + raise NotImplementedError + + return None - return None def ruff(argv: list[str]) -> None: - parser = argparse.ArgumentParser() - parser.add_argument( - '-i', - dest='paths', - help='specify paths to check', - default=[], - action='append', - ) - parser.add_argument( - '-e', - dest='exclude', - help='rules to ignore', - default=[], - action='append', - ) + parser = argparse.ArgumentParser() + parser.add_argument( + '-i', + dest='paths', + help='specify paths to check', + default=[], + action='append', + ) + parser.add_argument( + '-e', + dest='exclude', + help='rules to ignore', + default=[], + action='append', + ) - options, args = parser.parse_known_args(argv) + options, args = parser.parse_known_args(argv) - if len(options.paths) == 0: - options.paths.extend([ - '.', - 'dotfiles/.local/bin/commands', - ]) + if len(options.paths) == 0: + options.paths.extend( + [ + '.', + 'dotfiles/.local/bin/commands', + ] + ) - if len(options.exclude) == 0: - options.exclude.extend([ - 'E731', - 'E713', - 'E714', - 'E703', - ]) + if len(options.exclude) == 0: + options.exclude.extend( + [ + 'E731', + 'E713', + 'E714', + 'E703', + ] + ) - res = env([ - '-m', - 'ruff', - 'check', - *args, - '--output-format', 'json', - '--ignore', ','.join(options.exclude), - *options.paths, - ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + res = env( + [ + '-m', + 'ruff', + 'check', + *args, + '--output-format', + 'json', + '--ignore', + ','.join(options.exclude), + *options.paths, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) - assert not res is None + assert not res is None - errors = json.loads(res.stdout.decode('utf-8')) + errors = json.loads(res.stdout.decode('utf-8')) - g: dict[str, Any] = dict() - for o in errors: - if not o['filename'] in g: - g[o['filename']] = [] - g[o['filename']].append(o) + g: dict[str, Any] = dict() + for o in errors: + if not o['filename'] in g: + g[o['filename']] = [] + g[o['filename']].append(o) - h = { - k : len(v) - for k, v in g.items() - } + h = {k: len(v) for k, v in g.items()} - logger.info(json.dumps(errors, indent=4)) - logger.info(json.dumps(h, indent=4)) + logger.info(json.dumps(errors, indent=4)) + logger.info(json.dumps(h, indent=4)) def inside_env() -> bool: - try: - import numpy - return True - except Exception: - return False + try: + import numpy -#class Commands(enum.StrEnum): + return True + except Exception: + return False + + +# class Commands(enum.StrEnum): # js = 'js' # mypy = 'mypy' # env = 'env' @@ -172,83 +198,97 @@ def inside_env() -> bool: # argv, # ) + def host_deps(argv: list[str]) -> None: - if sys.platform in ['linux']: - subprocess.check_call(r''' + if sys.platform in ['linux']: + subprocess.check_call( + r""" exec yay -S $(cat requirements-archlinux.txt) - ''', shell=True,) - else: - raise NotImplementedError + """, + shell=True, + ) + else: + raise NotImplementedError -Command_args = ['js', 'mypy', 'env', 'ruff', 'm2', 'host_deps',] -Command : TypeAlias = Literal['js', 'mypy', 'env', 'ruff', 'm2', 'host_deps',] +Command_args = [ + 'js', + 'mypy', + 'env', + 'ruff', + 'm2', + 'host_deps', +] + +Command: TypeAlias = Literal[ + 'js', + 'mypy', + 'env', + 'ruff', + 'm2', + 'host_deps', +] + def run(argv: Optional[list[str]] = None) -> None: - logging.basicConfig( - level=logging.INFO, - format=( - '%(levelname)s:%(name)s:%(message)s' - ':%(process)d' - ':%(asctime)s' - ':%(pathname)s:%(funcName)s:%(lineno)s' - ), - ) + logging.basicConfig( + level=logging.INFO, + format=('%(levelname)s:%(name)s:%(message)s:%(process)d:%(asctime)s:%(pathname)s:%(funcName)s:%(lineno)s'), + ) - if argv is None: - argv = sys.argv[:] + if argv is None: + argv = sys.argv[:] + parser = argparse.ArgumentParser() + parser.add_argument( + 'command', + #'_command', + choices=[o for o in Command_args], + # required=True, + ) - parser = argparse.ArgumentParser() - parser.add_argument( - 'command', - #'_command', - choices=[ - o - for o in Command_args - ], - #required=True, - ) + options, args = parser.parse_known_args(argv[1:]) - options, args = parser.parse_known_args(argv[1:]) + assert options.command in Command_args - assert options.command in Command_args + if len(args) > 0 and args[0] == '--': + del args[0] - if len(args) > 0 and args[0] == '--': - del args[0] + # options.command = Commands(options._command) - #options.command = Commands(options._command) + if options.command == 'js': + js(args) + elif options.command == 'host_deps': + host_deps(args) + elif options.command == 'env': + env( + args, + mode='exec', + ) + # elif options.command == 'mypy': + # if not inside_env(): + # env( + # [ + # pathlib.Path(__file__).parent / 'm.py', + # *argv[1:], + # ], + # mode='exec' + # ) + # else: + # mypy(args) + elif options.command == 'ruff': + ruff(args) + elif options.command == 'm2': + if not inside_env(): + env(['--', '_m.py', 'm2', *args]) + return - if options.command == 'js': - js(args) - elif options.command == 'host_deps': - host_deps(args) - elif options.command == 'env': - env(args, mode='exec',) - # elif options.command == 'mypy': - # if not inside_env(): - # env( - # [ - # pathlib.Path(__file__).parent / 'm.py', - # *argv[1:], - # ], - # mode='exec' - # ) - # else: - # mypy(args) - elif options.command == 'ruff': - ruff(args) - elif options.command == 'm2': - if not inside_env(): - env(['--', '_m.py', 'm2', *args]) - return + import python.tasks.cython + + python.tasks.cython.mypyc_build(pathlib.Path('_m.py')) + else: + raise NotImplementedError - import python.tasks.cython - python.tasks.cython.mypyc_build( - pathlib.Path('_m.py') - ) - else: - raise NotImplementedError if __name__ == '__main__': - run() \ No newline at end of file + run() diff --git a/python/cli.py b/python/cli.py index 5d0becd..95a470e 100644 --- a/python/cli.py +++ b/python/cli.py @@ -10,7 +10,10 @@ import enum import argparse import dataclasses -from typing import (Optional, override,) +from typing import ( + Optional, + override, +) from online.fxreader.pr34.commands_typed.logging import setup as logging_setup @@ -24,183 +27,176 @@ logger = logging.getLogger(__name__) class Command(enum.StrEnum): - mypy = 'mypy' - pyright = 'pyright' - ruff = 'ruff' - deploy_wheel = 'deploy:wheel' - tests = 'tests' - meson_setup = 'meson:setup' + mypy = 'mypy' + pyright = 'pyright' + ruff = 'ruff' + deploy_wheel = 'deploy:wheel' + tests = 'tests' + meson_setup = 'meson:setup' + @dataclasses.dataclass class Settings( - _cli.DistSettings, + _cli.DistSettings, ): - base_dir: pathlib.Path = pathlib.Path(__file__).parent.parent - build_dir: pathlib.Path = base_dir / 'tmp' / 'build' - wheel_dir: pathlib.Path = base_dir / 'deps' / 'dist' - env_path: pathlib.Path = cli_bootstrap.BootstrapSettings.get().env_path - python_path: pathlib.Path = pathlib.Path(sys.executable) + base_dir: pathlib.Path = pathlib.Path(__file__).parent.parent + build_dir: pathlib.Path = base_dir / 'tmp' / 'build' + wheel_dir: pathlib.Path = base_dir / 'deps' / 'dist' + env_path: pathlib.Path = cli_bootstrap.BootstrapSettings.get().env_path + python_path: pathlib.Path = pathlib.Path(sys.executable) class CLI(_cli.CLI): - def __init__(self) -> None: - self.settings = Settings() - self._projects: dict[str, _cli.Project] = { - 'online.fxreader.pr34': _cli.Project( - source_dir=self.settings.base_dir / 'python', - build_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'build', - dest_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'install', - meson_path=self.settings.base_dir / 'python' / 'meson.build', - ) - } + def __init__(self) -> None: + self.settings = Settings() + self._projects: dict[str, _cli.Project] = { + 'online.fxreader.pr34': _cli.Project( + source_dir=self.settings.base_dir / 'python', + build_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'build', + dest_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'install', + meson_path=self.settings.base_dir / 'python' / 'meson.build', + ) + } - self._dependencies : dict[str, _cli.Dependency] = dict() + self._dependencies: dict[str, _cli.Dependency] = dict() - @override - @property - def dist_settings(self) -> _cli.DistSettings: - return self.settings + @override + @property + def dist_settings(self) -> _cli.DistSettings: + return self.settings - @override - @property - def projects(self) -> dict[str, _cli.Project]: - return self._projects + @override + @property + def projects(self) -> dict[str, _cli.Project]: + return self._projects - def mypy( - self, - argv: list[str], - ) -> None: - import online.fxreader.pr34.commands_typed.mypy as _mypy + def mypy( + self, + argv: list[str], + ) -> None: + import online.fxreader.pr34.commands_typed.mypy as _mypy - project = self._projects['online.fxreader.pr34'] + project = self._projects['online.fxreader.pr34'] - _mypy.run( - argv, - settings=_mypy.MypySettings( - paths=[ - #Settings.settings().project_root / 'dotfiles/.local/bin/commands', - # project.source_dir / 'm.py', - project.source_dir / '_m.py', - project.source_dir / 'online', - project.source_dir / 'cli.py', - project.source_dir / 'm.py', - # Settings.settings().project_root / 'deps/com.github.aiortc.aiortc/src', - #Settings.settings().project_root / 'm.py', - ], - max_errors={ - 'online/fxreader/pr34/commands_typed': 0, - # 'online/fxreader/pr34/commands': 0, - 'cli.py': 0, - 'm.py': 0, - '../deps/com.github.aiortc.aiortc/src/online_fxreader': 0, - '../deps/com.github.aiortc.aiortc/src/aiortc/contrib/signaling': 0 - } - ), - ) + _mypy.run( + argv, + settings=_mypy.MypySettings( + paths=[ + # Settings.settings().project_root / 'dotfiles/.local/bin/commands', + # project.source_dir / 'm.py', + project.source_dir / '_m.py', + project.source_dir / 'online', + project.source_dir / 'cli.py', + project.source_dir / 'm.py', + # Settings.settings().project_root / 'deps/com.github.aiortc.aiortc/src', + # Settings.settings().project_root / 'm.py', + ], + max_errors={ + 'online/fxreader/pr34/commands_typed': 0, + # 'online/fxreader/pr34/commands': 0, + 'cli.py': 0, + 'm.py': 0, + '../deps/com.github.aiortc.aiortc/src/online_fxreader': 0, + '../deps/com.github.aiortc.aiortc/src/aiortc/contrib/signaling': 0, + }, + ), + ) - @override - @property - def dependencies(self) -> dict[str, _cli.Dependency]: - return self._dependencies + @override + @property + def dependencies(self) -> dict[str, _cli.Dependency]: + return self._dependencies - def run(self, argv: Optional[list[str]] = None) -> None: - if argv is None: - argv = copy.deepcopy(sys.argv) + def run(self, argv: Optional[list[str]] = None) -> None: + if argv is None: + argv = copy.deepcopy(sys.argv) - parser = argparse.ArgumentParser() - parser.add_argument( - 'command', - choices=[ - o.value - for o in Command - ] - ) - parser.add_argument( - '-p', '--project', - choices=[ - o - for o in self.projects - ] - ) - parser.add_argument( - '-o', '--output_dir', - default=None, - help='wheel output dir for deploy:wheel', - ) - parser.add_argument( - '-f', '--force', - default=False, - action='store_true', - help='remove install dir, before installing, default = false', - ) + parser = argparse.ArgumentParser() + parser.add_argument('command', choices=[o.value for o in Command]) + parser.add_argument('-p', '--project', choices=[o for o in self.projects]) + parser.add_argument( + '-o', + '--output_dir', + default=None, + help='wheel output dir for deploy:wheel', + ) + parser.add_argument( + '-f', + '--force', + default=False, + action='store_true', + help='remove install dir, before installing, default = false', + ) - options, args = parser.parse_known_args(argv[1:]) + options, args = parser.parse_known_args(argv[1:]) - default_project : Optional[str] = None + default_project: Optional[str] = None - for k, v in self.projects.items(): - if ( - cli_bootstrap.paths_equal( - v.source_dir.resolve(), - # pathlib.Path(__file__).parent.resolve(), - pathlib.Path.cwd(), - ) - ): - default_project = k + for k, v in self.projects.items(): + if cli_bootstrap.paths_equal( + v.source_dir.resolve(), + # pathlib.Path(__file__).parent.resolve(), + pathlib.Path.cwd(), + ): + default_project = k - if options.project is None: - if not default_project is None: - options.project = default_project - else: - logger.error(dict(msg='not provided project name')) - raise NotImplementedError + if options.project is None: + if not default_project is None: + options.project = default_project + else: + logger.error(dict(msg='not provided project name')) + raise NotImplementedError - options.command = Command(options.command) + options.command = Command(options.command) - if options.command is Command.deploy_wheel: - assert not options.project is None + if options.command is Command.deploy_wheel: + assert not options.project is None - self.deploy_wheel( - project_name=options.project, - argv=args, - output_dir=options.output_dir, - mypy=True, - ruff=True, - pyright=True, - ) - elif options.command is Command.pyright: - self.pyright( - project_name=options.project, - argv=args, - ) - elif options.command is Command.ruff: - self.ruff( - project_name=options.project, - argv=args, - ) - elif options.command is Command.meson_setup: - assert not options.project is None + self.deploy_wheel( + project_name=options.project, + argv=args, + output_dir=options.output_dir, + mypy=True, + ruff=True, + pyright=True, + ) + elif options.command is Command.pyright: + self.pyright( + project_name=options.project, + argv=args, + ) + elif options.command is Command.ruff: + self.ruff( + project_name=options.project, + argv=args, + ) + elif options.command is Command.meson_setup: + assert not options.project is None + + self.meson_setup( + project_name=options.project, + argv=args, + force=options.force, + ) + elif options.command is Command.mypy: + self.mypy( + argv=args, + ) + elif options.command is Command.tests: + for k, v in self.projects.items(): + subprocess.check_call( + [ + sys.executable, + '-m', + 'unittest', + 'online.fxreader.pr34.tests.test_crypto', + *args, + ], + cwd=str(v.source_dir), + ) + else: + raise NotImplementedError - self.meson_setup( - project_name=options.project, - argv=args, - force=options.force, - ) - elif options.command is Command.mypy: - self.mypy( - argv=args, - ) - elif options.command is Command.tests: - for k, v in self.projects.items(): - subprocess.check_call([ - sys.executable, - '-m', - 'unittest', - 'online.fxreader.pr34.tests.test_crypto', - *args, - ], cwd=str(v.source_dir)) - else: - raise NotImplementedError if __name__ == '__main__': - CLI().run() + CLI().run() diff --git a/python/m.py b/python/m.py index 7d9e342..99da03a 100755 --- a/python/m.py +++ b/python/m.py @@ -10,329 +10,326 @@ import os import logging -from typing import (Optional, Any,) +from typing import ( + Optional, + Any, +) from typing_extensions import ( - Self, BinaryIO, + Self, + BinaryIO, ) logger = logging.getLogger(__name__) + def toml_load(f: BinaryIO) -> Any: - try: - import tomllib - return tomllib.load(f) - except: - pass + try: + import tomllib - try: - import tomli - return tomli.load(f) - except: - pass + return tomllib.load(f) + except: + pass + + try: + import tomli + + return tomli.load(f) + except: + pass + + raise NotImplementedError - raise NotImplementedError @dataclasses.dataclass class PyProject: - path: pathlib.Path - dependencies: dict[str, list[str]] - early_features: Optional[list[str]] = None - pip_find_links: Optional[list[pathlib.Path]] = None - runtime_libdirs: Optional[list[pathlib.Path]] = None - runtime_preload: Optional[list[pathlib.Path]] = None - requirements: dict[str, pathlib.Path] = dataclasses.field(default_factory=lambda : dict()) + path: pathlib.Path + dependencies: dict[str, list[str]] + early_features: Optional[list[str]] = None + pip_find_links: Optional[list[pathlib.Path]] = None + runtime_libdirs: Optional[list[pathlib.Path]] = None + runtime_preload: Optional[list[pathlib.Path]] = None + requirements: dict[str, pathlib.Path] = dataclasses.field(default_factory=lambda: dict()) + def pyproject_load( - d: pathlib.Path, + d: pathlib.Path, ) -> PyProject: - with io.open(d, 'rb') as f: - content = toml_load(f) + with io.open(d, 'rb') as f: + content = toml_load(f) - assert isinstance(content, dict) + assert isinstance(content, dict) - dependencies : dict[str, list[str]] = dict() + dependencies: dict[str, list[str]] = dict() - dependencies['default'] = content['project']['dependencies'] + dependencies['default'] = content['project']['dependencies'] - if ( - 'optional-dependencies' in content['project'] - ): - assert isinstance( - content['project']['optional-dependencies'], - dict - ) + if 'optional-dependencies' in content['project']: + assert isinstance(content['project']['optional-dependencies'], dict) - for k, v in content['project']['optional-dependencies'].items(): - assert isinstance(v, list) - assert isinstance(k, str) + for k, v in content['project']['optional-dependencies'].items(): + assert isinstance(v, list) + assert isinstance(k, str) - dependencies[k] = v + dependencies[k] = v + res = PyProject( + path=d, + dependencies=dependencies, + ) - res = PyProject( - path=d, - dependencies=dependencies, - ) + tool_name = 'online.fxreader.pr34'.replace('.', '-') - tool_name = 'online.fxreader.pr34'.replace('.', '-') + if 'tool' in content and isinstance(content['tool'], dict) and tool_name in content['tool'] and isinstance(content['tool'][tool_name], dict): + if 'early_features' in content['tool'][tool_name]: + res.early_features = content['tool'][tool_name]['early_features'] - if ( - 'tool' in content and - isinstance( - content['tool'], dict - ) and - tool_name in content['tool'] and - isinstance( - content['tool'][tool_name], - dict - ) - ): - if 'early_features' in content['tool'][tool_name]: - res.early_features = content['tool'][tool_name]['early_features'] + if 'pip_find_links' in content['tool'][tool_name]: + res.pip_find_links = [d.parent / pathlib.Path(o) for o in content['tool'][tool_name]['pip_find_links']] - if 'pip_find_links' in content['tool'][tool_name]: - res.pip_find_links = [ - d.parent / pathlib.Path(o) - for o in content['tool'][tool_name]['pip_find_links'] - ] + if 'runtime_libdirs' in content['tool'][tool_name]: + res.runtime_libdirs = [ + d.parent / pathlib.Path(o) + # pathlib.Path(o) + for o in content['tool'][tool_name]['runtime_libdirs'] + ] - if 'runtime_libdirs' in content['tool'][tool_name]: - res.runtime_libdirs = [ - d.parent / pathlib.Path(o) - # pathlib.Path(o) - for o in content['tool'][tool_name]['runtime_libdirs'] - ] + if 'runtime_preload' in content['tool'][tool_name]: + res.runtime_preload = [ + d.parent / pathlib.Path(o) + # pathlib.Path(o) + for o in content['tool'][tool_name]['runtime_preload'] + ] - if 'runtime_preload' in content['tool'][tool_name]: - res.runtime_preload = [ - d.parent / pathlib.Path(o) - # pathlib.Path(o) - for o in content['tool'][tool_name]['runtime_preload'] - ] + if 'requirements' in content['tool'][tool_name]: + assert isinstance(content['tool'][tool_name]['requirements'], dict) - if 'requirements' in content['tool'][tool_name]: - assert isinstance(content['tool'][tool_name]['requirements'], dict) + res.requirements = { + k: d.parent / pathlib.Path(v) + # pathlib.Path(o) + for k, v in content['tool'][tool_name]['requirements'].items() + } - res.requirements = { - k : d.parent / pathlib.Path(v) - # pathlib.Path(o) - for k, v in content['tool'][tool_name]['requirements'].items() - } + return res - return res @dataclasses.dataclass class BootstrapSettings: - env_path: pathlib.Path - python_path: pathlib.Path - base_dir: pathlib.Path - python_version: Optional[str] = dataclasses.field( - default_factory=lambda : os.environ.get( - 'PYTHON_VERSION', - '%d.%d' % ( - sys.version_info.major, - sys.version_info.minor, - ), - ).strip() - ) - uv_args: list[str] = dataclasses.field( - default_factory=lambda : os.environ.get( - 'UV_ARGS', - '--offline', - ).split(), - ) + env_path: pathlib.Path + python_path: pathlib.Path + base_dir: pathlib.Path + python_version: Optional[str] = dataclasses.field( + default_factory=lambda: os.environ.get( + 'PYTHON_VERSION', + '%d.%d' + % ( + sys.version_info.major, + sys.version_info.minor, + ), + ).strip() + ) + uv_args: list[str] = dataclasses.field( + default_factory=lambda: os.environ.get( + 'UV_ARGS', + '--offline', + ).split(), + ) - @classmethod - def get( - cls, - base_dir: Optional[pathlib.Path] = None, - ) -> Self: - if base_dir is None: - base_dir = pathlib.Path.cwd() + @classmethod + def get( + cls, + base_dir: Optional[pathlib.Path] = None, + ) -> Self: + if base_dir is None: + base_dir = pathlib.Path.cwd() - env_path = base_dir / '.venv' - python_path = env_path / 'bin' / 'python3' + env_path = base_dir / '.venv' + python_path = env_path / 'bin' / 'python3' + + return cls( + base_dir=base_dir, + env_path=env_path, + python_path=python_path, + ) - return cls( - base_dir=base_dir, - env_path=env_path, - python_path=python_path, - ) def env_bootstrap( - bootstrap_settings: BootstrapSettings, - pyproject: PyProject, + bootstrap_settings: BootstrapSettings, + pyproject: PyProject, ) -> None: - pip_find_links : list[pathlib.Path] = [] + pip_find_links: list[pathlib.Path] = [] - if not pyproject.pip_find_links is None: - pip_find_links.extend(pyproject.pip_find_links) + if not pyproject.pip_find_links is None: + pip_find_links.extend(pyproject.pip_find_links) - pip_find_links_args = sum([ - ['-f', str(o),] - for o in pip_find_links - ], []) + pip_find_links_args = sum( + [ + [ + '-f', + str(o), + ] + for o in pip_find_links + ], + [], + ) - features : list[str] = [] + features: list[str] = [] - if pyproject.early_features: - features.extend(pyproject.early_features) + if pyproject.early_features: + features.extend(pyproject.early_features) - requirements_python_version: Optional[str] = None - if not bootstrap_settings.python_version is None: - requirements_python_version = bootstrap_settings.python_version.replace('.', '_') + requirements_python_version: Optional[str] = None + if not bootstrap_settings.python_version is None: + requirements_python_version = bootstrap_settings.python_version.replace('.', '_') + + requirements_name = '_'.join(sorted(features)) + + if requirements_python_version: + requirements_name += '_' + requirements_python_version + + requirements_path: Optional[pathlib.Path] = None + + if requirements_name in pyproject.requirements: + requirements_path = pyproject.requirements[requirements_name] + else: + requirements_path = pyproject.path.parent / 'requirements.txt' + + requirements_in: list[str] = [] + + requirements_in.extend(['uv', 'pip', 'build', 'setuptools', 'meson-python', 'pybind11']) + + if pyproject.early_features: + early_dependencies = sum([pyproject.dependencies[o] for o in pyproject.early_features], []) + + logger.info( + dict( + early_dependencies=early_dependencies, + ) + ) + + requirements_in.extend(early_dependencies) + # if len(early_dependencies) > 0: + # subprocess.check_call([ + # bootstrap_settings.python_path, + # '-m', + # 'uv', 'pip', 'install', + # *pip_find_links_args, + # # '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'), + # *bootstrap_settings.uv_args, + # *early_dependencies, + # ]) + + if not requirements_path.exists(): + with tempfile.NamedTemporaryFile( + mode='w', + prefix='requirements', + suffix='.in', + ) as f: + f.write('\n'.join(requirements_in)) + f.flush() + + subprocess.check_call( + [ + 'uv', + 'pip', + 'compile', + '--generate-hashes', + *pip_find_links_args, + # '-p', + # bootstrap_settings.python_path, + *bootstrap_settings.uv_args, + '-o', + str(requirements_path), + f.name, + ] + ) + + uv_python_version: list[str] = [] + + if not bootstrap_settings.python_version is None: + uv_python_version.extend( + [ + '-p', + bootstrap_settings.python_version, + ] + ) + + subprocess.check_call( + [ + 'uv', + 'venv', + *uv_python_version, + *pip_find_links_args, + # '--seed', + *bootstrap_settings.uv_args, + str(bootstrap_settings.env_path), + ] + ) + + subprocess.check_call( + [ + 'uv', + 'pip', + 'install', + *pip_find_links_args, + '-p', + bootstrap_settings.python_path, + '--require-hashes', + *bootstrap_settings.uv_args, + '-r', + str(requirements_path), + ] + ) - requirements_name = '_'.join(sorted(features)) +def paths_equal(a: pathlib.Path | str, b: pathlib.Path | str) -> bool: + return os.path.abspath(str(a)) == os.path.abspath(str(b)) - if requirements_python_version: - requirements_name += '_' + requirements_python_version - - requirements_path : Optional[pathlib.Path] = None - - if requirements_name in pyproject.requirements: - requirements_path = pyproject.requirements[requirements_name] - else: - requirements_path = pyproject.path.parent / 'requirements.txt' - - requirements_in : list[str] = [] - - requirements_in.extend([ - 'uv', 'pip', 'build', 'setuptools', 'meson-python', 'pybind11' - ]) - - if pyproject.early_features: - early_dependencies = sum([ - pyproject.dependencies[o] - for o in pyproject.early_features - ], []) - - logger.info(dict( - early_dependencies=early_dependencies, - )) - - requirements_in.extend(early_dependencies) - # if len(early_dependencies) > 0: - # subprocess.check_call([ - # bootstrap_settings.python_path, - # '-m', - # 'uv', 'pip', 'install', - # *pip_find_links_args, - # # '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'), - # *bootstrap_settings.uv_args, - # *early_dependencies, - # ]) - - if not requirements_path.exists(): - with tempfile.NamedTemporaryFile( - mode='w', - prefix='requirements', - suffix='.in', - ) as f: - f.write( - '\n'.join(requirements_in) - ) - f.flush() - - subprocess.check_call([ - 'uv', - 'pip', - 'compile', - '--generate-hashes', - *pip_find_links_args, - # '-p', - # bootstrap_settings.python_path, - *bootstrap_settings.uv_args, - '-o', str(requirements_path), - f.name, - ]) - - uv_python_version: list[str] = [] - - if not bootstrap_settings.python_version is None: - uv_python_version.extend([ - '-p', bootstrap_settings.python_version, - ]) - - subprocess.check_call([ - 'uv', 'venv', - *uv_python_version, - *pip_find_links_args, - # '--seed', - *bootstrap_settings.uv_args, - str(bootstrap_settings.env_path) - ]) - - subprocess.check_call([ - 'uv', - 'pip', - 'install', - *pip_find_links_args, - '-p', - bootstrap_settings.python_path, - '--require-hashes', - *bootstrap_settings.uv_args, - '-r', str(requirements_path), - ]) - - -def paths_equal( - a: pathlib.Path | str, - b: pathlib.Path | str -) -> bool: - return ( - os.path.abspath(str(a)) == - os.path.abspath(str(b)) - ) def run( - d: Optional[pathlib.Path] = None, - cli_path: Optional[pathlib.Path] = None, + d: Optional[pathlib.Path] = None, + cli_path: Optional[pathlib.Path] = None, ) -> None: - if cli_path is None: - cli_path = pathlib.Path(__file__).parent / 'cli.py' + if cli_path is None: + cli_path = pathlib.Path(__file__).parent / 'cli.py' - if d is None: - d = pathlib.Path(__file__).parent / 'pyproject.toml' + if d is None: + d = pathlib.Path(__file__).parent / 'pyproject.toml' - bootstrap_settings = BootstrapSettings.get() + bootstrap_settings = BootstrapSettings.get() - pyproject : PyProject = pyproject_load( - d - ) + pyproject: PyProject = pyproject_load(d) - logging.basicConfig(level=logging.INFO) + logging.basicConfig(level=logging.INFO) - if not bootstrap_settings.env_path.exists(): - env_bootstrap( - bootstrap_settings=bootstrap_settings, - pyproject=pyproject, - ) + if not bootstrap_settings.env_path.exists(): + env_bootstrap( + bootstrap_settings=bootstrap_settings, + pyproject=pyproject, + ) - logger.info([sys.executable, sys.argv, bootstrap_settings.python_path]) + logger.info([sys.executable, sys.argv, bootstrap_settings.python_path]) - if not paths_equal(sys.executable, bootstrap_settings.python_path): - os.execv( - str(bootstrap_settings.python_path), - [ - str(bootstrap_settings.python_path), - *sys.argv, - ] - ) + if not paths_equal(sys.executable, bootstrap_settings.python_path): + os.execv( + str(bootstrap_settings.python_path), + [ + str(bootstrap_settings.python_path), + *sys.argv, + ], + ) + + os.execv( + str(bootstrap_settings.python_path), + [ + str(bootstrap_settings.python_path), + str(cli_path), + *sys.argv[1:], + ], + ) - os.execv( - str(bootstrap_settings.python_path), - [ - str(bootstrap_settings.python_path), - str( - cli_path - ), - *sys.argv[1:], - ] - ) if __name__ == '__main__': - run( - d=pathlib.Path(__file__).parent / 'pyproject.toml', - cli_path=pathlib.Path(__file__).parent / 'cli.py', - ) + run( + d=pathlib.Path(__file__).parent / 'pyproject.toml', + cli_path=pathlib.Path(__file__).parent / 'cli.py', + ) diff --git a/python/online/fxreader/pr34/commands.py b/python/online/fxreader/pr34/commands.py index 9871644..723df9d 100644 --- a/python/online/fxreader/pr34/commands.py +++ b/python/online/fxreader/pr34/commands.py @@ -25,8 +25,14 @@ import time import traceback from typing import ( - Literal, Optional, TypedDict, Callable, Generator, TypeAlias, Any, - cast, + Literal, + Optional, + TypedDict, + Callable, + Generator, + TypeAlias, + Any, + cast, ) @@ -34,1110 +40,1039 @@ logger = logging.getLogger(__name__) def custom_notify( - title: Optional[str]=None, - msg: Optional[str]=None, - timeout: Optional[int]=None, + title: Optional[str] = None, + msg: Optional[str] = None, + timeout: Optional[int] = None, ) -> None: - if timeout is None: - timeout = 5 + if timeout is None: + timeout = 5 - timeout2 = int(timeout * 1000) + timeout2 = int(timeout * 1000) - assert isinstance(timeout2, int) and timeout2 >= 500 + assert isinstance(timeout2, int) and timeout2 >= 500 - if title is None: - title = 'commands' + if title is None: + title = 'commands' - assert isinstance(title, str) and len(title) > 0 - assert isinstance(msg, str) and len(msg) > 0 + assert isinstance(title, str) and len(title) > 0 + assert isinstance(msg, str) and len(msg) > 0 - if sys.platform == 'darwin': - osascript_translate = functools.partial( - custom_translate, - check=lambda a, b: - not re.compile( - r'^[a-zA-Z0-9\<\>\/\(\)\s\.\,\:]*$' - )\ - .match(b) is None, - ) + if sys.platform == 'darwin': + osascript_translate = functools.partial( + custom_translate, + check=lambda a, b: not re.compile(r'^[a-zA-Z0-9\<\>\/\(\)\s\.\,\:]*$').match(b) is None, + ) + + subprocess.check_call( + [ + 'osascript', + '-e', + 'display notification "%s" with title "%s"' + % ( + osascript_translate(msg), + osascript_translate(title), + ), + ] + ) + else: + subprocess.check_call(['notify-send', '-t', '%d' % timeout2, title, msg[-128:]]) - subprocess.check_call([ - 'osascript', - '-e', - 'display notification "%s" with title "%s"' % ( - osascript_translate(msg), - osascript_translate(title), - ) - ]) - else: - subprocess.check_call([ - 'notify-send', - '-t', '%d' % timeout2, - title, - msg[-128:] - ]) class intercept_output_t: - class line_res_t(TypedDict): - aggregated: bool - line: bytes + class line_res_t(TypedDict): + aggregated: bool + line: bytes - class realtime_res_t(TypedDict): - aggregated: bool - data: bytes + class realtime_res_t(TypedDict): + aggregated: bool + data: bytes - class aggregated_res_t(TypedDict): - aggregated: bool - data: bytes - returncode: Optional[int] + class aggregated_res_t(TypedDict): + aggregated: bool + data: bytes + returncode: Optional[int] + + res_t: TypeAlias = line_res_t | realtime_res_t | aggregated_res_t - res_t: TypeAlias = line_res_t | realtime_res_t | aggregated_res_t def intercept_output( - current_subprocess: subprocess.Popen[bytes], - return_aggregated: Optional[bool]=None, - transform_callback: Optional[Callable[[bytes], Optional[bytes]]] =None, - real_time: Optional[bool]=None, - timeout: Optional[float]=None, - need_lines: Optional[bool]=None, -) -> Generator[intercept_output_t.res_t, None, None,]: - if real_time is None: - real_time = False + current_subprocess: subprocess.Popen[bytes], + return_aggregated: Optional[bool] = None, + transform_callback: Optional[Callable[[bytes], Optional[bytes]]] = None, + real_time: Optional[bool] = None, + timeout: Optional[float] = None, + need_lines: Optional[bool] = None, +) -> Generator[ + intercept_output_t.res_t, + None, + None, +]: + if real_time is None: + real_time = False - start_timestamp = datetime.datetime.now() + start_timestamp = datetime.datetime.now() - if not return_aggregated: - return_aggregated = False + if not return_aggregated: + return_aggregated = False - t1 = select.poll() + t1 = select.poll() - assert not current_subprocess.stdout is None + assert not current_subprocess.stdout is None - assert isinstance(current_subprocess.stdout, io.BufferedReader) + assert isinstance(current_subprocess.stdout, io.BufferedReader) - t1.register(current_subprocess.stdout, select.POLLIN) - #print([current_subprocess, current_subprocess.poll()]) - output: list[bytes] = [] - buffer: collections.deque[bytes] = collections.deque() - buffer_lines: collections.deque[bytes] = collections.deque() + t1.register(current_subprocess.stdout, select.POLLIN) + # print([current_subprocess, current_subprocess.poll()]) + output: list[bytes] = [] + buffer: collections.deque[bytes] = collections.deque() + buffer_lines: collections.deque[bytes] = collections.deque() - last_data = None + last_data = None - while not ( - not current_subprocess.poll() is None and \ - not last_data is None - ): - if not timeout is None and \ - (datetime.datetime.now() - start_timestamp).total_seconds() > timeout: - break + while not (not current_subprocess.poll() is None and not last_data is None): + if not timeout is None and (datetime.datetime.now() - start_timestamp).total_seconds() > timeout: + break - t2 = t1.poll(100) - if ( - len(t2) == 1 and (t2[0][1] & select.POLLIN) > 0 and \ - not (isinstance(last_data, bytes) and len(last_data) == 0) or - not current_subprocess.poll() is None - ): - t3 = current_subprocess.stdout.peek() + t2 = t1.poll(100) + if ( + len(t2) == 1 + and (t2[0][1] & select.POLLIN) > 0 + and not (isinstance(last_data, bytes) and len(last_data) == 0) + or not current_subprocess.poll() is None + ): + t3 = current_subprocess.stdout.peek() - t4: bytes = current_subprocess.stdout.read(len(t3)) - assert isinstance(t4, bytes) + t4: bytes = current_subprocess.stdout.read(len(t3)) + assert isinstance(t4, bytes) - last_data = t3 - output.append(t3) - if need_lines: - buffer.append(t3) + last_data = t3 + output.append(t3) + if need_lines: + buffer.append(t3) - if need_lines: - if b'\n' in t3: - t3_pos = t3.rfind(b'\n') - buffer_lines.extend([ - o + b'\n' - for o in b''.join( - list(buffer)[:-1] + [ - t3[:t3_pos] - ], - ).splitlines() - ]) - buffer.clear() - buffer.append(t3[t3_pos + 1:]) - while len(buffer_lines) > 0: - yield intercept_output_t.line_res_t( - aggregated=False, - line=buffer_lines.popleft(), - ) + if need_lines: + if b'\n' in t3: + t3_pos = t3.rfind(b'\n') + buffer_lines.extend( + [ + o + b'\n' + for o in b''.join( + list(buffer)[:-1] + [t3[:t3_pos]], + ).splitlines() + ] + ) + buffer.clear() + buffer.append(t3[t3_pos + 1 :]) + while len(buffer_lines) > 0: + yield intercept_output_t.line_res_t( + aggregated=False, + line=buffer_lines.popleft(), + ) - else: - yield dict( - data=t3, - aggregated=False, - ) - t6 = t3 - if not transform_callback is None: - t5 = transform_callback(t3) - if not t5 is None: - t6 = t5 + else: + yield dict( + data=t3, + aggregated=False, + ) + t6 = t3 + if not transform_callback is None: + t5 = transform_callback(t3) + if not t5 is None: + t6 = t5 - if len(t6) > 0: - os.write(sys.stdout.fileno(), t6) - elif real_time: - yield dict( - data=b'', - aggregated=False, - ) + if len(t6) > 0: + os.write(sys.stdout.fileno(), t6) + elif real_time: + yield dict( + data=b'', + aggregated=False, + ) + + if return_aggregated: + yield dict( + data=b''.join(output), + aggregated=True, + returncode=current_subprocess.poll(), + ) - if return_aggregated: - yield dict( - data=b''.join(output), - aggregated=True, - returncode=current_subprocess.poll(), - ) def player_metadata() -> Optional[str]: - for k in range(20): - try: - metadata = { - k : subprocess.check_output(['playerctl', 'metadata', k]).decode('utf-8').strip() - for k in ['artist', 'title'] - } - return '%s - %s' % (metadata['artist'], metadata['title']) - time.sleep(1.0) - except Exception: - continue + for k in range(20): + try: + metadata = {k: subprocess.check_output(['playerctl', 'metadata', k]).decode('utf-8').strip() for k in ['artist', 'title']} + return '%s - %s' % (metadata['artist'], metadata['title']) + time.sleep(1.0) + except Exception: + continue + + return None - return None class memory_stats_t: - class res_t(TypedDict): - mem_total: int - mem_used: int + class res_t(TypedDict): + mem_total: int + mem_used: int + def memory_stats() -> memory_stats_t.res_t: - if sys.platform == 'linux': - with io.BytesIO( - subprocess.check_output( - 'free', - shell=True - ) - ) as f: - t1 = f.read().decode('utf-8').splitlines() - mem_total = int(t1[1].strip().split()[1]) - mem_used = \ - int(t1[1].strip().split()[2]) + \ - int(t1[1].strip().split()[4]) + if sys.platform == 'linux': + with io.BytesIO(subprocess.check_output('free', shell=True)) as f: + t1 = f.read().decode('utf-8').splitlines() + mem_total = int(t1[1].strip().split()[1]) + mem_used = int(t1[1].strip().split()[2]) + int(t1[1].strip().split()[4]) - return dict( - mem_total=mem_total, - mem_used=mem_used, - ) - elif sys.platform == 'darwin': - sysctl_value = lambda name, custom_cast=int: \ - custom_cast( - subprocess.check_output( - 'sysctl -a | grep %s' % name, - shell=True, - ).decode('utf-8').split(':')[1] - ) + return dict( + mem_total=mem_total, + mem_used=mem_used, + ) + elif sys.platform == 'darwin': + sysctl_value = lambda name, custom_cast=int: custom_cast( + subprocess.check_output( + 'sysctl -a | grep %s' % name, + shell=True, + ) + .decode('utf-8') + .split(':')[1] + ) - vm_pagesize = sysctl_value('vm.pagesize') - mem_total = sysctl_value('hw.memsize') + vm_pagesize = sysctl_value('vm.pagesize') + mem_total = sysctl_value('hw.memsize') - t1 = subprocess.check_output('vm_stat').decode('utf-8') - t2 = [o.split(':') for o in t1.splitlines() if ':' in o] - t3 = { - o[0].replace(' ', '_').replace('-', '_').lower() \ - : \ - int(o[1].strip().rstrip('.')) - for o in t2 - if len(o) == 2 and len(o[0]) > 0 \ - and not re.compile(r'^\s*\d+\.\s*$').match(o[1]) is None \ - and not re.compile(r'^[a-zA-Z0-9\_\-\s]+$').match(o[0]) is None - } - mem_used = ( - t3['pages_active'] + \ - t3['pages_wired_down'] - ) * vm_pagesize + t1 = subprocess.check_output('vm_stat').decode('utf-8') + t2 = [o.split(':') for o in t1.splitlines() if ':' in o] + t3 = { + o[0].replace(' ', '_').replace('-', '_').lower(): int(o[1].strip().rstrip('.')) + for o in t2 + if len(o) == 2 + and len(o[0]) > 0 + and not re.compile(r'^\s*\d+\.\s*$').match(o[1]) is None + and not re.compile(r'^[a-zA-Z0-9\_\-\s]+$').match(o[0]) is None + } + mem_used = (t3['pages_active'] + t3['pages_wired_down']) * vm_pagesize - return dict( - mem_total=mem_total / 1024, - mem_used=mem_used / 1024, - ) - else: - raise NotImplementedError + return dict( + mem_total=mem_total / 1024, + mem_used=mem_used / 1024, + ) + else: + raise NotImplementedError -def chrome( - argv: list[str] -) -> int: - assert isinstance(argv, list) and all([isinstance(o, str) for o in argv]) - parser = optparse.OptionParser() - parser.add_option( - '--user_data_dir', - dest='user_data_dir', - default=None, - type=str, - ) - options, args = parser.parse_args(argv) +def chrome(argv: list[str]) -> int: + assert isinstance(argv, list) and all([isinstance(o, str) for o in argv]) + parser = optparse.OptionParser() + parser.add_option( + '--user_data_dir', + dest='user_data_dir', + default=None, + type=str, + ) - if options.user_data_dir is None: - options.user_data_dir = os.path.join( - os.environ['HOME'], - '.config', - 'google-chrome', - ) + options, args = parser.parse_args(argv) - #assert os.path.exists(options.user_data_dir) + if options.user_data_dir is None: + options.user_data_dir = os.path.join( + os.environ['HOME'], + '.config', + 'google-chrome', + ) + + # assert os.path.exists(options.user_data_dir) + + if sys.platform == 'linux': + return subprocess.check_call( + [ + 'google-chrome-stable', + '--enable-features=useOzonePlatform', + '--ozone-platform=wayland', + '--process-per-site', + '--user-data-dir=%s' % options.user_data_dir, + *args, + ] + ) + else: + raise NotImplementedError - if sys.platform == 'linux': - return subprocess.check_call([ - 'google-chrome-stable', - '--enable-features=useOzonePlatform', - '--ozone-platform=wayland', - '--process-per-site', - '--user-data-dir=%s' % options.user_data_dir, - *args, - ]) - else: - raise NotImplementedError def raise_not_implemented() -> None: - raise NotImplementedError + raise NotImplementedError + def eternal_oom(argv: list[str]) -> None: - import signal - import re - import time - import pprint - - assert isinstance(argv, list) and all([isinstance(o, str) for o in argv]) - parser = optparse.OptionParser() - parser.add_option( - '--cpu_wait', - dest='cpu_wait', - default=None, - type=float, - ) - parser.add_option( - '--mean_size', - dest='mean_size', - default=None, - type=int, - ) - parser.add_option( - '--one_shot_clean', - dest='one_shot_clean', - action='store_true', - default=None, - ) - parser.add_option( - '--cpu', - dest='cpu_json', - type=str, - default=None, - ) - parser.add_option( - '--one_shot_app', - dest='one_shot_app', - default=[], - action='append', - ) - parser.add_option( - '--period', - dest='period', - default=None, - type=float, - ) - parser.add_option( - '--memory_limit', - dest='memory_limit', - default=None, - type=float, - ) - parser.add_option( - '--cpu_limit', - dest='cpu_limit', - default=None, - type=float, - ) - parser.add_option( - '--debug', - dest='debug', - action='store_true', - default=False, - ) - options, args = parser.parse_args(argv) - - if not options.cpu_json is None: - options.cpu = json.loads(options.cpu_json) - else: - options.cpu = True - - self_pid = os.getpid() - - if isinstance(options.one_shot_clean, bool) and options.one_shot_clean: - if len(options.one_shot_app) == 0: - options.one_shot_app = ['chrome', 'teams'] - - config = dict( - chrome=(r'chrome.*type=renderer', r'^.*--extension-process.*$'), - teams=(r'teams.*type=renderer', None), - ) - - for app in options.one_shot_app: - p = config[app] - - try: - t1 = subprocess.check_output([ - 'pgrep', '-a', '-f', p[0] - ]).decode('utf-8') - except Exception: - continue - t2 = t1.splitlines() - if not p[1] is None: - t3 = [o for o in t2 if re.compile(p[1]).match(o) is None] - else: - t3 = t2 - t4 = [ - int(o.split()[0]) for o in t3 - ] - - for pid in t4: - if pid == self_pid: - raise NotImplementedError - - os.kill(pid, signal.SIGTERM) - - logging.info(json.dumps(dict( - apps=options.one_shot_app, - count=len(t4), - processes=[ - o.split()[:3] for o in t3 - ], - ))) - - if len(t4) > 0: - print( - '\n'.join([ - str(o.split()[:3]) for o in t3 - ]) - ) - return - - cpu_count = os.cpu_count() - assert isinstance(cpu_count, int) - - if options.period is None: - options.period = 1 - if options.memory_limit is None: - options.memory_limit = 3 * 1024 * 1024 - if options.cpu_limit is None: - options.cpu_limit = 0.6 * cpu_count - if options.cpu_wait is None: - options.cpu_wait = 10 - if options.mean_size is None: - options.mean_size = 30 - - if isinstance(options.memory_limit, float): - options.memory_limit = int(options.memory_limit) - - assert isinstance(options.memory_limit, int) \ - and options.memory_limit < memory_stats()['mem_total'] * 0.95 \ - and options.memory_limit > 512 * 1024 - - assert isinstance(options.cpu_limit, float) \ - and options.cpu_limit > 0.2 * cpu_count and \ - options.cpu_limit < cpu_count * 0.95 - - assert options.period >= 1 - - assert options.cpu_wait >= 10 - assert options.mean_size >= 16 - - def pandas_data_frame( - lines: list[str], - groups_regex: re.Pattern[str], - header_regex: re.Pattern[str], - extra_columns: dict[ - str, - Callable[ - [dict[str, str]], - Any - ] - ], - ) -> dict[str, list[Any]]: - header_match = re.compile(header_regex).search(lines[0]) - assert not header_match is None - header = header_match.groups() - rows = [] - for line in lines[1:]: - row_match = re.compile(groups_regex).search(line) - assert not row_match is None - rows.append(row_match.groups()) - - columns: dict[str, list[Any]] = { - column: [] - for column in header - } - - for row in rows: - for value, column in zip(row, header): - columns[column].append(value) - for column, transformation in extra_columns.items(): - columns[column] = [ - transformation( - { - k : v[index] - for k, v in columns.items() - } - ) - for index in range(len(rows)) - ] - - return columns - - def pandas_merge( - left: dict[str, list[Any]], - right: dict[str, list[Any]], - on: str, - ) -> dict[str, list[Any]]: - index : dict[str, dict[Any, list[int]]] = {} - - input_data_frames : list[ - tuple[ - str, - dict[str, list[Any]] - ] - ] = [ - ('left', left), - ('right', right), - ] - for index_name, data_frame in input_data_frames: - current_index: dict[Any, list[int]] = {} - for row_index, value in enumerate(data_frame[on]): - if not value in current_index: - current_index[value] = [] - current_index[value].append(row_index) - - index[index_name] = current_index - - class MergedDataFrame(TypedDict): - header: list[str] - columns: dict[str, list[Any]] - - merged_data_frame: MergedDataFrame = dict( - header=[ - column + '_x' - for column in left - ] + [ - column + '_y' - for column in right - ], - columns={}, - ) - - for column in merged_data_frame['header']: - merged_data_frame['columns'][column] = [] - - common_values: set[Any] = { - left_value - for left_value in index['left'] - if left_value in index['right'] - } - class RowMatch(TypedDict): - left_row_index: int - right_row_index: int - - common_rows: list[RowMatch] = sorted( - [ - dict( - left_row_index=index['left'][value][0], - right_row_index=index['right'][value][0], - ) - for value in common_values - ], - key=lambda x: x['left_row_index'], - ) - for common_row in common_rows: - row = sum([ - [ - values[ - common_row[ - cast( - Literal['left_row_index' | 'right_row_index'], - 'left_row_index' - if index_name == 'left' - else - 'right_row_index' - if index_name == 'right' - else raise_not_implemented() - ) - ] - ] - for column, values in data_frame.items() - ] - for index_name, data_frame in input_data_frames - ], []) - for column, value in zip(merged_data_frame['header'], row): - merged_data_frame['columns'][column].append(value) - - return merged_data_frame['columns'] - - def pandas_sort_values(data_frame, by, ascending): - assert len(by) == 1 - assert ascending is False - t1 = [ - o['row_index'] - for o in sorted( - [ - dict( - row_index=row_index, - value=value - ) - for row_index, value in enumerate(data_frame[by[0]]) - ], - key=lambda x: x['value'] - )[::-1] - ] - return { - column : [ - values[row_index] - for row_index in t1 - ] - for column, values in data_frame.items() - } - - def pandas_filter_values(data_frame, condition): - shape = [ - len(data_frame), - ] - if shape[0] > 0: - shape.append( - len(list(data_frame.values())[0]) - ) - t1 = [ - row_index - for row_index in range(shape[1]) - if condition( - { - column : values[row_index] - for column, values in data_frame.items() - } - ) - ] - return { - column : [ - values[row_index] - for row_index in t1 - ] - for column, values in data_frame.items() - } - - def pandas_row(data_frame, row_index): - return { - column : values[row_index] - for column, values in data_frame.items() - } - - def pandas_shape(data_frame): - columns_count = len(data_frame) - if columns_count > 0: - rows_count = len(data_frame[ - next(iter(data_frame.keys())) - ]) - else: - rows_count = 0 - - return [ - columns_count, - rows_count, - ] - - def ps_regex(groups_cnt): - assert groups_cnt >= 1 - return ''.join([ - r'^\s*', - r'([^\s]+)\s+' * (groups_cnt - 1), - r'([^\s]+)\s*$', - ]) - - def oom_get_processes(extra_filter=None,): - with io.BytesIO( - subprocess.check_output( - 'ps -e -o pid,rss,user,%cpu', - shell=True - ) - ) as f: - t1 = pandas_data_frame( - f.read().decode('utf-8').splitlines(), - ps_regex(4), - ps_regex(4), - dict( - PID=lambda row: int(row['PID']), - RSS=lambda row: int(row['RSS']), - CPU=lambda row: float(row['%CPU']), - ), - ) - del t1['%CPU'] - assert set(t1.keys()) == set(['PID', 'RSS', 'USER', 'CPU']) - - t5 = subprocess.check_output( - 'ps -e -o pid,args', - shell=True - ).decode('utf-8').splitlines() - t6 = pandas_data_frame( - t5, - r'^\s*(\d+)\s(.*)$', - r'^\s+(\w+)\s+(\w+)\s*$', - dict( - PID=lambda row: int(row['PID']) - ), - ) - - if not 'COMMAND' in t6: - if sys.platform == 'darwin' and 'ARGS' in t6: - t6['COMMAND'] = t6['ARGS'] - del t6['ARGS'] - else: - raise NotImplementedError - - assert set(t6.keys()) == set(['PID', 'COMMAND']) - t11 = pandas_merge(t1, t6, on='PID') - if extra_filter is None: - extra_filter = lambda *args : True - - t7 = pandas_filter_values( - t11, - lambda row: \ - row['PID_x'] != self_pid and \ - not 'freelancer' in row['COMMAND_y'] and \ - extra_filter(row) - ) - - t8 = pandas_sort_values( - t7, - by=['RSS_x'], - ascending=False - ) - t9 = pandas_sort_values( - t7, - by=['CPU_x'], - ascending=False - ) - t10 = sum(t9['CPU_x'], 0.0) / 100 - if options.debug: - pprint.pprint([t9['CPU_x'][:10], t10 * 100]) - - return dict( - by_mem=t8, - by_cpu=t9, - total_cpu=t10, - ) - - def oom_display_rows(current_dataframe): - print('\n'.join([ - ( - lambda row: \ - '% 8d\t% 6.3f GiB\t% 5.2f %%\t% 10s\t%s' % ( - row['PID_x'], - row['RSS_x'] / 1024 / 1024, - row['CPU_x'], - row['USER_x'], - row['COMMAND_y'], - ) - )( - pandas_row(current_dataframe, k) - ) - for k in range( - 0, - min( - 5, - pandas_shape(current_dataframe)[1], - ) - ) - ])) - - def oom_kill(pid): - assert isinstance(pid, int) - - try: - logging.info('%s oom_kill, pid %d' % ( - datetime.datetime.now().isoformat(), - pid, - )) - os.kill(pid, signal.SIGKILL) - except Exception: - logging.error(traceback.format_exc()) - custom_notify( - msg='oom_kill, failed to kill pid %d' % pid - ) - - def oom_status(): - print( - '\r%s %6.2f / %.2f %%, %6.2f / %.2f GiB' % ( - datetime.datetime.now().isoformat(), - oom_mean_cpu() / os.cpu_count() * 100, - options.cpu_limit / os.cpu_count() * 100, - memory_stats()['mem_used'] / 1024 / 1024, - options.memory_limit / 1024 / 1024, - ), - end='' - ) - - def first_check(): - current_memory_stats = memory_stats() - - t11 = oom_get_processes() - t8 = t11['by_mem'] - - if current_memory_stats['mem_used'] > options.memory_limit: - oom_display_rows(t8) - - if t11['total_cpu'] > options.cpu_limit: - oom_display_rows(t11['by_cpu']) - - free_before_oom = ( - options.memory_limit - current_memory_stats['mem_used'] - ) - - print( - 'available %5.2f %% out of %5.2f %% of cpu limit before OOC' % ( - (options.cpu_limit - t11['total_cpu']) * 100 / os.cpu_count(), - options.cpu_limit * 100 / os.cpu_count(), - ) - ) - - print( - '%5.2f GiB [%5.2f %%] out of %5.2f GiB of free memory before OOM' % ( - free_before_oom / 1024 / 1024, - free_before_oom / options.memory_limit * 100, - options.memory_limit / 1024 / 1024, - ) - ) - - del t8 - del t11 - - print('press Enter to start monitoring: ...', end='') - input() - print('\nstarted...') - - first_check() - - last_total_cpu = [] - - last_cpu_high = None - - def oom_add_cpu(total_cpu): - if options.debug: - pprint.pprint([total_cpu, last_total_cpu]) - last_total_cpu.append(total_cpu) - if len(last_total_cpu) > options.mean_size: - del last_total_cpu[-options.mean_size:] - - def oom_mean_cpu(): - return sum(last_total_cpu) / (len(last_total_cpu) + 1e-8) - - def oom_cpu_high(cpu_limit=None): - if cpu_limit is None: - cpu_limit = options.cpu_limit - - nonlocal last_cpu_high - - if oom_mean_cpu() > cpu_limit: - if last_cpu_high is None: - last_cpu_high = datetime.datetime.now().timestamp() - - if datetime.datetime.now().timestamp() - last_cpu_high > options.cpu_wait: - last_cpu_high = None - del last_total_cpu[:] - return True - - return False - - mem_used = None - mem_stat = None - - def oom_mem_high(memory_limit=None): - nonlocal mem_used - - if memory_limit is None: - memory_limit = options.memory_limit - - return mem_used > memory_limit - - while True: - mem_stat = memory_stats() - mem_used = mem_stat['mem_used'] - - if options.memory_limit < mem_stat['mem_total'] and not oom_mem_high( - mem_stat['mem_total'] - ( - mem_stat['mem_total'] - options.memory_limit - ) / 2 - ): - extra_filters = lambda row: ( - 'chrome' in row['COMMAND_y'] and '--type=renderer' in row['COMMAND_y'] - or not 'chrome' in row['COMMAND_y'] - ) - else: - extra_filters = None + import signal + import re + import time + import pprint + + assert isinstance(argv, list) and all([isinstance(o, str) for o in argv]) + parser = optparse.OptionParser() + parser.add_option( + '--cpu_wait', + dest='cpu_wait', + default=None, + type=float, + ) + parser.add_option( + '--mean_size', + dest='mean_size', + default=None, + type=int, + ) + parser.add_option( + '--one_shot_clean', + dest='one_shot_clean', + action='store_true', + default=None, + ) + parser.add_option( + '--cpu', + dest='cpu_json', + type=str, + default=None, + ) + parser.add_option( + '--one_shot_app', + dest='one_shot_app', + default=[], + action='append', + ) + parser.add_option( + '--period', + dest='period', + default=None, + type=float, + ) + parser.add_option( + '--memory_limit', + dest='memory_limit', + default=None, + type=float, + ) + parser.add_option( + '--cpu_limit', + dest='cpu_limit', + default=None, + type=float, + ) + parser.add_option( + '--debug', + dest='debug', + action='store_true', + default=False, + ) + options, args = parser.parse_args(argv) + + if not options.cpu_json is None: + options.cpu = json.loads(options.cpu_json) + else: + options.cpu = True + + self_pid = os.getpid() + + if isinstance(options.one_shot_clean, bool) and options.one_shot_clean: + if len(options.one_shot_app) == 0: + options.one_shot_app = ['chrome', 'teams'] + + config = dict( + chrome=(r'chrome.*type=renderer', r'^.*--extension-process.*$'), + teams=(r'teams.*type=renderer', None), + ) + + for app in options.one_shot_app: + p = config[app] + + try: + t1 = subprocess.check_output(['pgrep', '-a', '-f', p[0]]).decode('utf-8') + except Exception: + continue + t2 = t1.splitlines() + if not p[1] is None: + t3 = [o for o in t2 if re.compile(p[1]).match(o) is None] + else: + t3 = t2 + t4 = [int(o.split()[0]) for o in t3] + + for pid in t4: + if pid == self_pid: + raise NotImplementedError + + os.kill(pid, signal.SIGTERM) + + logging.info( + json.dumps( + dict( + apps=options.one_shot_app, + count=len(t4), + processes=[o.split()[:3] for o in t3], + ) + ) + ) + + if len(t4) > 0: + print('\n'.join([str(o.split()[:3]) for o in t3])) + return + + cpu_count = os.cpu_count() + assert isinstance(cpu_count, int) + + if options.period is None: + options.period = 1 + if options.memory_limit is None: + options.memory_limit = 3 * 1024 * 1024 + if options.cpu_limit is None: + options.cpu_limit = 0.6 * cpu_count + if options.cpu_wait is None: + options.cpu_wait = 10 + if options.mean_size is None: + options.mean_size = 30 + + if isinstance(options.memory_limit, float): + options.memory_limit = int(options.memory_limit) + + assert isinstance(options.memory_limit, int) and options.memory_limit < memory_stats()['mem_total'] * 0.95 and options.memory_limit > 512 * 1024 + + assert isinstance(options.cpu_limit, float) and options.cpu_limit > 0.2 * cpu_count and options.cpu_limit < cpu_count * 0.95 + + assert options.period >= 1 + + assert options.cpu_wait >= 10 + assert options.mean_size >= 16 + + def pandas_data_frame( + lines: list[str], + groups_regex: re.Pattern[str], + header_regex: re.Pattern[str], + extra_columns: dict[str, Callable[[dict[str, str]], Any]], + ) -> dict[str, list[Any]]: + header_match = re.compile(header_regex).search(lines[0]) + assert not header_match is None + header = header_match.groups() + rows = [] + for line in lines[1:]: + row_match = re.compile(groups_regex).search(line) + assert not row_match is None + rows.append(row_match.groups()) + + columns: dict[str, list[Any]] = {column: [] for column in header} + + for row in rows: + for value, column in zip(row, header): + columns[column].append(value) + for column, transformation in extra_columns.items(): + columns[column] = [transformation({k: v[index] for k, v in columns.items()}) for index in range(len(rows))] + + return columns + + def pandas_merge( + left: dict[str, list[Any]], + right: dict[str, list[Any]], + on: str, + ) -> dict[str, list[Any]]: + index: dict[str, dict[Any, list[int]]] = {} + + input_data_frames: list[tuple[str, dict[str, list[Any]]]] = [ + ('left', left), + ('right', right), + ] + for index_name, data_frame in input_data_frames: + current_index: dict[Any, list[int]] = {} + for row_index, value in enumerate(data_frame[on]): + if not value in current_index: + current_index[value] = [] + current_index[value].append(row_index) + + index[index_name] = current_index + + class MergedDataFrame(TypedDict): + header: list[str] + columns: dict[str, list[Any]] + + merged_data_frame: MergedDataFrame = dict( + header=[column + '_x' for column in left] + [column + '_y' for column in right], + columns={}, + ) + + for column in merged_data_frame['header']: + merged_data_frame['columns'][column] = [] + + common_values: set[Any] = {left_value for left_value in index['left'] if left_value in index['right']} + + class RowMatch(TypedDict): + left_row_index: int + right_row_index: int + + common_rows: list[RowMatch] = sorted( + [ + dict( + left_row_index=index['left'][value][0], + right_row_index=index['right'][value][0], + ) + for value in common_values + ], + key=lambda x: x['left_row_index'], + ) + for common_row in common_rows: + row = sum( + [ + [ + values[ + common_row[ + cast( + Literal['left_row_index' | 'right_row_index'], + 'left_row_index' if index_name == 'left' else 'right_row_index' if index_name == 'right' else raise_not_implemented(), + ) + ] + ] + for column, values in data_frame.items() + ] + for index_name, data_frame in input_data_frames + ], + [], + ) + for column, value in zip(merged_data_frame['header'], row): + merged_data_frame['columns'][column].append(value) + + return merged_data_frame['columns'] + + def pandas_sort_values(data_frame, by, ascending): + assert len(by) == 1 + assert ascending is False + t1 = [ + o['row_index'] + for o in sorted([dict(row_index=row_index, value=value) for row_index, value in enumerate(data_frame[by[0]])], key=lambda x: x['value'])[::-1] + ] + return {column: [values[row_index] for row_index in t1] for column, values in data_frame.items()} + + def pandas_filter_values(data_frame, condition): + shape = [ + len(data_frame), + ] + if shape[0] > 0: + shape.append(len(list(data_frame.values())[0])) + t1 = [row_index for row_index in range(shape[1]) if condition({column: values[row_index] for column, values in data_frame.items()})] + return {column: [values[row_index] for row_index in t1] for column, values in data_frame.items()} + + def pandas_row(data_frame, row_index): + return {column: values[row_index] for column, values in data_frame.items()} + + def pandas_shape(data_frame): + columns_count = len(data_frame) + if columns_count > 0: + rows_count = len(data_frame[next(iter(data_frame.keys()))]) + else: + rows_count = 0 + + return [ + columns_count, + rows_count, + ] + + def ps_regex(groups_cnt): + assert groups_cnt >= 1 + return ''.join( + [ + r'^\s*', + r'([^\s]+)\s+' * (groups_cnt - 1), + r'([^\s]+)\s*$', + ] + ) + + def oom_get_processes( + extra_filter=None, + ): + with io.BytesIO(subprocess.check_output('ps -e -o pid,rss,user,%cpu', shell=True)) as f: + t1 = pandas_data_frame( + f.read().decode('utf-8').splitlines(), + ps_regex(4), + ps_regex(4), + dict( + PID=lambda row: int(row['PID']), + RSS=lambda row: int(row['RSS']), + CPU=lambda row: float(row['%CPU']), + ), + ) + del t1['%CPU'] + assert set(t1.keys()) == set(['PID', 'RSS', 'USER', 'CPU']) + + t5 = subprocess.check_output('ps -e -o pid,args', shell=True).decode('utf-8').splitlines() + t6 = pandas_data_frame( + t5, + r'^\s*(\d+)\s(.*)$', + r'^\s+(\w+)\s+(\w+)\s*$', + dict(PID=lambda row: int(row['PID'])), + ) + + if not 'COMMAND' in t6: + if sys.platform == 'darwin' and 'ARGS' in t6: + t6['COMMAND'] = t6['ARGS'] + del t6['ARGS'] + else: + raise NotImplementedError + + assert set(t6.keys()) == set(['PID', 'COMMAND']) + t11 = pandas_merge(t1, t6, on='PID') + if extra_filter is None: + extra_filter = lambda *args: True + + t7 = pandas_filter_values(t11, lambda row: row['PID_x'] != self_pid and not 'freelancer' in row['COMMAND_y'] and extra_filter(row)) + + t8 = pandas_sort_values(t7, by=['RSS_x'], ascending=False) + t9 = pandas_sort_values(t7, by=['CPU_x'], ascending=False) + t10 = sum(t9['CPU_x'], 0.0) / 100 + if options.debug: + pprint.pprint([t9['CPU_x'][:10], t10 * 100]) + + return dict( + by_mem=t8, + by_cpu=t9, + total_cpu=t10, + ) + + def oom_display_rows(current_dataframe): + print( + '\n'.join( + [ + ( + lambda row: '% 8d\t% 6.3f GiB\t% 5.2f %%\t% 10s\t%s' + % ( + row['PID_x'], + row['RSS_x'] / 1024 / 1024, + row['CPU_x'], + row['USER_x'], + row['COMMAND_y'], + ) + )(pandas_row(current_dataframe, k)) + for k in range( + 0, + min( + 5, + pandas_shape(current_dataframe)[1], + ), + ) + ] + ) + ) + + def oom_kill(pid): + assert isinstance(pid, int) + + try: + logging.info( + '%s oom_kill, pid %d' + % ( + datetime.datetime.now().isoformat(), + pid, + ) + ) + os.kill(pid, signal.SIGKILL) + except Exception: + logging.error(traceback.format_exc()) + custom_notify(msg='oom_kill, failed to kill pid %d' % pid) + + def oom_status(): + print( + '\r%s %6.2f / %.2f %%, %6.2f / %.2f GiB' + % ( + datetime.datetime.now().isoformat(), + oom_mean_cpu() / os.cpu_count() * 100, + options.cpu_limit / os.cpu_count() * 100, + memory_stats()['mem_used'] / 1024 / 1024, + options.memory_limit / 1024 / 1024, + ), + end='', + ) + + def first_check(): + current_memory_stats = memory_stats() + + t11 = oom_get_processes() + t8 = t11['by_mem'] + + if current_memory_stats['mem_used'] > options.memory_limit: + oom_display_rows(t8) + + if t11['total_cpu'] > options.cpu_limit: + oom_display_rows(t11['by_cpu']) + + free_before_oom = options.memory_limit - current_memory_stats['mem_used'] + + print( + 'available %5.2f %% out of %5.2f %% of cpu limit before OOC' + % ( + (options.cpu_limit - t11['total_cpu']) * 100 / os.cpu_count(), + options.cpu_limit * 100 / os.cpu_count(), + ) + ) + + print( + '%5.2f GiB [%5.2f %%] out of %5.2f GiB of free memory before OOM' + % ( + free_before_oom / 1024 / 1024, + free_before_oom / options.memory_limit * 100, + options.memory_limit / 1024 / 1024, + ) + ) + + del t8 + del t11 + + print('press Enter to start monitoring: ...', end='') + input() + print('\nstarted...') + + first_check() + + last_total_cpu = [] + + last_cpu_high = None + + def oom_add_cpu(total_cpu): + if options.debug: + pprint.pprint([total_cpu, last_total_cpu]) + last_total_cpu.append(total_cpu) + if len(last_total_cpu) > options.mean_size: + del last_total_cpu[-options.mean_size :] + + def oom_mean_cpu(): + return sum(last_total_cpu) / (len(last_total_cpu) + 1e-8) + + def oom_cpu_high(cpu_limit=None): + if cpu_limit is None: + cpu_limit = options.cpu_limit + + nonlocal last_cpu_high + + if oom_mean_cpu() > cpu_limit: + if last_cpu_high is None: + last_cpu_high = datetime.datetime.now().timestamp() + + if datetime.datetime.now().timestamp() - last_cpu_high > options.cpu_wait: + last_cpu_high = None + del last_total_cpu[:] + return True + + return False + + mem_used = None + mem_stat = None + + def oom_mem_high(memory_limit=None): + nonlocal mem_used + + if memory_limit is None: + memory_limit = options.memory_limit + + return mem_used > memory_limit + + while True: + mem_stat = memory_stats() + mem_used = mem_stat['mem_used'] + + if options.memory_limit < mem_stat['mem_total'] and not oom_mem_high(mem_stat['mem_total'] - (mem_stat['mem_total'] - options.memory_limit) / 2): + extra_filters = lambda row: ('chrome' in row['COMMAND_y'] and '--type=renderer' in row['COMMAND_y'] or not 'chrome' in row['COMMAND_y']) + else: + extra_filters = None + + t11 = oom_get_processes(extra_filters) - t11 = oom_get_processes(extra_filters) + oom_add_cpu(t11['total_cpu']) - oom_add_cpu(t11['total_cpu']) + t8 = t11['by_mem'] + + t9 = t8 + t4 = lambda: oom_kill(t9['PID_x'][0]) + + oom_status() + + if oom_mem_high(): + print('\n', end='') + pprint.pprint( + [ + 'Killing [OOM]', + pandas_row(t9, 0), + mem_used, + ] + ) + t4() + + if options.cpu and oom_cpu_high(): + oom_display_rows(t11['by_cpu']) + print('\n', end='') + pprint.pprint( + [ + 'Killing [CPU]', + pandas_row(t11['by_cpu'], 0), + [options.cpu_limit, oom_mean_cpu(), t11['total_cpu']], + ] + ) + oom_kill(t11['by_cpu']['PID_x'][0]) + time.sleep(options.period) - t8 = t11['by_mem'] - - t9 = t8 - t4 = lambda : oom_kill(t9['PID_x'][0]) - - oom_status() - - if oom_mem_high(): - print('\n', end='') - pprint.pprint([ - 'Killing [OOM]', - pandas_row(t9, 0), - mem_used, - ]) - t4() - - if options.cpu and oom_cpu_high(): - oom_display_rows(t11['by_cpu']) - print('\n', end='') - pprint.pprint([ - 'Killing [CPU]', - pandas_row(t11['by_cpu'], 0), - [options.cpu_limit, oom_mean_cpu(), t11['total_cpu']], - ]) - oom_kill(t11['by_cpu']['PID_x'][0]) - time.sleep(options.period) def resilient_vlc(stream=None): - if stream is None: - streams_path = os.path.join( - os.environ['CACHE_PATH'], - 'resilient-vlc-streams.json' - ) + if stream is None: + streams_path = os.path.join(os.environ['CACHE_PATH'], 'resilient-vlc-streams.json') - if os.path.exists(streams_path): - with io.open( - streams_path, - 'r' - ) as f: - stream = json.load(f) - else: - raise RuntimeError( - 'not found, %s' % streams_path - ) + if os.path.exists(streams_path): + with io.open(streams_path, 'r') as f: + stream = json.load(f) + else: + raise RuntimeError('not found, %s' % streams_path) - if isinstance(stream, str): - stream = [stream] + if isinstance(stream, str): + stream = [stream] - if len(stream) == 0: - raise RuntimeError('no streams') + if len(stream) == 0: + raise RuntimeError('no streams') + + import subprocess + import time + + while True: + print('new start') + with subprocess.Popen( + [ + 'cvlc', + '--verbose', + '2', + *stream, + ], + stderr=subprocess.PIPE, + ) as p: + while p.returncode is None: + t1 = p.stderr.readline().decode('utf-8') + if len(t1) > 0: + print(t1) + if not all( + [ + o in t1 + for o in [ + 'prefetch stream error', + 'terror', + 'main interface error', + ] + ] + ) and any([o in t1 for o in ['pulse audio output debug: underflow']]): + print('shit') + p.kill() + while True: + try: + t2 = p.wait(timeout=1) + print(t2) + break + except Exception: + print('shit') + pass + time.sleep(1.0) - import subprocess - import time - while True: - print('new start') - with subprocess.Popen([ - 'cvlc', '--verbose', '2', *stream, - ], stderr=subprocess.PIPE) as p: - while p.returncode is None: - t1 = p.stderr.readline().decode('utf-8') - if len(t1) > 0: - print(t1) - if not all([ - o in t1 - for o in [ - 'prefetch stream error', - 'terror', - 'main interface error', - ] - ]) and any([ - o in t1 - for o in [ - 'pulse audio output debug: underflow' - ] - ]): - print('shit') - p.kill() - while True: - try: - t2 = p.wait(timeout=1) - print(t2) - break - except Exception: - print('shit') - pass - time.sleep(1.0) def sway_sock( - wait: bool = False, + wait: bool = False, ) -> Optional[str]: - while True: - import glob - uid = os.stat(os.environ['HOME']).st_uid - t1 = glob.glob( - os.path.join( - '/run', - 'user', - '%d' % uid, - 'sway-ipc.%d*.sock' % uid, - ) - ) - t2 = [ - os.stat(o).st_mtime - for o in t1 - ] - sorted_entries = sorted(enumerate(t1), key=lambda x: t2[x[0]]) - if len(sorted_entries) > 0: - t3 = sorted_entries[-1][0] - return t1[t3] - else: - if wait: - time.sleep(0.1) - continue - else: - return None + while True: + import glob + + uid = os.stat(os.environ['HOME']).st_uid + t1 = glob.glob( + os.path.join( + '/run', + 'user', + '%d' % uid, + 'sway-ipc.%d*.sock' % uid, + ) + ) + t2 = [os.stat(o).st_mtime for o in t1] + sorted_entries = sorted(enumerate(t1), key=lambda x: t2[x[0]]) + if len(sorted_entries) > 0: + t3 = sorted_entries[-1][0] + return t1[t3] + else: + if wait: + time.sleep(0.1) + continue + else: + return None + def eternal_firefox( - tabs=None, - profile=None, - group_name=None, - window_position=None, - debug=None, + tabs=None, + profile=None, + group_name=None, + window_position=None, + debug=None, ): - import os - import datetime - import pprint - import subprocess - import time - if debug is None: - debug = False - if tabs is None: - raise RuntimeError('no tabs provided') - if profile is None: - raise RuntimeError('no profile provided') - if group_name is None: - raise RuntimeError('no group provided') - if window_position is None: - #window_position = '1,600,0,600,540' - raise RuntimeError('no window-position provided') - while True: - os.system(r'''date''') - with subprocess.Popen([ - 'firefox', - '-P', profile, - *tabs, - ]) as p: - try: - if debug: - assert subprocess.check_call(['notify-send', '%s:Starting' % group_name]) == 0 + import os + import datetime + import pprint + import subprocess + import time - #t3 = '' - for k in range(300): - t1 = subprocess.check_output(r''' + if debug is None: + debug = False + if tabs is None: + raise RuntimeError('no tabs provided') + if profile is None: + raise RuntimeError('no profile provided') + if group_name is None: + raise RuntimeError('no group provided') + if window_position is None: + # window_position = '1,600,0,600,540' + raise RuntimeError('no window-position provided') + while True: + os.system(r"""date""") + with subprocess.Popen( + [ + 'firefox', + '-P', + profile, + *tabs, + ] + ) as p: + try: + if debug: + assert subprocess.check_call(['notify-send', '%s:Starting' % group_name]) == 0 + + # t3 = '' + for k in range(300): + t1 = subprocess.check_output( + r""" swaymsg -t get_tree | jq -r '..|try select(.pid== %d)' - ''' % p.pid, shell=True).decode('utf-8') - if len(t1) > 10: - break - #time.sleep(0.1) - #t1 = subprocess.check_output(['wmctrl', '-p', '-l']).decode('utf-8') - #t4 = [o for o in t1.splitlines() if str(p.pid) in o] - #if len(t4) == 1: - # t3 = t4[0] - # break + """ + % p.pid, + shell=True, + ).decode('utf-8') + if len(t1) > 10: + break + # time.sleep(0.1) + # t1 = subprocess.check_output(['wmctrl', '-p', '-l']).decode('utf-8') + # t4 = [o for o in t1.splitlines() if str(p.pid) in o] + # if len(t4) == 1: + # t3 = t4[0] + # break - #if t3 == '': - # raise RuntimeError + # if t3 == '': + # raise RuntimeError - #t2 = t3.split()[0] - #assert os.system('wmctrl -i -r %s -e %s' % (t2, window_position)) == 0 - #assert os.system('wmctrl -i -r %s -b add,below' % t2) == 0 - def reposition(): - t1 = lambda s: \ - s \ - .replace('{{PID}}', str(p.pid)) \ - .replace('{{X}}', str(window_position[1])) \ - .replace('{{Y}}', str(window_position[2])) \ - .replace('{{W}}', str(window_position[3])) \ - .replace('{{H}}', str(window_position[4])) \ - .replace('{{WORKSPACE}}', str(window_position[0])) + # t2 = t3.split()[0] + # assert os.system('wmctrl -i -r %s -e %s' % (t2, window_position)) == 0 + # assert os.system('wmctrl -i -r %s -b add,below' % t2) == 0 + def reposition(): + t1 = ( + lambda s: s.replace('{{PID}}', str(p.pid)) + .replace('{{X}}', str(window_position[1])) + .replace('{{Y}}', str(window_position[2])) + .replace('{{W}}', str(window_position[3])) + .replace('{{H}}', str(window_position[4])) + .replace('{{WORKSPACE}}', str(window_position[0])) + ) - assert os.system(t1(r''' + assert ( + os.system( + t1(r""" swaymsg '[pid="{{PID}}"] move window to workspace {{WORKSPACE}}' - ''')) == 0 + """) + ) + == 0 + ) - if window_position[1] != '' and window_position[2] != '': - assert os.system(t1(r''' + if window_position[1] != '' and window_position[2] != '': + assert ( + os.system( + t1(r""" swaymsg '[pid="{{PID}}"] floating enable' \ swaymsg '[pid="{{PID}}"] resize set width {{W}}px height {{H}}px' && \ swaymsg '[pid="{{PID}}"] move absolute position {{X}}px {{Y}}px' - ''')) == 0 - else: - assert os.system(t1(r''' + """) + ) + == 0 + ) + else: + assert ( + os.system( + t1(r""" swaymsg '[pid="{{PID}}"] floating disable' - ''')) == 0 + """) + ) + == 0 + ) + if False: + for tab in tabs[1:]: + time.sleep(10) + assert ( + subprocess.check_call( + [ + 'firefox', + '-P', + profile, + '--new-tab', + tab, + ] + ) + == 0 + ) - if False: - for tab in tabs[1:]: - time.sleep(10) - assert subprocess.check_call([ - 'firefox', - '-P', profile, - '--new-tab', - tab, - ]) == 0 + reposition() + if debug: + assert subprocess.check_call(['notify-send', '%s:Started' % group_name]) == 0 - reposition() + start = datetime.datetime.now() + is_to_restart = lambda: (datetime.datetime.now() - start).total_seconds() >= 900 * 4 + polling_count = 0 - if debug: - assert subprocess.check_call(['notify-send', '%s:Started' % group_name]) == 0 + while not is_to_restart(): + if polling_count == 0: + reposition() - start = datetime.datetime.now() - is_to_restart = lambda : (datetime.datetime.now() - start).total_seconds() >= 900 * 4 - polling_count = 0 + if not p.poll() is None: + break + time.sleep(10) + polling_count += 1 - while not is_to_restart(): - if polling_count == 0: - reposition() + if debug: + assert subprocess.check_call(['notify-send', '%s:Closing' % group_name]) == 0 - if not p.poll() is None: - break - time.sleep(10) - polling_count += 1 - - if debug: - assert subprocess.check_call(['notify-send', '%s:Closing' % group_name]) == 0 - - #assert os.system('wmctrl -i -c %s' % t2) == 0 - assert os.system(r''' + # assert os.system('wmctrl -i -c %s' % t2) == 0 + assert ( + os.system( + r""" swaymsg '[pid="%d"] kill' - ''' % (p.pid,)) == 0 + """ + % (p.pid,) + ) + == 0 + ) - except KeyboardInterrupt: - assert os.system(r''' + except KeyboardInterrupt: + assert ( + os.system( + r""" swaymsg '[pid="%d"] kill' - ''' % (p.pid,)) == 0 - break - except Exception: - import traceback - import pprint - pprint.pprint(traceback.format_exc()) - finally: - try: - p.wait(20) - except subprocess.TimeoutExpired: - pprint.pprint([p.pid, '20 seconds timeout', 'kill']) - p.kill() - if debug: - assert subprocess.check_call(['notify-send', '%s:Closed' % group_name]) == 0 + """ + % (p.pid,) + ) + == 0 + ) + break + except Exception: + import traceback + import pprint + + pprint.pprint(traceback.format_exc()) + finally: + try: + p.wait(20) + except subprocess.TimeoutExpired: + pprint.pprint([p.pid, '20 seconds timeout', 'kill']) + p.kill() + if debug: + assert subprocess.check_call(['notify-send', '%s:Closed' % group_name]) == 0 + def resilient_ethernet(ip_addr, ethernet_device): - subprocess.check_call( - r''' + subprocess.check_call( + r""" sudo sh -c '\ while true; \ do ping -c 3 -w 3 -W 1 {{IP_ADDR}} || (\ @@ -1147,170 +1082,146 @@ do ping -c 3 -w 3 -W 1 {{IP_ADDR}} || (\ ); \ sleep 10; clear; date; \ done' - '''.replace( - '{{IP_ADDR}}', - ip_addr - ).replace( - '{{ETHERNET_DEVICE}}}', - ethernet_device - ), - shell=True - ) + """.replace('{{IP_ADDR}}', ip_addr).replace('{{ETHERNET_DEVICE}}}', ethernet_device), + shell=True, + ) + def http_server(argv): - from .commands_typed import os as commands_os - assert isinstance(argv, list) and all([isinstance(o, str) for o in argv]) - parser = optparse.OptionParser() - parser.add_option( - '--public', - dest='public', - action='store_true', - default=False, - ) - parser.add_option( - '--force', - dest='force', - action='store_true', - default=False, - ) - parser.add_option( - '--token', - dest='token', - type=str, - default=None, - ) - parser.add_option( - '--port', - dest='port', - type='int', - default=80, - ) - parser.add_option( - '--no_docker', - dest='docker', - action='store_false', - default=None, - ) - parser.add_option( - '-H', '--header', - dest='response_headers', - type='str', - action='append', - default=[], - ) - parser.add_option( - '--host', - dest='host', - type='str', - default='127.0.0.1', - ) - parser.add_option( - '--prefix', - dest='prefix', - type='str', - default=None, - ) - options, args = parser.parse_args(argv) + from .commands_typed import os as commands_os - assert options.port >= 1 + assert isinstance(argv, list) and all([isinstance(o, str) for o in argv]) + parser = optparse.OptionParser() + parser.add_option( + '--public', + dest='public', + action='store_true', + default=False, + ) + parser.add_option( + '--force', + dest='force', + action='store_true', + default=False, + ) + parser.add_option( + '--token', + dest='token', + type=str, + default=None, + ) + parser.add_option( + '--port', + dest='port', + type='int', + default=80, + ) + parser.add_option( + '--no_docker', + dest='docker', + action='store_false', + default=None, + ) + parser.add_option( + '-H', + '--header', + dest='response_headers', + type='str', + action='append', + default=[], + ) + parser.add_option( + '--host', + dest='host', + type='str', + default='127.0.0.1', + ) + parser.add_option( + '--prefix', + dest='prefix', + type='str', + default=None, + ) + options, args = parser.parse_args(argv) - try: - if not options.docker and options.host == '0.0.0.0': - found : bool = False - for o in commands_os.interfaces_index(): - if found: - break - for o2 in o.addr_info: - if o2.family == 'inet' and o2.local != '127.0.0.1': - options.host = o2.local - logger.info(dict( - host=options.host, - msg='found', - )) - found = True - break + assert options.port >= 1 - assert not socket.inet_aton(options.host) is None - # subprocess.check_call([ - # 'ping', '-w', '1', - # options.host - # ]) - assert options.host in sum([ - [ - o2.local - for o2 in o.addr_info - ] - for o in commands_os.interfaces_index() - ], []) - except Exception: - raise RuntimeError('invalid ip address %s' % options.host) + try: + if not options.docker and options.host == '0.0.0.0': + found: bool = False + for o in commands_os.interfaces_index(): + if found: + break + for o2 in o.addr_info: + if o2.family == 'inet' and o2.local != '127.0.0.1': + options.host = o2.local + logger.info( + dict( + host=options.host, + msg='found', + ) + ) + found = True + break + assert not socket.inet_aton(options.host) is None + # subprocess.check_call([ + # 'ping', '-w', '1', + # options.host + # ]) + assert options.host in sum([[o2.local for o2 in o.addr_info] for o in commands_os.interfaces_index()], []) + except Exception: + raise RuntimeError('invalid ip address %s' % options.host) - if options.docker is None: - options.docker = True + if options.docker is None: + options.docker = True - index_section = 'autoindex on;' + index_section = 'autoindex on;' - if options.public: - options.token = 'public' + if options.public: + options.token = 'public' - location_section = 'location / {%s}' % index_section - else: - if options.token is None: - options.token = os.urandom(16).hex() + location_section = 'location / {%s}' % index_section + else: + if options.token is None: + options.token = os.urandom(16).hex() - if options.docker: - DATA_DIR = '/usr/share/nginx' - APP_DIR = '/app' - CONF_DIR = '/etc/nginx' - else: - DATA_DIR = '/opt/nginx/%s/data/' % options.token - CONF_DIR = '/opt/nginx/%s/conf/' % options.token - APP_DIR = os.path.abspath(os.path.curdir) + if options.docker: + DATA_DIR = '/usr/share/nginx' + APP_DIR = '/app' + CONF_DIR = '/etc/nginx' + else: + DATA_DIR = '/opt/nginx/%s/data/' % options.token + CONF_DIR = '/opt/nginx/%s/conf/' % options.token + APP_DIR = os.path.abspath(os.path.curdir) - if not options.public: - if not options.prefix is None: - path = options.prefix + options.token - else: - path = options.token + if not options.public: + if not options.prefix is None: + path = options.prefix + options.token + else: + path = options.token - logger.info( - 'access url is http://%s:%d/%s/' % ( - options.host, - options.port, - path, - ) - ) + logger.info( + 'access url is http://%s:%d/%s/' + % ( + options.host, + options.port, + path, + ) + ) - assert all([ - not re.compile( - r'^[A-Za-z-]+ [a-z0-9A-Z-\.]+$' - ).match(o) is None - for o in options.response_headers - ]) + assert all([not re.compile(r'^[A-Za-z-]+ [a-z0-9A-Z-\.]+$').match(o) is None for o in options.response_headers]) - location_section = ( - 'location / {' - 'deny all;' - '}' - 'location /%s/ {' - 'alias %s/;' - '%s' - '%s' - '}' - ) % ( - path, - APP_DIR, - '\n'.join([ - 'add_header %s;' % o - for o in options.response_headers - ]), - index_section - ) + location_section = ('location / {deny all;}location /%s/ {alias %s/;%s%s}') % ( + path, + APP_DIR, + '\n'.join(['add_header %s;' % o for o in options.response_headers]), + index_section, + ) - if options.docker: - subprocess.check_call( - r''' + if options.docker: + subprocess.check_call( + r""" sudo docker run \ -p %s:%d:80 \ -u root \ @@ -1319,38 +1230,37 @@ def http_server(argv): --log-driver none \ nginx:latest \ -c 'echo "server{listen 80; charset UTF-8; root /app; %s}" > /etc/nginx/conf.d/default.conf; nginx -g "daemon off;"' - ''' % ( - options.host, - options.port, - APP_DIR, - location_section, - ), - shell=True - ) - else: - if os.path.exists(CONF_DIR): - assert options.force - shutil.rmtree(CONF_DIR) + """ + % ( + options.host, + options.port, + APP_DIR, + location_section, + ), + shell=True, + ) + else: + if os.path.exists(CONF_DIR): + assert options.force + shutil.rmtree(CONF_DIR) - if os.path.exists(DATA_DIR): - assert options.force - shutil.rmtree(DATA_DIR) + if os.path.exists(DATA_DIR): + assert options.force + shutil.rmtree(DATA_DIR) - os.makedirs(CONF_DIR, exist_ok=True) - os.makedirs( - os.path.join( - CONF_DIR, - 'conf.d', - ), - exist_ok=True - ) - os.makedirs(DATA_DIR, exist_ok=True) + os.makedirs(CONF_DIR, exist_ok=True) + os.makedirs( + os.path.join( + CONF_DIR, + 'conf.d', + ), + exist_ok=True, + ) + os.makedirs(DATA_DIR, exist_ok=True) - with io.open( - os.path.join(CONF_DIR, 'nginx.conf'), - 'w' - ) as f: - f.write(r''' + with io.open(os.path.join(CONF_DIR, 'nginx.conf'), 'w') as f: + f.write( + r""" events { multi_accept on; @@ -1379,17 +1289,20 @@ http { '' close; } } - ''' % CONF_DIR) + """ + % CONF_DIR + ) - with io.open( - os.path.join( - CONF_DIR, - 'conf.d', - 'default.conf', - ), - 'w', - ) as f: - f.write(r''' + with io.open( + os.path.join( + CONF_DIR, + 'conf.d', + 'default.conf', + ), + 'w', + ) as f: + f.write( + r""" server { server_name %s; listen %d; @@ -1405,124 +1318,120 @@ server { root %s; %s } - ''' % ( - options.host, - options.port, - DATA_DIR, - location_section, - )) + """ + % ( + options.host, + options.port, + DATA_DIR, + location_section, + ) + ) - sys.stderr.flush() - sys.stdout.flush() + sys.stderr.flush() + sys.stdout.flush() + + os.execv( + '/usr/sbin/nginx', + [ + '/usr/sbin/nginx', + '-c', + os.path.join(CONF_DIR, 'nginx.conf'), + '-p', + DATA_DIR, + '-g', + 'daemon off;', + ], + ) - os.execv( - '/usr/sbin/nginx', - [ - '/usr/sbin/nginx', - '-c', - os.path.join(CONF_DIR, 'nginx.conf'), - '-p', DATA_DIR, - '-g', 'daemon off;', - ], - ) class pass_ssh_osx_t: - class kwargs_t: - class Mode(enum.StrEnum): - clipboard = 'clipboard' - qrcode = 'qrcode' + class kwargs_t: + class Mode(enum.StrEnum): + clipboard = 'clipboard' + qrcode = 'qrcode' + def pass_ssh_osx(argv): - assert isinstance(argv, list) and all([isinstance(o, str) for o in argv]) - parser = optparse.OptionParser() - parser.add_option( - '--list', - dest='list', - default=False, - action='store_true', - ) - parser.add_option( - '--pass_option', - dest='pass_option', - action='append', - default=[], - type=str, - help='pass secret path, like --pass_option google.com/login/password/v1', - ) - parser.add_option( - '--clipboard_copy', - dest='clipboard_copy', - default=None, - type=str, - ) + assert isinstance(argv, list) and all([isinstance(o, str) for o in argv]) + parser = optparse.OptionParser() + parser.add_option( + '--list', + dest='list', + default=False, + action='store_true', + ) + parser.add_option( + '--pass_option', + dest='pass_option', + action='append', + default=[], + type=str, + help='pass secret path, like --pass_option google.com/login/password/v1', + ) + parser.add_option( + '--clipboard_copy', + dest='clipboard_copy', + default=None, + type=str, + ) - parser.add_option( - '--mode', - dest='_mode', - choices=[ - o.value - for o in pass_ssh_osx_t.kwargs_t.Mode - ], - default=None, - help='a mode to retrieve the password', - ) - parser.add_option( - '--debug', - dest='debug', - action='store_true', - default=False, - ) - assert sys.platform in ['darwin', 'linux'] - options, args = parser.parse_args(argv) + parser.add_option( + '--mode', + dest='_mode', + choices=[o.value for o in pass_ssh_osx_t.kwargs_t.Mode], + default=None, + help='a mode to retrieve the password', + ) + parser.add_option( + '--debug', + dest='debug', + action='store_true', + default=False, + ) + assert sys.platform in ['darwin', 'linux'] + options, args = parser.parse_args(argv) - if options._mode is None: - options._mode = pass_ssh_osx_t.kwargs_t.Mode.clipboard.value + if options._mode is None: + options._mode = pass_ssh_osx_t.kwargs_t.Mode.clipboard.value - options.mode = pass_ssh_osx_t.kwargs_t.Mode(options._mode) + options.mode = pass_ssh_osx_t.kwargs_t.Mode(options._mode) - if options.clipboard_copy is None: - if sys.platform == 'linux': - options.clipboard_copy = 'wl-copy' - elif sys.platform == 'darwin': - options.clipboard_copy = 'pbcopy' - else: - raise NotImplementedError + if options.clipboard_copy is None: + if sys.platform == 'linux': + options.clipboard_copy = 'wl-copy' + elif sys.platform == 'darwin': + options.clipboard_copy = 'pbcopy' + else: + raise NotImplementedError - if len(args) == 0: - raise RuntimeError('ssh_command is required') + if len(args) == 0: + raise RuntimeError('ssh_command is required') - if options.debug: - print(options.pass_option) - pprint.pprint(args) + if options.debug: + print(options.pass_option) + pprint.pprint(args) - - - reset_gpg_agent = r''' + reset_gpg_agent = r""" gpgconf --kill gpg-agent && \ gpgconf --reload gpg-agent - ''' + """ - if not options.list: - t1 = options.pass_option - assert len(t1) > 0 + if not options.list: + t1 = options.pass_option + assert len(t1) > 0 - print( - 'select on of pass names\n%s' % '\n'.join([ - '%d: %s' % (k, v) - for k, v in enumerate(t1) - ]) - ) + print('select on of pass names\n%s' % '\n'.join(['%d: %s' % (k, v) for k, v in enumerate(t1)])) - while True: - try: - t2 = input() - t3 = int(t2) - assert t3 >= 0 and t3 < len(t1) - break - except AssertionError: - continue + while True: + try: + t2 = input() + t3 = int(t2) + assert t3 >= 0 and t3 < len(t1) + break + except AssertionError: + continue - command = r''' + command = r""" %s gpg \ --pinentry-mode=ask \ @@ -1531,521 +1440,516 @@ def pass_ssh_osx(argv): ~/.password-store/%s.gpg && \ echo -n '['$?']' && \ %s - ''' % ( - reset_gpg_agent, - t1[t3], - reset_gpg_agent, - ) - else: - command = 'pass list | less -R' + """ % ( + reset_gpg_agent, + t1[t3], + reset_gpg_agent, + ) + else: + command = 'pass list | less -R' - ssh_command = [ - 'ssh', '-C', - '-o', 'ConnectTimeout 10', - '-o', 'ServerAliveInterval 1', - *args, - '-t', - command, - ] + ssh_command = [ + 'ssh', + '-C', + '-o', + 'ConnectTimeout 10', + '-o', + 'ServerAliveInterval 1', + *args, + '-t', + command, + ] - if options.debug: - pprint.pprint( - dict( - ssh_command=ssh_command, - ) - ) + if options.debug: + pprint.pprint( + dict( + ssh_command=ssh_command, + ) + ) - if options.list: - subprocess.check_call(ssh_command) - else: - def clipboard_set(text): - with subprocess.Popen([ - options.clipboard_copy, - ], stdin=subprocess.PIPE) as p: - p.stdin.write(text.encode('utf-8')) - p.stdin.flush() - p.stdin.close() - p.wait(1) - assert p.poll() == 0 + if options.list: + subprocess.check_call(ssh_command) + else: - with subprocess.Popen( - ssh_command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) as p: - password = None - last_chunk = None + def clipboard_set(text): + with subprocess.Popen( + [ + options.clipboard_copy, + ], + stdin=subprocess.PIPE, + ) as p: + p.stdin.write(text.encode('utf-8')) + p.stdin.flush() + p.stdin.close() + p.wait(1) + assert p.poll() == 0 - hide_password = False - pinentry_delimeter = b'\x1b>' + with subprocess.Popen(ssh_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + password = None + last_chunk = None - def transform_callback(data): - nonlocal hide_password - nonlocal pinentry_delimeter + hide_password = False + pinentry_delimeter = b'\x1b>' - data2 = None + def transform_callback(data): + nonlocal hide_password + nonlocal pinentry_delimeter - if not last_chunk is None: - data = last_chunk['data'] + data + data2 = None - if hide_password: - data2 = b'' - elif pinentry_delimeter in data: - hide_password = True - pos = data.rfind(pinentry_delimeter) - if pos == -1: - data2 = data - else: - data2 = data[:pos + len(pinentry_delimeter)] - elif data == b'': - #return b'\r\n' - return b'' - else: - data2 = None + if not last_chunk is None: + data = last_chunk['data'] + data - return data2 + if hide_password: + data2 = b'' + elif pinentry_delimeter in data: + hide_password = True + pos = data.rfind(pinentry_delimeter) + if pos == -1: + data2 = data + else: + data2 = data[: pos + len(pinentry_delimeter)] + elif data == b'': + # return b'\r\n' + return b'' + else: + data2 = None - for chunk in intercept_output( - current_subprocess=p, - return_aggregated=True, - transform_callback=transform_callback, - real_time=True, - #timeout=10, - ): - if chunk['aggregated']: - last_chunk = chunk - break + return data2 - assert not p.poll() is None + for chunk in intercept_output( + current_subprocess=p, + return_aggregated=True, + transform_callback=transform_callback, + real_time=True, + # timeout=10, + ): + if chunk['aggregated']: + last_chunk = chunk + break - if p.poll() != 0: - logger.error(p.stderr.read()) - sys.exit(p.poll()) + assert not p.poll() is None - assert not last_chunk is None - assert last_chunk['returncode'] == 0 + if p.poll() != 0: + logger.error(p.stderr.read()) + sys.exit(p.poll()) - if options.debug: - pprint.pprint(last_chunk['data']) + assert not last_chunk is None + assert last_chunk['returncode'] == 0 - if last_chunk['data'].endswith('\r\n[0]'.encode('utf-8')) and \ - last_chunk['data'].rfind(pinentry_delimeter) != -1: - last_line = last_chunk['data'].splitlines()[-2] - else: - raise RuntimeError( - 'gpg failure %s' % str( - last_chunk['data'][ - max(last_chunk['data'].find(pinentry_delimeter), -128): - ] - ) - ) + if options.debug: + pprint.pprint(last_chunk['data']) - pos2 = last_line.rfind(pinentry_delimeter) - if pos2 == -1: - last_line2 = last_line - else: - last_line2 = last_line[ - pos2 + len(pinentry_delimeter): - ] + if last_chunk['data'].endswith('\r\n[0]'.encode('utf-8')) and last_chunk['data'].rfind(pinentry_delimeter) != -1: + last_line = last_chunk['data'].splitlines()[-2] + else: + raise RuntimeError('gpg failure %s' % str(last_chunk['data'][max(last_chunk['data'].find(pinentry_delimeter), -128) :])) - password = last_line2.decode('utf-8').rstrip('\r\n') - assert not password is None + pos2 = last_line.rfind(pinentry_delimeter) + if pos2 == -1: + last_line2 = last_line + else: + last_line2 = last_line[pos2 + len(pinentry_delimeter) :] + password = last_line2.decode('utf-8').rstrip('\r\n') + assert not password is None - if options.mode is pass_ssh_osx_t.kwargs_t.Mode.clipboard: - try: - clipboard_set(password) - get_time = lambda : datetime.datetime.now().timestamp() - start = get_time() - while True: - cur = get_time() - remains = 10 - (cur - start) - if remains <= 1e-8: - break - else: - print('\r%5.2fs remains' % remains, end='') - time.sleep(0.1) - except KeyboardInterrupt: - pass + if options.mode is pass_ssh_osx_t.kwargs_t.Mode.clipboard: + try: + clipboard_set(password) + get_time = lambda: datetime.datetime.now().timestamp() + start = get_time() + while True: + cur = get_time() + remains = 10 - (cur - start) + if remains <= 1e-8: + break + else: + print('\r%5.2fs remains' % remains, end='') + time.sleep(0.1) + except KeyboardInterrupt: + pass - clipboard_set('') - print('\rcleared cliboard\n', end='') - elif options.mode is pass_ssh_osx_t.kwargs_t.Mode.qrcode: - assert subprocess.run(r''' + clipboard_set('') + print('\rcleared cliboard\n', end='') + elif options.mode is pass_ssh_osx_t.kwargs_t.Mode.qrcode: + assert ( + subprocess.run( + r""" qrencode -t PNG -o - | feh - - ''', input=password.encode('utf-8'), shell=True).returncode == 0 - else: - raise NotImplementedError + """, + input=password.encode('utf-8'), + shell=True, + ).returncode + == 0 + ) + else: + raise NotImplementedError + def vpn(argv: list[str]) -> None: - python_path : list[str] + python_path: list[str] - if (pathlib.Path(__file__).parent / 'env3').exists(): - python_path = [ - str(pathlib.Path(__file__).parent / 'env3' / 'bin' / 'python3') - ] - elif (pathlib.Path(__file__).parent.parent.parent.parent / 'm').exists(): - python_path = [ - str(pathlib.Path(__file__).parent.parent.parent.parent / 'm'), - 'env', - '--', - ] - else: - python_path = [sys.executable] + if (pathlib.Path(__file__).parent / 'env3').exists(): + python_path = [str(pathlib.Path(__file__).parent / 'env3' / 'bin' / 'python3')] + elif (pathlib.Path(__file__).parent.parent.parent.parent / 'm').exists(): + python_path = [ + str(pathlib.Path(__file__).parent.parent.parent.parent / 'm'), + 'env', + '--', + ] + else: + python_path = [sys.executable] + + subprocess.check_call( + [ + 'sudo', + *python_path, + '-B', + '-Xfrozen_modules=off', + '-m', + 'online_fxreader.vpn.vpn', + *argv, + ] + ) - subprocess.check_call([ - 'sudo', - *python_path, - '-B', - '-Xfrozen_modules=off', - '-m', - 'online_fxreader.vpn.vpn', - *argv, - ]) def player_v1(folder_url, item_id): - #import sys - import urllib.parse - import re - import subprocess - import os - import tqdm - t4 = folder_url - t1 = subprocess.check_output(['curl', '-s', t4]).decode('utf-8') - t2 = re.compile(r"href=\"(.*\.mp3)\""); - t3 = [o.group(1) for o in t2.finditer(t1)]; - t5 = ['%s/%s' % (t4, o) for o in t3] - t6 = item_id - t9 = range(t6, len(t5)) - with tqdm.tqdm( - total=len(t5), - ) as progress_bar: - progress_bar.update(t6) - for k in t9: - t7 = t5[k] - t9 = urllib.parse.unquote(os.path.split(t7)[1]) - progress_bar.set_description('%03d %s' % (k, t9)) - with subprocess.Popen([ - 'ffprobe', - '-hide_banner', - '-i', - t7 - ], stderr=subprocess.PIPE, stdout=subprocess.PIPE) as p: - p.wait() - assert p.returncode == 0 - t8 = p.stderr.read().decode('utf-8') - assert isinstance(t8, str) - #print(t8) - with subprocess.Popen(['ffplay', '-hide_banner', '-nodisp', '-autoexit', '-loop', '1', t7], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) as p: - p.wait() - assert p.returncode == 0 - progress_bar.update(1) + # import sys + import urllib.parse + import re + import subprocess + import os + import tqdm + + t4 = folder_url + t1 = subprocess.check_output(['curl', '-s', t4]).decode('utf-8') + t2 = re.compile(r'href=\"(.*\.mp3)\"') + t3 = [o.group(1) for o in t2.finditer(t1)] + t5 = ['%s/%s' % (t4, o) for o in t3] + t6 = item_id + t9 = range(t6, len(t5)) + with tqdm.tqdm( + total=len(t5), + ) as progress_bar: + progress_bar.update(t6) + for k in t9: + t7 = t5[k] + t9 = urllib.parse.unquote(os.path.split(t7)[1]) + progress_bar.set_description('%03d %s' % (k, t9)) + with subprocess.Popen(['ffprobe', '-hide_banner', '-i', t7], stderr=subprocess.PIPE, stdout=subprocess.PIPE) as p: + p.wait() + assert p.returncode == 0 + t8 = p.stderr.read().decode('utf-8') + assert isinstance(t8, str) + # print(t8) + with subprocess.Popen( + ['ffplay', '-hide_banner', '-nodisp', '-autoexit', '-loop', '1', t7], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) as p: + p.wait() + assert p.returncode == 0 + progress_bar.update(1) + def numpy_linspace(a, b, count): - pos = a - step = (b - a) / count - steps = [] + pos = a + step = (b - a) / count + steps = [] - for i in range(count): - if i == 0: - pos = a - elif i == count - 1: - pos = b - else: - pos = a + i * step - steps.append(pos) + for i in range(count): + if i == 0: + pos = a + elif i == count - 1: + pos = b + else: + pos = a + i * step + steps.append(pos) + + return steps - return steps def pm_service(argv): - parser = optparse.OptionParser() - parser.add_option( - '--events', - dest='events', - default=[], - action='append', - help='pb,tp,kb', - ) - parser.add_option( - '--verbose', - dest='verbose', - type=str, - default=None, - help='true,false', - ) - options, args = parser.parse_args(argv) + parser = optparse.OptionParser() + parser.add_option( + '--events', + dest='events', + default=[], + action='append', + help='pb,tp,kb', + ) + parser.add_option( + '--verbose', + dest='verbose', + type=str, + default=None, + help='true,false', + ) + options, args = parser.parse_args(argv) - if options.verbose is None: - options.verbose = False - else: - val = json.loads(options.verbose) - assert isinstance(val, bool) - options.verbose = val + if options.verbose is None: + options.verbose = False + else: + val = json.loads(options.verbose) + assert isinstance(val, bool) + options.verbose = val - if len(options.events) == 0: - options.events.extend([ - 'pb', - #'tp', 'kb' - ]) + if len(options.events) == 0: + options.events.extend( + [ + 'pb', + #'tp', 'kb' + ] + ) - assert all([ - o in [ - 'pb','tp', 'kb' - ] - for o in options.events - ]) + assert all([o in ['pb', 'tp', 'kb'] for o in options.events]) - assert sys.platform == 'darwin' + assert sys.platform == 'darwin' - wu = 0 + wu = 0 - while True: - subprocess.check_call([ - 'osascript', - '-e', - 'tell application "Finder" to sleep' - ]) - subprocess.check_call( - ['pmset','sleepnow'], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) + while True: + subprocess.check_call(['osascript', '-e', 'tell application "Finder" to sleep']) + subprocess.check_call( + ['pmset', 'sleepnow'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) - wu += 1 + wu += 1 + sample = ( + r'2024-03-19 22:00:36.808589+0300 0x4caa7 Default 0x0 102' + r'0 powerd: [com.apple.powerd:assertions] Process WindowServer.156 TurnedOn ' + r'UserIsActive "com.apple.iohideventsystem.queue.tickle serviceID:10000267f ' + r'service:AppleMultitouchDevice product:Apple Internal Keyboard / Trackpad ' + r'eventType:11" age:00:00:00 id:38654742889 [System: PrevIdle DeclUser kDisp]' + ) + assert isinstance(sample, str) - sample = ( - r'2024-03-19 22:00:36.808589+0300 0x4caa7 Default 0x0 102' - r'0 powerd: [com.apple.powerd:assertions] Process WindowServer.156 TurnedOn ' - r'UserIsActive "com.apple.iohideventsystem.queue.tickle serviceID:10000267f ' - r'service:AppleMultitouchDevice product:Apple Internal Keyboard / Trackpad ' - r'eventType:11" age:00:00:00 id:38654742889 [System: PrevIdle DeclUser kDisp]' - ) - assert isinstance(sample, str) + action = None + with subprocess.Popen(['log', 'stream'], stdout=subprocess.PIPE) as p: + for chunk in intercept_output( + p, + return_aggregated=False, + need_lines=True, + transform_callback=lambda x: b'', + ): + line = chunk['line'].decode('utf-8') + # p.stdout.readline().decode('utf-8') + cmd = None + if 'powerd' in line: + cmd = line + if options.verbose: + logging.error(json.dumps(dict(line=cmd))) - action = None - with subprocess.Popen(['log', 'stream'], stdout=subprocess.PIPE) as p: - for chunk in intercept_output( - p, - return_aggregated=False, - need_lines=True, - transform_callback=lambda x: b'', - ): - line = chunk['line'].decode('utf-8') - #p.stdout.readline().decode('utf-8') - cmd = None - if 'powerd' in line: - cmd = line - if options.verbose: - logging.error(json.dumps(dict(line=cmd))) + # cmd = subprocess.check_output(r''' + # log stream | grep --line-buffered -i \ + # -E 'powerd.*TurnedOn.*UserIsActive' | head -n 1 + #''', shell=True).decode('utf-8') - #cmd = subprocess.check_output(r''' - # log stream | grep --line-buffered -i \ - # -E 'powerd.*TurnedOn.*UserIsActive' | head -n 1 - #''', shell=True).decode('utf-8') + if not cmd is None and ('TurnedOn' in cmd or 'PrevIdle' in cmd or 'PMRD: kIOMessageSystemWillPowerOn' in cmd): + if ( + ('AppleMultitouchDevice' in cmd and 'tp' in options.events) + or ('AppleACPIButton' in cmd and 'pb' in options.events) + or ('eventType:29' in cmd and 'kb' in options.events) + ): + action = 'wake-up' + break + else: + action = 'sleep' + break - if not cmd is None and ( - 'TurnedOn' in cmd or - 'PrevIdle' in cmd or - 'PMRD: kIOMessageSystemWillPowerOn' in cmd - ): - if ( - ('AppleMultitouchDevice' in cmd and 'tp' in options.events) or - ('AppleACPIButton' in cmd and 'pb' in options.events) or - ('eventType:29' in cmd and 'kb' in options.events) - ): - action = 'wake-up' - break - else: - action = 'sleep' - break + if options.verbose: + logging.error( + json.dumps( + dict( + cmd=cmd, + action=action, + ) + ) + ) + else: + print('\r%s wu : %d, la : %s' % (datetime.datetime.now().isoformat(), wu, action), end='') + if action == 'wake-up': + break + elif action == 'sleep': + continue + else: + raise NotImplementedError - if options.verbose: - logging.error(json.dumps(dict(cmd=cmd, action=action,))) - else: - print('\r%s wu : %d, la : %s' % ( - datetime.datetime.now().isoformat(), - wu, - action - ), end='') + print('') - if action == 'wake-up': - break - elif action == 'sleep': - continue - else: - raise NotImplementedError - - print('') def scrap_yt_music(argv: list[str]) -> None: - parser = optparse.OptionParser() - parser.add_option( - '--verbose', - dest='verbose', - type=str, - default=None, - help='true,false', - ) - parser.add_option( - '-l', - '--library_path', - dest='library_path', - type=str, - default=None, - ) - options, args = parser.parse_args(argv) + parser = optparse.OptionParser() + parser.add_option( + '--verbose', + dest='verbose', + type=str, + default=None, + help='true,false', + ) + parser.add_option( + '-l', + '--library_path', + dest='library_path', + type=str, + default=None, + ) + options, args = parser.parse_args(argv) - if options.library_path is None: - options.library_path = os.path.abspath(os.path.curdir) + if options.library_path is None: + options.library_path = os.path.abspath(os.path.curdir) - if options.verbose is None: - options.verbose = False - else: - val = json.loads(options.verbose) - assert isinstance(val, bool) - options.verbose = val + if options.verbose is None: + options.verbose = False + else: + val = json.loads(options.verbose) + assert isinstance(val, bool) + options.verbose = val - import aiohttp.web + import aiohttp.web - def http_events(context, res_cb): - data = [] + def http_events(context, res_cb): + data = [] - async def handle(request): - data.append(request.rel_url.query.copy()) + async def handle(request): + data.append(request.rel_url.query.copy()) - res_cb(event=data[-1], events=data) - if len(data) > 128: - del data[:128] + res_cb(event=data[-1], events=data) + if len(data) > 128: + del data[:128] - return aiohttp.web.Response(text='ok') + return aiohttp.web.Response(text='ok') - async def serve(): - logging.info('http_events starting') - app = aiohttp.web.Application() - app.add_routes([aiohttp.web.get('/status', handle)]) - runner = aiohttp.web.AppRunner(app, handle_signals=False,) - await runner.setup() - site = aiohttp.web.TCPSite(runner, host='127.0.0.1', port=8877) - await site.start() + async def serve(): + logging.info('http_events starting') + app = aiohttp.web.Application() + app.add_routes([aiohttp.web.get('/status', handle)]) + runner = aiohttp.web.AppRunner( + app, + handle_signals=False, + ) + await runner.setup() + site = aiohttp.web.TCPSite(runner, host='127.0.0.1', port=8877) + await site.start() - logging.info('http_events started') + logging.info('http_events started') - while True: - await asyncio.sleep(1) - if context['shutdown']: - break + while True: + await asyncio.sleep(1) + if context['shutdown']: + break - await runner.cleanup() + await runner.cleanup() - logging.info('http_events done') + logging.info('http_events done') - asyncio.run(serve()) + asyncio.run(serve()) - #aiohttp.web.run_app( - # app, - # host='127.0.0.1', - # port=8877, - # handle_signals=False, - #) + # aiohttp.web.run_app( + # app, + # host='127.0.0.1', + # port=8877, + # handle_signals=False, + # ) - #while True: - # data.append( - # subprocess.check_output(r''' - # nc -w 1 -l 127.0.0.1 8877 | head -n 1 - # ''', shell=True,) - # ) + # while True: + # data.append( + # subprocess.check_output(r''' + # nc -w 1 -l 127.0.0.1 8877 | head -n 1 + # ''', shell=True,) + # ) - def audio_recorder(context): - current_name = None + def audio_recorder(context): + current_name = None - p = None + p = None - try: - while True: - with context['track_cv']: - context['track_cv'].wait(1) + try: + while True: + with context['track_cv']: + context['track_cv'].wait(1) - if context['track_name'] != current_name: - logging.info('audio_record, track changed, started') - if not p is None: - logging.info('audio_record, track changed, terminating') - p.terminate() - p.wait() - p = None - logging.info('audio_record, track changed, terminated') - current_name = context['track_name'] + if context['track_name'] != current_name: + logging.info('audio_record, track changed, started') + if not p is None: + logging.info('audio_record, track changed, terminating') + p.terminate() + p.wait() + p = None + logging.info('audio_record, track changed, terminated') + current_name = context['track_name'] - if context['shutdown']: - if not p is None: - p.terminate() - break + if context['shutdown']: + if not p is None: + p.terminate() + break - if p is None and not current_name is None: - output_name = os.path.join( - options.library_path, - '%s.mp3' % current_name - ) - logging.info('audio_record, new recording') - p = subprocess.Popen( - ['sox', '-d', output_name], - stdout=subprocess.DEVNULL, - stdin=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - logging.info(json.dumps(dict(output_name=output_name))) - except Exception: - logging.error(traceback.format_exc()) - finally: - if not p is None: - p.terminate() + if p is None and not current_name is None: + output_name = os.path.join(options.library_path, '%s.mp3' % current_name) + logging.info('audio_record, new recording') + p = subprocess.Popen( + ['sox', '-d', output_name], + stdout=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + logging.info(json.dumps(dict(output_name=output_name))) + except Exception: + logging.error(traceback.format_exc()) + finally: + if not p is None: + p.terminate() - class Context(TypedDict): - http_on_event: Callable[..., None] - shutdown: bool - workers: list[threading.Thread] - track_cv: threading.Condition - main_cv: threading.Condition - track_name: Optional[str] + class Context(TypedDict): + http_on_event: Callable[..., None] + shutdown: bool + workers: list[threading.Thread] + track_cv: threading.Condition + main_cv: threading.Condition + track_name: Optional[str] - context: Context = dict( - http_on_event=lambda *args, **kwargs: None, - shutdown=False, - workers=[], - track_cv=threading.Condition(), - main_cv=threading.Condition(), - track_name=None, - ) + context: Context = dict( + http_on_event=lambda *args, **kwargs: None, + shutdown=False, + workers=[], + track_cv=threading.Condition(), + main_cv=threading.Condition(), + track_name=None, + ) - context['workers'].extend([ - threading.Thread( - target=functools.partial( - http_events, - context=context, - res_cb=lambda *args, **kwargs: context['http_on_event'](*args, **kwargs), - ) - ), - threading.Thread( - target=functools.partial( - audio_recorder, - context=context, - ) - ), - ]) + context['workers'].extend( + [ + threading.Thread( + target=functools.partial( + http_events, + context=context, + res_cb=lambda *args, **kwargs: context['http_on_event'](*args, **kwargs), + ) + ), + threading.Thread( + target=functools.partial( + audio_recorder, + context=context, + ) + ), + ] + ) - def http_on_event(event, events): - with context['track_cv']: - if 'title' in event and event['title'].strip() != '': - context['track_name'] = str(event['title'])[:128].replace('\n', '') - else: - context['track_name'] = None + def http_on_event(event, events): + with context['track_cv']: + if 'title' in event and event['title'].strip() != '': + context['track_name'] = str(event['title'])[:128].replace('\n', '') + else: + context['track_name'] = None - logging.info(event) + logging.info(event) - context['http_on_event'] = http_on_event + context['http_on_event'] = http_on_event - print(r''' + print(r""" https://github.com/ExistentialAudio/BlackHole/wiki/Multi-Output-Device#5-set-audio-output-to-multi-output-device Open Youtube Music, @@ -2085,558 +1989,607 @@ def scrap_yt_music(argv: list[str]) -> None: })(); ``` - ''') + """) - for w in context['workers']: - w.start() + for w in context['workers']: + w.start() - #context['main_cv'] = threading.Condition() + # context['main_cv'] = threading.Condition() - def on_interrupt(*args, **kwargs): - logging.info('on_interrupt') - with context['main_cv']: - context['main_cv'].notify() + def on_interrupt(*args, **kwargs): + logging.info('on_interrupt') + with context['main_cv']: + context['main_cv'].notify() - signal.signal( - signal.SIGINT, - on_interrupt, - ) - signal.signal( - signal.SIGTERM, - on_interrupt, - ) + signal.signal( + signal.SIGINT, + on_interrupt, + ) + signal.signal( + signal.SIGTERM, + on_interrupt, + ) - with context['main_cv']: - context['main_cv'].wait() + with context['main_cv']: + context['main_cv'].wait() - with context['main_cv']: - context['shutdown'] = True - context['main_cv'].notify() - with context['track_cv']: - context['track_cv'].notify() + with context['main_cv']: + context['shutdown'] = True + context['main_cv'].notify() + with context['track_cv']: + context['track_cv'].notify() + + for o in context['workers']: + o.join() - for o in context['workers']: - o.join() def loginctl(argv: list[str]) -> None: - parser = argparse.ArgumentParser() - parser.add_argument( - '--action', - choices=[ - 'lock-session', - ], - ) + parser = argparse.ArgumentParser() + parser.add_argument( + '--action', + choices=[ + 'lock-session', + ], + ) - options = parser.parse_args(argv) + options = parser.parse_args(argv) - if options.action == 'lock-session': - subprocess.check_call(r''' + if options.action == 'lock-session': + subprocess.check_call( + r""" loginctl list-sessions -j | jq -r ".[] | select(.uid==$UID) | \ .session" | loginctl lock-session - ''', shell=True, timeout=1) - else: - raise NotImplementedError + """, + shell=True, + timeout=1, + ) + else: + raise NotImplementedError + def desktop_services(argv): - parser = optparse.OptionParser() - parser.add_option( - '--background_image', - dest='background_image', - default=None, - type=str, - ) - parser.add_option( - '--cpufreq', - dest='cpufreq', - default=None, - type=int, - help='0 - mac book air (no turbo boost, max pct 30, every 4 seconds', - ) - parser.add_option( - '--cpufreq-action', - dest='cpufreq_action', - default=None, - choices=[ - 'performance', - 'powersave', - ], - #type=str, - ) - parser.add_option( - '--battery', - dest='battery', - default=None, - type=int, - help='0 - battery check with sleep <10%, every 10 seconds', - ) - parser.add_option( - '--backlight-increase', - dest='backlight_increase', - default=False, - action='store_true', - help='increase keyboard backlight', - ) - parser.add_option( - '--backlight-type', - dest='backlight_type', - default=[], - action='append', - help='backlight type, like keyboard, output', - ) - parser.add_option( - '--backlight-decrease', - dest='backlight_decrease', - default=False, - action='store_true', - help='decrease keyboard backlight', - ) - parser.add_option( - '--backlight_service', - dest='backlight_service', - action='store_true', - default=False, - help='enable backlight_service', - ) - parser.add_option( - '--polkit_service', - dest='polkit_service', - action='store_true', - default=False, - help='enable polkit_service', - ) + parser = optparse.OptionParser() + parser.add_option( + '--background_image', + dest='background_image', + default=None, + type=str, + ) + parser.add_option( + '--cpufreq', + dest='cpufreq', + default=None, + type=int, + help='0 - mac book air (no turbo boost, max pct 30, every 4 seconds', + ) + parser.add_option( + '--cpufreq-action', + dest='cpufreq_action', + default=None, + choices=[ + 'performance', + 'powersave', + ], + # type=str, + ) + parser.add_option( + '--battery', + dest='battery', + default=None, + type=int, + help='0 - battery check with sleep <10%, every 10 seconds', + ) + parser.add_option( + '--backlight-increase', + dest='backlight_increase', + default=False, + action='store_true', + help='increase keyboard backlight', + ) + parser.add_option( + '--backlight-type', + dest='backlight_type', + default=[], + action='append', + help='backlight type, like keyboard, output', + ) + parser.add_option( + '--backlight-decrease', + dest='backlight_decrease', + default=False, + action='store_true', + help='decrease keyboard backlight', + ) + parser.add_option( + '--backlight_service', + dest='backlight_service', + action='store_true', + default=False, + help='enable backlight_service', + ) + parser.add_option( + '--polkit_service', + dest='polkit_service', + action='store_true', + default=False, + help='enable polkit_service', + ) - options, args = parser.parse_args(argv) + options, args = parser.parse_args(argv) - class VLC: - @classmethod - def vlc_is_playing_fullscreen(cls): - import subprocess - import json - #import sys - #import pprint + class VLC: + @classmethod + def vlc_is_playing_fullscreen(cls): + import subprocess + import json + # import sys + # import pprint - t2 = [] - try: - t1 = subprocess.check_output(['swaymsg', '-t', 'get_tree']).decode('utf-8') - t2 = json.loads(t1) - except Exception: - logging.error(traceback.format_exc()) + t2 = [] + try: + t1 = subprocess.check_output(['swaymsg', '-t', 'get_tree']).decode('utf-8') + t2 = json.loads(t1) + except Exception: + logging.error(traceback.format_exc()) - def walk(o, cb): - if isinstance(o, dict): - cb(o) - for k, v in o.items(): - walk(v, cb,) - elif isinstance(o, list): - cb(o) - for o2 in o: - walk(o2, cb,) - else: - cb(o) + def walk(o, cb): + if isinstance(o, dict): + cb(o) + for k, v in o.items(): + walk( + v, + cb, + ) + elif isinstance(o, list): + cb(o) + for o2 in o: + walk( + o2, + cb, + ) + else: + cb(o) - t3 = [] + t3 = [] - walk(t2, lambda o: [ - t3.append(o) - if isinstance(o, dict) and \ - 'fullscreen_mode' in o and \ - o['fullscreen_mode'] == 1 and \ - 'window_properties' in o and \ - 'class' in o['window_properties'] and \ - o['window_properties']['class'] == 'vlc' - else None - ]) + walk( + t2, + lambda o: [ + t3.append(o) + if isinstance(o, dict) + and 'fullscreen_mode' in o + and o['fullscreen_mode'] == 1 + and 'window_properties' in o + and 'class' in o['window_properties'] + and o['window_properties']['class'] == 'vlc' + else None + ], + ) - t4 = False + t4 = False - try: - t4 = subprocess.check_output([ - 'playerctl', '-p', 'vlc', 'status' - ], timeout=1,).decode('utf-8').strip() == 'Playing' - except Exception: - logging.error(traceback.format_exc()) + try: + t4 = ( + subprocess.check_output( + ['playerctl', '-p', 'vlc', 'status'], + timeout=1, + ) + .decode('utf-8') + .strip() + == 'Playing' + ) + except Exception: + logging.error(traceback.format_exc()) - #pprint.pprint(t3) + # pprint.pprint(t3) - return len(t3) > 0 and t4 + return len(t3) > 0 and t4 + class Battery: + def __init__( + self, + should_start=None, + ): + if should_start is None: + should_start = False - class Battery: - def __init__(self, should_start=None,): - if should_start is None: - should_start = False + assert isinstance(should_start, bool) - assert isinstance(should_start, bool) + self.last_check = None + self.period = 10 + self.is_running = should_start - self.last_check = None - self.period = 10 - self.is_running = should_start + def check_is_needed(self): + now = datetime.datetime.now(tz=datetime.timezone.utc) - def check_is_needed(self): - now = datetime.datetime.now(tz=datetime.timezone.utc) + is_needed = None - is_needed = None + if self.last_check is None: + is_needed = True + else: + if (now - self.last_check).total_seconds() >= self.period: + is_needed = True + else: + is_needed = False - if self.last_check is None: - is_needed = True - else: - if ((now - self.last_check).total_seconds() >= self.period): - is_needed = True - else: - is_needed = False + if is_needed: + self.last_check = now - if is_needed: - self.last_check = now + return is_needed - return is_needed + def run(self): + while True: + self.check() - def run(self): - while True: - self.check() + time.sleep(self.period) - time.sleep(self.period) + def terminate(self): + self.is_running = False - def terminate(self): - self.is_running = False + def wait(self, *args, **kwargs): + if self.is_running: + raise NotImplementedError - def wait(self, *args, **kwargs): - if self.is_running: - raise NotImplementedError + def poll(self): + if self.is_running: + return None + else: + return 0 - def poll(self): - if self.is_running: - return None - else: - return 0 - - @property - def percentage_low(self) -> int: - try: - return int(subprocess.check_output(r''' + @property + def percentage_low(self) -> int: + try: + return int( + subprocess.check_output( + r""" cat /etc/UPower/UPower.conf | grep -Po '^PercentageLow=\d+' - ''', shell=True,).decode('utf-8').strip().split('=')[1]) - except: - logger.exception('') - return 15 + """, + shell=True, + ) + .decode('utf-8') + .strip() + .split('=')[1] + ) + except: + logger.exception('') + return 15 - @property - def percentage_critical(self) -> int: - try: - return int(subprocess.check_output(r''' + @property + def percentage_critical(self) -> int: + try: + return int( + subprocess.check_output( + r""" cat /etc/UPower/UPower.conf | grep -Po '^PercentageCritical=\d+' - ''', shell=True,).decode('utf-8').strip().split('=')[1]) - except: - logger.exception('') - return 10 + """, + shell=True, + ) + .decode('utf-8') + .strip() + .split('=')[1] + ) + except: + logger.exception('') + return 10 - def check(self): - try: - if not self.check_is_needed(): - return + def check(self): + try: + if not self.check_is_needed(): + return - t1 = subprocess.check_output( - ['upower', '-d'], - timeout=1, - ).decode('utf-8') - t2 = [ - o for o in t1.splitlines() if 'percentage' in o.lower() - ] - t4 = [ - o for o in t1.splitlines() if 'state' in o.lower() - ] - t3 = float(t2[0].split(':')[1].strip()[:-1]) - t5 = any(['discharging' in o.lower() for o in t4]) - if t3 < self.percentage_critical and t5: - logging.error(json.dumps(dict( - msg='too low', t3=t3, t5=t5 - ))) - subprocess.check_call(['systemctl', 'suspend']) - elif t3 < self.percentage_low and t5: - msg = 'battery near low' - logging.error(json.dumps(dict( - msg=msg, t3=t3, t5=t5 - ))) - subprocess.check_call([ - 'notify-send', '-t', '%d' % (5 * 1000), msg, '% 5.2f' % t3, - ]) - else: - pass - print( - '\r%s % 5.2f%% %s' % ( - datetime.datetime.now().isoformat(), t3, str(t5) - ), - end='' - ) - except Exception: - logging.error(traceback.format_exc()) + t1 = subprocess.check_output( + ['upower', '-d'], + timeout=1, + ).decode('utf-8') + t2 = [o for o in t1.splitlines() if 'percentage' in o.lower()] + t4 = [o for o in t1.splitlines() if 'state' in o.lower()] + t3 = float(t2[0].split(':')[1].strip()[:-1]) + t5 = any(['discharging' in o.lower() for o in t4]) + if t3 < self.percentage_critical and t5: + logging.error(json.dumps(dict(msg='too low', t3=t3, t5=t5))) + subprocess.check_call(['systemctl', 'suspend']) + elif t3 < self.percentage_low and t5: + msg = 'battery near low' + logging.error(json.dumps(dict(msg=msg, t3=t3, t5=t5))) + subprocess.check_call( + [ + 'notify-send', + '-t', + '%d' % (5 * 1000), + msg, + '% 5.2f' % t3, + ] + ) + else: + pass + print('\r%s % 5.2f%% %s' % (datetime.datetime.now().isoformat(), t3, str(t5)), end='') + except Exception: + logging.error(traceback.format_exc()) - class Backlight: - class Direction(enum.Enum): - increase = 'increase' - decrease = 'decrease' - absolute = 'absolute' - get_state = 'get_state' + class Backlight: + class Direction(enum.Enum): + increase = 'increase' + decrease = 'decrease' + absolute = 'absolute' + get_state = 'get_state' - class Mode(enum.Enum): - light = 'light' + class Mode(enum.Enum): + light = 'light' - def __init__(self): - self.state = [] - self.dpms = Backlight.dpms_get() + def __init__(self): + self.state = [] + self.dpms = Backlight.dpms_get() - @classmethod - def dpms_get(cls): - try: - t1 = subprocess.check_output( - ['swaymsg', '-r', '-t', 'get_outputs'], - timeout=1 - ) - t2 = t1.decode('utf-8') - t3 = json.loads(t2) - t4 = [ - dict( - id=o['id'], - name=o['name'], - dpms=o['dpms'], - ) - for o in t3 - ] + @classmethod + def dpms_get(cls): + try: + t1 = subprocess.check_output(['swaymsg', '-r', '-t', 'get_outputs'], timeout=1) + t2 = t1.decode('utf-8') + t3 = json.loads(t2) + t4 = [ + dict( + id=o['id'], + name=o['name'], + dpms=o['dpms'], + ) + for o in t3 + ] - return any([o['dpms'] for o in t4]) - except Exception: - return True + return any([o['dpms'] for o in t4]) + except Exception: + return True - def check(self): - try: - new_dpms = Backlight.dpms_get() - if new_dpms != self.dpms: - logging.info(json.dumps(dict( - module='backlight', - action='new_dpms', - dpms=self.dpms, - new_dpms=new_dpms, - ))) - if new_dpms: - Backlight.enable(self.state) - else: - self.state = Backlight.change( - Backlight.Direction.get_state, - ) - logging.info(json.dumps(dict( - state=pprint.pformat( - self.state, - width=1e+8, - compact=True, - ), - action='disable', - ))) - Backlight.disable() - self.dpms = new_dpms - except Exception: - logging.error(traceback.format_exc()) + def check(self): + try: + new_dpms = Backlight.dpms_get() + if new_dpms != self.dpms: + logging.info( + json.dumps( + dict( + module='backlight', + action='new_dpms', + dpms=self.dpms, + new_dpms=new_dpms, + ) + ) + ) + if new_dpms: + Backlight.enable(self.state) + else: + self.state = Backlight.change( + Backlight.Direction.get_state, + ) + logging.info( + json.dumps( + dict( + state=pprint.pformat( + self.state, + width=1e8, + compact=True, + ), + action='disable', + ) + ) + ) + Backlight.disable() + self.dpms = new_dpms + except Exception: + logging.error(traceback.format_exc()) - @classmethod - def get_state(cls): - raise NotImplementedError + @classmethod + def get_state(cls): + raise NotImplementedError - @classmethod - def set_state(cls): - raise NotImplementedError + @classmethod + def set_state(cls): + raise NotImplementedError - @classmethod - def disable(cls): - return cls.change( - cls.Direction.absolute, - 0, - ) + @classmethod + def disable(cls): + return cls.change( + cls.Direction.absolute, + 0, + ) - @classmethod - def enable(cls, state,): - res = [] - for device_state in state: - res.append( - cls.change( - direction=cls.Direction.absolute, - value=device_state['value'], - device_name=device_state['device_name'], - ) - ) - return res + @classmethod + def enable( + cls, + state, + ): + res = [] + for device_state in state: + res.append( + cls.change( + direction=cls.Direction.absolute, + value=device_state['value'], + device_name=device_state['device_name'], + ) + ) + return res - @classmethod - def change( - cls, - direction, - value=None, - devices=None, - device_name=None, - types=None, - ): - assert isinstance(direction, Backlight.Direction) + @classmethod + def change( + cls, + direction, + value=None, + devices=None, + device_name=None, + types=None, + ): + assert isinstance(direction, Backlight.Direction) - state = [] - devices_all = dict( - smc_kbd=dict( - sysfs_path='sysfs/leds/smc::kbd_backlight', - ), - intel_backlight=dict( - sysfs_path='sysfs/backlight/intel_backlight', - ), - ) + state = [] + devices_all = dict( + smc_kbd=dict( + sysfs_path='sysfs/leds/smc::kbd_backlight', + ), + intel_backlight=dict( + sysfs_path='sysfs/backlight/intel_backlight', + ), + ) - if devices is None: - devices = [] + if devices is None: + devices = [] - if not device_name is None: - devices.append(device_name) + if not device_name is None: + devices.append(device_name) - if len(devices) == 0: - if types is None: - types = [ - 'keyboard', - 'output', - ] + if len(devices) == 0: + if types is None: + types = [ + 'keyboard', + 'output', + ] - for current_type in types: - if current_type == 'keyboard': - devices.extend([ - 'smc_kbd' - ]) - elif current_type == 'output': - devices.extend([ - 'intel_backlight', - ]) - else: - raise NotImplementedError - else: - assert types is None + for current_type in types: + if current_type == 'keyboard': + devices.extend(['smc_kbd']) + elif current_type == 'output': + devices.extend( + [ + 'intel_backlight', + ] + ) + else: + raise NotImplementedError + else: + assert types is None - devices2 = list(set(devices)) + devices2 = list(set(devices)) - if sys.platform == 'linux': - assert all([ - o in devices_all - for o in devices2 - ]) + if sys.platform == 'linux': + assert all([o in devices_all for o in devices2]) - leds = \ - [ - o.strip() - for o in subprocess.check_output( - ['light', '-L'], - timeout=1, - )\ - .decode('utf-8')\ - .splitlines()[1:] - ] + leds = [ + o.strip() + for o in subprocess.check_output( + ['light', '-L'], + timeout=1, + ) + .decode('utf-8') + .splitlines()[1:] + ] + for current_device_name in devices2: + device = devices_all[current_device_name] - for current_device_name in devices2: - device = devices_all[current_device_name] + sysfs_path = device['sysfs_path'] - sysfs_path = device['sysfs_path'] + if not sysfs_path in leds: + raise NotImplementedError - if not sysfs_path in leds: - raise NotImplementedError + extra_args = [] + if value is None: + value = 20.0 - extra_args = [] - if value is None: - value = 20.0 + value2 = max(float(value), 0.0) - value2 = max(float(value), 0.0) + assert isinstance(value2, float) and value >= -1e-8 - assert isinstance(value2, float) and value >= -1e-8 + if direction == cls.Direction.increase: + extra_args.extend(['-A', '%f' % value2]) + elif direction == cls.Direction.decrease: + extra_args.extend(['-U', '%f' % value2]) + elif direction == cls.Direction.absolute: + extra_args.extend(['-S', '%f' % value2]) + elif direction == cls.Direction.get_state: + pass + else: + raise NotImplementedError - if direction == cls.Direction.increase: - extra_args.extend(['-A', '%f' % value2]) - elif direction == cls.Direction.decrease: - extra_args.extend(['-U', '%f' % value2]) - elif direction == cls.Direction.absolute: - extra_args.extend(['-S', '%f' % value2]) - elif direction == cls.Direction.get_state: - pass - else: - raise NotImplementedError + get_current = lambda: float( + subprocess.check_output( + [ + 'light', + '-G', + '-s', + sysfs_path, + ], + timeout=1, + ).decode('utf-8') + ) - get_current = lambda : float(subprocess.check_output([ - 'light', '-G', - '-s', sysfs_path, - ], timeout=1).decode('utf-8')) + if not (direction == cls.Direction.get_state): + old_value = get_current() - if not (direction == cls.Direction.get_state): - old_value = get_current() + value_steps = None - value_steps = None + if direction == cls.Direction.decrease: + value_steps = numpy_linspace( + old_value, + max(old_value - value2, 0), + 10, + ) + elif direction == cls.Direction.increase: + value_steps = numpy_linspace( + old_value, + min(old_value + value2, 100), + 10, + ) + elif direction == cls.Direction.absolute: + value_steps = numpy_linspace( + old_value, + min( + max( + 0, + value2, + ), + 100, + ), + 10, + ) + else: + raise NotImplementedError - if direction == cls.Direction.decrease: - value_steps = numpy_linspace( - old_value, - max(old_value - value2, 0), - 10, - ) - elif direction == cls.Direction.increase: - value_steps = numpy_linspace( - old_value, - min(old_value + value2, 100), - 10, - ) - elif direction == cls.Direction.absolute: - value_steps = numpy_linspace( - old_value, - min( - max( - 0, - value2, - ), - 100 - ), - 10, - ) - else: - raise NotImplementedError + for current_value in value_steps: + subprocess.check_call( + [ + 'light', + '-v', + '3', + '-s', + sysfs_path, + '-S', + '%f' % current_value, + ], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + time.sleep(0.05) - for current_value in value_steps: - subprocess.check_call( - [ - 'light', '-v', '3', - '-s', sysfs_path, - '-S', '%f' % current_value, - ], - stderr=subprocess.PIPE, - stdout=subprocess.PIPE - ) - time.sleep(0.05) + state.append( + dict( + mode=cls.Mode.light, + device_path=sysfs_path, + device_name=current_device_name, + value=get_current(), + ) + ) + else: + raise NotImplementedError - state.append( - dict( - mode=cls.Mode.light, - device_path=sysfs_path, - device_name=current_device_name, - value=get_current(), - ) - ) - else: - raise NotImplementedError + return state - return state + class Cpufreq: + @classmethod + def profile( + cls, + ) -> Literal[ + 'applesmc.768', + 'intel_pstate', + ]: + if os.path.exists('/sys/bus/platform/devices/applesmc.768'): + return 'applesmc.768' + if os.path.exists('/sys/devices/system/cpu/intel_pstate/no_turbo'): + return 'intel_pstate' + else: + raise NotImplementedError - class Cpufreq: - @classmethod - def profile(cls) -> Literal[ - 'applesmc.768', - 'intel_pstate', - ]: - if os.path.exists('/sys/bus/platform/devices/applesmc.768'): - return 'applesmc.768' - if os.path.exists('/sys/devices/system/cpu/intel_pstate/no_turbo'): - return 'intel_pstate' - else: - raise NotImplementedError - - @classmethod - def powersave(cls): - if cls.profile() == 'applesmc.768': - subprocess.check_call(r''' + @classmethod + def powersave(cls): + if cls.profile() == 'applesmc.768': + subprocess.check_call( + r""" #echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; #echo powersave | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; #echo 1 > /sys/bus/platform/devices/applesmc.768/fan1_manual; @@ -2649,21 +2602,27 @@ echo schedutil | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; echo 1 > /sys/bus/platform/devices/applesmc.768/fan1_manual; echo 2000 > /sys/bus/platform/devices/applesmc.768/fan1_output; - ''', shell=True) - elif cls.profile() == 'intel_pstate': - subprocess.check_call(r''' + """, + shell=True, + ) + elif cls.profile() == 'intel_pstate': + subprocess.check_call( + r""" echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; echo 40 > /sys/devices/system/cpu/intel_pstate/max_perf_pct; echo 900000 | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq; echo schedutil | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; - ''', shell=True) - else: - raise NotImplementedError + """, + shell=True, + ) + else: + raise NotImplementedError - @classmethod - def performance(cls): - if cls.profile() == 'applesmc.768': - subprocess.check_call(r''' + @classmethod + def performance(cls): + if cls.profile() == 'applesmc.768': + subprocess.check_call( + r""" #echo powersave | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; #echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; @@ -2674,92 +2633,102 @@ echo schedutil | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; echo 1 > /sys/bus/platform/devices/applesmc.768/fan1_manual; echo 6500 > /sys/bus/platform/devices/applesmc.768/fan1_output; - ''', shell=True) - elif cls.profile() == 'intel_pstate': - subprocess.check_call(r''' + """, + shell=True, + ) + elif cls.profile() == 'intel_pstate': + subprocess.check_call( + r""" echo powersave | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; echo 60 > /sys/devices/system/cpu/intel_pstate/max_perf_pct; echo 1200000 | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq; echo schedutil | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; - ''', shell=True) - else: - raise NotImplementedError + """, + shell=True, + ) + else: + raise NotImplementedError - if options.backlight_increase or options.backlight_decrease: - if options.backlight_increase: - direction = Backlight.Direction.increase - elif options.backlight_decrease: - direction = Backlight.Direction.decrease - else: - raise NotImplementedError + if options.backlight_increase or options.backlight_decrease: + if options.backlight_increase: + direction = Backlight.Direction.increase + elif options.backlight_decrease: + direction = Backlight.Direction.decrease + else: + raise NotImplementedError - Backlight.change( - direction=direction, - types=options.backlight_type, - ) + Backlight.change( + direction=direction, + types=options.backlight_type, + ) - return + return - elif not options.cpufreq_action is None: - if options.cpufreq_action == 'performance': - Cpufreq.performance() - elif options.cpufreq_action == 'powersave': - Cpufreq.powersave() - else: - raise NotImplementedError + elif not options.cpufreq_action is None: + if options.cpufreq_action == 'performance': + Cpufreq.performance() + elif options.cpufreq_action == 'powersave': + Cpufreq.powersave() + else: + raise NotImplementedError - return - else: - pass + return + else: + pass - #logger.info(dict( - # environ=os.environ, - # #session=subprocess.check_output(['loginctl', 'show-session'], timeout=1,), - #)) + # logger.info(dict( + # environ=os.environ, + # #session=subprocess.check_output(['loginctl', 'show-session'], timeout=1,), + # )) - sway_sock_res = sway_sock(True) + sway_sock_res = sway_sock(True) - if sway_sock_res: - os.environ['SWAYSOCK'] = sway_sock_res + if sway_sock_res: + os.environ['SWAYSOCK'] = sway_sock_res - assert all([ - env_name in os.environ - for env_name in [ - 'GTK_IM_MODULE', - 'XMODIFIERS', - 'QT_IM_MODULE', - #'I3SOCK', - #'XDG_SEAT', - 'SWAYSOCK', - 'WAYLAND_DISPLAY', - ] - ]) and os.environ['SWAYSOCK'] == sway_sock() - services = [] + assert ( + all( + [ + env_name in os.environ + for env_name in [ + 'GTK_IM_MODULE', + 'XMODIFIERS', + 'QT_IM_MODULE', + #'I3SOCK', + #'XDG_SEAT', + 'SWAYSOCK', + 'WAYLAND_DISPLAY', + ] + ] + ) + and os.environ['SWAYSOCK'] == sway_sock() + ) + services = [] - shutdown = False + shutdown = False - def on_interrupt(*args, **kwargs): - logging.info('blah') - nonlocal shutdown - shutdown = True + def on_interrupt(*args, **kwargs): + logging.info('blah') + nonlocal shutdown + shutdown = True - signal.signal( - signal.SIGINT, - on_interrupt, - ) - signal.signal( - signal.SIGTERM, - on_interrupt, - ) + signal.signal( + signal.SIGINT, + on_interrupt, + ) + signal.signal( + signal.SIGTERM, + on_interrupt, + ) - try: - if options.cpufreq == 0: - #logging.info('launching cpufreq, need sudo') - #subprocess.check_call(['sudo', 'whoami']) + try: + if options.cpufreq == 0: + # logging.info('launching cpufreq, need sudo') + # subprocess.check_call(['sudo', 'whoami']) - services.append( - subprocess.Popen( - r''' + services.append( + subprocess.Popen( + r""" exec sh -c 'echo cpufreq, user; whoami; while [[ -a /proc/{pid} ]]; do #echo passive > /sys/devices/system/cpu/intel_pstate/status; @@ -2773,46 +2742,44 @@ echo schedutil | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; done; sleep 10; done;' - '''.format(pid=os.getpid()), - shell=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - stdin=subprocess.DEVNULL, - ) - ) + """.format(pid=os.getpid()), + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + ) + ) - class start_swayidle: - def __init__(self): - swaylock_cmd = [ - 'swaylock', '-f', '-d', - ] - if not options.background_image is None: - swaylock_cmd.extend( - [ - '-i', - '"%s"' % options.background_image, - ] - ) + class start_swayidle: + def __init__(self): + swaylock_cmd = [ + 'swaylock', + '-f', + '-d', + ] + if not options.background_image is None: + swaylock_cmd.extend( + [ + '-i', + '"%s"' % options.background_image, + ] + ) - self.commands = dict( - swaylock_cmd2=' '.join(swaylock_cmd), - timeout1='echo timeout1; swaymsg "output * dpms off";', - lock='echo lock; pkill --signal SIGUSR1 swayidle;', - unlock='echo unlock; pkill --signal SIGINT swaylock; swaymsg "output * dpms on";', - unlock2='pkill --signal SIGINT swaylock;', - resume='echo resume; swaymsg "output * dpms on";', - before_sleep='echo before_sleep; commands loginctl lock-session;', - #before_sleep='echo blah;', - after_resume='echo after_resume; pkill --signal SIGUSR1 swayidle;', - ) - self.last_force_idle = None - self.commands.update( - timeout2='echo timeout2; {swaylock_cmd};'.format( - swaylock_cmd=self.commands['swaylock_cmd2'] - ) - ) - self.swayidle = subprocess.Popen( - r''' + self.commands = dict( + swaylock_cmd2=' '.join(swaylock_cmd), + timeout1='echo timeout1; swaymsg "output * dpms off";', + lock='echo lock; pkill --signal SIGUSR1 swayidle;', + unlock='echo unlock; pkill --signal SIGINT swaylock; swaymsg "output * dpms on";', + unlock2='pkill --signal SIGINT swaylock;', + resume='echo resume; swaymsg "output * dpms on";', + before_sleep='echo before_sleep; commands loginctl lock-session;', + # before_sleep='echo blah;', + after_resume='echo after_resume; pkill --signal SIGUSR1 swayidle;', + ) + self.last_force_idle = None + self.commands.update(timeout2='echo timeout2; {swaylock_cmd};'.format(swaylock_cmd=self.commands['swaylock_cmd2'])) + self.swayidle = subprocess.Popen( + r""" exec swayidle -d -w \ timeout 300 'echo t1; read;' \ resume 'echo t5; ' \ @@ -2822,1146 +2789,1121 @@ echo schedutil | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; unlock 'echo t3;' \ before-sleep 'echo t6; read;' \ after-resume 'echo t7; read;' 2>&1 - ''', - shell=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - restore_signals=False, - preexec_fn=lambda : os.setpgrp(), - ) - self.output = intercept_output( - self.swayidle, - real_time=True, - transform_callback=lambda x: [logging.info(x), b''][-1], - ) - self.events = [] - self.last_skip_loop = None - self.data = [] - if options.backlight_service: - self.backlight = Backlight() - else: - self.backlight = None - self.bg = None - self.bg_terminate = False + """, + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + restore_signals=False, + preexec_fn=lambda: os.setpgrp(), + ) + self.output = intercept_output( + self.swayidle, + real_time=True, + transform_callback=lambda x: [logging.info(x), b''][-1], + ) + self.events = [] + self.last_skip_loop = None + self.data = [] + if options.backlight_service: + self.backlight = Backlight() + else: + self.backlight = None + self.bg = None + self.bg_terminate = False - def skip_loop_long_ago(self): - if self.last_skip_loop is None or ( - datetime.datetime.now() - self.last_skip_loop - ).total_seconds() >= 30: - self.last_skip_loop = datetime.datetime.now() - return True - else: - return False + def skip_loop_long_ago(self): + if self.last_skip_loop is None or (datetime.datetime.now() - self.last_skip_loop).total_seconds() >= 30: + self.last_skip_loop = datetime.datetime.now() + return True + else: + return False - def background_check(self): - if ( - self.bg is None or \ - not self.bg.poll() is None - ) and not self.bg_terminate: - if not options.background_image is None: - self.bg = subprocess.Popen([ - 'swaybg', - '--output', - '*', - '--image', - options.background_image, - '--mode', - 'fill', - ]) + def background_check(self): + if (self.bg is None or not self.bg.poll() is None) and not self.bg_terminate: + if not options.background_image is None: + self.bg = subprocess.Popen( + [ + 'swaybg', + '--output', + '*', + '--image', + options.background_image, + '--mode', + 'fill', + ] + ) - def background_terminate(self, *args, **kwargs): - if not self.bg is None: - self.bg_terminate = True - self.bg.terminate(*args, **kwargs) + def background_terminate(self, *args, **kwargs): + if not self.bg is None: + self.bg_terminate = True + self.bg.terminate(*args, **kwargs) - def poll(self): - return self.swayidle.poll() + def poll(self): + return self.swayidle.poll() - def release_lock(self): - self.swayidle.stdin.write(b'\n') - self.swayidle.stdin.flush() + def release_lock(self): + self.swayidle.stdin.write(b'\n') + self.swayidle.stdin.flush() - def force_idle(self): - if self.last_force_idle is None or ( - datetime.datetime.now() - self.last_force_idle - ).total_seconds() >= 10: - self.last_force_idle = datetime.datetime.now() - return True - else: - return False + def force_idle(self): + if self.last_force_idle is None or (datetime.datetime.now() - self.last_force_idle).total_seconds() >= 10: + self.last_force_idle = datetime.datetime.now() + return True + else: + return False - def terminate(self, *args, **kwargs): - self.background_terminate() + def terminate(self, *args, **kwargs): + self.background_terminate() - return self.swayidle.terminate(*args, **kwargs) + return self.swayidle.terminate(*args, **kwargs) - def wait(self, *args, **kwargs): - return self.swayidle.wait(*args, **kwargs) + def wait(self, *args, **kwargs): + return self.swayidle.wait(*args, **kwargs) - def kill(self): - return self.swayidle.kill() + def kill(self): + return self.swayidle.kill() - def dpms(self, direction): - assert direction in ['on', 'off'] + def dpms(self, direction): + assert direction in ['on', 'off'] - raise NotImplementedError + raise NotImplementedError - def check(self): - new_events = [] + def check(self): + new_events = [] - class event_t(enum.Enum): - idle_state = 'idle state' - active_state = 'active state' + class event_t(enum.Enum): + idle_state = 'idle state' + active_state = 'active state' - while True: - if self.output is None: - break + while True: + if self.output is None: + break - chunk = next(self.output) + chunk = next(self.output) - if chunk['aggregated']: - self.output = None - continue + if chunk['aggregated']: + self.output = None + continue - if len(chunk['data']) == 0: - break + if len(chunk['data']) == 0: + break - self.data.append(chunk) + self.data.append(chunk) - if b'\n' in chunk['data']: - total = b''.join([ - o['data'] - for o in self.data - ]).decode('utf-8') - sep_pos = total.rfind('\n') - lines = total[:sep_pos].splitlines() - self.data = [ - dict( - data=total[sep_pos:].encode('utf-8'), - aggregated=False, - ) - ] - for line in lines: - if event_t.idle_state.value in line: - line = event_t.idle_state.value - elif event_t.active_state.value in line: - line = event_t.active_state.value - else: - pass + if b'\n' in chunk['data']: + total = b''.join([o['data'] for o in self.data]).decode('utf-8') + sep_pos = total.rfind('\n') + lines = total[:sep_pos].splitlines() + self.data = [ + dict( + data=total[sep_pos:].encode('utf-8'), + aggregated=False, + ) + ] + for line in lines: + if event_t.idle_state.value in line: + line = event_t.idle_state.value + elif event_t.active_state.value in line: + line = event_t.active_state.value + else: + pass - if line in [ - 't1', 't2', 't3', 't4', - 't5', 't5', 't6', 't7', - event_t.idle_state.value, - event_t.active_state.value, - ]: - new_events.append(line) + if line in [ + 't1', + 't2', + 't3', + 't4', + 't5', + 't5', + 't6', + 't7', + event_t.idle_state.value, + event_t.active_state.value, + ]: + new_events.append(line) - def retry(cb, cnt=None): - if cnt is None: - cnt = 10 + def retry(cb, cnt=None): + if cnt is None: + cnt = 10 - i = 0 - while True: - logging.info('retry i = %d, cnt = %d' % (i, cnt)) + i = 0 + while True: + logging.info('retry i = %d, cnt = %d' % (i, cnt)) - if not ( - subprocess.call(['swaymsg', '-t', 'get_version']) == 0 - ): - continue + if not (subprocess.call(['swaymsg', '-t', 'get_version']) == 0): + continue - if cb() == 0: - break + if cb() == 0: + break - time.sleep(0.5) + time.sleep(0.5) - i += 1 + i += 1 - if ( - len(new_events) > 0 or \ - len(self.events) > 0 and \ - self.skip_loop_long_ago() - ): - self.events.extend(new_events) + if len(new_events) > 0 or len(self.events) > 0 and self.skip_loop_long_ago(): + self.events.extend(new_events) - skip_loop = False + skip_loop = False - if ( - all([ - o in ['t1', 't4'] - for o in self.events - ]) and \ - VLC.vlc_is_playing_fullscreen() and \ - self.backlight.dpms - ): - skip_loop = True - logging.info( - 'skip loop, %s' % ( - [ - json.dumps(self.events), - self.backlight.dpms, - VLC.vlc_is_playing_fullscreen(), - self.events, - new_events, - ], - ) - ) - elif ( - len(new_events) == 0 and \ - len(self.events) > 1 and \ - all([ - o in ['t1', 't4'] - for o in self.events - ]) - ): - self.events = ['t4'] - elif len(self.events) > 1 and ( - self.events == ['t1', 't4', 't5', 't5'] or \ - self.events == ['t1', 't5', 't5'] or \ - self.events == ['t1', 't5'] - ): - for o in new_events: - self.release_lock() + if all([o in ['t1', 't4'] for o in self.events]) and VLC.vlc_is_playing_fullscreen() and self.backlight.dpms: + skip_loop = True + logging.info( + 'skip loop, %s' + % ( + [ + json.dumps(self.events), + self.backlight.dpms, + VLC.vlc_is_playing_fullscreen(), + self.events, + new_events, + ], + ) + ) + elif len(new_events) == 0 and len(self.events) > 1 and all([o in ['t1', 't4'] for o in self.events]): + self.events = ['t4'] + elif len(self.events) > 1 and (self.events == ['t1', 't4', 't5', 't5'] or self.events == ['t1', 't5', 't5'] or self.events == ['t1', 't5']): + for o in new_events: + self.release_lock() - self.events = [] + self.events = [] - for o in self.events: - if skip_loop: - self.release_lock() - continue + for o in self.events: + if skip_loop: + self.release_lock() + continue - if o == 't1': - #if self.force_idle(): - # subprocess.check_call(self.commands['lock'], shell=True) - logging.info('started t1') - if self.force_idle(): - subprocess.check_call(self.commands['timeout1'], shell=True) - logging.info('done t1') - self.release_lock() - elif o == 't2': - logging.info('started lock') - if self.force_idle(): - custom_notify( - title='swayidle', - msg='loginctl lock started', - ) - while True: - if not subprocess.call( - self.commands['lock'], shell=True - ) == 0: - continue - if not subprocess.call( - self.commands['timeout2'], shell=True - ) == 0: - #continue - pass - if not subprocess.call( - self.commands['timeout1'], shell=True - ) == 0: - continue - break - logging.info('done lock') - self.release_lock() - elif o == 't3': - pass - elif o == 't4': - logging.info('started t4') - if self.force_idle(): - subprocess.check_call(self.commands['lock'], shell=True) - subprocess.call(self.commands['timeout2'], shell=True) - subprocess.check_call(self.commands['timeout1'], shell=True) - logging.info('done t4') - self.release_lock() - elif o == 't5': - logging.info('started timeout resume') - if self.force_idle(): - subprocess.check_call(self.commands['lock'], shell=True) - retry( - lambda: subprocess.call(self.commands['resume'], shell=True), - ) - logging.info('done timeout resume') - elif o == 't6': - logging.info('started before-sleep') - if self.force_idle(): - subprocess.call(self.commands['timeout2'], shell=True), - subprocess.check_call(self.commands['timeout1'], shell=True), - logging.info('done before-sleep') - self.release_lock() - elif o == 't7': - logging.info('started after-resume') - #if self.force_idle(): - #subprocess.check_call(self.commands['lock'], shell=True) - while True: - if subprocess.call( - self.commands['resume'], - shell=True - ) == 0: - break - else: - time.sleep(0.5) - logging.info('done after-resume') - self.release_lock() - elif o in [ - event_t.idle_state.value, - event_t.active_state.value, - ]: - logging.info(json.dumps(dict(o=o))) - else: - logging.error(json.dumps(dict(o=o))) - raise NotImplementedError + if o == 't1': + # if self.force_idle(): + # subprocess.check_call(self.commands['lock'], shell=True) + logging.info('started t1') + if self.force_idle(): + subprocess.check_call(self.commands['timeout1'], shell=True) + logging.info('done t1') + self.release_lock() + elif o == 't2': + logging.info('started lock') + if self.force_idle(): + custom_notify( + title='swayidle', + msg='loginctl lock started', + ) + while True: + if not subprocess.call(self.commands['lock'], shell=True) == 0: + continue + if not subprocess.call(self.commands['timeout2'], shell=True) == 0: + # continue + pass + if not subprocess.call(self.commands['timeout1'], shell=True) == 0: + continue + break + logging.info('done lock') + self.release_lock() + elif o == 't3': + pass + elif o == 't4': + logging.info('started t4') + if self.force_idle(): + subprocess.check_call(self.commands['lock'], shell=True) + subprocess.call(self.commands['timeout2'], shell=True) + subprocess.check_call(self.commands['timeout1'], shell=True) + logging.info('done t4') + self.release_lock() + elif o == 't5': + logging.info('started timeout resume') + if self.force_idle(): + subprocess.check_call(self.commands['lock'], shell=True) + retry( + lambda: subprocess.call(self.commands['resume'], shell=True), + ) + logging.info('done timeout resume') + elif o == 't6': + logging.info('started before-sleep') + if self.force_idle(): + (subprocess.call(self.commands['timeout2'], shell=True),) + (subprocess.check_call(self.commands['timeout1'], shell=True),) + logging.info('done before-sleep') + self.release_lock() + elif o == 't7': + logging.info('started after-resume') + # if self.force_idle(): + # subprocess.check_call(self.commands['lock'], shell=True) + while True: + if subprocess.call(self.commands['resume'], shell=True) == 0: + break + else: + time.sleep(0.5) + logging.info('done after-resume') + self.release_lock() + elif o in [ + event_t.idle_state.value, + event_t.active_state.value, + ]: + logging.info(json.dumps(dict(o=o))) + else: + logging.error(json.dumps(dict(o=o))) + raise NotImplementedError - if not skip_loop: - pprint.pprint(self.events) - del self.events[:] + if not skip_loop: + pprint.pprint(self.events) + del self.events[:] - if not self.backlight is None: - self.backlight.check() + if not self.backlight is None: + self.backlight.check() - self.background_check() + self.background_check() - if options.polkit_service: - services.extend([ - subprocess.Popen( - ['/usr/lib/polkit-gnome/polkit-gnome-authentication-agent-1'] - ) - ]) + if options.polkit_service: + services.extend([subprocess.Popen(['/usr/lib/polkit-gnome/polkit-gnome-authentication-agent-1'])]) - services.extend([ - subprocess.Popen( - ['ibus-daemon'], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - stdin=subprocess.DEVNULL, - ), - start_swayidle(), - ]) + services.extend( + [ + subprocess.Popen( + ['ibus-daemon'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + ), + start_swayidle(), + ] + ) - if not options.battery is None: - assert options.battery in [0] + if not options.battery is None: + assert options.battery in [0] - logging.info('launching battery') - services.append( - Battery( - should_start=True, - ) - ) + logging.info('launching battery') + services.append( + Battery( + should_start=True, + ) + ) + while True: + if shutdown: + logging.info('shutdown') + break - while True: - if shutdown: - logging.info('shutdown') - break + if all([not o.poll() is None for o in services]): + logging.info('done') + break - if all([not o.poll() is None for o in services]): - logging.info('done') - break + for o in services: + if hasattr(o, 'check'): + o.check() - for o in services: - if hasattr(o, 'check'): - o.check() - - time.sleep(0.1) - except Exception: - logging.error(traceback.format_exc()) - finally: - for o in services: - try: - o.terminate() - o.wait(timeout=10) - except Exception: - logging.error(traceback.format_exc()) - logging.error('killed %s' % str(o.__dict__)) - o.kill() + time.sleep(0.1) + except Exception: + logging.error(traceback.format_exc()) + finally: + for o in services: + try: + o.terminate() + o.wait(timeout=10) + except Exception: + logging.error(traceback.format_exc()) + logging.error('killed %s' % str(o.__dict__)) + o.kill() def suspend_timer(argv): - import datetime; - import subprocess; - import time; - #import sys; - if len(argv) == 0: - print("enter HH:MM"); - t3 = input().strip() - else: - t3 = argv[0] - t2 = datetime.datetime.strptime(t3, "%H:%M").time() - while True: - t1 = datetime.datetime.now() - if ((t1.hour, t1.minute) >= (t2.hour, t2.minute)): - break - else: - t3 = [ - (t2.hour - t1.hour), - t2.minute - t1.minute - ] - if t3[1] < 0: - t3[1] += 60 - t3[0] -= 1 - print("\r%s, %02d:%02d" % ( - t1, - *t3, - ), end="") - time.sleep(1) - print("suspend computer at %s" % t1.isoformat()) - subprocess.check_call(["systemctl", "suspend"]); + import datetime + import subprocess + import time + + # import sys; + if len(argv) == 0: + print('enter HH:MM') + t3 = input().strip() + else: + t3 = argv[0] + t2 = datetime.datetime.strptime(t3, '%H:%M').time() + while True: + t1 = datetime.datetime.now() + if (t1.hour, t1.minute) >= (t2.hour, t2.minute): + break + else: + t3 = [(t2.hour - t1.hour), t2.minute - t1.minute] + if t3[1] < 0: + t3[1] += 60 + t3[0] -= 1 + print( + '\r%s, %02d:%02d' + % ( + t1, + *t3, + ), + end='', + ) + time.sleep(1) + print('suspend computer at %s' % t1.isoformat()) + subprocess.check_call(['systemctl', 'suspend']) + def gnome_shortcuts(argv: list[str]) -> None: - parser = optparse.OptionParser() - parser.add_option( - '-a', '--add', - action='store_true', - default=None, - ) - parser.add_option( - '-l', '--list', - action='store_true', - default=None, - ) + parser = optparse.OptionParser() + parser.add_option( + '-a', + '--add', + action='store_true', + default=None, + ) + parser.add_option( + '-l', + '--list', + action='store_true', + default=None, + ) - options, args = parser.parse_args(argv) + options, args = parser.parse_args(argv) - def commands_ids() -> list[str]: - bindings = subprocess.check_output([ - 'gsettings', 'get', 'org.gnome.settings-daemon.plugins.media-keys', - 'custom-keybindings', - ]).decode('utf-8').strip().replace('\'', '"',) - if bindings == '@as []': - t1 = [] - else: - t1 = json.loads(bindings) + def commands_ids() -> list[str]: + bindings = ( + subprocess.check_output( + [ + 'gsettings', + 'get', + 'org.gnome.settings-daemon.plugins.media-keys', + 'custom-keybindings', + ] + ) + .decode('utf-8') + .strip() + .replace( + "'", + '"', + ) + ) + if bindings == '@as []': + t1 = [] + else: + t1 = json.loads(bindings) - return t1 + return t1 - def add_command(name, command, binding): - command_id = len(commands_ids()) + def add_command(name, command, binding): + command_id = len(commands_ids()) - for cmd in [ - ( - 'gsettings', 'set', 'org.gnome.settings-daemon.plugins.media-keys', - 'custom-keybindings', '[%s]' % ','.join([ - "'%s'" % \ - ( - '/org/gnome/settings-daemon/plugins/media-keys' - '/custom-keybindings/custom%d/' - ) % o - for o in range(command_id + 1) - ]), - ), - ( - 'gsettings', 'set', - ( - 'org.gnome.settings-daemon.plugins.media-keys.custom-keybinding' - ':/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom%d/' - ) % command_id, - 'name', name, - ), - ( - 'gsettings', 'set', - ( - 'org.gnome.settings-daemon.plugins.media-keys.custom-keybinding' - ':/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom%d/' - ) % command_id, - 'command', command, - ), - ( - 'gsettings', 'set', - ( - 'org.gnome.settings-daemon.plugins.media-keys.custom-keybinding' - ':/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom%d/' - ) % command_id, - 'binding', binding, - ), - ]: - subprocess.check_call(cmd) + for cmd in [ + ( + 'gsettings', + 'set', + 'org.gnome.settings-daemon.plugins.media-keys', + 'custom-keybindings', + '[%s]' % ','.join(["'%s'" % ('/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom%d/') % o for o in range(command_id + 1)]), + ), + ( + 'gsettings', + 'set', + ('org.gnome.settings-daemon.plugins.media-keys.custom-keybinding:/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom%d/') + % command_id, + 'name', + name, + ), + ( + 'gsettings', + 'set', + ('org.gnome.settings-daemon.plugins.media-keys.custom-keybinding:/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom%d/') + % command_id, + 'command', + command, + ), + ( + 'gsettings', + 'set', + ('org.gnome.settings-daemon.plugins.media-keys.custom-keybinding:/org/gnome/settings-daemon/plugins/media-keys/custom-keybindings/custom%d/') + % command_id, + 'binding', + binding, + ), + ]: + subprocess.check_call(cmd) - if options.list: - t1 = commands_ids() + if options.list: + t1 = commands_ids() - t2 = [ - { - k : json.loads(subprocess.check_output([ - 'gsettings', 'get', - ( - 'org.gnome.settings-daemon.plugins.media-keys.custom-keybinding:%s' - ) % o, - k, - ]).decode('utf-8').replace('\'', '"',)) - for k in ['name', 'binding', 'command'] - } - for o in t1 - ] + t2 = [ + { + k: json.loads( + subprocess.check_output( + [ + 'gsettings', + 'get', + ('org.gnome.settings-daemon.plugins.media-keys.custom-keybinding:%s') % o, + k, + ] + ) + .decode('utf-8') + .replace( + "'", + '"', + ) + ) + for k in ['name', 'binding', 'command'] + } + for o in t1 + ] - pprint.pprint(t2) - elif options.add: - add_command(*args) - else: - raise NotImplementedError + pprint.pprint(t2) + elif options.add: + add_command(*args) + else: + raise NotImplementedError def socat_ssh(argv): - parser = optparse.OptionParser() - parser.add_option( - '--local_port', - dest='local_port', - default=None, - type=int, - ) - parser.add_option( - '--ssh_key', - dest='ssh_key', - default=None, - type=str, - ) - parser.add_option( - '--socat_verbose', - dest='socat_verbose', - action='store_true', - default=False, - ) - parser.add_option( - '--ssh_host', - dest='ssh_host', - default=None, - type=str, - ) - parser.add_option( - '--target_port', - dest='target_port', - default=None, - type=int, - ) - parser.add_option( - '--ssh_command', - dest='ssh_command', - default=None, - type=str, - ) - parser.add_option( - '--gateway_command', - dest='gateway_command', - default=None, - type=str, - help=( - 'a shell command that forwards ssh socket data ' - 'somewhere else, like ' - 'busybox nc 127.0.0.1 $(cat remote-ssh.port)' - ), - ) - options, args = parser.parse_args(argv) + parser = optparse.OptionParser() + parser.add_option( + '--local_port', + dest='local_port', + default=None, + type=int, + ) + parser.add_option( + '--ssh_key', + dest='ssh_key', + default=None, + type=str, + ) + parser.add_option( + '--socat_verbose', + dest='socat_verbose', + action='store_true', + default=False, + ) + parser.add_option( + '--ssh_host', + dest='ssh_host', + default=None, + type=str, + ) + parser.add_option( + '--target_port', + dest='target_port', + default=None, + type=int, + ) + parser.add_option( + '--ssh_command', + dest='ssh_command', + default=None, + type=str, + ) + parser.add_option( + '--gateway_command', + dest='gateway_command', + default=None, + type=str, + help=('a shell command that forwards ssh socket data somewhere else, like busybox nc 127.0.0.1 $(cat remote-ssh.port)'), + ) + options, args = parser.parse_args(argv) - if options.ssh_command is None: - ssh_command = ['ssh', '-T', '-C'] - else: - ssh_command = options.ssh_command.split() + if options.ssh_command is None: + ssh_command = ['ssh', '-T', '-C'] + else: + ssh_command = options.ssh_command.split() - if not options.ssh_key is None: - subprocess.check_call(['ssh-add', options.ssh_key]) - ssh_command.extend([ - '-i', options.ssh_key, - ]) + if not options.ssh_key is None: + subprocess.check_call(['ssh-add', options.ssh_key]) + ssh_command.extend( + [ + '-i', + options.ssh_key, + ] + ) - if not options.ssh_host is None: - ssh_command.extend([options.ssh_host]) + if not options.ssh_host is None: + ssh_command.extend([options.ssh_host]) - restart = False + restart = False - def on_interrupt(*args, **kwargs): - nonlocal restart - restart = True + def on_interrupt(*args, **kwargs): + nonlocal restart + restart = True + socat_command = ['socat'] - socat_command = ['socat'] + if options.socat_verbose: + socat_command.extend(['-v']) - if options.socat_verbose: - socat_command.extend(['-v']) + socat_command.extend( + [ + 'tcp-listen:%d,fork,bind=127.0.0.1' % (options.local_port,), + ] + ) - socat_command.extend([ - 'tcp-listen:%d,fork,bind=127.0.0.1' % ( - options.local_port, - ), - ]) + signal.signal( + signal.SIGINT, + on_interrupt, + ) + signal.signal( + signal.SIGTERM, + on_interrupt, + ) - signal.signal( - signal.SIGINT, - on_interrupt, - ) - signal.signal( - signal.SIGTERM, - on_interrupt, - ) + gateway = None + p = None - gateway = None - p = None - - while True: - if gateway is None: - gateway = tempfile.NamedTemporaryFile(suffix='.sh', mode='w') - gateway.write( - r''' + while True: + if gateway is None: + gateway = tempfile.NamedTemporaryFile(suffix='.sh', mode='w') + gateway.write( + r""" exec %s - ''' % ' '.join( - ssh_command + [options.gateway_command] - ) - ) - gateway.flush() + """ + % ' '.join(ssh_command + [options.gateway_command]) + ) + gateway.flush() - if p is None: - p = subprocess.Popen( - socat_command + [ - 'EXEC:sh %s' % gateway.name, - ] - ) + if p is None: + p = subprocess.Popen( + socat_command + + [ + 'EXEC:sh %s' % gateway.name, + ] + ) - time.sleep(1) + time.sleep(1) - if restart: - try: - p.terminate() - p.wait(10) - except Exception: - p.kill() + if restart: + try: + p.terminate() + p.wait(10) + except Exception: + p.kill() - restart = False + restart = False - if not p.poll() is None: - p = None + if not p.poll() is None: + p = None - if not gateway is None: - os.path.unlink(gateway.name) - if not p is None: - p.terminate() + if not gateway is None: + os.path.unlink(gateway.name) + if not p is None: + p.terminate() def share_wifi(argv): - parser = optparse.OptionParser() - parser.add_option( - '--to-wifi', - dest='to_wifi', - default=None, - type=str, - ) - parser.add_option( - '--from-eth', - dest='from_eth', - default=None, - type=str, - ) - parser.add_option( - '--channel', - dest='channel', - default=None, - type=int, - ) - parser.add_option( - '--ap-name', - dest='ap_name', - default=None, - type=str, - ) - parser.add_option( - '--restart-delay', - dest='restart_delay', - default=None, - type=int, - ) - options, args = parser.parse_args(argv) + parser = optparse.OptionParser() + parser.add_option( + '--to-wifi', + dest='to_wifi', + default=None, + type=str, + ) + parser.add_option( + '--from-eth', + dest='from_eth', + default=None, + type=str, + ) + parser.add_option( + '--channel', + dest='channel', + default=None, + type=int, + ) + parser.add_option( + '--ap-name', + dest='ap_name', + default=None, + type=str, + ) + parser.add_option( + '--restart-delay', + dest='restart_delay', + default=None, + type=int, + ) + options, args = parser.parse_args(argv) - if options.restart_delay is None: - options.restart_delay = 2 + if options.restart_delay is None: + options.restart_delay = 2 + assert not options.to_wifi is None + assert not options.from_eth is None + assert not options.ap_name is None + assert options.restart_delay >= 1 - assert not options.to_wifi is None - assert not options.from_eth is None - assert not options.ap_name is None - assert options.restart_delay >= 1 + print('enter password:') - print('enter password:') + pw = subprocess.check_output('read -s PW; echo -n $PW', shell=True).decode('utf-8') + if len(pw) == 0: + pw = subprocess.check_output('pwgen -syn 20 1', shell=True).decode('utf-8').strip() + with subprocess.Popen(['qrencode', '-t', 'UTF8'], stdin=subprocess.PIPE) as p: + p.stdin.write(pw.encode('utf-8')) + p.stdin.flush() + p.stdin.close() + try: + p.wait(5) + except Exception as exception: + p.kill() + raise exception - pw = subprocess.check_output( - 'read -s PW; echo -n $PW', - shell=True - ).decode('utf-8') - if len(pw) == 0: - pw = subprocess.check_output( - 'pwgen -syn 20 1', - shell=True - ).decode('utf-8').strip() - with subprocess.Popen( - ['qrencode', '-t', 'UTF8'], - stdin=subprocess.PIPE - ) as p: - p.stdin.write(pw.encode('utf-8')) - p.stdin.flush() - p.stdin.close() - try: - p.wait(5) - except Exception as exception: - p.kill() - raise exception + last_timestamp = datetime.datetime.now() + hostapd = None + restart = False + shutdown = False - last_timestamp = datetime.datetime.now() - hostapd = None - restart = False - shutdown = False + def on_interrupt(sig, *args, **kwargs): + nonlocal restart + nonlocal shutdown - def on_interrupt(sig, *args, **kwargs): - nonlocal restart - nonlocal shutdown + if sig == signal.SIGINT: + restart = True + elif sig == signal.SIGTERM: + shutdown = True + else: + raise NotImplementedError - if sig == signal.SIGINT: - restart = True - elif sig == signal.SIGTERM: - shutdown = True - else: - raise NotImplementedError + signal.signal( + signal.SIGINT, + on_interrupt, + ) + signal.signal( + signal.SIGTERM, + on_interrupt, + ) - signal.signal( - signal.SIGINT, - on_interrupt, - ) - signal.signal( - signal.SIGTERM, - on_interrupt, - ) + hostapd_args = [ + 'create_ap', + '--hostapd-timestamps', + options.to_wifi, + options.from_eth, + options.ap_name, + pw, + ] + if not options.channel is None: + hostapd_args.extend(['-c', '%d' % options.channel]) - hostapd_args = [ - 'create_ap', - '--hostapd-timestamps', - options.to_wifi, - options.from_eth, - options.ap_name, - pw, - ] - if not options.channel is None: - hostapd_args.extend(['-c', '%d' % options.channel]) + while True: + try: + if hostapd is None: + print('\n%s, start new' % last_timestamp) + hostapd = subprocess.Popen(hostapd_args) + else: + if restart or shutdown: + print('\n%s, shutdown current' % last_timestamp) + os.kill(hostapd.pid, signal.SIGINT) + try: + hostapd.wait(20) + except Exception: + hostapd.terminate() + restart = False - while True: - try: - if hostapd is None: - print('\n%s, start new' % last_timestamp) - hostapd = subprocess.Popen(hostapd_args) - else: - if restart or shutdown: - print('\n%s, shutdown current' % last_timestamp) - os.kill( - hostapd.pid, - signal.SIGINT - ) - try: - hostapd.wait(20) - except Exception: - hostapd.terminate() - restart = False + if not hostapd.poll() is None: + hostapd = None - if not hostapd.poll() is None: - hostapd = None + if shutdown: + break - if shutdown: - break + if (datetime.datetime.now() - last_timestamp).total_seconds() > options.restart_delay: + restart = True - if ( - datetime.datetime.now() - last_timestamp - ).total_seconds() > options.restart_delay: - restart = True + last_timestamp = datetime.datetime.now() + except Exception: + print(traceback.format_exc()) + restart = True + finally: + time.sleep(1) - last_timestamp = datetime.datetime.now() - except Exception: - print(traceback.format_exc()) - restart = True - finally: - time.sleep(1) def status(argv): - import inspect - import textwrap + import inspect + import textwrap - assert isinstance(argv, list) and all([isinstance(o, str) for o in argv]) + assert isinstance(argv, list) and all([isinstance(o, str) for o in argv]) - class c1(optparse.IndentedHelpFormatter): - def format_option(self, *args, **kwargs): - f1 = lambda text, width: '\n'.join([ - textwrap.fill('\t' + o, width, replace_whitespace=False) - for o in text.splitlines() - ]).splitlines() - t1 = inspect.getsource(optparse.IndentedHelpFormatter.format_option) - t2 = '\n'.join([o[4:] for o in t1.splitlines()[:]]).replace( - 'textwrap.wrap', 'f1', - ).replace('format_option', 'f2') - exec(t2, dict(f1=f1), locals()) - return locals()['f2'](self, *args, **kwargs) + class c1(optparse.IndentedHelpFormatter): + def format_option(self, *args, **kwargs): + f1 = lambda text, width: '\n'.join([textwrap.fill('\t' + o, width, replace_whitespace=False) for o in text.splitlines()]).splitlines() + t1 = inspect.getsource(optparse.IndentedHelpFormatter.format_option) + t2 = ( + '\n'.join([o[4:] for o in t1.splitlines()[:]]) + .replace( + 'textwrap.wrap', + 'f1', + ) + .replace('format_option', 'f2') + ) + exec(t2, dict(f1=f1), locals()) + return locals()['f2'](self, *args, **kwargs) - parser = optparse.OptionParser( - formatter=c1( - width=None, - ), - ) - parser.add_option( - '--sh', - dest='sh', - default=[], - action='append', - type=str, - ) - parser.add_option( - '--timeout', - dest='timeout', - default=None, - type=float, - ) - parser.add_option( - '--config', - dest='config', - default=None, - type=str, - help=''.join([ - '.json file with array of strings, each is a shell command ', - 'that outputs a separate status text value, ', - 'like\n', - r''' + parser = optparse.OptionParser( + formatter=c1( + width=None, + ), + ) + parser.add_option( + '--sh', + dest='sh', + default=[], + action='append', + type=str, + ) + parser.add_option( + '--timeout', + dest='timeout', + default=None, + type=float, + ) + parser.add_option( + '--config', + dest='config', + default=None, + type=str, + help=''.join( + [ + '.json file with array of strings, each is a shell command ', + 'that outputs a separate status text value, ', + 'like\n', + r""" ping -w 1 -i 0.02 -c 3 | tail -n 2| head -n 1 | grep -Po $'time\\s+.*$' sensors -j | jq -r '.\"coretemp-isa-0000\".\"Package id 0\".temp1_input|tostring + \" C\"' printf '%d RPM' $(cat /sys/devices/platform/applesmc.768/fan1_input) printf '% 3.0f%%' $(upower -d | grep -Po 'percentage:\\s+\\d+(\\.\\d+)?%' | grep -Po '\\d+(\\.\\d+)?' | head -n 1) - '''.strip() - ]) - ) - options, args = parser.parse_args(argv) + """.strip(), + ] + ), + ) + options, args = parser.parse_args(argv) - if options.timeout is None: - options.timeout = 0.5 + if options.timeout is None: + options.timeout = 0.5 - timeout2 = max(options.timeout, 0.0) + timeout2 = max(options.timeout, 0.0) - assert timeout2 >= 0.0 and timeout2 <= 4 + assert timeout2 >= 0.0 and timeout2 <= 4 - config = dict() - try: - if not options.config is None: - with io.open(options.config, 'r') as f: - config.update( - json.load(f) - ) - except Exception: - logging.error(traceback.format_exc()) - pass + config = dict() + try: + if not options.config is None: + with io.open(options.config, 'r') as f: + config.update(json.load(f)) + except Exception: + logging.error(traceback.format_exc()) + pass - options.sh.extend( - config.get('sh', []) - ) + options.sh.extend(config.get('sh', [])) - t1 = [] + t1 = [] - for sh_index, o in enumerate([ - *options.sh, - *[ - r''' + for sh_index, o in enumerate( + [ + *options.sh, + *[ + r""" A=$(free -h | grep -P Mem: | grep -Po '[\w\.\d]+'); echo -n $A | awk '{print $2, $7}'; - ''', - r''' + """, + r""" date +'%Y-%m-%d %l:%M:%S %p'; - ''', - ], - ]): - try: - t1.append( - subprocess.check_output( - o, - shell=True, - timeout=timeout2, - ).decode('utf-8').strip() - ) - except Exception: - t1.append('fail %d' % sh_index) + """, + ], + ] + ): + try: + t1.append( + subprocess.check_output( + o, + shell=True, + timeout=timeout2, + ) + .decode('utf-8') + .strip() + ) + except Exception: + t1.append('fail %d' % sh_index) - t3 = ' | '.join(t1).replace('\n\r', '') + t3 = ' | '.join(t1).replace('\n\r', '') - sys.stdout.write(t3) - sys.stdout.flush() + sys.stdout.write(t3) + sys.stdout.flush() -def custom_translate(current_string, check, none_char=None,): - if none_char is None: - none_char = '.' - class A: - def __getitem__(self, k): - t1 = chr(k) +def custom_translate( + current_string, + check, + none_char=None, +): + if none_char is None: + none_char = '.' + + class A: + def __getitem__(self, k): + t1 = chr(k) + + t2 = check(k, t1) + if isinstance(t2, bool): + if t2: + return t1 + else: + return none_char + elif isinstance(t2, str): + return t2 + + return current_string.translate(A()) - t2 = check(k, t1) - if isinstance(t2, bool): - if t2: - return t1 - else: - return none_char - elif isinstance(t2, str): - return t2 - return current_string.translate( - A() - ) def media_keys(argv): - assert isinstance(argv, list) and all([isinstance(o, str) for o in argv]) - parser = argparse.ArgumentParser() - parser.add_argument( - #'-c', '--command', - 'command', - #dest='command', - type=str, - default=None, - ) + assert isinstance(argv, list) and all([isinstance(o, str) for o in argv]) + parser = argparse.ArgumentParser() + parser.add_argument( + #'-c', '--command', + 'command', + # dest='command', + type=str, + default=None, + ) - options, args = parser.parse_known_args(argv) + options, args = parser.parse_known_args(argv) - if options.command is None and len(args) > 0: - assert len(args) == 1 - options.command = args[0] + if options.command is None and len(args) > 0: + assert len(args) == 1 + options.command = args[0] - assert options.command in [ - 'media-play-pause', - 'media-next', - 'media-forward-seconds', - 'media-backward-seconds', - 'media-prev', - 'media-lower-volume', - 'media-raise-volume', - 'media-toggle-volume', - ] + assert options.command in [ + 'media-play-pause', + 'media-next', + 'media-forward-seconds', + 'media-backward-seconds', + 'media-prev', + 'media-lower-volume', + 'media-raise-volume', + 'media-toggle-volume', + ] - msg = None + msg = None - mode = None - is_mocp = lambda : \ - subprocess.call([ - 'pgrep', - '-u', os.environ['USER'], - 'mocp', - ], stdout=subprocess.PIPE) == 0 + mode = None + is_mocp = ( + lambda: subprocess.call( + [ + 'pgrep', + '-u', + os.environ['USER'], + 'mocp', + ], + stdout=subprocess.PIPE, + ) + == 0 + ) - def mocp_info() -> str: - t1 = subprocess.check_output(['mocp', '-i']) - t3 = t1.decode('utf-8') - t2 : dict[str, str] = dict([ - (lambda o2: (o2[0], o2[1]))(o.split(':')) - #tuple(o.split(':')[:2]) - for o in t3.splitlines() - ]) + def mocp_info() -> str: + t1 = subprocess.check_output(['mocp', '-i']) + t3 = t1.decode('utf-8') + t2: dict[str, str] = dict( + [ + (lambda o2: (o2[0], o2[1]))(o.split(':')) + # tuple(o.split(':')[:2]) + for o in t3.splitlines() + ] + ) - return t2['Title'].strip()[:128] + return t2['Title'].strip()[:128] - if is_mocp(): - mode = 'mocp' - else: - mode = 'playerctl' + if is_mocp(): + mode = 'mocp' + else: + mode = 'playerctl' - if options.command == 'media-play-pause': - if mode == 'mocp': - subprocess.check_call(['mocp', '--toggle-pause']) - msg = mocp_info() - elif mode == 'playerctl': - subprocess.check_call(['playerctl', 'play-pause']) - msg = player_metadata() - else: - raise NotImplementedError - elif options.command == 'media-next': - if mode == 'mocp': - subprocess.check_call(['mocp', '--next']) - msg = mocp_info() - elif mode == 'playerctl': - subprocess.check_call(['playerctl', 'next']) - msg = player_metadata() - else: - raise NotImplementedError - elif options.command == 'media-backward-seconds': - if mode == 'mocp': - raise NotImplementedError - elif mode == 'playerctl': - pos = float(subprocess.check_output(['playerctl', 'position']).decode('utf-8')) - subprocess.check_call([ - 'playerctl', - 'position', - '%f' % (pos - float(args[0])) - ]) - #msg = player_metadata() - else: - raise NotImplementedError - elif options.command == 'media-forward-seconds': - if mode == 'mocp': - raise NotImplementedError - elif mode == 'playerctl': - pos = float(subprocess.check_output(['playerctl', 'position']).decode('utf-8')) - subprocess.check_call([ - 'playerctl', - 'position', - '%f' % (pos + float(args[0])) - ]) - #msg = player_metadata() - else: - raise NotImplementedError - elif options.command == 'media-prev': - if mode == 'mocp': - subprocess.check_call(['mocp', '--previous']) - msg = mocp_info() - elif mode == 'playerctl': - subprocess.check_call(['playerctl', 'previous']) - msg = player_metadata() - else: - raise NotImplementedError - elif options.command == 'media-lower-volume': - subprocess.check_call([ - 'pactl', - 'set-sink-volume', - '@DEFAULT_SINK@', - '-5%' - ]) - msg = subprocess.check_output([ - 'pactl', - 'get-sink-volume', - '@DEFAULT_SINK@' - ]).decode('utf-8').strip() - elif options.command == 'media-toggle-volume': - subprocess.check_call([ - 'pactl', - 'set-sink-mute', - '@DEFAULT_SINK@', - 'toggle', - ]) - msg = subprocess.check_output([ - 'pactl', - 'get-sink-volume', - '@DEFAULT_SINK@' - ]).decode('utf-8').strip() - elif options.command == 'media-raise-volume': - subprocess.check_call([ - 'pactl', - 'set-sink-volume', - '@DEFAULT_SINK@', - '+5%' - ]) - msg = subprocess.check_output([ - 'pactl', - 'get-sink-volume', - '@DEFAULT_SINK@' - ]).decode('utf-8').strip() - else: - raise NotImplementedError + if options.command == 'media-play-pause': + if mode == 'mocp': + subprocess.check_call(['mocp', '--toggle-pause']) + msg = mocp_info() + elif mode == 'playerctl': + subprocess.check_call(['playerctl', 'play-pause']) + msg = player_metadata() + else: + raise NotImplementedError + elif options.command == 'media-next': + if mode == 'mocp': + subprocess.check_call(['mocp', '--next']) + msg = mocp_info() + elif mode == 'playerctl': + subprocess.check_call(['playerctl', 'next']) + msg = player_metadata() + else: + raise NotImplementedError + elif options.command == 'media-backward-seconds': + if mode == 'mocp': + raise NotImplementedError + elif mode == 'playerctl': + pos = float(subprocess.check_output(['playerctl', 'position']).decode('utf-8')) + subprocess.check_call(['playerctl', 'position', '%f' % (pos - float(args[0]))]) + # msg = player_metadata() + else: + raise NotImplementedError + elif options.command == 'media-forward-seconds': + if mode == 'mocp': + raise NotImplementedError + elif mode == 'playerctl': + pos = float(subprocess.check_output(['playerctl', 'position']).decode('utf-8')) + subprocess.check_call(['playerctl', 'position', '%f' % (pos + float(args[0]))]) + # msg = player_metadata() + else: + raise NotImplementedError + elif options.command == 'media-prev': + if mode == 'mocp': + subprocess.check_call(['mocp', '--previous']) + msg = mocp_info() + elif mode == 'playerctl': + subprocess.check_call(['playerctl', 'previous']) + msg = player_metadata() + else: + raise NotImplementedError + elif options.command == 'media-lower-volume': + subprocess.check_call(['pactl', 'set-sink-volume', '@DEFAULT_SINK@', '-5%']) + msg = subprocess.check_output(['pactl', 'get-sink-volume', '@DEFAULT_SINK@']).decode('utf-8').strip() + elif options.command == 'media-toggle-volume': + subprocess.check_call( + [ + 'pactl', + 'set-sink-mute', + '@DEFAULT_SINK@', + 'toggle', + ] + ) + msg = subprocess.check_output(['pactl', 'get-sink-volume', '@DEFAULT_SINK@']).decode('utf-8').strip() + elif options.command == 'media-raise-volume': + subprocess.check_call(['pactl', 'set-sink-volume', '@DEFAULT_SINK@', '+5%']) + msg = subprocess.check_output(['pactl', 'get-sink-volume', '@DEFAULT_SINK@']).decode('utf-8').strip() + else: + raise NotImplementedError - logging.info( - json.dumps( - dict( - command=options.command, - msg=msg, - mode=mode, - ), - ensure_ascii=False - ) - ) + logging.info( + json.dumps( + dict( + command=options.command, + msg=msg, + mode=mode, + ), + ensure_ascii=False, + ) + ) + + return dict( + msg=msg, + ) - return dict( - msg=msg, - ) def install(argv: list[str]) -> None: - parser = argparse.ArgumentParser() - parser.add_argument( - '-r', - dest='recursive', - ) - parser.add_argument( - '-s', '--source', - help='source file/dir to install (copy with permissions preserving)', - dest='source', - type=pathlib.Path, - required=True, - ) - parser.add_argument( - '-p', '--relative', - type=pathlib.Path, - help='copy source relative to path, allows to strip extra components', - dest='relative', - default=None, - ) - parser.add_argument( - '-t', '--target', - type=pathlib.Path, - help='target file/dir to install (copy with permissions preserving)', - dest='target', - required=True, - ) - parser.add_argument( - '-f', '--overwrite', - help='overwrite if target is present', - dest='overwrite', - action='store_true', - default=False, - ) + parser = argparse.ArgumentParser() + parser.add_argument( + '-r', + dest='recursive', + ) + parser.add_argument( + '-s', + '--source', + help='source file/dir to install (copy with permissions preserving)', + dest='source', + type=pathlib.Path, + required=True, + ) + parser.add_argument( + '-p', + '--relative', + type=pathlib.Path, + help='copy source relative to path, allows to strip extra components', + dest='relative', + default=None, + ) + parser.add_argument( + '-t', + '--target', + type=pathlib.Path, + help='target file/dir to install (copy with permissions preserving)', + dest='target', + required=True, + ) + parser.add_argument( + '-f', + '--overwrite', + help='overwrite if target is present', + dest='overwrite', + action='store_true', + default=False, + ) - options, args = parser.parse_known_args(argv) + options, args = parser.parse_known_args(argv) - if options.relative: - relative_source = options.source.relative_to(options.relative) - else: - if options.source.is_absolute(): - relative_source = options.source.relative_to('/') - else: - relative_source = options.source + if options.relative: + relative_source = options.source.relative_to(options.relative) + else: + if options.source.is_absolute(): + relative_source = options.source.relative_to('/') + else: + relative_source = options.source - logger.info(dict(source=options.source, target=options.source)) + logger.info(dict(source=options.source, target=options.source)) - final_target = options.target / relative_source + final_target = options.target / relative_source - logger.info(dict(final_target=final_target, relative_source=relative_source)) + logger.info(dict(final_target=final_target, relative_source=relative_source)) - if final_target.exists(): - if not options.overwrite: - raise NotImplementedError + if final_target.exists(): + if not options.overwrite: + raise NotImplementedError - if final_target.is_dir(): - shutil.rmtree(final_target) - else: - os.unlink(final_target) + if final_target.is_dir(): + shutil.rmtree(final_target) + else: + os.unlink(final_target) - if options.source.is_dir() and not options.recursive: - raise NotImplementedError + if options.source.is_dir() and not options.recursive: + raise NotImplementedError - if options.source.is_dir(): - os.makedirs(final_target, exist_ok=True,) - else: - os.makedirs(final_target.parent, exist_ok=True,) + if options.source.is_dir(): + os.makedirs( + final_target, + exist_ok=True, + ) + else: + os.makedirs( + final_target.parent, + exist_ok=True, + ) - subprocess.check_call([ - 'cp', '-rp', - str(options.source), - '-T', - str(final_target), - ]) + subprocess.check_call( + [ + 'cp', + '-rp', + str(options.source), + '-T', + str(final_target), + ] + ) - #shutil.copy( - # options.source, - # final_target.parent, - #) + # shutil.copy( + # options.source, + # final_target.parent, + # ) + + logger.info(dict(msg='done')) - logger.info(dict(msg='done')) def backup(argv: list[str]) -> None: - parser = argparse.ArgumentParser() - parser.add_argument( - '-r', '--recipients', - #type=list[str], - action='append', - default=[], - ) - parser.add_argument( - '-o', '--output_dir', - #type=list[str], - required=True, - ) + parser = argparse.ArgumentParser() + parser.add_argument( + '-r', + '--recipients', + # type=list[str], + action='append', + default=[], + ) + parser.add_argument( + '-o', + '--output_dir', + # type=list[str], + required=True, + ) - options, args = parser.parse_known_args(argv) + options, args = parser.parse_known_args(argv) - assert len(options.recipients) > 0 + assert len(options.recipients) > 0 - subprocess.check_call(r''' + subprocess.check_call( + r""" (\ mkdir -p ~/.cache/; \ P=~/.cache/"secrets-$(date -Iseconds).tar.xz.gpg"; \ @@ -3975,228 +3917,225 @@ def backup(argv: list[str]) -> None: ';\ sync; \ ) - '''.replace( - '{GPG_ARGS}', - ' '.join([ - '-r %s' % o - for o in options.recipients - ]), - ).replace( - '{OUTPUT_DIR}', - options.output_dir, - ), shell=True) + """.replace( + '{GPG_ARGS}', + ' '.join(['-r %s' % o for o in options.recipients]), + ).replace( + '{OUTPUT_DIR}', + options.output_dir, + ), + shell=True, + ) class Command(enum.StrEnum): - media = 'media' - status = 'status' - http_server = 'http-server' - pass_ssh_osx = 'pass-ssh-osx' - wl_screenshot = 'wl-screenshot' - chrome = 'chrome' - eternal_oom = 'eternal-oom' - resilient_vlc = 'resilient-vlc' - eternal_firefox = 'eternal-firefox' - install = 'install' - resilient_ethernet = 'resilient-ethernet' - player = 'player' - share_wifi = 'share-wifi' - socat_ssh = 'socat-ssh' - gnome_shortcuts = 'gnome-shortcuts' - sway_sock = 'sway_sock' - loginctl = 'loginctl' - suspend_timer = 'suspend-timer' - desktop_services = 'desktop-services' - pm_service = 'pm-service' - scrap_yt_music = 'scrap-yt-music' - vpn = 'vpn' - backup = 'backup' - pip_resolve = 'pip_resolve' - pip_check_conflicts = 'pip_check_conflicts' + media = 'media' + status = 'status' + http_server = 'http-server' + pass_ssh_osx = 'pass-ssh-osx' + wl_screenshot = 'wl-screenshot' + chrome = 'chrome' + eternal_oom = 'eternal-oom' + resilient_vlc = 'resilient-vlc' + eternal_firefox = 'eternal-firefox' + install = 'install' + resilient_ethernet = 'resilient-ethernet' + player = 'player' + share_wifi = 'share-wifi' + socat_ssh = 'socat-ssh' + gnome_shortcuts = 'gnome-shortcuts' + sway_sock = 'sway_sock' + loginctl = 'loginctl' + suspend_timer = 'suspend-timer' + desktop_services = 'desktop-services' + pm_service = 'pm-service' + scrap_yt_music = 'scrap-yt-music' + vpn = 'vpn' + backup = 'backup' + pip_resolve = 'pip_resolve' + pip_check_conflicts = 'pip_check_conflicts' + def pip_check_conflicts( - args: list[str], -) -> None: - from .commands_typed.pip import pip_check_conflicts - from .commands_typed.argparse import parse_args as pr34_parse_args - parser = argparse.ArgumentParser() - parser.add_argument( - '-p', - dest='venv_path', - type=pathlib.Path, - help='venv path', - default=None, - ) + args: list[str], +) -> None: + from .commands_typed.pip import pip_check_conflicts + from .commands_typed.argparse import parse_args as pr34_parse_args - options, argv = pr34_parse_args(parser, args) + parser = argparse.ArgumentParser() + parser.add_argument( + '-p', + dest='venv_path', + type=pathlib.Path, + help='venv path', + default=None, + ) - res = pip_check_conflicts(options.venv_path) - logger.info(dict(res=res)) + options, argv = pr34_parse_args(parser, args) + + res = pip_check_conflicts(options.venv_path) + logger.info(dict(res=res)) + + assert res.status == 'ok' - assert res.status == 'ok' def pip_resolve( - args: list[str], + args: list[str], ) -> None: - from online.fxreader.pr34.commands_typed.pip import pip_resolve, pip_resolve_t + from online.fxreader.pr34.commands_typed.pip import pip_resolve, pip_resolve_t - parser = argparse.ArgumentParser() - parser.add_argument( - '-m', '--mode', - choices=[ - o.value - for o in pip_resolve_t.kwargs_t.mode_t - ], - required=True, - ) - parser.add_argument( - '-r', '--requirement', - default=[], - dest='requirements', - type=str, - action='append', - help=r''' + parser = argparse.ArgumentParser() + parser.add_argument( + '-m', + '--mode', + choices=[o.value for o in pip_resolve_t.kwargs_t.mode_t], + required=True, + ) + parser.add_argument( + '-r', + '--requirement', + default=[], + dest='requirements', + type=str, + action='append', + help=r""" requirement, can be multiple in a single parameter, all of them are to be split by whitespace and printed into a temp file, that is fed into uv pip compile - ''', - ) + """, + ) + + options, argv = parser.parse_known_args(args) + + requirements: Optional[list[str]] = [] + + for o in options.requirements: + requirements.extend(o.split()) + + if len(requirements) == 0: + requirements = None + + options.mode = pip_resolve_t.kwargs_t.mode_t(options.mode) + + resolve_res = pip_resolve( + argv, + mode=options.mode, + requirements=requirements, + ) + + assert not resolve_res.txt is None + + sys.stdout.write(resolve_res.txt) + sys.stdout.flush() - options, argv = parser.parse_known_args(args) +def commands_cli(argv: Optional[list[str]] = None) -> int: + if argv is None: + argv = sys.argv[1:] - requirements : Optional[list[str]] = [] + from online.fxreader.pr34.commands_typed.logging import setup as logging_setup - for o in options.requirements: - requirements.extend(o.split()) + logging_setup() + # logging.getLogger().setLevel(logging.INFO) + # logger.setLevel(logging.INFO) + # handler = logging.StreamHandler(sys.stderr) + # logging.getLogger().addHandler(handler) - if len(requirements) == 0: - requirements = None + msg: Optional[str] = None - options.mode = pip_resolve_t.kwargs_t.mode_t( - options.mode - ) + try: + if len(argv) > 0 and argv[0].startswith('media'): + msg = media_keys(argv).get('msg') + else: + parser = argparse.ArgumentParser( + #'online_fxreader.commands' + ) + parser.add_argument( + '_command', + choices=[o.value for o in Command], + ) - resolve_res = pip_resolve( - argv, - mode=options.mode, - requirements=requirements, - ) + options, args = parser.parse_known_args() + options.command = Command(options._command) - assert not resolve_res.txt is None - - sys.stdout.write(resolve_res.txt) - sys.stdout.flush() - -def commands_cli( - argv: Optional[list[str]] = None -) -> int: - if argv is None: - argv = sys.argv[1:] - - from online.fxreader.pr34.commands_typed.logging import setup as logging_setup - logging_setup() - #logging.getLogger().setLevel(logging.INFO) - #logger.setLevel(logging.INFO) - #handler = logging.StreamHandler(sys.stderr) - #logging.getLogger().addHandler(handler) - - msg : Optional[str] = None - - try: - if len(argv) > 0 and argv[0].startswith('media'): - msg = media_keys(argv).get('msg') - else: - parser = argparse.ArgumentParser( - #'online_fxreader.commands' - ) - parser.add_argument( - '_command', - choices=[ - o.value - for o in Command - ], - ) - - options, args = parser.parse_known_args() - options.command = Command(options._command) - - - if options.command is Command.status: - status(args) - elif options.command is Command.http_server: - http_server(args) - elif options.command is Command.pass_ssh_osx: - pass_ssh_osx(args) - elif options.command is Command.wl_screenshot: - subprocess.check_call(r''' + if options.command is Command.status: + status(args) + elif options.command is Command.http_server: + http_server(args) + elif options.command is Command.pass_ssh_osx: + pass_ssh_osx(args) + elif options.command is Command.wl_screenshot: + subprocess.check_call( + r""" grim -g "$(slurp)" - | wl-copy - ''', shell=True) - elif options.command is Command.chrome: - chrome(args) - elif options.command is Command.eternal_oom: - eternal_oom(args) - elif options.command is Command.resilient_vlc: - resilient_vlc(args) - elif options.command is Command.eternal_firefox: - eternal_firefox( - profile=sys.argv[2], - group_name=sys.argv[3], - window_position=json.loads(sys.argv[4]), - debug=json.loads(sys.argv[5]), - tabs=sys.argv[6:], - ) - elif options.command is Command.install: - install(args) - elif options.command is Command.resilient_ethernet: - resilient_ethernet( - ip_addr=sys.argv[2], - ethernet_device=sys.argv[3], - ) - elif options.command is Command.player: - player_v1( - folder_url=sys.argv[2], - item_id=int(sys.argv[3]), - ) - elif options.command is Command.share_wifi: - share_wifi(args) - elif options.command is Command.socat_ssh: - socat_ssh(args) - elif options.command is Command.gnome_shortcuts: - gnome_shortcuts(args) - elif options.command is Command.sway_sock: - print(sway_sock()) - elif options.command is Command.loginctl: - loginctl(args) - elif options.command is Command.suspend_timer: - suspend_timer(args) - elif options.command is Command.desktop_services: - desktop_services(args) - elif options.command is Command.pip_resolve: - pip_resolve(args) - elif options.command is Command.pip_check_conflicts: - pip_check_conflicts(args) - elif options.command is Command.pm_service: - pm_service(args) - elif options.command is Command.backup: - backup(args) - elif options.command is Command.scrap_yt_music: - scrap_yt_music(args) - elif options.command is Command.vpn: - vpn(args) - else: - raise NotImplementedError - except SystemExit: - raise - except Exception: - msg = 'not implemented\n%s' % traceback.format_exc() - logging.error(msg) - finally: - if not msg is None: - custom_notify(msg=msg) + """, + shell=True, + ) + elif options.command is Command.chrome: + chrome(args) + elif options.command is Command.eternal_oom: + eternal_oom(args) + elif options.command is Command.resilient_vlc: + resilient_vlc(args) + elif options.command is Command.eternal_firefox: + eternal_firefox( + profile=sys.argv[2], + group_name=sys.argv[3], + window_position=json.loads(sys.argv[4]), + debug=json.loads(sys.argv[5]), + tabs=sys.argv[6:], + ) + elif options.command is Command.install: + install(args) + elif options.command is Command.resilient_ethernet: + resilient_ethernet( + ip_addr=sys.argv[2], + ethernet_device=sys.argv[3], + ) + elif options.command is Command.player: + player_v1( + folder_url=sys.argv[2], + item_id=int(sys.argv[3]), + ) + elif options.command is Command.share_wifi: + share_wifi(args) + elif options.command is Command.socat_ssh: + socat_ssh(args) + elif options.command is Command.gnome_shortcuts: + gnome_shortcuts(args) + elif options.command is Command.sway_sock: + print(sway_sock()) + elif options.command is Command.loginctl: + loginctl(args) + elif options.command is Command.suspend_timer: + suspend_timer(args) + elif options.command is Command.desktop_services: + desktop_services(args) + elif options.command is Command.pip_resolve: + pip_resolve(args) + elif options.command is Command.pip_check_conflicts: + pip_check_conflicts(args) + elif options.command is Command.pm_service: + pm_service(args) + elif options.command is Command.backup: + backup(args) + elif options.command is Command.scrap_yt_music: + scrap_yt_music(args) + elif options.command is Command.vpn: + vpn(args) + else: + raise NotImplementedError + except SystemExit: + raise + except Exception: + msg = 'not implemented\n%s' % traceback.format_exc() + logging.error(msg) + finally: + if not msg is None: + custom_notify(msg=msg) if __name__ == '__main__': - sys.exit(commands_cli()) + sys.exit(commands_cli()) diff --git a/python/online/fxreader/pr34/commands_typed/argparse.py b/python/online/fxreader/pr34/commands_typed/argparse.py index 229cfe0..ee09011 100644 --- a/python/online/fxreader/pr34/commands_typed/argparse.py +++ b/python/online/fxreader/pr34/commands_typed/argparse.py @@ -1,27 +1,28 @@ -__all__ = ( - 'parse_args', -) +__all__ = ('parse_args',) import sys import argparse -from typing import (Optional,) +from typing import ( + Optional, +) + def parse_args( - parser: argparse.ArgumentParser, - args: Optional[list[str]] = None, + parser: argparse.ArgumentParser, + args: Optional[list[str]] = None, ) -> tuple[argparse.Namespace, list[str]]: - if args is None: - args = sys.argv[1:] + if args is None: + args = sys.argv[1:] - argv : list[str] = [] + argv: list[str] = [] - for i, o in enumerate(args): - if o == '--': - argv.extend(args[i + 1:]) + for i, o in enumerate(args): + if o == '--': + argv.extend(args[i + 1 :]) - del args[i:] + del args[i:] - break + break - return parser.parse_args(args), argv + return parser.parse_args(args), argv diff --git a/python/online/fxreader/pr34/commands_typed/asyncio.py b/python/online/fxreader/pr34/commands_typed/asyncio.py index b69718f..9cbd40b 100644 --- a/python/online/fxreader/pr34/commands_typed/asyncio.py +++ b/python/online/fxreader/pr34/commands_typed/asyncio.py @@ -1,14 +1,23 @@ import logging import asyncio -from typing import (Any,) +from typing import ( + Any, +) logger = logging.getLogger(__name__) -def handle_task_result(fut: asyncio.Future[Any]) -> None: - try: - fut.result() - logger.debug(dict(fut=fut, msg='done'), stacklevel=2,) - except: - logger.exception('', stacklevel=2,) +def handle_task_result(fut: asyncio.Future[Any]) -> None: + try: + fut.result() + + logger.debug( + dict(fut=fut, msg='done'), + stacklevel=2, + ) + except: + logger.exception( + '', + stacklevel=2, + ) diff --git a/python/online/fxreader/pr34/commands_typed/cli.py b/python/online/fxreader/pr34/commands_typed/cli.py index 638134a..a225f94 100644 --- a/python/online/fxreader/pr34/commands_typed/cli.py +++ b/python/online/fxreader/pr34/commands_typed/cli.py @@ -14,752 +14,763 @@ import argparse from .os import shutil_which from typing import ( - Optional, - Literal, - Any, - MutableMapping, - Mapping, - IO, + Optional, + Literal, + Any, + MutableMapping, + Mapping, + IO, ) from typing_extensions import ( - cast, - Callable, + cast, + Callable, ) logger = logging.getLogger(__name__) + @dataclasses.dataclass class Project: - source_dir : pathlib.Path - build_dir : pathlib.Path - dest_dir : pathlib.Path - meson_path: Optional[pathlib.Path] = None + source_dir: pathlib.Path + build_dir: pathlib.Path + dest_dir: pathlib.Path + meson_path: Optional[pathlib.Path] = None + @dataclasses.dataclass class PyProject: - @dataclasses.dataclass - class Tool: - @dataclasses.dataclass - class Meson: - @dataclasses.dataclass - class Args: - install: list[str] + @dataclasses.dataclass + class Tool: + @dataclasses.dataclass + class Meson: + @dataclasses.dataclass + class Args: + install: list[str] - args: Args + args: Args - @dataclasses.dataclass - class MesonPython: - @dataclasses.dataclass - class Args: - install: list[str] + @dataclasses.dataclass + class MesonPython: + @dataclasses.dataclass + class Args: + install: list[str] - args: Args + args: Args - meson: Optional[Meson] = None + meson: Optional[Meson] = None - meson_python: Optional[MesonPython] = pydantic.Field( - alias='meson-python', - default=None, - ) + meson_python: Optional[MesonPython] = pydantic.Field( + alias='meson-python', + default=None, + ) + + tool: Optional[Tool] = None - tool: Optional[Tool] = None @dataclasses.dataclass class Dependency: - name: str - mode : Literal['pyproject', 'meson', 'meson-python', 'm'] - source_path : pathlib.Path - args: Optional[list[str]] = None + name: str + mode: Literal['pyproject', 'meson', 'meson-python', 'm'] + source_path: pathlib.Path + args: Optional[list[str]] = None + @dataclasses.dataclass class DistSettings: - wheel_dir : pathlib.Path - python_path: pathlib.Path - env_path: pathlib.Path + wheel_dir: pathlib.Path + python_path: pathlib.Path + env_path: pathlib.Path + class CLI(abc.ABC): - @property - @abc.abstractmethod - def dist_settings(self) -> DistSettings: - raise NotImplementedError - - @property - @abc.abstractmethod - def projects(self) -> dict[str, Project]: - raise NotImplementedError - - @property - @abc.abstractmethod - def dependencies(self) -> dict[str, Dependency]: - raise NotImplementedError - - def mypy( - self, - argv: list[str] - ) -> None: - from . import mypy as _mypy - - _mypy.run( - argv, - ) - - def ruff( - self, - project_name: str, - argv: list[str], - ) -> None: - project = self.projects[project_name] - - if len(argv) == 0: - argv = ['check', '.',] - - cmd = [ - str(self.dist_settings.python_path), - '-m', - 'ruff', - '--config', str(project.source_dir / 'pyproject.toml'), - *argv, - ] - - logger.info(dict( - cmd=cmd, - )) - - subprocess.check_call(cmd) - - def pyright( - self, - project_name: str, - argv: list[str], - ) -> None: - project = self.projects[project_name] - - if len(argv) == 0: - argv = ['--threads', '3'] - - cmd = [ - str(self.dist_settings.python_path), - '-m', - 'pyright', - '--pythonpath', str(self.dist_settings.python_path), - '-p', str(project.source_dir / 'pyproject.toml'), - *argv, - ] - - logger.info(cmd) - - subprocess.check_call(cmd) - - def pip_sync( - self, - project: str, - features: list[str], - ) -> None: - from . import cli_bootstrap - - pyproject = cli_bootstrap.pyproject_load( - self.projects[project].source_dir / 'pyproject.toml' - ) - - dependencies = sum([ - pyproject.dependencies[o] - for o in features - ], cast(list[str], [])) - - pip_find_links : list[pathlib.Path] = [] - - if not pyproject.pip_find_links is None: - pip_find_links.extend(pyproject.pip_find_links) - - - logger.info(dict( - dependencies=dependencies, - )) - - if len(dependencies) > 0: - subprocess.check_call([ - self.dist_settings.python_path, - '-m', - 'uv', 'pip', 'install', - *sum([ - ['-f', str(o),] - for o in pip_find_links - ], cast(list[str], [])), - # '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'), - '--offline', - *dependencies, - ]) - - def deploy_fetch_dist( - self, - force: bool, - ) -> None: - for k, d in self.dependencies.items(): - whl_glob = self.dist_settings.wheel_dir / ('*%s*.whl' % d.name.replace('.', '_')) - if len(glob.glob( - str(whl_glob) - )) == 0 or force: - if d.source_path.exists(): - def whl_files_get() -> list[dict[str, Any]]: - return [ - dict( - path=o, - stat=os.stat(o).st_mtime, - ) - for o in glob.glob( - str(whl_glob) - ) - ] - - present_files = whl_files_get() - - if d.mode == 'm': - if (d.source_path / 'm.py').exists(): - cmd = [ - sys.executable, - str(d.source_path / 'm.py'), - 'deploy:wheel', - '-o', - str(self.dist_settings.wheel_dir), - ] - - if not d.args is None: - cmd.extend(d.args) - - subprocess.check_call( - cmd, - cwd=d.source_path, - ) - else: - raise NotImplementedError - - updated_files = whl_files_get() - - def index_get(o: dict[str, Any]) -> tuple[Any, ...]: - return (o['path'], o['stat']) - - present_files_index = { - index_get(o) : o - for o in present_files - } - - new_files : list[dict[str, Any]] = [] - - for o in updated_files: - entry_index = index_get(o) - - if not entry_index in present_files_index: - new_files.append(o) - - if len(new_files) == 0: - raise NotImplementedError - - latest_file = sorted( - new_files, - key=lambda x: x['stat'] - )[-1] - - subprocess.check_call([ - self.dist_settings.python_path, - '-m', 'pip', - 'install', - latest_file['path'], - ]) - - @property - def pkg_config_path(self,) -> set[pathlib.Path]: - return { - pathlib.Path(o) - for o in glob.glob( - str(self.dist_settings.env_path / 'lib' / 'python*' / '**' / 'pkgconfig'), - recursive=True, - ) - } - - def deploy_wheel( - self, - project_name: str, - argv: Optional[list[str]] = None, - output_dir: Optional[pathlib.Path] = None, - force: Optional[bool] = None, - env: Optional[dict[str, str]] = None, - mypy: bool = False, - ruff: bool = False, - pyright: bool = False, - tests: bool = False, - ) -> None: - project = self.projects[project_name] - - # subprocess.check_call([ - # sys.argv[0], - # # sys.executable, - # '-p', options.project, - # Command.meson_setup.value, - # ]) - - if argv is None: - argv = [] - - # assert argv is None or len(argv) == 0 - - if not project.meson_path is None: - if tests: - self.meson_test( - project_name=project_name, - ) - - self.meson_install( - project_name=project_name, - force=force, - ) - - if ruff: - self.ruff( - project_name=project_name, - argv=['format', '--check',], - ) - - self.ruff( - project_name=project_name, - argv=[], - ) - - if mypy: - self.mypy([]) - - if pyright: - self.pyright( - project_name=project_name, - argv=[], - ) - - if env is None: - env = dict() - - extra_args: list[str] = [] - - if len(self.third_party_roots) > 0: - extra_args.extend([ - '-Csetup-args=%s' % ( - '-Dthird_party_roots=%s' % str(o.absolute()) - ) - for o in self.third_party_roots - ]) - - cmd = [ - sys.executable, - '-m', - 'build', - '-w', '-n', - *extra_args, - '-Csetup-args=-Dmodes=pyproject', - '-Cbuild-dir=%s' % str(project.build_dir / 'pyproject'), - '-Csetup-args=-Dinstall_path=%s' % str(project.dest_dir), - # '-Cbuild-dir=%s' % str(project.build_dir), - str(project.source_dir), - *argv, - ] - - if not output_dir is None: - cmd.extend(['-o', str(output_dir)]) - - logger.info(dict(env=env)) - - subprocess.check_call( - cmd, - env=dict(list(os.environ.items())) | env, - ) - - if not project.meson_path is None: - if tests: - subprocess.check_call( - [ - 'ninja', - '-C', - str(project.build_dir / 'pyproject'), - 'test', - ] - ) - - def meson_install( - self, - project_name: str, - force: Optional[bool] = None, - argv: Optional[list[str]] = None, - ) -> None: - from . import cli_bootstrap - - project = self.projects[project_name] - - if force is None: - force = False - - if argv is None: - argv = [] - - if force and project.dest_dir.exists(): - shutil.rmtree(project.dest_dir) - - pyproject = cli_bootstrap.pyproject_load( - project.source_dir / 'pyproject.toml' - ) - - pyproject_tool = pydantic.RootModel[ - PyProject.Tool - ].model_validate(pyproject.tool).root - - if ( - pyproject_tool.meson and - pyproject_tool.meson.args and - pyproject_tool.meson.args.install - ): - argv = pyproject_tool.meson.args.install + argv - - cmd = [ - shutil_which('meson', True,), - 'install', - '-C', - str(project.build_dir / 'meson'), - '--destdir', str(project.dest_dir), - *argv, - ] - - logger.info(dict( - cmd=cmd, - )) - - subprocess.check_call(cmd) - - for o in glob.glob( - str(project.dest_dir / 'lib' / 'pkgconfig' / '*.pc'), - recursive=True, - ): - logger.info(dict( - path=o, - action='patch prefix', - )) - - with io.open(o, 'r') as f: - content = f.read() - - with io.open(o, 'w') as f: - f.write( - content.replace('prefix=/', 'prefix=${pcfiledir}/../../') - ) - def ninja( - self, - project_name: str, - argv: Optional[list[str]] = None, - env: Optional[dict[str, str]] = None, - ) -> None: - project = self.projects[project_name] - - if argv is None: - argv = [] - - if env is None: - env = dict() - - logger.info(dict(env=env)) - - subprocess.check_call( - [ - shutil_which('ninja', True), - '-C', - str(project.build_dir / 'meson'), - *argv, - ], - env=dict(list(os.environ.items())) | env, - ) - - def meson_test( - self, - project_name: str, - argv: Optional[list[str]] = None, - ) -> None: - project = self.projects[project_name] - - if argv is None: - argv = [] - - subprocess.check_call([ - shutil_which('meson', True,), - 'test', - '-C', - project.build_dir / 'meson', - *argv, - ]) - - - def meson_compile( - self, - project_name: str, - argv: Optional[list[str]] = None, - ) -> None: - project = self.projects[project_name] - - if argv is None: - argv = [] - - subprocess.check_call([ - shutil_which('meson', True,), - 'compile', - '-C', - project.build_dir / 'meson', - *argv, - ]) - - @property - def third_party_roots(self) -> list[pathlib.Path]: - return [] - - def meson_setup( - self, - project_name: str, - force: bool, - argv: Optional[list[str]] = None, - env: Optional[dict[str, str]] = None, - # third_party_roots: Optional[list[pathlib.Path]] = None, - ) -> None: - project = self.projects[project_name] - - if argv is None: - argv = [] - - if env is None: - env = dict() - - logger.info(dict(env=env)) - - if force: - if (project.build_dir / 'meson').exists(): - logger.info(dict(action='removing build dir', path=project.build_dir / 'meson')) - shutil.rmtree(project.build_dir / 'meson') - - extra_args : list[str] = [] - - if len(self.third_party_roots) > 0: - extra_args.extend([ - '-Dthird_party_roots=%s' % str(o.absolute()) - for o in self.third_party_roots - ]) - - cmd = [ - shutil_which('meson', True,), - 'setup', - str(project.source_dir), - str(project.build_dir / 'meson'), - '-Dmodes=["meson"]', - *extra_args, - # '-Dpkgconfig.relocatable=true', - '-Dprefix=/', - *argv, - ] - - logger.info(dict(cmd=cmd)) - - subprocess.check_call( - cmd, - env=dict(list(os.environ.items())) | env, - ) - - def venv_compile( - self, - project_name: str, - # force: bool, - argv: Optional[list[str]] = None, - ) -> None: - from . import cli_bootstrap - from . import argparse as pr34_argparse - - project = self.projects[project_name] - - bootstrap_settings = cli_bootstrap.BootstrapSettings.get() - - parser = argparse.ArgumentParser() - parser.add_argument( - '-f', - dest='features', - action='append', - default=[], - # type=pathlib.Path, - type=str, - ) - parser.add_argument( - '-p', - dest='python_version', - default=bootstrap_settings.python_version, - # type=pathlib.Path, - type=str, - ) - - options, args = pr34_argparse.parse_args( - parser, - argv, - ) - - pyproject = cli_bootstrap.pyproject_load( - project.source_dir / 'pyproject.toml' - ) - - dependencies = sum([ - pyproject.dependencies[o] - for o in options.features - ], cast(list[str], [])) - - pip_find_links : list[pathlib.Path] = [] - - if not pyproject.pip_find_links is None: - pip_find_links.extend([ - o - for o in pyproject.pip_find_links - if o.exists() - ]) - - requirements_name_get_res = cli_bootstrap.requirements_name_get( - source_dir=project.source_dir, - features=options.features, - python_version=options.python_version, - requirements=pyproject.requirements, - ) - - logger.info(dict( - dependencies=dependencies, - requirements_name_get_res=requirements_name_get_res, - )) - - with io.open( - requirements_name_get_res.not_compiled, - 'w', - ) as f: - f.write( - '\n'.join(dependencies), - ) - f.flush() - - if len(dependencies) > 0: - cmd = [ - str(self.dist_settings.python_path), - '-m', - 'uv', 'pip', 'compile', - *args, - '--python-version', options.python_version, - *sum([ - ['-f', str(o),] - for o in pip_find_links - ], cast(list[str], [])), - '--generate-hashes', - str(requirements_name_get_res.not_compiled), - '-o', - str(requirements_name_get_res.compiled), - ] - - logger.info(dict( - cmd=cmd, - )) - subprocess.check_call(cmd) - - def module_switch( - self, - project_name: str, - # force: bool, - argv: Optional[list[str]] = None, - ) -> None: - import tomlkit - import tomlkit.container - import tomlkit.items - - from . import cli_bootstrap - from . import argparse as pr34_argparse - - project = self.projects[project_name] - - parser = argparse.ArgumentParser() - parser.add_argument( - '-m', - dest='module', - # choices=[ - # o.name - # for o in project.modules - # ], - required=True, - # type=pathlib.Path, - type=str, - ) - - parser.add_argument( - '-f', - dest='file', - default=pathlib.Path('pyproject.common.toml'), - # type=pathlib.Path, - type=pathlib.Path, - ) - - options, args = pr34_argparse.parse_args( - parser, - argv, - ) - - if not options.file.is_absolute(): - options.file = project.source_dir / options.file - - pyproject = cli_bootstrap.pyproject_load( - options.file, - ) - - assert options.module in [ - o.name - for o in pyproject.modules - ] - - modules: dict[ - str, - cli_bootstrap.PyProject.Module - ] = { - o.name : o - for o in pyproject.modules - } - - module = modules[options.module] - - with io.open(options.file, 'rb') as f: - pyproject2 = tomlkit.load(f) - - with io.open( - project.source_dir / 'pyproject.toml', - 'w', - ) as f: - p = pyproject2['project'] - assert isinstance(p, tomlkit.items.Table) - p['name'] = module.name - - if not pyproject2['tool']: - pyproject2['tool'] = [] - - if not 'tool' in pyproject2: - pyproject2['tool'] = dict() - - pyproject_tool = pyproject2['tool'] - # assert isinstance(pyproject_tool, tomlkit.items.Array) - assert isinstance(pyproject_tool, MutableMapping) - - for k, v in module.tool.items(): - assert not k in pyproject_tool - pyproject_tool[k] = v - - del p - del pyproject_tool - - cast( - Callable[[Mapping[Any, Any], IO[str] | IO[bytes]], None], - getattr(tomlkit, 'dump'), - )( - pyproject2, - f - ) - - del pyproject2 - del module - - # raise NotImplementedError + @property + @abc.abstractmethod + def dist_settings(self) -> DistSettings: + raise NotImplementedError + + @property + @abc.abstractmethod + def projects(self) -> dict[str, Project]: + raise NotImplementedError + + @property + @abc.abstractmethod + def dependencies(self) -> dict[str, Dependency]: + raise NotImplementedError + + def mypy(self, argv: list[str]) -> None: + from . import mypy as _mypy + + _mypy.run( + argv, + ) + + def ruff( + self, + project_name: str, + argv: list[str], + ) -> None: + project = self.projects[project_name] + + if len(argv) == 0: + argv = [ + 'check', + '.', + ] + + cmd = [ + str(self.dist_settings.python_path), + '-m', + 'ruff', + '--config', + str(project.source_dir / 'pyproject.toml'), + *argv, + ] + + logger.info( + dict( + cmd=cmd, + ) + ) + + subprocess.check_call(cmd) + + def pyright( + self, + project_name: str, + argv: list[str], + ) -> None: + project = self.projects[project_name] + + if len(argv) == 0: + argv = ['--threads', '3'] + + cmd = [ + str(self.dist_settings.python_path), + '-m', + 'pyright', + '--pythonpath', + str(self.dist_settings.python_path), + '-p', + str(project.source_dir / 'pyproject.toml'), + *argv, + ] + + logger.info(cmd) + + subprocess.check_call(cmd) + + def pip_sync( + self, + project: str, + features: list[str], + ) -> None: + from . import cli_bootstrap + + pyproject = cli_bootstrap.pyproject_load(self.projects[project].source_dir / 'pyproject.toml') + + dependencies = sum([pyproject.dependencies[o] for o in features], cast(list[str], [])) + + pip_find_links: list[pathlib.Path] = [] + + if not pyproject.pip_find_links is None: + pip_find_links.extend(pyproject.pip_find_links) + + logger.info( + dict( + dependencies=dependencies, + ) + ) + + if len(dependencies) > 0: + subprocess.check_call( + [ + self.dist_settings.python_path, + '-m', + 'uv', + 'pip', + 'install', + *sum( + [ + [ + '-f', + str(o), + ] + for o in pip_find_links + ], + cast(list[str], []), + ), + # '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'), + '--offline', + *dependencies, + ] + ) + + def deploy_fetch_dist( + self, + force: bool, + ) -> None: + for k, d in self.dependencies.items(): + whl_glob = self.dist_settings.wheel_dir / ('*%s*.whl' % d.name.replace('.', '_')) + if len(glob.glob(str(whl_glob))) == 0 or force: + if d.source_path.exists(): + + def whl_files_get() -> list[dict[str, Any]]: + return [ + dict( + path=o, + stat=os.stat(o).st_mtime, + ) + for o in glob.glob(str(whl_glob)) + ] + + present_files = whl_files_get() + + if d.mode == 'm': + if (d.source_path / 'm.py').exists(): + cmd = [ + sys.executable, + str(d.source_path / 'm.py'), + 'deploy:wheel', + '-o', + str(self.dist_settings.wheel_dir), + ] + + if not d.args is None: + cmd.extend(d.args) + + subprocess.check_call( + cmd, + cwd=d.source_path, + ) + else: + raise NotImplementedError + + updated_files = whl_files_get() + + def index_get(o: dict[str, Any]) -> tuple[Any, ...]: + return (o['path'], o['stat']) + + present_files_index = {index_get(o): o for o in present_files} + + new_files: list[dict[str, Any]] = [] + + for o in updated_files: + entry_index = index_get(o) + + if not entry_index in present_files_index: + new_files.append(o) + + if len(new_files) == 0: + raise NotImplementedError + + latest_file = sorted(new_files, key=lambda x: x['stat'])[-1] + + subprocess.check_call( + [ + self.dist_settings.python_path, + '-m', + 'pip', + 'install', + latest_file['path'], + ] + ) + + @property + def pkg_config_path( + self, + ) -> set[pathlib.Path]: + return { + pathlib.Path(o) + for o in glob.glob( + str(self.dist_settings.env_path / 'lib' / 'python*' / '**' / 'pkgconfig'), + recursive=True, + ) + } + + def deploy_wheel( + self, + project_name: str, + argv: Optional[list[str]] = None, + output_dir: Optional[pathlib.Path] = None, + force: Optional[bool] = None, + env: Optional[dict[str, str]] = None, + mypy: bool = False, + ruff: bool = False, + pyright: bool = False, + tests: bool = False, + ) -> None: + project = self.projects[project_name] + + # subprocess.check_call([ + # sys.argv[0], + # # sys.executable, + # '-p', options.project, + # Command.meson_setup.value, + # ]) + + if argv is None: + argv = [] + + # assert argv is None or len(argv) == 0 + + if not project.meson_path is None: + if tests: + self.meson_test( + project_name=project_name, + ) + + self.meson_install( + project_name=project_name, + force=force, + ) + + if ruff: + self.ruff( + project_name=project_name, + argv=[ + 'format', + '--check', + ], + ) + + self.ruff( + project_name=project_name, + argv=[], + ) + + if mypy: + self.mypy([]) + + if pyright: + self.pyright( + project_name=project_name, + argv=[], + ) + + if env is None: + env = dict() + + extra_args: list[str] = [] + + if len(self.third_party_roots) > 0: + extra_args.extend(['-Csetup-args=%s' % ('-Dthird_party_roots=%s' % str(o.absolute())) for o in self.third_party_roots]) + + cmd = [ + sys.executable, + '-m', + 'build', + '-w', + '-n', + *extra_args, + '-Csetup-args=-Dmodes=pyproject', + '-Cbuild-dir=%s' % str(project.build_dir / 'pyproject'), + '-Csetup-args=-Dinstall_path=%s' % str(project.dest_dir), + # '-Cbuild-dir=%s' % str(project.build_dir), + str(project.source_dir), + *argv, + ] + + if not output_dir is None: + cmd.extend(['-o', str(output_dir)]) + + logger.info(dict(env=env)) + + subprocess.check_call( + cmd, + env=dict(list(os.environ.items())) | env, + ) + + if not project.meson_path is None: + if tests: + subprocess.check_call( + [ + 'ninja', + '-C', + str(project.build_dir / 'pyproject'), + 'test', + ] + ) + + def meson_install( + self, + project_name: str, + force: Optional[bool] = None, + argv: Optional[list[str]] = None, + ) -> None: + from . import cli_bootstrap + + project = self.projects[project_name] + + if force is None: + force = False + + if argv is None: + argv = [] + + if force and project.dest_dir.exists(): + shutil.rmtree(project.dest_dir) + + pyproject = cli_bootstrap.pyproject_load(project.source_dir / 'pyproject.toml') + + pyproject_tool = pydantic.RootModel[PyProject.Tool].model_validate(pyproject.tool).root + + if pyproject_tool.meson and pyproject_tool.meson.args and pyproject_tool.meson.args.install: + argv = pyproject_tool.meson.args.install + argv + + cmd = [ + shutil_which( + 'meson', + True, + ), + 'install', + '-C', + str(project.build_dir / 'meson'), + '--destdir', + str(project.dest_dir), + *argv, + ] + + logger.info( + dict( + cmd=cmd, + ) + ) + + subprocess.check_call(cmd) + + for o in glob.glob( + str(project.dest_dir / 'lib' / 'pkgconfig' / '*.pc'), + recursive=True, + ): + logger.info( + dict( + path=o, + action='patch prefix', + ) + ) + + with io.open(o, 'r') as f: + content = f.read() + + with io.open(o, 'w') as f: + f.write(content.replace('prefix=/', 'prefix=${pcfiledir}/../../')) + + def ninja( + self, + project_name: str, + argv: Optional[list[str]] = None, + env: Optional[dict[str, str]] = None, + ) -> None: + project = self.projects[project_name] + + if argv is None: + argv = [] + + if env is None: + env = dict() + + logger.info(dict(env=env)) + + subprocess.check_call( + [ + shutil_which('ninja', True), + '-C', + str(project.build_dir / 'meson'), + *argv, + ], + env=dict(list(os.environ.items())) | env, + ) + + def meson_test( + self, + project_name: str, + argv: Optional[list[str]] = None, + ) -> None: + project = self.projects[project_name] + + if argv is None: + argv = [] + + subprocess.check_call( + [ + shutil_which( + 'meson', + True, + ), + 'test', + '-C', + project.build_dir / 'meson', + *argv, + ] + ) + + def meson_compile( + self, + project_name: str, + argv: Optional[list[str]] = None, + ) -> None: + project = self.projects[project_name] + + if argv is None: + argv = [] + + subprocess.check_call( + [ + shutil_which( + 'meson', + True, + ), + 'compile', + '-C', + project.build_dir / 'meson', + *argv, + ] + ) + + @property + def third_party_roots(self) -> list[pathlib.Path]: + return [] + + def meson_setup( + self, + project_name: str, + force: bool, + argv: Optional[list[str]] = None, + env: Optional[dict[str, str]] = None, + # third_party_roots: Optional[list[pathlib.Path]] = None, + ) -> None: + project = self.projects[project_name] + + if argv is None: + argv = [] + + if env is None: + env = dict() + + logger.info(dict(env=env)) + + if force: + if (project.build_dir / 'meson').exists(): + logger.info(dict(action='removing build dir', path=project.build_dir / 'meson')) + shutil.rmtree(project.build_dir / 'meson') + + extra_args: list[str] = [] + + if len(self.third_party_roots) > 0: + extra_args.extend(['-Dthird_party_roots=%s' % str(o.absolute()) for o in self.third_party_roots]) + + cmd = [ + shutil_which( + 'meson', + True, + ), + 'setup', + str(project.source_dir), + str(project.build_dir / 'meson'), + '-Dmodes=["meson"]', + *extra_args, + # '-Dpkgconfig.relocatable=true', + '-Dprefix=/', + *argv, + ] + + logger.info(dict(cmd=cmd)) + + subprocess.check_call( + cmd, + env=dict(list(os.environ.items())) | env, + ) + + def venv_compile( + self, + project_name: str, + # force: bool, + argv: Optional[list[str]] = None, + ) -> None: + from . import cli_bootstrap + from . import argparse as pr34_argparse + + project = self.projects[project_name] + + bootstrap_settings = cli_bootstrap.BootstrapSettings.get() + + parser = argparse.ArgumentParser() + parser.add_argument( + '-f', + dest='features', + action='append', + default=[], + # type=pathlib.Path, + type=str, + ) + parser.add_argument( + '-p', + dest='python_version', + default=bootstrap_settings.python_version, + # type=pathlib.Path, + type=str, + ) + + options, args = pr34_argparse.parse_args( + parser, + argv, + ) + + pyproject = cli_bootstrap.pyproject_load(project.source_dir / 'pyproject.toml') + + dependencies = sum([pyproject.dependencies[o] for o in options.features], cast(list[str], [])) + + pip_find_links: list[pathlib.Path] = [] + + if not pyproject.pip_find_links is None: + pip_find_links.extend([o for o in pyproject.pip_find_links if o.exists()]) + + requirements_name_get_res = cli_bootstrap.requirements_name_get( + source_dir=project.source_dir, + features=options.features, + python_version=options.python_version, + requirements=pyproject.requirements, + ) + + logger.info( + dict( + dependencies=dependencies, + requirements_name_get_res=requirements_name_get_res, + ) + ) + + with io.open( + requirements_name_get_res.not_compiled, + 'w', + ) as f: + f.write( + '\n'.join(dependencies), + ) + f.flush() + + if len(dependencies) > 0: + cmd = [ + str(self.dist_settings.python_path), + '-m', + 'uv', + 'pip', + 'compile', + *args, + '--python-version', + options.python_version, + *sum( + [ + [ + '-f', + str(o), + ] + for o in pip_find_links + ], + cast(list[str], []), + ), + '--generate-hashes', + str(requirements_name_get_res.not_compiled), + '-o', + str(requirements_name_get_res.compiled), + ] + + logger.info( + dict( + cmd=cmd, + ) + ) + subprocess.check_call(cmd) + + def module_switch( + self, + project_name: str, + # force: bool, + argv: Optional[list[str]] = None, + ) -> None: + import tomlkit + import tomlkit.container + import tomlkit.items + + from . import cli_bootstrap + from . import argparse as pr34_argparse + + project = self.projects[project_name] + + parser = argparse.ArgumentParser() + parser.add_argument( + '-m', + dest='module', + # choices=[ + # o.name + # for o in project.modules + # ], + required=True, + # type=pathlib.Path, + type=str, + ) + + parser.add_argument( + '-f', + dest='file', + default=pathlib.Path('pyproject.common.toml'), + # type=pathlib.Path, + type=pathlib.Path, + ) + + options, args = pr34_argparse.parse_args( + parser, + argv, + ) + + if not options.file.is_absolute(): + options.file = project.source_dir / options.file + + pyproject = cli_bootstrap.pyproject_load( + options.file, + ) + + assert options.module in [o.name for o in pyproject.modules] + + modules: dict[str, cli_bootstrap.PyProject.Module] = {o.name: o for o in pyproject.modules} + + module = modules[options.module] + + with io.open(options.file, 'rb') as f: + pyproject2 = tomlkit.load(f) + + with io.open( + project.source_dir / 'pyproject.toml', + 'w', + ) as f: + p = pyproject2['project'] + assert isinstance(p, tomlkit.items.Table) + p['name'] = module.name + + if not pyproject2['tool']: + pyproject2['tool'] = [] + + if not 'tool' in pyproject2: + pyproject2['tool'] = dict() + + pyproject_tool = pyproject2['tool'] + # assert isinstance(pyproject_tool, tomlkit.items.Array) + assert isinstance(pyproject_tool, MutableMapping) + + for k, v in module.tool.items(): + assert not k in pyproject_tool + pyproject_tool[k] = v + + del p + del pyproject_tool + + cast( + Callable[[Mapping[Any, Any], IO[str] | IO[bytes]], None], + getattr(tomlkit, 'dump'), + )(pyproject2, f) + + del pyproject2 + del module + + # raise NotImplementedError diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index cec9b65..087fcd5 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -11,532 +11,519 @@ import os import logging -from typing import (Optional, Any, cast, Type, TypeVar,) +from typing import ( + Optional, + Any, + cast, + Type, + TypeVar, +) from typing_extensions import ( - Self, BinaryIO, overload, + Self, + BinaryIO, + overload, ) logger = logging.getLogger(__name__) + def toml_load(f: BinaryIO) -> Any: - try: - import tomllib - return tomllib.load(f) - except: - pass + try: + import tomllib - try: - import tomli - return tomli.load(f) - except: - pass + return tomllib.load(f) + except: + pass + + try: + import tomli + + return tomli.load(f) + except: + pass + + raise NotImplementedError - raise NotImplementedError @dataclasses.dataclass class PyProject: - @dataclasses.dataclass - class Module: - name: str - meson: Optional[pathlib.Path] = None - tool: dict[str, Any] = dataclasses.field(default_factory=lambda : dict()) + @dataclasses.dataclass + class Module: + name: str + meson: Optional[pathlib.Path] = None + tool: dict[str, Any] = dataclasses.field(default_factory=lambda: dict()) + path: pathlib.Path + dependencies: dict[str, list[str]] + early_features: Optional[list[str]] = None + pip_find_links: Optional[list[pathlib.Path]] = None + runtime_libdirs: Optional[list[pathlib.Path]] = None + runtime_preload: Optional[list[pathlib.Path]] = None + requirements: dict[str, pathlib.Path] = dataclasses.field(default_factory=lambda: dict()) - path: pathlib.Path - dependencies: dict[str, list[str]] - early_features: Optional[list[str]] = None - pip_find_links: Optional[list[pathlib.Path]] = None - runtime_libdirs: Optional[list[pathlib.Path]] = None - runtime_preload: Optional[list[pathlib.Path]] = None - requirements: dict[str, pathlib.Path] = dataclasses.field(default_factory=lambda : dict()) + modules: list[Module] = dataclasses.field( + default_factory=lambda: [], + ) - modules: list[Module] = dataclasses.field( - default_factory=lambda : [], - ) + tool: dict[str, Any] = dataclasses.field( + default_factory=lambda: dict(), + ) - tool: dict[str, Any] = dataclasses.field( - default_factory=lambda : dict(), - ) Key = TypeVar('Key') Value = TypeVar('Value') + @overload def check_dict( - value: Any, - KT: Type[Key], - VT: Type[Value], + value: Any, + KT: Type[Key], + VT: Type[Value], ) -> dict[Key, Value]: ... + @overload def check_dict( - value: Any, - KT: Type[Key], + value: Any, + KT: Type[Key], ) -> dict[Key, Any]: ... + def check_dict( - value: Any, - KT: Type[Key], - VT: Optional[Type[Value]] = None, + value: Any, + KT: Type[Key], + VT: Optional[Type[Value]] = None, ) -> dict[Key, Value]: - assert isinstance(value, dict) - value2 = cast(dict[Any, Any], value) + assert isinstance(value, dict) + value2 = cast(dict[Any, Any], value) - assert all([ - isinstance(k, KT) and ( - VT is None or - isinstance(v, VT) - ) - for k, v in value2.items() - ]) + assert all([isinstance(k, KT) and (VT is None or isinstance(v, VT)) for k, v in value2.items()]) + + if VT is None: + return cast( + dict[Key, Any], + value, + ) + else: + return cast( + dict[Key, Value], + value, + ) - if VT is None: - return cast( - dict[Key, Any], - value, - ) - else: - return cast( - dict[Key, Value], - value, - ) @overload def check_list( - value: Any, - VT: Type[Value], + value: Any, + VT: Type[Value], ) -> list[Value]: ... + @overload def check_list( - value: Any, + value: Any, ) -> list[Any]: ... + def check_list( - value: Any, - VT: Optional[Type[Value]] = None, + value: Any, + VT: Optional[Type[Value]] = None, ) -> list[Value] | list[Any]: - assert isinstance(value, list) - value2 = cast(list[Any], value) + assert isinstance(value, list) + value2 = cast(list[Any], value) - assert all([ - ( - VT is None or - isinstance(o, VT) - ) - for o in value2 - ]) + assert all([(VT is None or isinstance(o, VT)) for o in value2]) + + if VT is None: + return cast( + list[Any], + value, + ) + else: + return cast( + list[Value], + value, + ) - if VT is None: - return cast( - list[Any], - value, - ) - else: - return cast( - list[Value], - value, - ) def pyproject_load( - d: pathlib.Path, + d: pathlib.Path, ) -> PyProject: - with io.open(d, 'rb') as f: - content = toml_load(f) + with io.open(d, 'rb') as f: + content = toml_load(f) - assert isinstance(content, dict) + assert isinstance(content, dict) - dependencies : dict[str, list[str]] = dict() + dependencies: dict[str, list[str]] = dict() - dependencies['default'] = content['project']['dependencies'] + dependencies['default'] = content['project']['dependencies'] - if ( - 'optional-dependencies' in content['project'] - ): - assert isinstance( - content['project']['optional-dependencies'], - dict - ) + if 'optional-dependencies' in content['project']: + assert isinstance(content['project']['optional-dependencies'], dict) - for k, v in check_dict( - check_dict( - check_dict( - content, - str, - # Any, - )['project'], - str, - # Any, - )['optional-dependencies'], - str, - list[Any], - ).items(): - # assert isinstance(v, list) - # assert isinstance(k, str) + for k, v in check_dict( + check_dict( + check_dict( + content, + str, + # Any, + )['project'], + str, + # Any, + )['optional-dependencies'], + str, + list[Any], + ).items(): + # assert isinstance(v, list) + # assert isinstance(k, str) - dependencies[k] = v + dependencies[k] = v + res = PyProject( + path=d, + dependencies=dependencies, + ) - res = PyProject( - path=d, - dependencies=dependencies, - ) + tool_name = 'online.fxreader.pr34'.replace('.', '-') - tool_name = 'online.fxreader.pr34'.replace('.', '-') + if 'tool' in content: + res.tool = check_dict( + content['tool'], + str, + ) - if ( - 'tool' in content - ): - res.tool = check_dict( - content['tool'], - str, - ) + if 'tool' in content and isinstance(content['tool'], dict) and tool_name in content['tool'] and isinstance(content['tool'][tool_name], dict): + pr34_tool = check_dict( + check_dict( + content['tool'], + str, + )[tool_name], + str, + ) - if ( - 'tool' in content and - isinstance( - content['tool'], dict - ) and - tool_name in content['tool'] and - isinstance( - content['tool'][tool_name], - dict - ) - ): - pr34_tool = check_dict( - check_dict( - content['tool'], - str, - )[tool_name], - str - ) + if 'early_features' in pr34_tool: + res.early_features = pr34_tool['early_features'] - if 'early_features' in pr34_tool: - res.early_features = pr34_tool['early_features'] + if 'pip_find_links' in pr34_tool: + res.pip_find_links = [d.parent / pathlib.Path(o) for o in pr34_tool['pip_find_links']] - if 'pip_find_links' in pr34_tool: - res.pip_find_links = [ - d.parent / pathlib.Path(o) - for o in pr34_tool['pip_find_links'] - ] + if 'runtime_libdirs' in pr34_tool: + res.runtime_libdirs = [ + d.parent / pathlib.Path(o) + # pathlib.Path(o) + for o in pr34_tool['runtime_libdirs'] + ] - if 'runtime_libdirs' in pr34_tool: - res.runtime_libdirs = [ - d.parent / pathlib.Path(o) - # pathlib.Path(o) - for o in pr34_tool['runtime_libdirs'] - ] + if 'runtime_preload' in pr34_tool: + res.runtime_preload = [ + d.parent / pathlib.Path(o) + # pathlib.Path(o) + for o in pr34_tool['runtime_preload'] + ] - if 'runtime_preload' in pr34_tool: - res.runtime_preload = [ - d.parent / pathlib.Path(o) - # pathlib.Path(o) - for o in pr34_tool['runtime_preload'] - ] + if 'requirements' in pr34_tool: + res.requirements = { + k: d.parent / pathlib.Path(v) + # pathlib.Path(o) + for k, v in check_dict(pr34_tool['requirements'], str, str).items() + } - if 'requirements' in pr34_tool: - res.requirements = { - k : d.parent / pathlib.Path(v) - # pathlib.Path(o) - for k, v in check_dict( - pr34_tool['requirements'], - str, - str - ).items() - } + if 'modules' in pr34_tool: + modules = check_list(pr34_tool['modules']) + # res.modules = [] - if 'modules' in pr34_tool: - modules = check_list( - pr34_tool['modules'] - ) - # res.modules = [] + for o in modules: + assert isinstance(o, dict) + assert 'name' in o and isinstance(o['name'], str) - for o in modules: - assert isinstance(o, dict) - assert 'name' in o and isinstance(o['name'], str) + module = PyProject.Module( + name=o['name'], + ) - module = PyProject.Module( - name=o['name'], - ) + if 'meson' in o: + assert 'meson' in o and isinstance(o['meson'], str) - if 'meson' in o: - assert 'meson' in o and isinstance(o['meson'], str) + module.meson = pathlib.Path(o['meson']) - module.meson = pathlib.Path(o['meson']) + if 'tool' in o: + module.tool.update( + check_dict( + o['tool'], + str, + ) + ) - if 'tool' in o: - module.tool.update( - check_dict( - o['tool'], - str, - ) - ) + res.modules.append(module) - res.modules.append(module) + return res - return res @dataclasses.dataclass class BootstrapSettings: - env_path: pathlib.Path - python_path: pathlib.Path - base_dir: pathlib.Path - python_version: Optional[str] = dataclasses.field( - default_factory=lambda : os.environ.get( - 'PYTHON_VERSION', - '%d.%d' % ( - sys.version_info.major, - sys.version_info.minor, - ), - ).strip() - ) - pip_check_conflicts: Optional[bool] = dataclasses.field( - default_factory=lambda : os.environ.get( - 'PIP_CHECK_CONFLICTS', - json.dumps(True) - ) in [json.dumps(True)], - ) - uv_args: list[str] = dataclasses.field( - default_factory=lambda : os.environ.get( - 'UV_ARGS', - '--offline', - ).split(), - ) + env_path: pathlib.Path + python_path: pathlib.Path + base_dir: pathlib.Path + python_version: Optional[str] = dataclasses.field( + default_factory=lambda: os.environ.get( + 'PYTHON_VERSION', + '%d.%d' + % ( + sys.version_info.major, + sys.version_info.minor, + ), + ).strip() + ) + pip_check_conflicts: Optional[bool] = dataclasses.field( + default_factory=lambda: os.environ.get('PIP_CHECK_CONFLICTS', json.dumps(True)) in [json.dumps(True)], + ) + uv_args: list[str] = dataclasses.field( + default_factory=lambda: os.environ.get( + 'UV_ARGS', + '--offline', + ).split(), + ) - @classmethod - def get( - cls, - base_dir: Optional[pathlib.Path] = None, - ) -> Self: - if base_dir is None: - base_dir = pathlib.Path.cwd() + @classmethod + def get( + cls, + base_dir: Optional[pathlib.Path] = None, + ) -> Self: + if base_dir is None: + base_dir = pathlib.Path.cwd() - env_path: Optional[pathlib.Path] = None - if 'ENV_PATH' in os.environ: - env_path = pathlib.Path(os.environ['ENV_PATH']) - else: - env_path = base_dir / '.venv' + env_path: Optional[pathlib.Path] = None + if 'ENV_PATH' in os.environ: + env_path = pathlib.Path(os.environ['ENV_PATH']) + else: + env_path = base_dir / '.venv' - python_path = env_path / 'bin' / 'python3' + python_path = env_path / 'bin' / 'python3' + + return cls( + base_dir=base_dir, + env_path=env_path, + python_path=python_path, + ) - return cls( - base_dir=base_dir, - env_path=env_path, - python_path=python_path, - ) class requirements_name_get_t: - @dataclasses.dataclass - class res_t: - not_compiled : pathlib.Path - compiled: pathlib.Path - name: str + @dataclasses.dataclass + class res_t: + not_compiled: pathlib.Path + compiled: pathlib.Path + name: str + def requirements_name_get( - source_dir: pathlib.Path, - python_version: Optional[str], - features: list[str], - requirements: dict[str, pathlib.Path], + source_dir: pathlib.Path, + python_version: Optional[str], + features: list[str], + requirements: dict[str, pathlib.Path], ) -> requirements_name_get_t.res_t: - requirements_python_version: Optional[str] = None - if not python_version is None: - requirements_python_version = \ - python_version.replace('.', '_') + requirements_python_version: Optional[str] = None + if not python_version is None: + requirements_python_version = python_version.replace('.', '_') - requirements_name = '_'.join(sorted(features)) + requirements_name = '_'.join(sorted(features)) - if requirements_python_version: - requirements_name += '_' + requirements_python_version + if requirements_python_version: + requirements_name += '_' + requirements_python_version - requirements_path : Optional[pathlib.Path] = None + requirements_path: Optional[pathlib.Path] = None - if requirements_name in requirements: - requirements_path = requirements[requirements_name] - else: - requirements_path = source_dir / 'requirements.txt' + if requirements_name in requirements: + requirements_path = requirements[requirements_name] + else: + requirements_path = source_dir / 'requirements.txt' - requirements_path_in = requirements_path.parent / ( - requirements_path.stem + '.in' - ) + requirements_path_in = requirements_path.parent / (requirements_path.stem + '.in') - requirements_in : list[str] = [] + requirements_in: list[str] = [] + + return requirements_name_get_t.res_t( + not_compiled=requirements_path_in, + compiled=requirements_path, + name=requirements_name, + ) - return requirements_name_get_t.res_t( - not_compiled=requirements_path_in, - compiled=requirements_path, - name=requirements_name, - ) def env_bootstrap( - bootstrap_settings: BootstrapSettings, - pyproject: PyProject, + bootstrap_settings: BootstrapSettings, + pyproject: PyProject, ) -> None: - pip_find_links : list[pathlib.Path] = [] + pip_find_links: list[pathlib.Path] = [] - if not pyproject.pip_find_links is None: - pip_find_links.extend(pyproject.pip_find_links) + if not pyproject.pip_find_links is None: + pip_find_links.extend(pyproject.pip_find_links) - pip_find_links_args = sum([ - ['-f', str(o),] - for o in pip_find_links - ], cast(list[str], [])) + pip_find_links_args = sum( + [ + [ + '-f', + str(o), + ] + for o in pip_find_links + ], + cast(list[str], []), + ) - features : list[str] = [] + features: list[str] = [] - if pyproject.early_features: - features.extend(pyproject.early_features) + if pyproject.early_features: + features.extend(pyproject.early_features) - requirements_name_get_res = requirements_name_get( - python_version=bootstrap_settings.python_version, - features=features, - requirements=pyproject.requirements, - source_dir=pyproject.path.parent, - ) - requirements_path = requirements_name_get_res.compiled + requirements_name_get_res = requirements_name_get( + python_version=bootstrap_settings.python_version, + features=features, + requirements=pyproject.requirements, + source_dir=pyproject.path.parent, + ) + requirements_path = requirements_name_get_res.compiled - requirements_in : list[str] = [] + requirements_in: list[str] = [] - requirements_in.extend([ - 'uv', 'pip', 'build', 'setuptools', 'meson-python', 'pybind11' - ]) + requirements_in.extend(['uv', 'pip', 'build', 'setuptools', 'meson-python', 'pybind11']) - if pyproject.early_features: - early_dependencies = sum([ - pyproject.dependencies[o] - for o in pyproject.early_features - ], cast(list[str], [])) + if pyproject.early_features: + early_dependencies = sum([pyproject.dependencies[o] for o in pyproject.early_features], cast(list[str], [])) - logger.info(dict( - requirements_name_get_res=requirements_name_get_res, - early_dependencies=early_dependencies, - )) + logger.info( + dict( + requirements_name_get_res=requirements_name_get_res, + early_dependencies=early_dependencies, + ) + ) - requirements_in.extend(early_dependencies) - # if len(early_dependencies) > 0: - # subprocess.check_call([ - # bootstrap_settings.python_path, - # '-m', - # 'uv', 'pip', 'install', - # *pip_find_links_args, - # # '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'), - # *bootstrap_settings.uv_args, - # *early_dependencies, - # ]) + requirements_in.extend(early_dependencies) + # if len(early_dependencies) > 0: + # subprocess.check_call([ + # bootstrap_settings.python_path, + # '-m', + # 'uv', 'pip', 'install', + # *pip_find_links_args, + # # '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'), + # *bootstrap_settings.uv_args, + # *early_dependencies, + # ]) - if not requirements_path.exists(): - with tempfile.NamedTemporaryFile( - mode='w', - prefix='requirements', - suffix='.in', - ) as f: - f.write( - '\n'.join(requirements_in) - ) - f.flush() + if not requirements_path.exists(): + with tempfile.NamedTemporaryFile( + mode='w', + prefix='requirements', + suffix='.in', + ) as f: + f.write('\n'.join(requirements_in)) + f.flush() - subprocess.check_call([ - 'uv', - 'pip', - 'compile', - '--generate-hashes', - *pip_find_links_args, - # '-p', - # bootstrap_settings.python_path, - *bootstrap_settings.uv_args, - '-o', str(requirements_path), - f.name, - ]) + subprocess.check_call( + [ + 'uv', + 'pip', + 'compile', + '--generate-hashes', + *pip_find_links_args, + # '-p', + # bootstrap_settings.python_path, + *bootstrap_settings.uv_args, + '-o', + str(requirements_path), + f.name, + ] + ) - uv_python_version: list[str] = [] + uv_python_version: list[str] = [] - if not bootstrap_settings.python_version is None: - uv_python_version.extend([ - '-p', bootstrap_settings.python_version, - ]) + if not bootstrap_settings.python_version is None: + uv_python_version.extend( + [ + '-p', + bootstrap_settings.python_version, + ] + ) - subprocess.check_call([ - 'uv', 'venv', - *uv_python_version, - *pip_find_links_args, - # '--seed', - *bootstrap_settings.uv_args, - str(bootstrap_settings.env_path) - ]) + subprocess.check_call( + [ + 'uv', + 'venv', + *uv_python_version, + *pip_find_links_args, + # '--seed', + *bootstrap_settings.uv_args, + str(bootstrap_settings.env_path), + ] + ) - subprocess.check_call([ - 'uv', - 'pip', - 'install', - *pip_find_links_args, - '-p', - bootstrap_settings.python_path, - '--require-hashes', - *bootstrap_settings.uv_args, - '-r', str(requirements_path), - ]) + subprocess.check_call( + [ + 'uv', + 'pip', + 'install', + *pip_find_links_args, + '-p', + bootstrap_settings.python_path, + '--require-hashes', + *bootstrap_settings.uv_args, + '-r', + str(requirements_path), + ] + ) - if bootstrap_settings.pip_check_conflicts: - subprocess.check_call([ - bootstrap_settings.python_path, - '-m', - 'online.fxreader.pr34.commands', - 'pip_check_conflicts', - ]) + if bootstrap_settings.pip_check_conflicts: + subprocess.check_call( + [ + bootstrap_settings.python_path, + '-m', + 'online.fxreader.pr34.commands', + 'pip_check_conflicts', + ] + ) + + +def paths_equal(a: pathlib.Path | str, b: pathlib.Path | str) -> bool: + return os.path.abspath(str(a)) == os.path.abspath(str(b)) -def paths_equal( - a: pathlib.Path | str, - b: pathlib.Path | str -) -> bool: - return ( - os.path.abspath(str(a)) == - os.path.abspath(str(b)) - ) def run( - d: Optional[pathlib.Path] = None, - cli_path: Optional[pathlib.Path] = None, + d: Optional[pathlib.Path] = None, + cli_path: Optional[pathlib.Path] = None, ) -> None: - if cli_path is None: - cli_path = pathlib.Path(__file__).parent / 'cli.py' + if cli_path is None: + cli_path = pathlib.Path(__file__).parent / 'cli.py' - if d is None: - d = pathlib.Path(__file__).parent / 'pyproject.toml' + if d is None: + d = pathlib.Path(__file__).parent / 'pyproject.toml' - bootstrap_settings = BootstrapSettings.get() + bootstrap_settings = BootstrapSettings.get() - pyproject : PyProject = pyproject_load( - d - ) + pyproject: PyProject = pyproject_load(d) - logging.basicConfig(level=logging.INFO) + logging.basicConfig(level=logging.INFO) - if not bootstrap_settings.env_path.exists(): - env_bootstrap( - bootstrap_settings=bootstrap_settings, - pyproject=pyproject, - ) + if not bootstrap_settings.env_path.exists(): + env_bootstrap( + bootstrap_settings=bootstrap_settings, + pyproject=pyproject, + ) - logger.info([sys.executable, sys.argv, bootstrap_settings.python_path]) + logger.info([sys.executable, sys.argv, bootstrap_settings.python_path]) - if not paths_equal(sys.executable, bootstrap_settings.python_path): - os.execv( - str(bootstrap_settings.python_path), - [ - str(bootstrap_settings.python_path), - *sys.argv, - ] - ) + if not paths_equal(sys.executable, bootstrap_settings.python_path): + os.execv( + str(bootstrap_settings.python_path), + [ + str(bootstrap_settings.python_path), + *sys.argv, + ], + ) + + os.execv( + str(bootstrap_settings.python_path), + [ + str(bootstrap_settings.python_path), + str(cli_path), + *sys.argv[1:], + ], + ) - os.execv( - str(bootstrap_settings.python_path), - [ - str(bootstrap_settings.python_path), - str( - cli_path - ), - *sys.argv[1:], - ] - ) if __name__ == '__main__': - run() + run() diff --git a/python/online/fxreader/pr34/commands_typed/crypto.py b/python/online/fxreader/pr34/commands_typed/crypto.py index d76f8f8..327d1cb 100644 --- a/python/online/fxreader/pr34/commands_typed/crypto.py +++ b/python/online/fxreader/pr34/commands_typed/crypto.py @@ -4,88 +4,95 @@ import os import cryptography.hazmat.primitives.kdf.scrypt import cryptography.exceptions -from typing import (Literal, overload, Optional,) +from typing import ( + Literal, + overload, + Optional, +) + class PasswordUtils: - @overload - @classmethod - def secret_hash( - cls, - secret: str | bytes, - mode: Literal['base64'], - salt: Optional[bytes] = None, - ) -> tuple[str, str]: ... + @overload + @classmethod + def secret_hash( + cls, + secret: str | bytes, + mode: Literal['base64'], + salt: Optional[bytes] = None, + ) -> tuple[str, str]: ... - @overload - @classmethod - def secret_hash( - cls, - secret: str | bytes, - mode: Literal['bytes'], - salt: Optional[bytes] = None, - ) -> tuple[bytes, bytes]: ... + @overload + @classmethod + def secret_hash( + cls, + secret: str | bytes, + mode: Literal['bytes'], + salt: Optional[bytes] = None, + ) -> tuple[bytes, bytes]: ... - @classmethod - def secret_hash( - cls, - secret: str | bytes, - mode: Literal['bytes', 'base64'], - salt: Optional[bytes] = None, - ) -> tuple[str, str] | tuple[bytes, bytes]: - if salt is None: - salt = os.urandom(16) + @classmethod + def secret_hash( + cls, + secret: str | bytes, + mode: Literal['bytes', 'base64'], + salt: Optional[bytes] = None, + ) -> tuple[str, str] | tuple[bytes, bytes]: + if salt is None: + salt = os.urandom(16) - if isinstance(secret, str): - secret = secret.encode('utf-8') - # derive - kdf = cls._scrypt_init(salt=salt) + if isinstance(secret, str): + secret = secret.encode('utf-8') + # derive + kdf = cls._scrypt_init(salt=salt) - hashed_secret = kdf.derive(secret) + hashed_secret = kdf.derive(secret) - if mode == 'bytes': - return (salt, hashed_secret) - elif mode == 'base64': - res_tuple = tuple(( - base64.b64encode(o).decode('utf-8') - for o in (salt, hashed_secret,) - )) - return (res_tuple[0], res_tuple[1]) - else: - raise NotImplementedError + if mode == 'bytes': + return (salt, hashed_secret) + elif mode == 'base64': + res_tuple = tuple( + ( + base64.b64encode(o).decode('utf-8') + for o in ( + salt, + hashed_secret, + ) + ) + ) + return (res_tuple[0], res_tuple[1]) + else: + raise NotImplementedError - @classmethod - def _scrypt_init( - cls, - salt: bytes - ) -> cryptography.hazmat.primitives.kdf.scrypt.Scrypt: - return cryptography.hazmat.primitives.kdf.scrypt.Scrypt( - salt=salt, - length=32, - n=2**14, - r=8, - p=1, - ) + @classmethod + def _scrypt_init(cls, salt: bytes) -> cryptography.hazmat.primitives.kdf.scrypt.Scrypt: + return cryptography.hazmat.primitives.kdf.scrypt.Scrypt( + salt=salt, + length=32, + n=2**14, + r=8, + p=1, + ) - @classmethod - def secret_check( - cls, - secret: str | bytes, - salt: str | bytes, - hashed_secret: str | bytes, - ) -> bool: - if isinstance(salt, str): - salt = base64.b64decode(salt) + @classmethod + def secret_check( + cls, + secret: str | bytes, + salt: str | bytes, + hashed_secret: str | bytes, + ) -> bool: + if isinstance(salt, str): + salt = base64.b64decode(salt) - if isinstance(secret, str): - secret = secret.encode('utf-8') + if isinstance(secret, str): + secret = secret.encode('utf-8') - if isinstance(hashed_secret, str): - hashed_secret = base64.b64decode(hashed_secret) + if isinstance(hashed_secret, str): + hashed_secret = base64.b64decode(hashed_secret) - kdf = cls._scrypt_init(salt=salt) + kdf = cls._scrypt_init(salt=salt) - try: - kdf.verify(secret, hashed_secret) - return True - except cryptography.exceptions.InvalidKey: - return False + try: + kdf.verify(secret, hashed_secret) + return True + except cryptography.exceptions.InvalidKey: + return False diff --git a/python/online/fxreader/pr34/commands_typed/debug.py b/python/online/fxreader/pr34/commands_typed/debug.py index 045407e..0f37e4f 100644 --- a/python/online/fxreader/pr34/commands_typed/debug.py +++ b/python/online/fxreader/pr34/commands_typed/debug.py @@ -1,35 +1,39 @@ import os import logging -from typing import (Optional,) +from typing import ( + Optional, +) logger = logging.getLogger(__name__) + class DebugPy: - @classmethod - def set_trace( - cls, - host: Optional[str] = None, - port: Optional[int] = None, - wait: Optional[bool] = None, - ) -> None: - if host is None: - host = '127.0.0.1' - if port is None: - port = 4444 - if wait is None: - wait = True + @classmethod + def set_trace( + cls, + host: Optional[str] = None, + port: Optional[int] = None, + wait: Optional[bool] = None, + ) -> None: + if host is None: + host = '127.0.0.1' + if port is None: + port = 4444 + if wait is None: + wait = True - import debugpy + import debugpy - if os.environ.get('DEBUGPY_RUNNING') != 'true': - logger.info('debugpy init') - import debugpy - debugpy.listen((host, port)) - os.environ['DEBUGPY_RUNNING'] = 'true' + if os.environ.get('DEBUGPY_RUNNING') != 'true': + logger.info('debugpy init') + import debugpy - if wait: - debugpy.wait_for_client() - debugpy.breakpoint() + debugpy.listen((host, port)) + os.environ['DEBUGPY_RUNNING'] = 'true' - logger.info('debugpy done') \ No newline at end of file + if wait: + debugpy.wait_for_client() + debugpy.breakpoint() + + logger.info('debugpy done') diff --git a/python/online/fxreader/pr34/commands_typed/logging.py b/python/online/fxreader/pr34/commands_typed/logging.py index eb76c6d..afdd4c1 100644 --- a/python/online/fxreader/pr34/commands_typed/logging.py +++ b/python/online/fxreader/pr34/commands_typed/logging.py @@ -1,16 +1,14 @@ import logging -from typing import (Optional,) +from typing import ( + Optional, +) + def setup(level: Optional[int] = None) -> None: - if level is None: - level = logging.INFO + if level is None: + level = logging.INFO - logging.basicConfig( - level=level, - format=( - '%(levelname)s:%(name)s:%(message)s' - ':%(process)d' - ':%(asctime)s' - ':%(pathname)s:%(funcName)s:%(lineno)s' - ), - ) \ No newline at end of file + logging.basicConfig( + level=level, + format=('%(levelname)s:%(name)s:%(message)s:%(process)d:%(asctime)s:%(pathname)s:%(funcName)s:%(lineno)s'), + ) diff --git a/python/online/fxreader/pr34/commands_typed/mypy.py b/python/online/fxreader/pr34/commands_typed/mypy.py index 2e5ad9d..82bb7d8 100644 --- a/python/online/fxreader/pr34/commands_typed/mypy.py +++ b/python/online/fxreader/pr34/commands_typed/mypy.py @@ -9,208 +9,232 @@ import logging import sys import argparse -from pydantic import (Field,) +from pydantic import ( + Field, +) -from typing import (ClassVar, Generator, Annotated, Optional, Any,) +from typing import ( + ClassVar, + Generator, + Annotated, + Optional, + Any, +) logger = logging.getLogger(__name__) + @pydantic.dataclasses.dataclass class MypyFormatEntry: - name : str - value : str + name: str + value: str - def __eq__(self, other: object) -> bool: - if not isinstance(other, type(self)): - raise NotImplementedError + def __eq__(self, other: object) -> bool: + if not isinstance(other, type(self)): + raise NotImplementedError + + return self.value == other.value - return self.value == other.value class MypyFormat: - vscode : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='vscode', value='vscode') - json : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='json', value='json') + vscode: ClassVar[MypyFormatEntry] = MypyFormatEntry(name='vscode', value='vscode') + json: ClassVar[MypyFormatEntry] = MypyFormatEntry(name='json', value='json') + @classmethod + def from_value(cls, value: str) -> MypyFormatEntry: + for e in cls.entries(): + if value == e.value: + return e - @classmethod - def from_value(cls, value: str) -> MypyFormatEntry: - for e in cls.entries(): - if value == e.value: - return e + raise NotImplementedError - raise NotImplementedError + @classmethod + def entries( + cls, + ) -> Generator[ + MypyFormatEntry, + None, + None, + ]: + for o in dir(cls): + e = getattr(cls, o) + if not isinstance(e, MypyFormatEntry): + continue - @classmethod - def entries(cls) -> Generator[MypyFormatEntry, None, None,]: - for o in dir(cls): - e = getattr(cls, o) - if not isinstance(e, MypyFormatEntry): - continue + yield e - yield e class MypySettings(pydantic_settings.BaseSettings): - model_config = pydantic_settings.SettingsConfigDict( - env_prefix='online_fxreader_pr34_mypy_', - case_sensitive=False, - ) + model_config = pydantic_settings.SettingsConfigDict( + env_prefix='online_fxreader_pr34_mypy_', + case_sensitive=False, + ) + + config_path: pathlib.Path = pathlib.Path.cwd() / '.mypy.ini' + max_errors: dict[str, int] = dict() + paths: Annotated[list[pathlib.Path], Field(default_factory=lambda: ['.'])] - config_path : pathlib.Path = pathlib.Path.cwd() / '.mypy.ini' - max_errors : dict[str, int] = dict() - paths : Annotated[list[pathlib.Path], Field(default_factory=lambda : ['.'])] def run( - argv: Optional[list[str]] = None, - settings: Optional[MypySettings] = None, + argv: Optional[list[str]] = None, + settings: Optional[MypySettings] = None, ) -> None: - if argv is None: - argv = [] + if argv is None: + argv = [] - if settings is None: - settings = MypySettings.model_validate(dict()) + if settings is None: + settings = MypySettings.model_validate(dict()) - parser = argparse.ArgumentParser() - parser.add_argument( - '-q', '--quiet', - dest='quiet', - action='store_true', - help='do not print anything if the program is correct according to max_errors limits', - default=False, - ) - parser.add_argument( - '-i', - dest='paths', - help='specify paths to check', - default=[], - action='append', - ) - parser.add_argument( - '-f', '--format', - dest='_format', - help='output format of errors', - default=MypyFormat.json.value, - choices=[ - o.value - for o in MypyFormat.entries() - ], - ) - options, args = parser.parse_known_args(argv) + parser = argparse.ArgumentParser() + parser.add_argument( + '-q', + '--quiet', + dest='quiet', + action='store_true', + help='do not print anything if the program is correct according to max_errors limits', + default=False, + ) + parser.add_argument( + '-i', + dest='paths', + help='specify paths to check', + default=[], + action='append', + ) + parser.add_argument( + '-f', + '--format', + dest='_format', + help='output format of errors', + default=MypyFormat.json.value, + choices=[o.value for o in MypyFormat.entries()], + ) + options, args = parser.parse_known_args(argv) - if len(args) > 0 and args[0] == '--': - del args[0] + if len(args) > 0 and args[0] == '--': + del args[0] - options.format = MypyFormat.from_value(options._format) + options.format = MypyFormat.from_value(options._format) - if len(options.paths) == 0: - options.paths.extend(settings.paths) + if len(options.paths) == 0: + options.paths.extend(settings.paths) - started_at = datetime.datetime.now() + started_at = datetime.datetime.now() - mypy_cmd = [ - sys.executable, - '-m', - 'mypy', - '--config-file', str(settings.config_path), - '--strict', - '-O', - 'json', - *args, - *options.paths, - ] + mypy_cmd = [ + sys.executable, + '-m', + 'mypy', + '--config-file', + str(settings.config_path), + '--strict', + '-O', + 'json', + *args, + *options.paths, + ] + logger.info(dict(cmd=mypy_cmd)) - logger.info(dict(cmd=mypy_cmd)) + res = subprocess.run( + mypy_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) - res = subprocess.run( - mypy_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + done_at = datetime.datetime.now() - done_at = datetime.datetime.now() + try: + assert not res.returncode is None - try: - assert not res.returncode is None + errors = sorted( + [json.loads(o) for o in res.stdout.decode('utf-8').splitlines() if not o.strip() == ''], + key=lambda x: ( + x.get('file', ''), + x.get('line', 0), + ), + ) - errors = sorted([ - json.loads(o) - for o in res.stdout.decode('utf-8').splitlines() - if not o.strip() == '' - ], key=lambda x: ( - x.get('file', ''), - x.get('line', 0), - )) + if not options.quiet: + if (len(res.stderr)) > 0: + logger.error(res.stderr.decode('utf-8')) + except: + logger.exception('') + logger.error(res.stdout.decode('utf-8')) + logger.error(res.stderr.decode('utf-8')) + sys.exit(res.returncode) - if not options.quiet: - if (len(res.stderr)) > 0: - logger.error(res.stderr.decode('utf-8')) - except: - logger.exception('') - logger.error(res.stdout.decode('utf-8')) - logger.error(res.stderr.decode('utf-8')) - sys.exit(res.returncode) + g: dict[str, Any] = dict() + for o in errors: + if not o['file'] in g: + g[o['file']] = [] + g[o['file']].append(o) + h = { + k: len(v) + for k, v in sorted( + list(g.items()), + key=lambda x: x[0], + ) + } - g : dict[str, Any] = dict() - for o in errors: - if not o['file'] in g: - g[o['file']] = [] - g[o['file']].append(o) + mentioned_paths = marisa_trie.Trie(list(h)) - h = { - k : len(v) - for k, v in sorted( - list(g.items()), - key=lambda x: x[0], - ) - } + violated_limits: dict[str, str] = dict() - mentioned_paths = marisa_trie.Trie(list(h)) + for k, v in settings.max_errors.items(): + matching_paths = mentioned_paths.keys(k) + total_errors = sum([h[o] for o in matching_paths], 0) - violated_limits : dict[str, str] = dict() + if total_errors > v: + violated_limits[k] = '%s - [%s]: has %d errors > %d' % ( + k, + ', '.join(matching_paths), + total_errors, + v, + ) - for k, v in settings.max_errors.items(): - matching_paths = mentioned_paths.keys(k) - total_errors = sum([ - h[o] - for o in matching_paths - ], 0) + if len(violated_limits) > 0 or not options.quiet: + if options.format == MypyFormat.vscode: + for o in errors: + sys.stdout.write( + '[%s] %s:%d,%d %s - %s - %s\n' + % ( + o['severity'], + o['file'], + o['line'], + o['column'], + o['message'], + o['hint'], + o['code'], + ) + ) + sys.stdout.flush() + # logger.info(json.dumps(errors, indent=4)) + else: + logger.info(json.dumps(errors, indent=4)) - if total_errors > v: - violated_limits[k] = '%s - [%s]: has %d errors > %d' % ( - k, ', '.join(matching_paths), total_errors, v, - ) + # if len(violated_limits) > 0: + # logger.info(json.dumps(violated_limits, indent=4)) + logger.info( + json.dumps( + dict( + max_errors=settings.max_errors, + violated_limits=violated_limits, + histogram=h, + elapsed=(done_at - started_at).total_seconds(), + ), + indent=4, + ) + ) - if len(violated_limits) > 0 or not options.quiet: - if options.format == MypyFormat.vscode: - for o in errors: - sys.stdout.write('[%s] %s:%d,%d %s - %s - %s\n' % ( - o['severity'], - o['file'], - o['line'], - o['column'], - o['message'], - o['hint'], - o['code'], - )) - sys.stdout.flush() - #logger.info(json.dumps(errors, indent=4)) - else: - logger.info(json.dumps(errors, indent=4)) + if len(violated_limits) > 0: + sys.exit(1) - #if len(violated_limits) > 0: - # logger.info(json.dumps(violated_limits, indent=4)) - logger.info(json.dumps(dict( - max_errors=settings.max_errors, - violated_limits=violated_limits, - histogram=h, - elapsed=(done_at - started_at).total_seconds(), - ), indent=4)) - - if len(violated_limits) > 0: - sys.exit(1) if __name__ == '__main__': - from . import logging as _logging - _logging.setup() - run(sys.argv[1:]) + from . import logging as _logging + + _logging.setup() + run(sys.argv[1:]) diff --git a/python/online/fxreader/pr34/commands_typed/os.py b/python/online/fxreader/pr34/commands_typed/os.py index ee2dfbe..68fcd9c 100644 --- a/python/online/fxreader/pr34/commands_typed/os.py +++ b/python/online/fxreader/pr34/commands_typed/os.py @@ -11,112 +11,115 @@ import dataclasses logger = logging.getLogger(__name__) -from typing import (overload, Optional, Literal, Any, Annotated,) +from typing import ( + overload, + Optional, + Literal, + Any, + Annotated, +) from .cli_bootstrap import PyProject + @overload def shutil_which( - name: str, - raise_on_failure: Literal[True], + name: str, + raise_on_failure: Literal[True], ) -> str: ... + @overload def shutil_which( - name: str, - raise_on_failure: bool, + name: str, + raise_on_failure: bool, ) -> Optional[str]: ... + def shutil_which( - name: str, - raise_on_failure: bool, + name: str, + raise_on_failure: bool, ) -> Optional[str]: - res = shutil.which(name) - if res is None and raise_on_failure: - raise NotImplementedError - else: - return res + res = shutil.which(name) + if res is None and raise_on_failure: + raise NotImplementedError + else: + return res + def runtime_libdirs_init( - project: PyProject, + project: PyProject, ) -> None: - if sys.platform == 'linux': - ld_library_path : list[pathlib.Path] = [ - o - for o in [ - *[ - o.absolute() - for o in ( - project.runtime_libdirs - if project.runtime_libdirs - else [] - ) - ], - *[ - pathlib.Path(o) - for o in os.environ.get( - 'LD_LIBRARY_PATH', - '' - ).split(os.path.pathsep) - if o != '' - ] - ] - ] + if sys.platform == 'linux': + ld_library_path: list[pathlib.Path] = [ + o + for o in [ + *[o.absolute() for o in (project.runtime_libdirs if project.runtime_libdirs else [])], + *[pathlib.Path(o) for o in os.environ.get('LD_LIBRARY_PATH', '').split(os.path.pathsep) if o != ''], + ] + ] - ld_library_path_present : list[pathlib.Path] = [] + ld_library_path_present: list[pathlib.Path] = [] - for o in ld_library_path: - if not o.exists(): - logger.warning(dict( - ld_library_path=o, - msg='not found', - )) + for o in ld_library_path: + if not o.exists(): + logger.warning( + dict( + ld_library_path=o, + msg='not found', + ) + ) - ld_library_path_present.append(o) + ld_library_path_present.append(o) - os.environ.update( - LD_LIBRARY_PATH=os.path.pathsep.join([ - str(o) for o in ld_library_path_present - ]) - ) + os.environ.update(LD_LIBRARY_PATH=os.path.pathsep.join([str(o) for o in ld_library_path_present])) - for preload_path in (project.runtime_preload or []): - for preload_found in glob.glob(str( - preload_path.parent / ('lib%s.so' % preload_path.name) - )): - logger.info(dict( - preload_path=preload_path, preload_found=preload_found, - # lib_path=o, - msg='load_library', - )) + for preload_path in project.runtime_preload or []: + for preload_found in glob.glob(str(preload_path.parent / ('lib%s.so' % preload_path.name))): + logger.info( + dict( + preload_path=preload_path, + preload_found=preload_found, + # lib_path=o, + msg='load_library', + ) + ) + + ctypes.cdll.LoadLibrary(preload_found) + else: + raise NotImplementedError - ctypes.cdll.LoadLibrary(preload_found) - else: - raise NotImplementedError class interfaces_index_t: - @dataclasses.dataclass - class Interface: - @dataclasses.dataclass - class AddrInfo: - family: str - local: str + @dataclasses.dataclass + class Interface: + @dataclasses.dataclass + class AddrInfo: + family: str + local: str + + name: Annotated[ + str, + pydantic.Field( + alias='ifname', + ), + ] + addr_info: list[AddrInfo] - name: Annotated[ - str, - pydantic.Field( - alias='ifname', - ) - ] - addr_info: list[AddrInfo] def interfaces_index() -> list[interfaces_index_t.Interface]: - res = pydantic.RootModel[ - list[interfaces_index_t.Interface] - ].model_validate_json( - subprocess.check_output([ - 'ip', '-j', 'addr', - ]).decode('utf-8') - ).root + res = ( + pydantic.RootModel[list[interfaces_index_t.Interface]] + .model_validate_json( + subprocess.check_output( + [ + 'ip', + '-j', + 'addr', + ] + ).decode('utf-8') + ) + .root + ) - return res + return res diff --git a/python/online/fxreader/pr34/commands_typed/pip.py b/python/online/fxreader/pr34/commands_typed/pip.py index 9953aa7..f1dca2f 100644 --- a/python/online/fxreader/pr34/commands_typed/pip.py +++ b/python/online/fxreader/pr34/commands_typed/pip.py @@ -11,609 +11,567 @@ import unittest.mock import logging import typing -from typing_extensions import (cast, Protocol,) +from typing_extensions import ( + cast, + Protocol, +) if typing.TYPE_CHECKING: - import pip._internal.commands.show - import pip._internal.commands.download - import pip._internal.cli.main_parser - import pip._internal.models.index - import pip._internal.utils.temp_dir - import pip._internal.cli.main - import pip._internal.network.download - import pip._internal.resolution.base - import pip._internal.resolution.resolvelib.resolver - import pip._internal.operations.prepare - import pip._internal.index.package_finder - from pip._internal.req.req_install import InstallRequirement + import pip._internal.commands.show + import pip._internal.commands.download + import pip._internal.cli.main_parser + import pip._internal.models.index + import pip._internal.utils.temp_dir + import pip._internal.cli.main + import pip._internal.network.download + import pip._internal.resolution.base + import pip._internal.resolution.resolvelib.resolver + import pip._internal.operations.prepare + import pip._internal.index.package_finder + from pip._internal.req.req_install import InstallRequirement from typing import ( - Literal, Optional, Iterable, Any, + Literal, + Optional, + Iterable, + Any, ) logger = logging.getLogger(__name__) class pip_show_t: - class res_t: - class package_info_t: - pass + class res_t: + class package_info_t: + pass + def pip_show( - argv: list[str], + argv: list[str], ) -> list[ - # 'pip._internal.commands.show._PackageInfo' - pip_show_t.res_t.package_info_t, + # 'pip._internal.commands.show._PackageInfo' + pip_show_t.res_t.package_info_t, ]: - import pip._internal.commands.show - return cast( - list[pip_show_t.res_t.package_info_t], - list( - pip._internal.commands.show.search_packages_info( - argv, - ) - ) - ) + import pip._internal.commands.show + + return cast( + list[pip_show_t.res_t.package_info_t], + list( + pip._internal.commands.show.search_packages_info( + argv, + ) + ), + ) class pip_resolve_t: - class build_package_finder_t(Protocol): - def __call__( - self, - options: Any, - session: Any, - target_python: Any, - ignore_requires_python: Any, - ) -> 'pip._internal.index.package_finder.PackageFinder': ... + class build_package_finder_t(Protocol): + def __call__( + self, + options: Any, + session: Any, + target_python: Any, + ignore_requires_python: Any, + ) -> 'pip._internal.index.package_finder.PackageFinder': ... - class complete_partial_requirements_t(Protocol): - def __call__( - self, - _self: 'pip._internal.resolution.resolvelib.resolver.Resolver', - partially_downloaded_reqs: Iterable['InstallRequirement',], - parallel_builds: bool = False, - ) -> None: ... + class complete_partial_requirements_t(Protocol): + def __call__( + self, + _self: 'pip._internal.resolution.resolvelib.resolver.Resolver', + partially_downloaded_reqs: Iterable['InstallRequirement',], + parallel_builds: bool = False, + ) -> None: ... - class kwargs_t: - class mode_t(enum.StrEnum): - copy_paste = "copy_paste" - monkey_patch = "monkey_patch" - uv_pip_freeze = "uv_pip_freeze" - uv_pip_compile = "uv_pip_compile" + class kwargs_t: + class mode_t(enum.StrEnum): + copy_paste = 'copy_paste' + monkey_patch = 'monkey_patch' + uv_pip_freeze = 'uv_pip_freeze' + uv_pip_compile = 'uv_pip_compile' - @dataclasses.dataclass - class res_t: - @dataclasses.dataclass - class download_info_t: - url: str - sha256: str - constraint: str + @dataclasses.dataclass + class res_t: + @dataclasses.dataclass + class download_info_t: + url: str + sha256: str + constraint: str - txt: Optional[str] = None - entries: Optional[list[download_info_t]] = None + txt: Optional[str] = None + entries: Optional[list[download_info_t]] = None -def pip_resolve_entries_to_txt( - entries: list[pip_resolve_t.res_t.download_info_t] -) -> str: - return '\n'.join([ - '#%s\n%s %s' % ( - o.url, - o.constraint, - ' '.join([ - '--hash=sha256:%s' % o2 - for o2 in o.sha256 - ]) - ) - for o in entries - ]) +def pip_resolve_entries_to_txt(entries: list[pip_resolve_t.res_t.download_info_t]) -> str: + return '\n'.join(['#%s\n%s %s' % (o.url, o.constraint, ' '.join(['--hash=sha256:%s' % o2 for o2 in o.sha256])) for o in entries]) + def pip_resolve( - argv: list[str], - mode: pip_resolve_t.kwargs_t.mode_t, - requirements: Optional[list[str]] = None, + argv: list[str], + mode: pip_resolve_t.kwargs_t.mode_t, + requirements: Optional[list[str]] = None, ) -> pip_resolve_t.res_t: - if mode is pip_resolve_t.kwargs_t.mode_t.copy_paste: - import pip._internal.commands.show - import pip._internal.commands.download - import pip._internal.cli.cmdoptions - import pip._internal.cli.main_parser - import pip._internal.models.index - import pip._internal.utils.temp_dir - import pip._internal.cli.main - import pip._internal.network.download - import pip._internal.resolution.base - import pip._internal.req.req_install - import pip._internal.resolution.resolvelib.resolver - import pip._internal.operations.prepare - import pip._internal.utils.temp_dir - import pip._internal.operations.build.build_tracker - import pip._internal.models.direct_url + if mode is pip_resolve_t.kwargs_t.mode_t.copy_paste: + import pip._internal.commands.show + import pip._internal.commands.download + import pip._internal.cli.cmdoptions + import pip._internal.cli.main_parser + import pip._internal.models.index + import pip._internal.utils.temp_dir + import pip._internal.cli.main + import pip._internal.network.download + import pip._internal.resolution.base + import pip._internal.req.req_install + import pip._internal.resolution.resolvelib.resolver + import pip._internal.operations.prepare + import pip._internal.utils.temp_dir + import pip._internal.operations.build.build_tracker + import pip._internal.models.direct_url - with contextlib.ExitStack() as stack: - stack.enter_context(pip._internal.utils.temp_dir.global_tempdir_manager()) + with contextlib.ExitStack() as stack: + stack.enter_context(pip._internal.utils.temp_dir.global_tempdir_manager()) - t2 = pip._internal.cli.main_parser.create_main_parser() + t2 = pip._internal.cli.main_parser.create_main_parser() - t3 = t2.parse_args(["download"]) - t1 = pip._internal.commands.download.DownloadCommand("blah", "shit") + t3 = t2.parse_args(['download']) + t1 = pip._internal.commands.download.DownloadCommand('blah', 'shit') - stack.enter_context(t1.main_context()) + stack.enter_context(t1.main_context()) - # options = pip._internal.commands.download.Values() - options = t3[0] - options.python_version = None - options.platforms = [] - options.abis = [] - options.implementation = [] - options.format_control = None - options.ignore_dependencies = None - options.index_url = pip._internal.models.index.PyPI.simple_url - options.extra_index_urls = [] - options.no_index = None - options.find_links = [] - options.pre = None - options.prefer_binary = True - options.only_binary = True - options.constraints = [] - options.use_pep517 = None - options.editables = [] - options.requirements = [] - options.src_dir = str(pathlib.Path(__file__).parent) - options.build_isolation = None - options.check_build_deps = None - options.progress_bar = True - options.require_hashes = None - options.ignore_requires_python = False - # options.cache_dir - pip._internal.cli.cmdoptions.check_dist_restriction(options) - # t1._in_main_context = True - session = t1.get_default_session(options) - target_python = pip._internal.cli.cmdoptions.make_target_python(options) - finder = cast( - pip_resolve_t.build_package_finder_t, - getattr( - t1, - '_build_package_finder' - ) - )( - options=options, - session=session, - target_python=target_python, - ignore_requires_python=options.ignore_requires_python, - ) + # options = pip._internal.commands.download.Values() + options = t3[0] + options.python_version = None + options.platforms = [] + options.abis = [] + options.implementation = [] + options.format_control = None + options.ignore_dependencies = None + options.index_url = pip._internal.models.index.PyPI.simple_url + options.extra_index_urls = [] + options.no_index = None + options.find_links = [] + options.pre = None + options.prefer_binary = True + options.only_binary = True + options.constraints = [] + options.use_pep517 = None + options.editables = [] + options.requirements = [] + options.src_dir = str(pathlib.Path(__file__).parent) + options.build_isolation = None + options.check_build_deps = None + options.progress_bar = True + options.require_hashes = None + options.ignore_requires_python = False + # options.cache_dir + pip._internal.cli.cmdoptions.check_dist_restriction(options) + # t1._in_main_context = True + session = t1.get_default_session(options) + target_python = pip._internal.cli.cmdoptions.make_target_python(options) + finder = cast(pip_resolve_t.build_package_finder_t, getattr(t1, '_build_package_finder'))( + options=options, + session=session, + target_python=target_python, + ignore_requires_python=options.ignore_requires_python, + ) - build_tracker = t1.enter_context( - pip._internal.operations.build.build_tracker.get_build_tracker() - ) - reqs = t1.get_requirements( - [ - #'pip', 'uv', 'ipython', - *argv, - ], - options, - finder, - session, - ) - pip._internal.req.req_install.check_legacy_setup_py_options(options, reqs) - directory = pip._internal.utils.temp_dir.TempDirectory( - delete=True, kind="download", globally_managed=True - ) - preparer = t1.make_requirement_preparer( - temp_build_dir=directory, - options=options, - build_tracker=build_tracker, - session=session, - finder=finder, - download_dir=None, - use_user_site=False, - verbosity=False, - ) - resolver = t1.make_resolver( - preparer=preparer, - finder=finder, - options=options, - ignore_requires_python=options.ignore_requires_python, - use_pep517=options.use_pep517, - py_version_info=options.python_version, - ) - t1.trace_basic_info(finder) - requirement_set = resolver.resolve(reqs, check_supported_wheels=True) + build_tracker = t1.enter_context(pip._internal.operations.build.build_tracker.get_build_tracker()) + reqs = t1.get_requirements( + [ + #'pip', 'uv', 'ipython', + *argv, + ], + options, + finder, + session, + ) + pip._internal.req.req_install.check_legacy_setup_py_options(options, reqs) + directory = pip._internal.utils.temp_dir.TempDirectory(delete=True, kind='download', globally_managed=True) + preparer = t1.make_requirement_preparer( + temp_build_dir=directory, + options=options, + build_tracker=build_tracker, + session=session, + finder=finder, + download_dir=None, + use_user_site=False, + verbosity=False, + ) + resolver = t1.make_resolver( + preparer=preparer, + finder=finder, + options=options, + ignore_requires_python=options.ignore_requires_python, + use_pep517=options.use_pep517, + py_version_info=options.python_version, + ) + t1.trace_basic_info(finder) + requirement_set = resolver.resolve(reqs, check_supported_wheels=True) - res = pip_resolve_t.res_t() + res = pip_resolve_t.res_t() - res.entries = [] + res.entries = [] - for k, v in requirement_set.requirements.items(): - assert not v.download_info is None - assert isinstance( - v.download_info.info, - pip._internal.models.direct_url.ArchiveInfo, - ) - assert not v.download_info.info.hashes is None + for k, v in requirement_set.requirements.items(): + assert not v.download_info is None + assert isinstance( + v.download_info.info, + pip._internal.models.direct_url.ArchiveInfo, + ) + assert not v.download_info.info.hashes is None - res.entries.append( - pip_resolve_t.res_t.download_info_t( - constraint=k, - sha256=v.download_info.info.hashes["sha256"], - url=v.download_info.url, - ) - ) + res.entries.append( + pip_resolve_t.res_t.download_info_t( + constraint=k, + sha256=v.download_info.info.hashes['sha256'], + url=v.download_info.url, + ) + ) - res.txt = pip_resolve_entries_to_txt( - res.entries - ) + res.txt = pip_resolve_entries_to_txt(res.entries) - return res - elif mode is pip_resolve_t.kwargs_t.mode_t.monkey_patch: - import pip._internal.commands.show - import pip._internal.commands.download - import pip._internal.cli.main_parser - import pip._internal.models.index - import pip._internal.models.link - from pip._internal.models.link import ( - Link, - ) - import pip._internal.utils.temp_dir - from pip._internal.metadata.base import ( - BaseDistribution, - ) - import pip._internal.cli.main - import pip._internal.network.download - import pip._internal.resolution.base - import pip._internal.resolution.resolvelib.resolver - import pip._internal.operations.prepare - from pip._internal.network.download import ( - Downloader, - ) - from pip._internal.operations.prepare import ( - File, - ) - from pip._internal.req.req_set import RequirementSet - from pip._internal.utils.hashes import Hashes - from pip._internal.req.req_install import InstallRequirement + return res + elif mode is pip_resolve_t.kwargs_t.mode_t.monkey_patch: + import pip._internal.commands.show + import pip._internal.commands.download + import pip._internal.cli.main_parser + import pip._internal.models.index + import pip._internal.models.link + from pip._internal.models.link import ( + Link, + ) + import pip._internal.utils.temp_dir + from pip._internal.metadata.base import ( + BaseDistribution, + ) + import pip._internal.cli.main + import pip._internal.network.download + import pip._internal.resolution.base + import pip._internal.resolution.resolvelib.resolver + import pip._internal.operations.prepare + from pip._internal.network.download import ( + Downloader, + ) + from pip._internal.operations.prepare import ( + File, + ) + from pip._internal.req.req_set import RequirementSet + from pip._internal.utils.hashes import Hashes + from pip._internal.req.req_install import InstallRequirement - from . import cli_bootstrap + from . import cli_bootstrap - downloader_call_def = pip._internal.network.download.Downloader.__call__ + downloader_call_def = pip._internal.network.download.Downloader.__call__ - def downloader_call( - _self: pip._internal.network.download.Downloader, - link: pip._internal.models.link.Link, - location: str, - ) -> tuple[str, str]: - logger.info( - dict( - url=link.url, - ) - ) + def downloader_call( + _self: pip._internal.network.download.Downloader, + link: pip._internal.models.link.Link, + location: str, + ) -> tuple[str, str]: + logger.info( + dict( + url=link.url, + ) + ) - return downloader_call_def( - _self, - link, location, - ) + return downloader_call_def( + _self, + link, + location, + ) - batch_downloader_call_def = ( - pip._internal.network.download.BatchDownloader.__call__ - ) + batch_downloader_call_def = pip._internal.network.download.BatchDownloader.__call__ - def batch_downloader_call( - _self: pip._internal.network.download.BatchDownloader, - links: Iterable[pip._internal.models.link.Link], - location: str, - ) -> Iterable[ - tuple[ - pip._internal.models.link.Link, - tuple[str, str] - ] - ]: - # print(args) + def batch_downloader_call( + _self: pip._internal.network.download.BatchDownloader, + links: Iterable[pip._internal.models.link.Link], + location: str, + ) -> Iterable[tuple[pip._internal.models.link.Link, tuple[str, str]]]: + # print(args) - logger.info( - dict( - links=links, - location=location, - ) - ) + logger.info( + dict( + links=links, + location=location, + ) + ) - return [ - (o, ("/dev/null", '')) - for o in links - ] + return [(o, ('/dev/null', '')) for o in links] - # base_resolver_resolve_def = pip._internal.resolution.base.BaseResolver.resolve - base_resolver_resolve_def = ( - pip._internal.resolution.resolvelib.resolver.Resolver.resolve - ) + # base_resolver_resolve_def = pip._internal.resolution.base.BaseResolver.resolve + base_resolver_resolve_def = pip._internal.resolution.resolvelib.resolver.Resolver.resolve - result_requirements : list[ - RequirementSet | InstallRequirement - ] = [] + result_requirements: list[RequirementSet | InstallRequirement] = [] - def base_resolver_resolve( - _self: pip._internal.resolution.resolvelib.resolver.Resolver, - root_reqs: list[ - InstallRequirement, - ], - check_supported_wheels: bool, - ) -> RequirementSet: - # print(args, kwargs) + def base_resolver_resolve( + _self: pip._internal.resolution.resolvelib.resolver.Resolver, + root_reqs: list[InstallRequirement,], + check_supported_wheels: bool, + ) -> RequirementSet: + # print(args, kwargs) - res = base_resolver_resolve_def( - _self, - root_reqs, - check_supported_wheels - ) + res = base_resolver_resolve_def(_self, root_reqs, check_supported_wheels) - result_requirements.append(res) - raise NotImplementedError - return res + result_requirements.append(res) + raise NotImplementedError + return res - get_http_url_def = pip._internal.operations.prepare.get_http_url + get_http_url_def = pip._internal.operations.prepare.get_http_url - def get_http_url( - link: Link, - download: Downloader, - download_dir: Optional[str] = None, - hashes: Optional[Hashes] = None, - ) -> File: - logger.info( - dict( - url=link.url, - hashes=hashes, - ) - ) + def get_http_url( + link: Link, + download: Downloader, + download_dir: Optional[str] = None, + hashes: Optional[Hashes] = None, + ) -> File: + logger.info( + dict( + url=link.url, + hashes=hashes, + ) + ) - if link.url.endswith(".whl"): - print("blah") - hashes = None + if link.url.endswith('.whl'): + print('blah') + hashes = None - return File( - "/dev/null", - '', - ) - else: - return get_http_url_def( - link, - download, - download_dir, - hashes - ) + return File( + '/dev/null', + '', + ) + else: + return get_http_url_def(link, download, download_dir, hashes) - prepare_linked_requirements_more_def = pip._internal.operations.prepare.RequirementPreparer.prepare_linked_requirements_more + prepare_linked_requirements_more_def = pip._internal.operations.prepare.RequirementPreparer.prepare_linked_requirements_more - def prepare_linked_requirements_more( - _self: pip._internal.resolution.resolvelib.resolver.Resolver, - reqs: Iterable[InstallRequirement], - parallel_builds: bool = False, - ) -> None: - result_requirements.extend( - reqs - ) - raise NotImplementedError + def prepare_linked_requirements_more( + _self: pip._internal.resolution.resolvelib.resolver.Resolver, + reqs: Iterable[InstallRequirement], + parallel_builds: bool = False, + ) -> None: + result_requirements.extend(reqs) + raise NotImplementedError - _complete_partial_requirements_def = cast( - pip_resolve_t.complete_partial_requirements_t, - getattr( - pip._internal.operations.prepare.RequirementPreparer, - '_complete_partial_requirements', - ) - ) + _complete_partial_requirements_def = cast( + pip_resolve_t.complete_partial_requirements_t, + getattr( + pip._internal.operations.prepare.RequirementPreparer, + '_complete_partial_requirements', + ), + ) - def _complete_partial_requirements( - _self: pip._internal.resolution.resolvelib.resolver.Resolver, - partially_downloaded_reqs: Iterable[InstallRequirement], - parallel_builds: bool = False, - ) -> None: - result_requirements.extend( - partially_downloaded_reqs - ) - raise NotImplementedError + def _complete_partial_requirements( + _self: pip._internal.resolution.resolvelib.resolver.Resolver, + partially_downloaded_reqs: Iterable[InstallRequirement], + parallel_builds: bool = False, + ) -> None: + result_requirements.extend(partially_downloaded_reqs) + raise NotImplementedError - patches : list[Any] = [] + patches: list[Any] = [] - patches.append( - unittest.mock.patch.object( - pip._internal.network.download.Downloader, "__call__", downloader_call - ) - ) - # patches.append( - # unittest.mock.patch.object( - # pip._internal.network.download.BatchDownloader, - # '__call__', - # batch_downloader_call - # ) - # ) - # patches.append( - # unittest.mock.patch.object( - # pip._internal.resolution.base.BaseResolver, 'resolve', base_resolver_resolve)) + patches.append(unittest.mock.patch.object(pip._internal.network.download.Downloader, '__call__', downloader_call)) + # patches.append( + # unittest.mock.patch.object( + # pip._internal.network.download.BatchDownloader, + # '__call__', + # batch_downloader_call + # ) + # ) + # patches.append( + # unittest.mock.patch.object( + # pip._internal.resolution.base.BaseResolver, 'resolve', base_resolver_resolve)) - patches.append( - unittest.mock.patch.object( - pip._internal.resolution.resolvelib.resolver.Resolver, - "resolve", - base_resolver_resolve, - ) - ) - patches.append( - unittest.mock.patch.object( - pip._internal.operations.prepare, - "get_http_url", - get_http_url, - ) - ) - patches.append( - unittest.mock.patch.object( - pip._internal.operations.prepare.RequirementPreparer, - "prepare_linked_requirements_more", - prepare_linked_requirements_more, - ) - ) - # patches.append( - # unittest.mock.patch.object( - # pip._internal.operations.prepare.RequirementPreparer, - # '_complete_partial_requirements', - # _complete_partial_requirements - # ) - # ) + patches.append( + unittest.mock.patch.object( + pip._internal.resolution.resolvelib.resolver.Resolver, + 'resolve', + base_resolver_resolve, + ) + ) + patches.append( + unittest.mock.patch.object( + pip._internal.operations.prepare, + 'get_http_url', + get_http_url, + ) + ) + patches.append( + unittest.mock.patch.object( + pip._internal.operations.prepare.RequirementPreparer, + 'prepare_linked_requirements_more', + prepare_linked_requirements_more, + ) + ) + # patches.append( + # unittest.mock.patch.object( + # pip._internal.operations.prepare.RequirementPreparer, + # '_complete_partial_requirements', + # _complete_partial_requirements + # ) + # ) - with contextlib.ExitStack() as stack: - for p in patches: - stack.enter_context(p) + with contextlib.ExitStack() as stack: + for p in patches: + stack.enter_context(p) - pip._internal.cli.main.main( - [ - "download", - "-q", - "--no-cache", - "-d", - "/dev/null", - *argv, - # 'numpy', - ] - ) + pip._internal.cli.main.main( + [ + 'download', + '-q', + '--no-cache', + '-d', + '/dev/null', + *argv, + # 'numpy', + ] + ) - # return sum([ - # [ - # pip_resolve_t.res_t.download_info_t( - # constraint=k, - # sha256=v.download_info.info.hashes['sha256'], - # url=v.download_info.url, - # ) - # for k, v in o.requirements.items() - # ] - # for o in result_requirements - # ], []) - logger.warning(result_requirements) + # return sum([ + # [ + # pip_resolve_t.res_t.download_info_t( + # constraint=k, + # sha256=v.download_info.info.hashes['sha256'], + # url=v.download_info.url, + # ) + # for k, v in o.requirements.items() + # ] + # for o in result_requirements + # ], []) + logger.warning(result_requirements) - res = pip_resolve_t.res_t() + res = pip_resolve_t.res_t() - res.entries = [] + res.entries = [] - for o in result_requirements: - assert isinstance(o, InstallRequirement) + for o in result_requirements: + assert isinstance(o, InstallRequirement) - sha256_hashes = cli_bootstrap.check_list( - cli_bootstrap.check_dict( - getattr(o.hashes(), '_allowed'), - str, - list[str], - )['sha256'], - str, - ) + sha256_hashes = cli_bootstrap.check_list( + cli_bootstrap.check_dict( + getattr(o.hashes(), '_allowed'), + str, + list[str], + )['sha256'], + str, + ) - assert len(sha256_hashes) == 1 - assert not o.link is None + assert len(sha256_hashes) == 1 + assert not o.link is None - res.entries.append( - pip_resolve_t.res_t.download_info_t( - constraint=str(o.req), - sha256=sha256_hashes[0], - url=o.link.url, - ) - ) + res.entries.append( + pip_resolve_t.res_t.download_info_t( + constraint=str(o.req), + sha256=sha256_hashes[0], + url=o.link.url, + ) + ) - res.txt = pip_resolve_entries_to_txt( - res.entries - ) + res.txt = pip_resolve_entries_to_txt(res.entries) - return res - elif mode is pip_resolve_t.kwargs_t.mode_t.uv_pip_freeze: - assert len(argv) == 0 + return res + elif mode is pip_resolve_t.kwargs_t.mode_t.uv_pip_freeze: + assert len(argv) == 0 - pip_freeze = subprocess.check_output( - [ - sys.executable, - "-m", - "uv", - "pip", - "freeze", - ], - ).decode('utf-8') - pip_compile = subprocess.check_output( - [ - sys.executable, '-m', - 'uv', 'pip', 'compile', - '--generate-hashes', - '-', + pip_freeze = subprocess.check_output( + [ + sys.executable, + '-m', + 'uv', + 'pip', + 'freeze', + ], + ).decode('utf-8') + pip_compile = subprocess.check_output( + [ + sys.executable, + '-m', + 'uv', + 'pip', + 'compile', + '--generate-hashes', + '-', + ], + input=pip_freeze.encode('utf-8'), + ).decode('utf-8') - ], - input=pip_freeze.encode('utf-8') - ).decode('utf-8') + return pip_resolve_t.res_t( + txt=pip_compile, + ) + elif mode is pip_resolve_t.kwargs_t.mode_t.uv_pip_compile: + with contextlib.ExitStack() as stack: + if not requirements is None: + # assert len(argv) == 0 - return pip_resolve_t.res_t( - txt=pip_compile, - ) - elif mode is pip_resolve_t.kwargs_t.mode_t.uv_pip_compile: - with contextlib.ExitStack() as stack: - if not requirements is None: - # assert len(argv) == 0 + f = stack.enter_context( + tempfile.NamedTemporaryFile( + suffix='.txt', + ) + ) + f.write(('\n'.join(requirements)).encode('utf-8')) + f.flush() - f = stack.enter_context( - tempfile.NamedTemporaryFile( - suffix='.txt', - ) - ) - f.write( - ('\n'.join(requirements)).encode('utf-8') - ) - f.flush() + argv.append(f.name) - argv.append(f.name) + if argv[0] == '--': + del argv[0] - if argv[0] == '--': - del argv[0] + pip_compile = subprocess.check_output( + [ + sys.executable, + '-m', + 'uv', + 'pip', + 'compile', + '--generate-hashes', + *argv, + ], + ).decode('utf-8') - pip_compile = subprocess.check_output( - [ - sys.executable, '-m', - 'uv', 'pip', 'compile', - '--generate-hashes', - *argv, - ], - ).decode('utf-8') + return pip_resolve_t.res_t( + txt=pip_compile, + ) + else: + raise NotImplementedError - return pip_resolve_t.res_t( - txt=pip_compile, - ) - else: - raise NotImplementedError class pip_check_conflicts_t: - @dataclasses.dataclass - class res_t: - status : Literal['ok', 'error'] - duplicates: list[str] + @dataclasses.dataclass + class res_t: + status: Literal['ok', 'error'] + duplicates: list[str] + def pip_check_conflicts( - venv_path: Optional[pathlib.Path] = None, + venv_path: Optional[pathlib.Path] = None, ) -> pip_check_conflicts_t.res_t: - assert sys.platform == 'linux' - if venv_path is None: - venv_path = pathlib.Path( - sys.executable, - ).parent / '..' + assert sys.platform == 'linux' + if venv_path is None: + venv_path = ( + pathlib.Path( + sys.executable, + ).parent + / '..' + ) - # records = glob.glob( - # str(venv_path / '*' / 'site-packages' / '*.dist-info' / 'RECORD'), - # recursive=True, - # ) + # records = glob.glob( + # str(venv_path / '*' / 'site-packages' / '*.dist-info' / 'RECORD'), + # recursive=True, + # ) - duplicates = [ - line - for line in subprocess.check_output(r''' + duplicates = [ + line + for line in subprocess.check_output( + r""" cat $(find $VENV_PATH/lib/*/*/*.dist-info/RECORD) | sort | uniq -c | (grep -v -P '^\s+1\s'; true;) - ''', shell=True, env=dict( - VENV_PATH=str(venv_path), - )).decode('utf-8').splitlines() - if line.strip() != '' - ] + """, + shell=True, + env=dict( + VENV_PATH=str(venv_path), + ), + ) + .decode('utf-8') + .splitlines() + if line.strip() != '' + ] - return pip_check_conflicts_t.res_t( - status=( - 'error' - if len(duplicates) > 0 - else 'ok' - ), - duplicates=duplicates - ) + return pip_check_conflicts_t.res_t(status=('error' if len(duplicates) > 0 else 'ok'), duplicates=duplicates) diff --git a/python/online/fxreader/pr34/commands_typed/typing.py b/python/online/fxreader/pr34/commands_typed/typing.py index 517898c..66011df 100644 --- a/python/online/fxreader/pr34/commands_typed/typing.py +++ b/python/online/fxreader/pr34/commands_typed/typing.py @@ -6,22 +6,23 @@ from typing import Any from typing_extensions import Protocol from abc import abstractmethod -C = typing.TypeVar("C", bound="Comparable") +C = typing.TypeVar('C', bound='Comparable') + class Comparable(Protocol): - @abstractmethod - def __eq__(self, other: Any) -> bool: - pass + @abstractmethod + def __eq__(self, other: Any) -> bool: + pass - @abstractmethod - def __lt__(self: C, other: C) -> bool: - pass + @abstractmethod + def __lt__(self: C, other: C) -> bool: + pass - def __gt__(self: C, other: C) -> bool: - return (not self < other) and self != other + def __gt__(self: C, other: C) -> bool: + return (not self < other) and self != other - def __le__(self: C, other: C) -> bool: - return self < other or self == other + def __le__(self: C, other: C) -> bool: + return self < other or self == other - def __ge__(self: C, other: C) -> bool: - return (not self < other) + def __ge__(self: C, other: C) -> bool: + return not self < other diff --git a/python/online/fxreader/pr34/tasks/ble.py b/python/online/fxreader/pr34/tasks/ble.py index 8a25a8c..48aa22c 100644 --- a/python/online/fxreader/pr34/tasks/ble.py +++ b/python/online/fxreader/pr34/tasks/ble.py @@ -5,121 +5,107 @@ import pprint async def f1(): - devices = await bleak.BleakScanner.discover() - return devices + devices = await bleak.BleakScanner.discover() + return devices + async def f2(device, timeout=None): - if timeout is None: - timeout = 1.0 + if timeout is None: + timeout = 1.0 - assert isinstance(timeout, float) and timeout >= 1e-8 + assert isinstance(timeout, float) and timeout >= 1e-8 + + p = await bleak.BleakClient( + device, + timeout=timeout, + ).__aenter__() + return p - p = await bleak.BleakClient( - device, - timeout=timeout, - ).__aenter__() - return p async def f3(client): - t1 = [ - dict( - service=o.__dict__, - characteristics=[ - o2.__dict__ - for o2 in o.characteristics - ] - ) - for o in client.services - ] - return t1 + t1 = [dict(service=o.__dict__, characteristics=[o2.__dict__ for o2 in o.characteristics]) for o in client.services] + return t1 + async def f5( - name_check=None, + name_check=None, ): - t2 = [] + t2 = [] - attempt = 0 + attempt = 0 - while True: - t1 = await f1() - pprint.pprint([o.__dict__ for o in t1]) + while True: + t1 = await f1() + pprint.pprint([o.__dict__ for o in t1]) - if not name_check is None: - assert inspect.isfunction(name_check) + if not name_check is None: + assert inspect.isfunction(name_check) - t5 = { - i : o.details[0].name() - for i, o in enumerate(t1) - } + t5 = {i: o.details[0].name() for i, o in enumerate(t1)} - t2.extend( - [ - t1[k] - for k, v in t5.items() - if isinstance(v, str) and name_check(v) - ] - ) - else: - t2.extend(t1) + t2.extend([t1[k] for k, v in t5.items() if isinstance(v, str) and name_check(v)]) + else: + t2.extend(t1) - if len(t2) > 0: - break + if len(t2) > 0: + break - attempt += 1 - print('\rattempt #%d' % attempt, end='') + attempt += 1 + print('\rattempt #%d' % attempt, end='') + + return t2 - return t2 async def f4( - timeout=None, - characteristics=None, - operations=None, - name_check=None, + timeout=None, + characteristics=None, + operations=None, + name_check=None, ): - if isinstance(name_check, str): - assert name_check in [ - 'watch fit', - ] - name_check2 = lambda current_name: name_check.lower() in current_name.lower() - else: - name_check2 = name_check + if isinstance(name_check, str): + assert name_check in [ + 'watch fit', + ] + name_check2 = lambda current_name: name_check.lower() in current_name.lower() + else: + name_check2 = name_check - assert not name_check2 is None + assert not name_check2 is None - if characteristics is None: - characteristics = [ - '0000ffd1-0000-1000-8000-00805f9b34fb', - ] + if characteristics is None: + characteristics = [ + '0000ffd1-0000-1000-8000-00805f9b34fb', + ] - t2 = await f5( - name_check=name_check2, - ) + t2 = await f5( + name_check=name_check2, + ) - if len(t2) == 0: - print('not found') - return + if len(t2) == 0: + print('not found') + return - t3 = None - try: - t3 = await f2(t2[0], timeout=timeout) - t4 = await f3(t3) - pprint.pprint(t4) + t3 = None + try: + t3 = await f2(t2[0], timeout=timeout) + t4 = await f3(t3) + pprint.pprint(t4) - if not operations is None and inspect.isfunction(operations): - await operations( - client=t3, - t4=t4, - ) - else: - t6 = {} - for o in characteristics: - try: - t7 = await t3.read_gatt_char(o) - except Exception as exception: - print(traceback.format_exc()) - t7 = None - t6[o] = t7 - pprint.pprint(t6) - finally: - if not t3 is None: - await t3.disconnect() + if not operations is None and inspect.isfunction(operations): + await operations( + client=t3, + t4=t4, + ) + else: + t6 = {} + for o in characteristics: + try: + t7 = await t3.read_gatt_char(o) + except Exception as exception: + print(traceback.format_exc()) + t7 = None + t6[o] = t7 + pprint.pprint(t6) + finally: + if not t3 is None: + await t3.disconnect() diff --git a/python/online/fxreader/pr34/tasks/cython.py b/python/online/fxreader/pr34/tasks/cython.py index 1ad6aa6..a0b4f3f 100644 --- a/python/online/fxreader/pr34/tasks/cython.py +++ b/python/online/fxreader/pr34/tasks/cython.py @@ -10,162 +10,149 @@ import threading import cython import datetime -from typing import (Any, Optional, TypeVar, Type, cast) +from typing import Any, Optional, TypeVar, Type, cast # from scoping import scoping as s -def test( - _id: int, - T: float, - a: numpy.ndarray[Any, numpy.dtype[numpy.int32]], -) -> None: - with cython.nogil: - #if True: - started_at = datetime.datetime.now() - print('started') - def elapsed() -> float: - return (datetime.datetime.now() - started_at).total_seconds() - #a = 0 - while elapsed() < T: - #a += 1 - for k in range(1024 * 1024): - a[_id] += 1 - print(['done', started_at, elapsed(), a[_id]]) +def test( + _id: int, + T: float, + a: numpy.ndarray[Any, numpy.dtype[numpy.int32]], +) -> None: + with cython.nogil: + # if True: + started_at = datetime.datetime.now() + print('started') + + def elapsed() -> float: + return (datetime.datetime.now() - started_at).total_seconds() + + # a = 0 + while elapsed() < T: + # a += 1 + for k in range(1024 * 1024): + a[_id] += 1 + + print(['done', started_at, elapsed(), a[_id]]) + M = TypeVar('M', bound=Type[Any]) + def build(content: str, module: M) -> M: - import pathlib - import tempfile - import hashlib - import Cython.Build.Inline + import pathlib + import tempfile + import hashlib + import Cython.Build.Inline - sha256sum = hashlib.sha256(content.encode('utf-8')).digest().hex() + sha256sum = hashlib.sha256(content.encode('utf-8')).digest().hex() - output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute() + output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute() + if not output_dir.exists() or True: + os.makedirs(str(output_dir), exist_ok=True) - if not output_dir.exists() or True: - os.makedirs(str(output_dir), exist_ok=True) + source_path = output_dir / ('_%s.pyx' % sha256sum) + if not source_path.exists(): + with io.open(str(source_path), 'w') as f: + f.write(content) - source_path = output_dir / ('_%s.pyx' % sha256sum) - if not source_path.exists(): - with io.open(str(source_path), 'w') as f: - f.write(content) + t1 = Cython.Build.Inline._get_build_extension() + t1.extensions = Cython.Build.cythonize(str(source_path)) + t1.build_temp = str(pathlib.Path('/')) + t1.build_lib = str(output_dir) + # t2 = Cython.Build.Inline.Extension( + # name=sha256sum, + # ) + t1.run() - t1 = Cython.Build.Inline._get_build_extension() - t1.extensions = Cython.Build.cythonize(str(source_path)) - t1.build_temp = str(pathlib.Path('/')) - t1.build_lib = str(output_dir) - #t2 = Cython.Build.Inline.Extension( - # name=sha256sum, - #) - t1.run() + return cast(M, Cython.Build.Inline.load_dynamic('_%s' % sha256sum, glob.glob(str(output_dir / ('_%s*.so' % sha256sum)))[0])) - return cast( - M, - Cython.Build.Inline.load_dynamic( - '_%s' % sha256sum, - glob.glob( - str(output_dir / ('_%s*.so' % sha256sum)) - )[0] - ) - ) + raise NotImplementedError - raise NotImplementedError def mypyc_build(file_path: pathlib.Path) -> Any: - import pathlib - import tempfile - import hashlib - import mypyc.build - import Cython.Build.Inline + import pathlib + import tempfile + import hashlib + import mypyc.build + import Cython.Build.Inline - assert isinstance(file_path, pathlib.Path) + assert isinstance(file_path, pathlib.Path) - #sha256sum = hashlib.sha256(content.encode('utf-8')).digest().hex() + # sha256sum = hashlib.sha256(content.encode('utf-8')).digest().hex() - #output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute() - output_dir = pathlib.Path('.') / 'tmp' / 'mypyc' - sha256sum = file_path.stem - lib_pattern = file_path.parent / ('%s.cpython*.so' % sha256sum) - lib_dir = pathlib.Path('.') + # output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute() + output_dir = pathlib.Path('.') / 'tmp' / 'mypyc' + sha256sum = file_path.stem + lib_pattern = file_path.parent / ('%s.cpython*.so' % sha256sum) + lib_dir = pathlib.Path('.') + def lib_path_glob(path: str | pathlib.Path) -> Optional[pathlib.Path]: + res: list[str] = glob.glob(str(path)) - def lib_path_glob(path: str | pathlib.Path) -> Optional[pathlib.Path]: - res : list[str] = glob.glob(str(path)) + if len(res) == 0: + return None + else: + return pathlib.Path(res[0]) - if len(res) == 0: - return None - else: - return pathlib.Path(res[0]) + need_build: bool = False - need_build : bool = False + lib_path: Optional[pathlib.Path] = None - lib_path : Optional[pathlib.Path] = None + lib_path = lib_path_glob(lib_pattern) - lib_path = lib_path_glob(lib_pattern) + if not lib_path is None: + t2 = file_path.stat() + t3 = lib_path.stat() + if t3.st_mtime < t2.st_mtime: + need_build = True - if not lib_path is None: - t2 = file_path.stat() - t3 = lib_path.stat() - if t3.st_mtime < t2.st_mtime: - need_build = True + del t2 + del t3 + else: + need_build = True - del t2 - del t3 - else: - need_build = True + if need_build: + for o in [ + output_dir, + output_dir / 'build' / file_path.parent, + ]: + os.makedirs(str(o), exist_ok=True) + # source_path = output_dir / ('_%s.py' % sha256sum) + source_path = file_path + # with io.open(str(source_path), 'w') as f: + # f.write(content) + t1 = Cython.Build.Inline._get_build_extension() + t1.extensions = mypyc.build.mypycify([str(source_path)], target_dir=str(output_dir / 'build')) + t1.build_temp = str(output_dir) + t1.build_lib = str(lib_dir) + # t2 = Cython.Build.Inline.Extension( + # name=sha256sum, + # ) + t1.run() - if need_build: - for o in [ - output_dir, - output_dir / 'build' / file_path.parent, - ]: - os.makedirs( - str(o), - exist_ok=True - ) - #source_path = output_dir / ('_%s.py' % sha256sum) - source_path = file_path - #with io.open(str(source_path), 'w') as f: - # f.write(content) + lib_path = lib_path_glob(lib_pattern) - t1 = Cython.Build.Inline._get_build_extension() - t1.extensions = mypyc.build.mypycify( - [str(source_path)], - target_dir=str(output_dir / 'build') - ) - t1.build_temp = str(output_dir) - t1.build_lib = str(lib_dir) - #t2 = Cython.Build.Inline.Extension( - # name=sha256sum, - #) - t1.run() + return Cython.Build.Inline.load_dynamic( + #'_%s' % sha256sum, + # t1.extensions[0].name, + file_path.stem, + str(lib_path), + ) - lib_path = lib_path_glob(lib_pattern) + raise NotImplementedError - return Cython.Build.Inline.load_dynamic( - #'_%s' % sha256sum, - #t1.extensions[0].name, - file_path.stem, - str(lib_path), - ) - - raise NotImplementedError class Source: - @staticmethod - def test2( - _a : numpy.ndarray[Any, numpy.dtype[numpy.int64]], - _id : numpy.dtype[numpy.int32] | int, - T : float=16 - ) -> int: - raise NotImplementedError + @staticmethod + def test2(_a: numpy.ndarray[Any, numpy.dtype[numpy.int64]], _id: numpy.dtype[numpy.int32] | int, T: float = 16) -> int: + raise NotImplementedError -source = build(r''' +source = build( + r""" cimport cython @cython.boundscheck(False) @@ -226,52 +213,52 @@ def test2(long long [:] _a, int _id, double T=16) -> int: return _a[_id] -''', Source) +""", + Source, +) -def test_cython(N: int=4, T:int=16) -> None: - #a = [0] * N - a = numpy.zeros((N,), dtype=numpy.int64) - t = [ - threading.Thread( - target=functools.partial( - source.test2, - a, - k, - T, - ) - ) - for k in range(N) - ] +def test_cython(N: int = 4, T: int = 16) -> None: + # a = [0] * N + a = numpy.zeros((N,), dtype=numpy.int64) - for o in t: - o.start() - for o in t: - o.join() + t = [ + threading.Thread( + target=functools.partial( + source.test2, + a, + k, + T, + ) + ) + for k in range(N) + ] - #cython_module['test2'](a, 0) + for o in t: + o.start() + for o in t: + o.join() -def test_mypyc(N: int=4, W:int=35) -> None: - cython2 = mypyc_build( - (pathlib.Path(__file__).parent / 'cython2.py').relative_to( - pathlib.Path.cwd() - ) - ) + # cython_module['test2'](a, 0) - # from .cython2 import fib - #a = [0] * N - t = [ - threading.Thread( - target=functools.partial( - cython2.fib, - W, - ) - ) - for k in range(N) - ] +def test_mypyc(N: int = 4, W: int = 35) -> None: + cython2 = mypyc_build((pathlib.Path(__file__).parent / 'cython2.py').relative_to(pathlib.Path.cwd())) - for o in t: - o.start() - for o in t: - o.join() + # from .cython2 import fib + + # a = [0] * N + t = [ + threading.Thread( + target=functools.partial( + cython2.fib, + W, + ) + ) + for k in range(N) + ] + + for o in t: + o.start() + for o in t: + o.join() diff --git a/python/online/fxreader/pr34/tasks/cython2.py b/python/online/fxreader/pr34/tasks/cython2.py index 7b2fb48..a3baa86 100644 --- a/python/online/fxreader/pr34/tasks/cython2.py +++ b/python/online/fxreader/pr34/tasks/cython2.py @@ -1,10 +1,12 @@ import time + def fib(n: int) -> int: - if n <= 1: - return n - else: - return fib(n - 2) + fib(n - 1) + if n <= 1: + return n + else: + return fib(n - 2) + fib(n - 1) + t0 = time.time() fib(32) diff --git a/python/online/fxreader/pr34/tasks/jigsaw_toxic.py b/python/online/fxreader/pr34/tasks/jigsaw_toxic.py index 8cfe331..2daac22 100644 --- a/python/online/fxreader/pr34/tasks/jigsaw_toxic.py +++ b/python/online/fxreader/pr34/tasks/jigsaw_toxic.py @@ -5,378 +5,334 @@ import os def kernel_1_sample_scrap( - max_articles=None, + max_articles=None, ): - if max_articles is None: - max_articles = 1 + if max_articles is None: + max_articles = 1 - with requests.get( - 'https://dev.to', - ) as p: - t10 = p.content.decode('utf-8') - t11 = pyquery.PyQuery(t10) - t13 = t11('.crayons-story__title > a') - t12 = [ - pyquery.PyQuery(o).attr('href') - for o in t13 - ] - pprint.pprint(t12) - t14 = [ - 'https://dev.to/%s' % o - for o in t12 - ] + with requests.get( + 'https://dev.to', + ) as p: + t10 = p.content.decode('utf-8') + t11 = pyquery.PyQuery(t10) + t13 = t11('.crayons-story__title > a') + t12 = [pyquery.PyQuery(o).attr('href') for o in t13] + pprint.pprint(t12) + t14 = ['https://dev.to/%s' % o for o in t12] - t8 = [] - for t7 in t14[:max_articles]: - with requests.get( - t7, - ) as p: - t1 = p.content.decode('utf-8') - t2 = pyquery.PyQuery(t1) - t3 = t2('.comment__content') - t6 = [] - for o in t3: - t4 = pyquery.PyQuery(o) - t5 = t4('.comment__header > a').attr['href'] - t9 = t4('.comment__body').text() - t6.append( - dict( - author=t5, - text=t9, - ) - ) + t8 = [] + for t7 in t14[:max_articles]: + with requests.get( + t7, + ) as p: + t1 = p.content.decode('utf-8') + t2 = pyquery.PyQuery(t1) + t3 = t2('.comment__content') + t6 = [] + for o in t3: + t4 = pyquery.PyQuery(o) + t5 = t4('.comment__header > a').attr['href'] + t9 = t4('.comment__body').text() + t6.append( + dict( + author=t5, + text=t9, + ) + ) - #pprint.pprint(t3) - pprint.pprint(t6) - t8.append( - dict( - article=t7, - comments=t6, - ) - ) + # pprint.pprint(t3) + pprint.pprint(t6) + t8.append( + dict( + article=t7, + comments=t6, + ) + ) - pprint.pprint(t8) + pprint.pprint(t8) + + return dict( + t1=t1, + t2=t2, + t3=t3, + t6=t6, + t8=t8, + t12=t12, + ) - return dict( - t1=t1, - t2=t2, - t3=t3, - t6=t6, - t8=t8, - t12=t12, - ) def kernel_2(): - import numpy as np # linear algebra - import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) - from tqdm import tqdm - from sklearn.model_selection import train_test_split - import tensorflow as tf - from keras.models import Sequential - from keras.layers.recurrent import LSTM, GRU,SimpleRNN - from keras.layers.core import Dense, Activation, Dropout - from keras.layers.embeddings import Embedding - from keras.layers.normalization import BatchNormalization - from keras.utils import np_utils - from sklearn import preprocessing, decomposition, model_selection, metrics, pipeline - from keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D - from keras.preprocessing import sequence, text - from keras.callbacks import EarlyStopping + import numpy as np # linear algebra + import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) + from tqdm import tqdm + from sklearn.model_selection import train_test_split + import tensorflow as tf + from keras.models import Sequential + from keras.layers.recurrent import LSTM, GRU, SimpleRNN + from keras.layers.core import Dense, Activation, Dropout + from keras.layers.embeddings import Embedding + from keras.layers.normalization import BatchNormalization + from keras.utils import np_utils + from sklearn import preprocessing, decomposition, model_selection, metrics, pipeline + from keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D + from keras.preprocessing import sequence, text + from keras.callbacks import EarlyStopping + import matplotlib.pyplot as plt + import seaborn as sns - import matplotlib.pyplot as plt - import seaborn as sns - #%matplotlib inline - from plotly import graph_objs as go - import plotly.express as px - import plotly.figure_factory as ff + # %matplotlib inline + from plotly import graph_objs as go + import plotly.express as px + import plotly.figure_factory as ff - # %% [markdown] - # # Configuring TPU's - # - # For this version of Notebook we will be using TPU's as we have to built a BERT Model + # %% [markdown] + # # Configuring TPU's + # + # For this version of Notebook we will be using TPU's as we have to built a BERT Model - # %% [code] - # Detect hardware, return appropriate distribution strategy - try: - # TPU detection. No parameters necessary if TPU_NAME environment variable is - # set: this is always the case on Kaggle. - tpu = tf.distribute.cluster_resolver.TPUClusterResolver() - print('Running on TPU ', tpu.master()) - except ValueError: - tpu = None + # %% [code] + # Detect hardware, return appropriate distribution strategy + try: + # TPU detection. No parameters necessary if TPU_NAME environment variable is + # set: this is always the case on Kaggle. + tpu = tf.distribute.cluster_resolver.TPUClusterResolver() + print('Running on TPU ', tpu.master()) + except ValueError: + tpu = None - if tpu: - tf.config.experimental_connect_to_cluster(tpu) - tf.tpu.experimental.initialize_tpu_system(tpu) - strategy = tf.distribute.experimental.TPUStrategy(tpu) - else: - # Default distribution strategy in Tensorflow. Works on CPU and single GPU. - strategy = tf.distribute.get_strategy() + if tpu: + tf.config.experimental_connect_to_cluster(tpu) + tf.tpu.experimental.initialize_tpu_system(tpu) + strategy = tf.distribute.experimental.TPUStrategy(tpu) + else: + # Default distribution strategy in Tensorflow. Works on CPU and single GPU. + strategy = tf.distribute.get_strategy() - print("REPLICAS: ", strategy.num_replicas_in_sync) + print('REPLICAS: ', strategy.num_replicas_in_sync) - # %% [code] - train = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv') - validation = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv') - test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv') + # %% [code] + train = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv') + validation = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv') + test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv') - # %% [markdown] - # We will drop the other columns and approach this problem as a Binary Classification Problem and also we will have our exercise done on a smaller subsection of the dataset(only 12000 data points) to make it easier to train the models + # %% [markdown] + # We will drop the other columns and approach this problem as a Binary Classification Problem and also we will have our exercise done on a smaller subsection of the dataset(only 12000 data points) to make it easier to train the models - # %% [code] - train.drop(['severe_toxic','obscene','threat','insult','identity_hate'],axis=1,inplace=True) + # %% [code] + train.drop(['severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'], axis=1, inplace=True) - # %% [code] - train = train.loc[:12000,:] - train.shape + # %% [code] + train = train.loc[:12000, :] + train.shape - # %% [markdown] - # We will check the maximum number of words that can be present in a comment , this will help us in padding later + # %% [markdown] + # We will check the maximum number of words that can be present in a comment , this will help us in padding later - # %% [code] - train['comment_text'].apply(lambda x:len(str(x).split())).max() + # %% [code] + train['comment_text'].apply(lambda x: len(str(x).split())).max() + # %% [markdown] + # ### Data Preparation - # %% [markdown] - # ### Data Preparation + # %% [code] + xtrain, xvalid, ytrain, yvalid = train_test_split( + train.comment_text.values, train.toxic.values, stratify=train.toxic.values, random_state=42, test_size=0.2, shuffle=True + ) - # %% [code] - xtrain, xvalid, ytrain, yvalid = train_test_split(train.comment_text.values, train.toxic.values, - stratify=train.toxic.values, - random_state=42, - test_size=0.2, shuffle=True) + # %% [markdown] + # # Before We Begin + # + # Before we Begin If you are a complete starter with NLP and never worked with text data, I am attaching a few kernels that will serve as a starting point of your journey + # * https://www.kaggle.com/arthurtok/spooky-nlp-and-topic-modelling-tutorial + # * https://www.kaggle.com/abhishek/approaching-almost-any-nlp-problem-on-kaggle + # + # If you want a more basic dataset to practice with here is another kernel which I wrote: + # * https://www.kaggle.com/tanulsingh077/what-s-cooking + # + # Below are some Resources to get started with basic level Neural Networks, It will help us to easily understand the upcoming parts + # * https://www.youtube.com/watch?v=aircAruvnKk&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv + # * https://www.youtube.com/watch?v=IHZwWFHWa-w&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=2 + # * https://www.youtube.com/watch?v=Ilg3gGewQ5U&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=3 + # * https://www.youtube.com/watch?v=tIeHLnjs5U8&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=4 + # + # For Learning how to visualize test data and what to use view: + # * https://www.kaggle.com/tanulsingh077/twitter-sentiment-extaction-analysis-eda-and-model + # * https://www.kaggle.com/jagangupta/stop-the-s-toxic-comments-eda - # %% [markdown] - # # Before We Begin - # - # Before we Begin If you are a complete starter with NLP and never worked with text data, I am attaching a few kernels that will serve as a starting point of your journey - # * https://www.kaggle.com/arthurtok/spooky-nlp-and-topic-modelling-tutorial - # * https://www.kaggle.com/abhishek/approaching-almost-any-nlp-problem-on-kaggle - # - # If you want a more basic dataset to practice with here is another kernel which I wrote: - # * https://www.kaggle.com/tanulsingh077/what-s-cooking - # - # Below are some Resources to get started with basic level Neural Networks, It will help us to easily understand the upcoming parts - # * https://www.youtube.com/watch?v=aircAruvnKk&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv - # * https://www.youtube.com/watch?v=IHZwWFHWa-w&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=2 - # * https://www.youtube.com/watch?v=Ilg3gGewQ5U&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=3 - # * https://www.youtube.com/watch?v=tIeHLnjs5U8&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=4 - # - # For Learning how to visualize test data and what to use view: - # * https://www.kaggle.com/tanulsingh077/twitter-sentiment-extaction-analysis-eda-and-model - # * https://www.kaggle.com/jagangupta/stop-the-s-toxic-comments-eda + # %% [markdown] + # # Simple RNN + # + # ## Basic Overview + # + # What is a RNN? + # + # Recurrent Neural Network(RNN) are a type of Neural Network where the output from previous step are fed as input to the current step. In traditional neural networks, all the inputs and outputs are independent of each other, but in cases like when it is required to predict the next word of a sentence, the previous words are required and hence there is a need to remember the previous words. Thus RNN came into existence, which solved this issue with the help of a Hidden Layer. + # + # Why RNN's? + # + # https://www.quora.com/Why-do-we-use-an-RNN-instead-of-a-simple-neural-network + # + # ## In-Depth Understanding + # + # * https://medium.com/mindorks/understanding-the-recurrent-neural-network-44d593f112a2 + # * https://www.youtube.com/watch?v=2E65LDnM2cA&list=PL1F3ABbhcqa3BBWo170U4Ev2wfsF7FN8l + # * https://www.d2l.ai/chapter_recurrent-neural-networks/rnn.html + # + # ## Code Implementation + # + # So first I will implement the and then I will explain the code step by step - # %% [markdown] - # # Simple RNN - # - # ## Basic Overview - # - # What is a RNN? - # - # Recurrent Neural Network(RNN) are a type of Neural Network where the output from previous step are fed as input to the current step. In traditional neural networks, all the inputs and outputs are independent of each other, but in cases like when it is required to predict the next word of a sentence, the previous words are required and hence there is a need to remember the previous words. Thus RNN came into existence, which solved this issue with the help of a Hidden Layer. - # - # Why RNN's? - # - # https://www.quora.com/Why-do-we-use-an-RNN-instead-of-a-simple-neural-network - # - # ## In-Depth Understanding - # - # * https://medium.com/mindorks/understanding-the-recurrent-neural-network-44d593f112a2 - # * https://www.youtube.com/watch?v=2E65LDnM2cA&list=PL1F3ABbhcqa3BBWo170U4Ev2wfsF7FN8l - # * https://www.d2l.ai/chapter_recurrent-neural-networks/rnn.html - # - # ## Code Implementation - # - # So first I will implement the and then I will explain the code step by step + # %% [code] + # using keras tokenizer here + token = text.Tokenizer(num_words=None) + max_len = 1500 - # %% [code] - # using keras tokenizer here - token = text.Tokenizer(num_words=None) - max_len = 1500 + token.fit_on_texts(list(xtrain) + list(xvalid)) + xtrain_seq = token.texts_to_sequences(xtrain) + xvalid_seq = token.texts_to_sequences(xvalid) - token.fit_on_texts(list(xtrain) + list(xvalid)) - xtrain_seq = token.texts_to_sequences(xtrain) - xvalid_seq = token.texts_to_sequences(xvalid) + # zero pad the sequences + xtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len) + xvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len) - #zero pad the sequences - xtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len) - xvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len) + word_index = token.word_index - word_index = token.word_index + # %% [code] + # %%time + with strategy.scope(): + # A simpleRNN without any pretrained embeddings and one dense layer + model = Sequential() + model.add(Embedding(len(word_index) + 1, 300, input_length=max_len)) + model.add(SimpleRNN(100)) + model.add(Dense(1, activation='sigmoid')) + model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) - # %% [code] - #%%time - with strategy.scope(): - # A simpleRNN without any pretrained embeddings and one dense layer - model = Sequential() - model.add(Embedding(len(word_index) + 1, - 300, - input_length=max_len)) - model.add(SimpleRNN(100)) - model.add(Dense(1, activation='sigmoid')) - model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) + model.summary() - model.summary() - - return dict( - model=model, - xtrain_pad=xtrain_pad, - strategy=strategy, - xvalid_pad=xvalid_pad, - xtrain_seq=xtrain_seq, - token=token, - max_len=max_len, - xtrain=xtrain, - xvalid=xvalid, - ytrain=ytrain, - yvalid=yvalid, - ) + return dict( + model=model, + xtrain_pad=xtrain_pad, + strategy=strategy, + xvalid_pad=xvalid_pad, + xtrain_seq=xtrain_seq, + token=token, + max_len=max_len, + xtrain=xtrain, + xvalid=xvalid, + ytrain=ytrain, + yvalid=yvalid, + ) def kernel_3( - o_2, - nb_epochs=None, + o_2, + nb_epochs=None, ): - if nb_epochs is None: - nb_epochs = 5 + if nb_epochs is None: + nb_epochs = 5 - # %% [markdown] - # Writing a function for getting auc score for validation + # %% [markdown] + # Writing a function for getting auc score for validation - # %% [code] - def roc_auc(predictions,target): - import sklearn.metrics - ''' + # %% [code] + def roc_auc(predictions, target): + import sklearn.metrics + + """ This methods returns the AUC Score when given the Predictions and Labels - ''' + """ - fpr, tpr, thresholds = sklearn.metrics.roc_curve(target, predictions) - roc_auc = sklearn.metrics.auc(fpr, tpr) - return roc_auc + fpr, tpr, thresholds = sklearn.metrics.roc_curve(target, predictions) + roc_auc = sklearn.metrics.auc(fpr, tpr) + return roc_auc - # %% [code] - if os.path.exists('model.h5'): - o_2['model'].load_weights('model.h5') - else: - o_2['model'].fit( - o_2['xtrain_pad'], - o_2['ytrain'], - nb_epoch=nb_epochs, - batch_size=64*o_2['strategy'].num_replicas_in_sync - ) #Multiplying by Strategy to run on TPU's - o_2['model'].save_weights('model.h5') + # %% [code] + if os.path.exists('model.h5'): + o_2['model'].load_weights('model.h5') + else: + o_2['model'].fit( + o_2['xtrain_pad'], o_2['ytrain'], nb_epoch=nb_epochs, batch_size=64 * o_2['strategy'].num_replicas_in_sync + ) # Multiplying by Strategy to run on TPU's + o_2['model'].save_weights('model.h5') - # %% [code] - scores = o_2['model'].predict(o_2['xvalid_pad']) - print( - "Auc: %.2f%%" % ( - roc_auc( - scores, - o_2['yvalid'] - ) - ) - ) + # %% [code] + scores = o_2['model'].predict(o_2['xvalid_pad']) + print('Auc: %.2f%%' % (roc_auc(scores, o_2['yvalid']))) - # %% [code] - scores_model = [] - scores_model.append( - { - 'Model': 'SimpleRNN', - 'AUC_Score': roc_auc( - scores, - o_2['yvalid'] - ) - } - ) + # %% [code] + scores_model = [] + scores_model.append({'Model': 'SimpleRNN', 'AUC_Score': roc_auc(scores, o_2['yvalid'])}) - # %% [markdown] - # ## Code Explanantion - # * Tokenization

- # So if you have watched the videos and referred to the links, you would know that in an RNN we input a sentence word by word. We represent every word as one hot vectors of dimensions : Numbers of words in Vocab +1.
- # What keras Tokenizer does is , it takes all the unique words in the corpus,forms a dictionary with words as keys and their number of occurences as values,it then sorts the dictionary in descending order of counts. It then assigns the first value 1 , second value 2 and so on. So let's suppose word 'the' occured the most in the corpus then it will assigned index 1 and vector representing 'the' would be a one-hot vector with value 1 at position 1 and rest zereos.
- # Try printing first 2 elements of xtrain_seq you will see every word is represented as a digit now + # %% [markdown] + # ## Code Explanantion + # * Tokenization

+ # So if you have watched the videos and referred to the links, you would know that in an RNN we input a sentence word by word. We represent every word as one hot vectors of dimensions : Numbers of words in Vocab +1.
+ # What keras Tokenizer does is , it takes all the unique words in the corpus,forms a dictionary with words as keys and their number of occurences as values,it then sorts the dictionary in descending order of counts. It then assigns the first value 1 , second value 2 and so on. So let's suppose word 'the' occured the most in the corpus then it will assigned index 1 and vector representing 'the' would be a one-hot vector with value 1 at position 1 and rest zereos.
+ # Try printing first 2 elements of xtrain_seq you will see every word is represented as a digit now + + # %% [code] + o_2['xtrain_seq'][:1] - # %% [code] - o_2['xtrain_seq'][:1] def kernel_4( - o_2, - input_texts=None, + o_2, + input_texts=None, ): - import keras.preprocessing.sequence + import keras.preprocessing.sequence - if input_texts is None: - input_texts = [ - 'blahb blahb blah', - 'Hello World!', - 'This is very good!', - 'A very non toxic comment! This is so polite and polished one!' - ] + if input_texts is None: + input_texts = ['blahb blahb blah', 'Hello World!', 'This is very good!', 'A very non toxic comment! This is so polite and polished one!'] - t6 = [] - for o in input_texts: - t1 = o - t2 = o_2['token'].texts_to_sequences( - [t1], - ) - t3 = keras.preprocessing.sequence.pad_sequences( - t2, - maxlen=o_2['max_len'] - ) - t4 = o_2['model'].predict( - t3, - ) - t6.append( - dict( - text=o, - score=t4[0][0], - ) - ) - pprint.pprint( - dict( - t1=t1, - t2=t2, - t3=t3, - t4=t4, - ) - ) - pprint.pprint(t6) + t6 = [] + for o in input_texts: + t1 = o + t2 = o_2['token'].texts_to_sequences( + [t1], + ) + t3 = keras.preprocessing.sequence.pad_sequences(t2, maxlen=o_2['max_len']) + t4 = o_2['model'].predict( + t3, + ) + t6.append( + dict( + text=o, + score=t4[0][0], + ) + ) + pprint.pprint( + dict( + t1=t1, + t2=t2, + t3=t3, + t4=t4, + ) + ) + pprint.pprint(t6) + + return dict( + t6=t6, + ) - return dict( - t6=t6, - ) def kernel_5( - o_1=None, - o_2=None, + o_1=None, + o_2=None, ): - if o_1 is None: - o_1 = kernel_1_sample_scrap(max_articles=50) + if o_1 is None: + o_1 = kernel_1_sample_scrap(max_articles=50) - if o_2 is None: - o_2 = kernel_2() - o_3 = kernel_3( - o_2=o_2, - nb_epochs=1 - ) + if o_2 is None: + o_2 = kernel_2() + o_3 = kernel_3(o_2=o_2, nb_epochs=1) - t1 = sum( - [ - [ - o['text'] for o in o2['comments'] - ] for o2 in o_1['t8'] - ], [] - ) + t1 = sum([[o['text'] for o in o2['comments']] for o2 in o_1['t8']], []) - t2 = kernel_4( - o_2=o_2, - input_texts=t1 - ) + t2 = kernel_4(o_2=o_2, input_texts=t1) - t3 = sorted( - t2['t6'], - key=lambda x: x['score'], - ) - pprint.pprint(t3) + t3 = sorted( + t2['t6'], + key=lambda x: x['score'], + ) + pprint.pprint(t3) diff --git a/python/online/fxreader/pr34/tasks/mlb_player.py b/python/online/fxreader/pr34/tasks/mlb_player.py index 38780c9..35701ef 100644 --- a/python/online/fxreader/pr34/tasks/mlb_player.py +++ b/python/online/fxreader/pr34/tasks/mlb_player.py @@ -13,157 +13,119 @@ import pandas import pickle import subprocess + def kernel_1(): - t4 = 'kernel_1-t3.dat' + t4 = 'kernel_1-t3.dat' - def preprocess(t4): - t1 = '/kaggle/input/mlb-player-digital-engagement-forecasting' - t2 = glob.glob( - os.path.join( - t1, - '*.csv' - ) - ) + def preprocess(t4): + t1 = '/kaggle/input/mlb-player-digital-engagement-forecasting' + t2 = glob.glob(os.path.join(t1, '*.csv')) - t3 = { - o : pandas.read_csv(o).to_xarray() - for o in t2 - } + t3 = {o: pandas.read_csv(o).to_xarray() for o in t2} - with io.open(t4, 'wb') as f: - pickle.dump(t3, f) + with io.open(t4, 'wb') as f: + pickle.dump(t3, f) - if not os.path.exists(t4): - preprocess(t4=t4) + if not os.path.exists(t4): + preprocess(t4=t4) - with io.open(t4, 'rb') as f: - t3 = pickle.load(f) + with io.open(t4, 'rb') as f: + t3 = pickle.load(f) + return dict( + t3=t3, + ) - return dict( - t3=t3, - ) def kernel_2( - o_1=None, + o_1=None, ): - t1 = {} + t1 = {} - for k in [ - 'playerTwitterFollowers', - 'teamTwitterFollowers', - 'games', - 'events' - ]: - t4 = '%s.nc' % k - if not os.path.exists(t4): - print('started %s' % t4) - t2 = '/kaggle/input/mlb-player-digital-engagement-forecasting/train.csv' - t3 = pandas.DataFrame( - sum( - [ - json.loads(o) - for o in o_1['t3'][t2][k].values - if isinstance(o, str) - ], - [] - ) - ).to_xarray() - t3.to_netcdf(t4) - print('cached %s' % t4) + for k in ['playerTwitterFollowers', 'teamTwitterFollowers', 'games', 'events']: + t4 = '%s.nc' % k + if not os.path.exists(t4): + print('started %s' % t4) + t2 = '/kaggle/input/mlb-player-digital-engagement-forecasting/train.csv' + t3 = pandas.DataFrame(sum([json.loads(o) for o in o_1['t3'][t2][k].values if isinstance(o, str)], [])).to_xarray() + t3.to_netcdf(t4) + print('cached %s' % t4) - if k == 'events': - t5 = '%s-v2.nc' % k - if not os.path.exists(t5): - t2 = xarray.load_dataset(t4) - t3 = t2.sel( - index=numpy.arange( - 2017653 - 10 * 1000, - 2017653 + 1 - ) - ) - t3.to_netcdf(t5) - t1[k] = xarray.load_dataset(t5) - print('loaded %s' % t5) - else: - t1[k] = xarray.load_dataset(t4) - print('loaded %s' % t4) + if k == 'events': + t5 = '%s-v2.nc' % k + if not os.path.exists(t5): + t2 = xarray.load_dataset(t4) + t3 = t2.sel(index=numpy.arange(2017653 - 10 * 1000, 2017653 + 1)) + t3.to_netcdf(t5) + t1[k] = xarray.load_dataset(t5) + print('loaded %s' % t5) + else: + t1[k] = xarray.load_dataset(t4) + print('loaded %s' % t4) + return dict( + t1=t1, + ) - return dict( - t1=t1, - ) def kernel_3(should_exist=None): - if should_exist is None: - should_exist = False + if should_exist is None: + should_exist = False - t3 = [ - ('playerTwitterFollowers', None), - ('teamTwitterFollowers', None), - ('games', None), - ('events', 'events-v2.nc'), - ] + t3 = [ + ('playerTwitterFollowers', None), + ('teamTwitterFollowers', None), + ('games', None), + ('events', 'events-v2.nc'), + ] - o_1 = None - o_2 = None + o_1 = None + o_2 = None - t4 = '/kaggle/input/garbage' - t5 = {} - for k, v in t3: - if v is None: - t1 = os.path.join( - t4, - '%s.nc' % k, - ) - else: - t1 = os.path.join( - t4, - v, - ) + t4 = '/kaggle/input/garbage' + t5 = {} + for k, v in t3: + if v is None: + t1 = os.path.join( + t4, + '%s.nc' % k, + ) + else: + t1 = os.path.join( + t4, + v, + ) - if os.path.exists(t1): - t2 = xarray.load_dataset(t1) - else: - if should_exist: - pprint.pprint([k, v, t1]) - raise NotImplementedError + if os.path.exists(t1): + t2 = xarray.load_dataset(t1) + else: + if should_exist: + pprint.pprint([k, v, t1]) + raise NotImplementedError - if o_1 is None: - o_1 = kernel_1() - if o_2 is None: - o_2 = kernel_2( - o_1=o_1 - ) + if o_1 is None: + o_1 = kernel_1() + if o_2 is None: + o_2 = kernel_2(o_1=o_1) - t2 = o_2['events'] - t5[k] = t2 + t2 = o_2['events'] + t5[k] = t2 + + return dict( + t5=t5, + ) - return dict( - t5=t5, - ) def kernel_4( - o_3=None, + o_3=None, ): - [ - print( - o_3['t5']['events'].to_dataframe().iloc[k].to_json(indent=4) - ) - for k in range(-10, -1) - ] + [print(o_3['t5']['events'].to_dataframe().iloc[k].to_json(indent=4)) for k in range(-10, -1)] - [ - print( - o_3['t5']['games'].to_dataframe().iloc[k].to_json(indent=4) - ) - for k in range(-10, -1) - ] + [print(o_3['t5']['games'].to_dataframe().iloc[k].to_json(indent=4)) for k in range(-10, -1)] + t4 = 'https://www.youtube.com/watch?v=reaC7BHgL3M' - t4 = 'https://www.youtube.com/watch?v=reaC7BHgL3M' - - r""" + r""" { "gamePk":634280, "gameType":"R", @@ -200,1034 +162,1059 @@ def kernel_4( } """ - t1 = numpy.where(o_3['t5']['events']['gamePk'] == 634280)[0] - t5 = o_3['t5']['events'].index.data - t6 = t5[t1] - t2 = o_3['t5']['events'].sel(index=t6) - t3 = o_3['t5']['games'].to_dataframe().iloc[-2].to_dict() - pprint.pprint(t3) - assert t3['gamePk'] == 634280 + t1 = numpy.where(o_3['t5']['events']['gamePk'] == 634280)[0] + t5 = o_3['t5']['events'].index.data + t6 = t5[t1] + t2 = o_3['t5']['events'].sel(index=t6) + t3 = o_3['t5']['games'].to_dataframe().iloc[-2].to_dict() + pprint.pprint(t3) + assert t3['gamePk'] == 634280 - t7 = 'https://www.youtube.com/watch?v=T0MUK91ZWys' + t7 = 'https://www.youtube.com/watch?v=T0MUK91ZWys' + + return dict( + t2=t2, + t3=t3, + t4=t4, + t7=t7, + ) - return dict( - t2=t2, - t3=t3, - t4=t4, - t7=t7, - ) def kernel_5(o_4): - for o in [o_4['t4'], o_4['t7']]: - subprocess.check_call( - [ - 'youtube-dl', - '-f', - '18', - o, - ] - ) + for o in [o_4['t4'], o_4['t7']]: + subprocess.check_call( + [ + 'youtube-dl', + '-f', + '18', + o, + ] + ) + def kernel_12(): - import easyocr - t6 = easyocr.Reader(['en']) + import easyocr + + t6 = easyocr.Reader(['en']) + + return dict( + t6=t6, + ) - return dict( - t6=t6, - ) def kernel_6( - o_7=None, - o_10=None, - o_12=None, - max_frames=None, + o_7=None, + o_10=None, + o_12=None, + max_frames=None, ): - if max_frames is None: - max_frames = 10 + if max_frames is None: + max_frames = 10 - import tqdm - import cv2 + import tqdm + import cv2 - t1 = glob.glob('*.mp4') + t1 = glob.glob('*.mp4') - t8 = [] - for o in t1: - t7 = [] - t2 = None - try: - t2 = cv2.VideoCapture(o) - for k in tqdm.tqdm(range(max_frames)): - t3 = t2.read() - assert t3[0] - t4 = t3[1] - if not o_12 is None: - t5 = o_12['t6'].readtext(t4) - else: - t5 = None + t8 = [] + for o in t1: + t7 = [] + t2 = None + try: + t2 = cv2.VideoCapture(o) + for k in tqdm.tqdm(range(max_frames)): + t3 = t2.read() + assert t3[0] + t4 = t3[1] + if not o_12 is None: + t5 = o_12['t6'].readtext(t4) + else: + t5 = None - if not o_7 is None: - t10 = o_7['estimate_pose'](t4) - else: - t10 = None - if not o_10 is None: - t11 = o_10['model'](t4).pandas().xywhn - else: - t11 = None + if not o_7 is None: + t10 = o_7['estimate_pose'](t4) + else: + t10 = None + if not o_10 is None: + t11 = o_10['model'](t4).pandas().xywhn + else: + t11 = None - t7.append( - dict( - frame_id=k, - t5=t5, - t10=t10, - t11=t11, - ), - ) - finally: - if not t2 is None: - t2.release() - t8.append( - dict( - video_path=o, - frames=t7, - ) - ) + t7.append( + dict( + frame_id=k, + t5=t5, + t10=t10, + t11=t11, + ), + ) + finally: + if not t2 is None: + t2.release() + t8.append( + dict( + video_path=o, + frames=t7, + ) + ) - t9 = [] - for o in t1: - cap = None + t9 = [] + for o in t1: + cap = None - try: - cap = cv2.VideoCapture(o) - fps = cap.get(cv2.CAP_PROP_FPS) # OpenCV2 version 2 used "CV_CAP_PROP_FPS" - frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - duration = frame_count/fps - finally: - if not cap is None: - cap.release() + try: + cap = cv2.VideoCapture(o) + fps = cap.get(cv2.CAP_PROP_FPS) # OpenCV2 version 2 used "CV_CAP_PROP_FPS" + frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + duration = frame_count / fps + finally: + if not cap is None: + cap.release() - t9.append( - dict( - video_path=o, - fps=fps, - frame_count=frame_count, - duration=duration, - ) - ) + t9.append( + dict( + video_path=o, + fps=fps, + frame_count=frame_count, + duration=duration, + ) + ) + return dict( + t8=t8, + t9=t9, + ) - return dict( - t8=t8, - t9=t9, - ) def kernel_7( - use_gpu=None, + use_gpu=None, ): - #!/usr/bin/env python - # coding: utf-8 - - # - # - # NOTE: Turn on Internet and GPU + #!/usr/bin/env python + # coding: utf-8 + + # + # + # NOTE: Turn on Internet and GPU + + # The code hidden below handles all the imports and function definitions (the heavy lifting). If you're a beginner I'd advice you skip this for now. When you are able to understand the rest of the code, come back here and understand each function to get a deeper knowledge. + + # In[1]: + + # !/usr/bin/env python3 + # coding=utf-8 + # author=dave.fang@outlook.com + # create=20171225 + + import os + import pprint + import cv2 + import sys + import math + import time + import tempfile + import numpy as np + import matplotlib.pyplot as plt + + import torch + import torch.nn as nn + import torch.nn.parallel + import torch.backends.cudnn as cudnn + import torch.optim as optim + import torchvision.transforms as transforms + import torchvision.datasets as datasets + import torchvision.models + + from torch.autograd import Variable + + from scipy.ndimage.filters import gaussian_filter + + # get_ipython().run_line_magic('matplotlib', 'inline') + # get_ipython().run_line_magic('config', "InlineBackend.figure_format = 'retina'") + + # find connection in the specified sequence, center 29 is in the position 15 + limb_seq = [ + [2, 3], + [2, 6], + [3, 4], + [4, 5], + [6, 7], + [7, 8], + [2, 9], + [9, 10], + [10, 11], + [2, 12], + [12, 13], + [13, 14], + [2, 1], + [1, 15], + [15, 17], + [1, 16], + [16, 18], + [3, 17], + [6, 18], + ] + + # the middle joints heatmap correpondence + map_ids = [ + [31, 32], + [39, 40], + [33, 34], + [35, 36], + [41, 42], + [43, 44], + [19, 20], + [21, 22], + [23, 24], + [25, 26], + [27, 28], + [29, 30], + [47, 48], + [49, 50], + [53, 54], + [51, 52], + [55, 56], + [37, 38], + [45, 46], + ] + + # these are the colours for the 18 body points + colors = [ + [255, 0, 0], + [255, 85, 0], + [255, 170, 0], + [255, 255, 0], + [170, 255, 0], + [85, 255, 0], + [0, 255, 0], + [0, 255, 85], + [0, 255, 170], + [0, 255, 255], + [0, 170, 255], + [0, 85, 255], + [0, 0, 255], + [85, 0, 255], + [170, 0, 255], + [255, 0, 255], + [255, 0, 170], + [255, 0, 85], + ] + + class PoseEstimation(nn.Module): + def __init__(self, model_dict): + super(PoseEstimation, self).__init__() + + self.model0 = model_dict['block_0'] + self.model1_1 = model_dict['block1_1'] + self.model2_1 = model_dict['block2_1'] + self.model3_1 = model_dict['block3_1'] + self.model4_1 = model_dict['block4_1'] + self.model5_1 = model_dict['block5_1'] + self.model6_1 = model_dict['block6_1'] + + self.model1_2 = model_dict['block1_2'] + self.model2_2 = model_dict['block2_2'] + self.model3_2 = model_dict['block3_2'] + self.model4_2 = model_dict['block4_2'] + self.model5_2 = model_dict['block5_2'] + self.model6_2 = model_dict['block6_2'] + + def forward(self, x): + out1 = self.model0(x) + + out1_1 = self.model1_1(out1) + out1_2 = self.model1_2(out1) + out2 = torch.cat([out1_1, out1_2, out1], 1) + + out2_1 = self.model2_1(out2) + out2_2 = self.model2_2(out2) + out3 = torch.cat([out2_1, out2_2, out1], 1) + + out3_1 = self.model3_1(out3) + out3_2 = self.model3_2(out3) + out4 = torch.cat([out3_1, out3_2, out1], 1) + + out4_1 = self.model4_1(out4) + out4_2 = self.model4_2(out4) + out5 = torch.cat([out4_1, out4_2, out1], 1) + + out5_1 = self.model5_1(out5) + out5_2 = self.model5_2(out5) + out6 = torch.cat([out5_1, out5_2, out1], 1) + + out6_1 = self.model6_1(out6) + out6_2 = self.model6_2(out6) + + return out6_1, out6_2 + + def make_layers(layer_dict): + layers = [] + + for i in range(len(layer_dict) - 1): + layer = layer_dict[i] + for k in layer: + v = layer[k] + if 'pool' in k: + layers += [nn.MaxPool2d(kernel_size=v[0], stride=v[1], padding=v[2])] + else: + conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1], kernel_size=v[2], stride=v[3], padding=v[4]) + layers += [conv2d, nn.ReLU(inplace=True)] + layer = list(layer_dict[-1].keys()) + k = layer[0] + v = layer_dict[-1][k] + + conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1], kernel_size=v[2], stride=v[3], padding=v[4]) + layers += [conv2d] + + return nn.Sequential(*layers) + + def get_pose_model(): + blocks = {} + + block_0 = [ + {'conv1_1': [3, 64, 3, 1, 1]}, + {'conv1_2': [64, 64, 3, 1, 1]}, + {'pool1_stage1': [2, 2, 0]}, + {'conv2_1': [64, 128, 3, 1, 1]}, + {'conv2_2': [128, 128, 3, 1, 1]}, + {'pool2_stage1': [2, 2, 0]}, + {'conv3_1': [128, 256, 3, 1, 1]}, + {'conv3_2': [256, 256, 3, 1, 1]}, + {'conv3_3': [256, 256, 3, 1, 1]}, + {'conv3_4': [256, 256, 3, 1, 1]}, + {'pool3_stage1': [2, 2, 0]}, + {'conv4_1': [256, 512, 3, 1, 1]}, + {'conv4_2': [512, 512, 3, 1, 1]}, + {'conv4_3_CPM': [512, 256, 3, 1, 1]}, + {'conv4_4_CPM': [256, 128, 3, 1, 1]}, + ] + + blocks['block1_1'] = [ + {'conv5_1_CPM_L1': [128, 128, 3, 1, 1]}, + {'conv5_2_CPM_L1': [128, 128, 3, 1, 1]}, + {'conv5_3_CPM_L1': [128, 128, 3, 1, 1]}, + {'conv5_4_CPM_L1': [128, 512, 1, 1, 0]}, + {'conv5_5_CPM_L1': [512, 38, 1, 1, 0]}, + ] + + blocks['block1_2'] = [ + {'conv5_1_CPM_L2': [128, 128, 3, 1, 1]}, + {'conv5_2_CPM_L2': [128, 128, 3, 1, 1]}, + {'conv5_3_CPM_L2': [128, 128, 3, 1, 1]}, + {'conv5_4_CPM_L2': [128, 512, 1, 1, 0]}, + {'conv5_5_CPM_L2': [512, 19, 1, 1, 0]}, + ] + + for i in range(2, 7): + blocks['block%d_1' % i] = [ + {'Mconv1_stage%d_L1' % i: [185, 128, 7, 1, 3]}, + {'Mconv2_stage%d_L1' % i: [128, 128, 7, 1, 3]}, + {'Mconv3_stage%d_L1' % i: [128, 128, 7, 1, 3]}, + {'Mconv4_stage%d_L1' % i: [128, 128, 7, 1, 3]}, + {'Mconv5_stage%d_L1' % i: [128, 128, 7, 1, 3]}, + {'Mconv6_stage%d_L1' % i: [128, 128, 1, 1, 0]}, + {'Mconv7_stage%d_L1' % i: [128, 38, 1, 1, 0]}, + ] + blocks['block%d_2' % i] = [ + {'Mconv1_stage%d_L2' % i: [185, 128, 7, 1, 3]}, + {'Mconv2_stage%d_L2' % i: [128, 128, 7, 1, 3]}, + {'Mconv3_stage%d_L2' % i: [128, 128, 7, 1, 3]}, + {'Mconv4_stage%d_L2' % i: [128, 128, 7, 1, 3]}, + {'Mconv5_stage%d_L2' % i: [128, 128, 7, 1, 3]}, + {'Mconv6_stage%d_L2' % i: [128, 128, 1, 1, 0]}, + {'Mconv7_stage%d_L2' % i: [128, 19, 1, 1, 0]}, + ] + + layers = [] + for block in block_0: + # print(block) + for key in block: + v = block[key] + if 'pool' in key: + layers += [nn.MaxPool2d(kernel_size=v[0], stride=v[1], padding=v[2])] + else: + conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1], kernel_size=v[2], stride=v[3], padding=v[4]) + layers += [conv2d, nn.ReLU(inplace=True)] + + models = {'block_0': nn.Sequential(*layers)} + + for k in blocks: + v = blocks[k] + models[k] = make_layers(v) + + return PoseEstimation(models) + + def get_paf_and_heatmap(model, img_raw, scale_search, param_stride=8, box_size=368): + multiplier = [scale * box_size / img_raw.shape[0] for scale in scale_search] + + heatmap_avg = torch.zeros((len(multiplier), 19, img_raw.shape[0], img_raw.shape[1])).cuda() + paf_avg = torch.zeros((len(multiplier), 38, img_raw.shape[0], img_raw.shape[1])).cuda() + + for i, scale in enumerate(multiplier): + img_test = cv2.resize(img_raw, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) + img_test_pad, pad = pad_right_down_corner(img_test, param_stride, param_stride) + img_test_pad = np.transpose(np.float32(img_test_pad[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5 + + feed = Variable(torch.from_numpy(img_test_pad)).cuda() + output1, output2 = model(feed) + + # print(output1.size()) + # print(output2.size()) + + heatmap = nn.UpsamplingBilinear2d((img_raw.shape[0], img_raw.shape[1])).cuda()(output2) + + paf = nn.UpsamplingBilinear2d((img_raw.shape[0], img_raw.shape[1])).cuda()(output1) + + heatmap_avg[i] = heatmap[0].data + paf_avg[i] = paf[0].data + + heatmap_avg = torch.transpose(torch.transpose(torch.squeeze(torch.mean(heatmap_avg, 0)), 0, 1), 1, 2).cuda() + heatmap_avg = heatmap_avg.cpu().numpy() + + paf_avg = torch.transpose(torch.transpose(torch.squeeze(torch.mean(paf_avg, 0)), 0, 1), 1, 2).cuda() + paf_avg = paf_avg.cpu().numpy() + + return paf_avg, heatmap_avg + + def extract_heatmap_info(heatmap_avg, param_thre1=0.1): + all_peaks = [] + peak_counter = 0 + + for part in range(18): + map_ori = heatmap_avg[:, :, part] + map_gau = gaussian_filter(map_ori, sigma=3) + + map_left = np.zeros(map_gau.shape) + map_left[1:, :] = map_gau[:-1, :] + map_right = np.zeros(map_gau.shape) + map_right[:-1, :] = map_gau[1:, :] + map_up = np.zeros(map_gau.shape) + map_up[:, 1:] = map_gau[:, :-1] + map_down = np.zeros(map_gau.shape) + map_down[:, :-1] = map_gau[:, 1:] + + peaks_binary = np.logical_and.reduce((map_gau >= map_left, map_gau >= map_right, map_gau >= map_up, map_gau >= map_down, map_gau > param_thre1)) + + peaks = zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0]) # note reverse + peaks = list(peaks) + peaks_with_score = [x + (map_ori[x[1], x[0]],) for x in peaks] + ids = range(peak_counter, peak_counter + len(peaks)) + peaks_with_score_and_id = [peaks_with_score[i] + (ids[i],) for i in range(len(ids))] + + all_peaks.append(peaks_with_score_and_id) + peak_counter += len(peaks) + + return all_peaks + + def extract_paf_info(img_raw, paf_avg, all_peaks, param_thre2=0.05, param_thre3=0.5): + connection_all = [] + special_k = [] + mid_num = 10 + + for k in range(len(map_ids)): + score_mid = paf_avg[:, :, [x - 19 for x in map_ids[k]]] + candA = all_peaks[limb_seq[k][0] - 1] + candB = all_peaks[limb_seq[k][1] - 1] + nA = len(candA) + nB = len(candB) + if nA != 0 and nB != 0: + connection_candidate = [] + for i in range(nA): + for j in range(nB): + vec = np.subtract(candB[j][:2], candA[i][:2]) + norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1]) + if norm < 1e-8: + raise ZeroDivisionError + vec = np.divide(vec, norm) + + startend = zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), np.linspace(candA[i][1], candB[j][1], num=mid_num)) + startend = list(startend) + + vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] for I in range(len(startend))]) + vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] for I in range(len(startend))]) + + score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1]) + score_with_dist_prior = sum(score_midpts) / len(score_midpts) + score_with_dist_prior += min(0.5 * img_raw.shape[0] / norm - 1, 0) + + criterion1 = len(np.nonzero(score_midpts > param_thre2)[0]) > 0.8 * len(score_midpts) + criterion2 = score_with_dist_prior > 0 + if criterion1 and criterion2: + connection_candidate.append([i, j, score_with_dist_prior, score_with_dist_prior + candA[i][2] + candB[j][2]]) + + connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True) + connection = np.zeros((0, 5)) + for c in range(len(connection_candidate)): + i, j, s = connection_candidate[c][0:3] + if i not in connection[:, 3] and j not in connection[:, 4]: + connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]]) + if len(connection) >= min(nA, nB): + break + + connection_all.append(connection) + else: + special_k.append(k) + connection_all.append([]) + + return special_k, connection_all + + def get_subsets(connection_all, special_k, all_peaks): + # last number in each row is the total parts number of that person + # the second last number in each row is the score of the overall configuration + subset = -1 * np.ones((0, 20)) + candidate = np.array([item for sublist in all_peaks for item in sublist]) + + for k in range(len(map_ids)): + if k not in special_k: + partAs = connection_all[k][:, 0] + partBs = connection_all[k][:, 1] + indexA, indexB = np.array(limb_seq[k]) - 1 + + for i in range(len(connection_all[k])): # = 1:size(temp,1) + found = 0 + subset_idx = [-1, -1] + for j in range(len(subset)): # 1:size(subset,1): + if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]: + subset_idx[found] = j + found += 1 + + if found == 1: + j = subset_idx[0] + if subset[j][indexB] != partBs[i]: + subset[j][indexB] = partBs[i] + subset[j][-1] += 1 + subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] + elif found == 2: # if found 2 and disjoint, merge them + j1, j2 = subset_idx + print('found = 2') + membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2] + if len(np.nonzero(membership == 2)[0]) == 0: # merge + subset[j1][:-2] += subset[j2][:-2] + 1 + subset[j1][-2:] += subset[j2][-2:] + subset[j1][-2] += connection_all[k][i][2] + subset = np.delete(subset, j2, 0) + else: # as like found == 1 + subset[j1][indexB] = partBs[i] + subset[j1][-1] += 1 + subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] + + # if find no partA in the subset, create a new subset + elif not found and k < 17: + row = -1 * np.ones(20) + row[indexA] = partAs[i] + row[indexB] = partBs[i] + row[-1] = 2 + row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2] + subset = np.vstack([subset, row]) + return subset, candidate + + def draw_key_point(subset, all_peaks, img_raw): + del_ids = [] + for i in range(len(subset)): + if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4: + del_ids.append(i) + subset = np.delete(subset, del_ids, axis=0) + + img_canvas = img_raw.copy() # B,G,R order + + for i in range(18): + for j in range(len(all_peaks[i])): + cv2.circle(img_canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1) + + return subset, img_canvas + + def link_key_point(img_canvas, candidate, subset, stickwidth=4): + for i in range(17): + for n in range(len(subset)): + index = subset[n][np.array(limb_seq[i]) - 1] + if -1 in index: + continue + cur_canvas = img_canvas.copy() + Y = candidate[index.astype(int), 0] + X = candidate[index.astype(int), 1] + mX = np.mean(X) + mY = np.mean(Y) + length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5 + angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) + polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1) + cv2.fillConvexPoly(cur_canvas, polygon, colors[i]) + img_canvas = cv2.addWeighted(img_canvas, 0.4, cur_canvas, 0.6, 0) + + return img_canvas + + def pad_right_down_corner(img, stride, pad_value): + h = img.shape[0] + w = img.shape[1] + + pad = 4 * [None] + pad[0] = 0 # up + pad[1] = 0 # left + pad[2] = 0 if (h % stride == 0) else stride - (h % stride) # down + pad[3] = 0 if (w % stride == 0) else stride - (w % stride) # right + + img_padded = img + pad_up = np.tile(img_padded[0:1, :, :] * 0 + pad_value, (pad[0], 1, 1)) + img_padded = np.concatenate((pad_up, img_padded), axis=0) + pad_left = np.tile(img_padded[:, 0:1, :] * 0 + pad_value, (1, pad[1], 1)) + img_padded = np.concatenate((pad_left, img_padded), axis=1) + pad_down = np.tile(img_padded[-2:-1, :, :] * 0 + pad_value, (pad[2], 1, 1)) + img_padded = np.concatenate((img_padded, pad_down), axis=0) + pad_right = np.tile(img_padded[:, -2:-1, :] * 0 + pad_value, (1, pad[3], 1)) + img_padded = np.concatenate((img_padded, pad_right), axis=1) + + return img_padded, pad + + if __name__ == '__main__': + print(get_pose_model()) + + # First let's download the pre-trained model. + + # In[2]: + + # Using gdown to download the model directly from Google Drive + + # assert os.system(' conda install -y gdown') == 0 + import gdown + + # In[3]: + + model = 'coco_pose_iter_440000.pth.tar' + if not os.path.exists(model): + url = 'https://drive.google.com/u/0/uc?export=download&confirm=f_Ix&id=0B1asvDK18cu_MmY1ZkpaOUhhRHM' + gdown.download(url, model, quiet=False) + + # In[4]: + + state_dict = torch.load(model)['state_dict'] # getting the pre-trained model's parameters + # A state_dict is simply a Python dictionary object that maps each layer to its parameter tensor. + + model_pose = get_pose_model() # building the model (see fn. defn. above). To see the architecture, see below cell. + model_pose.load_state_dict(state_dict) # Loading the parameters (weights, biases) into the model. + + model_pose.float() # I'm not sure why this is used. No difference if you remove it. + + if use_gpu is None: + use_gpu = True + + if use_gpu: + model_pose.cuda() + model_pose = torch.nn.DataParallel(model_pose, device_ids=range(torch.cuda.device_count())) + cudnn.benchmark = True + + def estimate_pose( + img_ori, + name=None, + scale_param=None, + display=None, + ): + if display is None: + display = True + + if scale_param is None: + scale_param = [0.5, 1.0, 1.5, 2.0] + + if display: + if name is None: + name = tempfile.mktemp( + dir='/kaggle/working', + suffix='.png', + ) + pprint.pprint( + ['estimate_pose', dict(name=name)], + ) + + # People might be at different scales in the image, perform inference at multiple scales to boost results + + # Predict Heatmaps for approximate joint position + # Use Part Affinity Fields (PAF's) as guidance to link joints to form skeleton + # PAF's are just unit vectors along the limb encoding the direction of the limb + # A dot product of possible joint connection will be high if actual limb else low - # The code hidden below handles all the imports and function definitions (the heavy lifting). If you're a beginner I'd advice you skip this for now. When you are able to understand the rest of the code, come back here and understand each function to get a deeper knowledge. + img_canvas = None + img_points = None - # In[1]: + try: + paf_info, heatmap_info = get_paf_and_heatmap(model_pose, img_ori, scale_param) + peaks = extract_heatmap_info(heatmap_info) + sp_k, con_all = extract_paf_info(img_ori, paf_info, peaks) + subsets, candidates = get_subsets(con_all, sp_k, peaks) - # !/usr/bin/env python3 - # coding=utf-8 - # author=dave.fang@outlook.com - # create=20171225 - - import os - import pprint - import cv2 - import sys - import math - import time - import tempfile - import numpy as np - import matplotlib.pyplot as plt - - import torch - import torch.nn as nn - import torch.nn.parallel - import torch.backends.cudnn as cudnn - import torch.optim as optim - import torchvision.transforms as transforms - import torchvision.datasets as datasets - import torchvision.models - - from torch.autograd import Variable - - from scipy.ndimage.filters import gaussian_filter - - #get_ipython().run_line_magic('matplotlib', 'inline') - #get_ipython().run_line_magic('config', "InlineBackend.figure_format = 'retina'") + subsets, img_points = draw_key_point(subsets, peaks, img_ori) + img_canvas = link_key_point(img_points, candidates, subsets) + except ZeroDivisionError: + pprint.pprint('zero de') - # find connection in the specified sequence, center 29 is in the position 15 - limb_seq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], - [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], [1, 15], [15, 17], - [1, 16], [16, 18], [3, 17], [6, 18]] - - # the middle joints heatmap correpondence - map_ids = [[31, 32], [39, 40], [33, 34], [35, 36], [41, 42], [43, 44], [19, 20], [21, 22], - [23, 24], [25, 26], [27, 28], [29, 30], [47, 48], [49, 50], [53, 54], [51, 52], - [55, 56], [37, 38], [45, 46]] - - # these are the colours for the 18 body points - colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], - [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], - [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]] - - - class PoseEstimation(nn.Module): - def __init__(self, model_dict): - super(PoseEstimation, self).__init__() - - self.model0 = model_dict['block_0'] - self.model1_1 = model_dict['block1_1'] - self.model2_1 = model_dict['block2_1'] - self.model3_1 = model_dict['block3_1'] - self.model4_1 = model_dict['block4_1'] - self.model5_1 = model_dict['block5_1'] - self.model6_1 = model_dict['block6_1'] - - self.model1_2 = model_dict['block1_2'] - self.model2_2 = model_dict['block2_2'] - self.model3_2 = model_dict['block3_2'] - self.model4_2 = model_dict['block4_2'] - self.model5_2 = model_dict['block5_2'] - self.model6_2 = model_dict['block6_2'] - - def forward(self, x): - out1 = self.model0(x) - - out1_1 = self.model1_1(out1) - out1_2 = self.model1_2(out1) - out2 = torch.cat([out1_1, out1_2, out1], 1) - - out2_1 = self.model2_1(out2) - out2_2 = self.model2_2(out2) - out3 = torch.cat([out2_1, out2_2, out1], 1) - - out3_1 = self.model3_1(out3) - out3_2 = self.model3_2(out3) - out4 = torch.cat([out3_1, out3_2, out1], 1) - - out4_1 = self.model4_1(out4) - out4_2 = self.model4_2(out4) - out5 = torch.cat([out4_1, out4_2, out1], 1) - - out5_1 = self.model5_1(out5) - out5_2 = self.model5_2(out5) - out6 = torch.cat([out5_1, out5_2, out1], 1) - - out6_1 = self.model6_1(out6) - out6_2 = self.model6_2(out6) - - return out6_1, out6_2 - - - def make_layers(layer_dict): - layers = [] - - for i in range(len(layer_dict) - 1): - layer = layer_dict[i] - for k in layer: - v = layer[k] - if 'pool' in k: - layers += [nn.MaxPool2d(kernel_size=v[0], stride=v[1], padding=v[2])] - else: - conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1], kernel_size=v[2], stride=v[3], padding=v[4]) - layers += [conv2d, nn.ReLU(inplace=True)] - layer = list(layer_dict[-1].keys()) - k = layer[0] - v = layer_dict[-1][k] - - conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1], kernel_size=v[2], stride=v[3], padding=v[4]) - layers += [conv2d] + img_points = img_ori.copy() + img_canvas = img_ori.copy() - return nn.Sequential(*layers) - - - def get_pose_model(): - blocks = {} - - block_0 = [{'conv1_1': [3, 64, 3, 1, 1]}, {'conv1_2': [64, 64, 3, 1, 1]}, {'pool1_stage1': [2, 2, 0]}, - {'conv2_1': [64, 128, 3, 1, 1]}, {'conv2_2': [128, 128, 3, 1, 1]}, {'pool2_stage1': [2, 2, 0]}, - {'conv3_1': [128, 256, 3, 1, 1]}, {'conv3_2': [256, 256, 3, 1, 1]}, {'conv3_3': [256, 256, 3, 1, 1]}, - {'conv3_4': [256, 256, 3, 1, 1]}, {'pool3_stage1': [2, 2, 0]}, {'conv4_1': [256, 512, 3, 1, 1]}, - {'conv4_2': [512, 512, 3, 1, 1]}, {'conv4_3_CPM': [512, 256, 3, 1, 1]}, - {'conv4_4_CPM': [256, 128, 3, 1, 1]}] - - blocks['block1_1'] = [{'conv5_1_CPM_L1': [128, 128, 3, 1, 1]}, {'conv5_2_CPM_L1': [128, 128, 3, 1, 1]}, - {'conv5_3_CPM_L1': [128, 128, 3, 1, 1]}, {'conv5_4_CPM_L1': [128, 512, 1, 1, 0]}, - {'conv5_5_CPM_L1': [512, 38, 1, 1, 0]}] - - blocks['block1_2'] = [{'conv5_1_CPM_L2': [128, 128, 3, 1, 1]}, {'conv5_2_CPM_L2': [128, 128, 3, 1, 1]}, - {'conv5_3_CPM_L2': [128, 128, 3, 1, 1]}, {'conv5_4_CPM_L2': [128, 512, 1, 1, 0]}, - {'conv5_5_CPM_L2': [512, 19, 1, 1, 0]}] - - for i in range(2, 7): - blocks['block%d_1' % i] = [{'Mconv1_stage%d_L1' % i: [185, 128, 7, 1, 3]}, - {'Mconv2_stage%d_L1' % i: [128, 128, 7, 1, 3]}, - {'Mconv3_stage%d_L1' % i: [128, 128, 7, 1, 3]}, - {'Mconv4_stage%d_L1' % i: [128, 128, 7, 1, 3]}, - {'Mconv5_stage%d_L1' % i: [128, 128, 7, 1, 3]}, - {'Mconv6_stage%d_L1' % i: [128, 128, 1, 1, 0]}, - {'Mconv7_stage%d_L1' % i: [128, 38, 1, 1, 0]}] - blocks['block%d_2' % i] = [{'Mconv1_stage%d_L2' % i: [185, 128, 7, 1, 3]}, - {'Mconv2_stage%d_L2' % i: [128, 128, 7, 1, 3]}, - {'Mconv3_stage%d_L2' % i: [128, 128, 7, 1, 3]}, - {'Mconv4_stage%d_L2' % i: [128, 128, 7, 1, 3]}, - {'Mconv5_stage%d_L2' % i: [128, 128, 7, 1, 3]}, - {'Mconv6_stage%d_L2' % i: [128, 128, 1, 1, 0]}, - {'Mconv7_stage%d_L2' % i: [128, 19, 1, 1, 0]}] - - layers = [] - for block in block_0: - # print(block) - for key in block: - v = block[key] - if 'pool' in key: - layers += [nn.MaxPool2d(kernel_size=v[0], stride=v[1], padding=v[2])] - else: - conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1], kernel_size=v[2], stride=v[3], padding=v[4]) - layers += [conv2d, nn.ReLU(inplace=True)] - - models = { - 'block_0': nn.Sequential(*layers) - } - - for k in blocks: - v = blocks[k] - models[k] = make_layers(v) - - return PoseEstimation(models) - - - def get_paf_and_heatmap(model, img_raw, scale_search, param_stride=8, box_size=368): - multiplier = [scale * box_size / img_raw.shape[0] for scale in scale_search] - - heatmap_avg = torch.zeros((len(multiplier), 19, img_raw.shape[0], img_raw.shape[1])).cuda() - paf_avg = torch.zeros((len(multiplier), 38, img_raw.shape[0], img_raw.shape[1])).cuda() - - for i, scale in enumerate(multiplier): - img_test = cv2.resize(img_raw, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) - img_test_pad, pad = pad_right_down_corner(img_test, param_stride, param_stride) - img_test_pad = np.transpose(np.float32(img_test_pad[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5 - - feed = Variable(torch.from_numpy(img_test_pad)).cuda() - output1, output2 = model(feed) - - #print(output1.size()) - #print(output2.size()) - - heatmap = nn.UpsamplingBilinear2d((img_raw.shape[0], img_raw.shape[1])).cuda()(output2) - - paf = nn.UpsamplingBilinear2d((img_raw.shape[0], img_raw.shape[1])).cuda()(output1) - - heatmap_avg[i] = heatmap[0].data - paf_avg[i] = paf[0].data - - heatmap_avg = torch.transpose(torch.transpose(torch.squeeze(torch.mean(heatmap_avg, 0)), 0, 1), 1, 2).cuda() - heatmap_avg = heatmap_avg.cpu().numpy() - - paf_avg = torch.transpose(torch.transpose(torch.squeeze(torch.mean(paf_avg, 0)), 0, 1), 1, 2).cuda() - paf_avg = paf_avg.cpu().numpy() - - return paf_avg, heatmap_avg - - - def extract_heatmap_info(heatmap_avg, param_thre1=0.1): - all_peaks = [] - peak_counter = 0 - - for part in range(18): - map_ori = heatmap_avg[:, :, part] - map_gau = gaussian_filter(map_ori, sigma=3) - - map_left = np.zeros(map_gau.shape) - map_left[1:, :] = map_gau[:-1, :] - map_right = np.zeros(map_gau.shape) - map_right[:-1, :] = map_gau[1:, :] - map_up = np.zeros(map_gau.shape) - map_up[:, 1:] = map_gau[:, :-1] - map_down = np.zeros(map_gau.shape) - map_down[:, :-1] = map_gau[:, 1:] - - peaks_binary = np.logical_and.reduce( - (map_gau >= map_left, map_gau >= map_right, map_gau >= map_up, - map_gau >= map_down, map_gau > param_thre1)) - - peaks = zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0]) # note reverse - peaks = list(peaks) - peaks_with_score = [x + (map_ori[x[1], x[0]],) for x in peaks] - ids = range(peak_counter, peak_counter + len(peaks)) - peaks_with_score_and_id = [peaks_with_score[i] + (ids[i],) for i in range(len(ids))] - - all_peaks.append(peaks_with_score_and_id) - peak_counter += len(peaks) - - return all_peaks - - - def extract_paf_info(img_raw, paf_avg, all_peaks, param_thre2=0.05, param_thre3=0.5): - connection_all = [] - special_k = [] - mid_num = 10 - - for k in range(len(map_ids)): - score_mid = paf_avg[:, :, [x - 19 for x in map_ids[k]]] - candA = all_peaks[limb_seq[k][0] - 1] - candB = all_peaks[limb_seq[k][1] - 1] - nA = len(candA) - nB = len(candB) - if nA != 0 and nB != 0: - connection_candidate = [] - for i in range(nA): - for j in range(nB): - vec = np.subtract(candB[j][:2], candA[i][:2]) - norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1]) - if norm < 1e-8: - raise ZeroDivisionError - vec = np.divide(vec, norm) - - startend = zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), - np.linspace(candA[i][1], candB[j][1], num=mid_num)) - startend = list(startend) - - vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] - for I in range(len(startend))]) - vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] - for I in range(len(startend))]) - - score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1]) - score_with_dist_prior = sum(score_midpts) / len(score_midpts) - score_with_dist_prior += min(0.5 * img_raw.shape[0] / norm - 1, 0) - - criterion1 = len(np.nonzero(score_midpts > param_thre2)[0]) > 0.8 * len(score_midpts) - criterion2 = score_with_dist_prior > 0 - if criterion1 and criterion2: - connection_candidate.append( - [i, j, score_with_dist_prior, score_with_dist_prior + candA[i][2] + candB[j][2]]) - - connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True) - connection = np.zeros((0, 5)) - for c in range(len(connection_candidate)): - i, j, s = connection_candidate[c][0:3] - if i not in connection[:, 3] and j not in connection[:, 4]: - connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]]) - if len(connection) >= min(nA, nB): - break - - connection_all.append(connection) - else: - special_k.append(k) - connection_all.append([]) - - return special_k, connection_all - - - def get_subsets(connection_all, special_k, all_peaks): - # last number in each row is the total parts number of that person - # the second last number in each row is the score of the overall configuration - subset = -1 * np.ones((0, 20)) - candidate = np.array([item for sublist in all_peaks for item in sublist]) - - for k in range(len(map_ids)): - if k not in special_k: - partAs = connection_all[k][:, 0] - partBs = connection_all[k][:, 1] - indexA, indexB = np.array(limb_seq[k]) - 1 - - for i in range(len(connection_all[k])): # = 1:size(temp,1) - found = 0 - subset_idx = [-1, -1] - for j in range(len(subset)): # 1:size(subset,1): - if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]: - subset_idx[found] = j - found += 1 - - if found == 1: - j = subset_idx[0] - if (subset[j][indexB] != partBs[i]): - subset[j][indexB] = partBs[i] - subset[j][-1] += 1 - subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] - elif found == 2: # if found 2 and disjoint, merge them - j1, j2 = subset_idx - print("found = 2") - membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2] - if len(np.nonzero(membership == 2)[0]) == 0: # merge - subset[j1][:-2] += (subset[j2][:-2] + 1) - subset[j1][-2:] += subset[j2][-2:] - subset[j1][-2] += connection_all[k][i][2] - subset = np.delete(subset, j2, 0) - else: # as like found == 1 - subset[j1][indexB] = partBs[i] - subset[j1][-1] += 1 - subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] - - # if find no partA in the subset, create a new subset - elif not found and k < 17: - row = -1 * np.ones(20) - row[indexA] = partAs[i] - row[indexB] = partBs[i] - row[-1] = 2 - row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2] - subset = np.vstack([subset, row]) - return subset, candidate - - - def draw_key_point(subset, all_peaks, img_raw): - del_ids = [] - for i in range(len(subset)): - if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4: - del_ids.append(i) - subset = np.delete(subset, del_ids, axis=0) - - img_canvas = img_raw.copy() # B,G,R order - - for i in range(18): - for j in range(len(all_peaks[i])): - cv2.circle(img_canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1) - - return subset, img_canvas - - - def link_key_point(img_canvas, candidate, subset, stickwidth=4): - for i in range(17): - for n in range(len(subset)): - index = subset[n][np.array(limb_seq[i]) - 1] - if -1 in index: - continue - cur_canvas = img_canvas.copy() - Y = candidate[index.astype(int), 0] - X = candidate[index.astype(int), 1] - mX = np.mean(X) - mY = np.mean(Y) - length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5 - angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) - polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1) - cv2.fillConvexPoly(cur_canvas, polygon, colors[i]) - img_canvas = cv2.addWeighted(img_canvas, 0.4, cur_canvas, 0.6, 0) - - return img_canvas - - def pad_right_down_corner(img, stride, pad_value): - h = img.shape[0] - w = img.shape[1] - - pad = 4 * [None] - pad[0] = 0 # up - pad[1] = 0 # left - pad[2] = 0 if (h % stride == 0) else stride - (h % stride) # down - pad[3] = 0 if (w % stride == 0) else stride - (w % stride) # right - - img_padded = img - pad_up = np.tile(img_padded[0:1, :, :] * 0 + pad_value, (pad[0], 1, 1)) - img_padded = np.concatenate((pad_up, img_padded), axis=0) - pad_left = np.tile(img_padded[:, 0:1, :] * 0 + pad_value, (1, pad[1], 1)) - img_padded = np.concatenate((pad_left, img_padded), axis=1) - pad_down = np.tile(img_padded[-2:-1, :, :] * 0 + pad_value, (pad[2], 1, 1)) - img_padded = np.concatenate((img_padded, pad_down), axis=0) - pad_right = np.tile(img_padded[:, -2:-1, :] * 0 + pad_value, (1, pad[3], 1)) - img_padded = np.concatenate((img_padded, pad_right), axis=1) - - return img_padded, pad - - - if __name__ == '__main__': - print(get_pose_model()) - - - # First let's download the pre-trained model. - - # In[2]: - - - # Using gdown to download the model directly from Google Drive + # After predicting Heatmaps and PAF's, proceeed to link joints correctly + if display: + f = plt.figure(figsize=(15, 10)) - #assert os.system(' conda install -y gdown') == 0 - import gdown + plt.subplot(1, 2, 1) + plt.imshow(img_points[..., ::-1]) + plt.subplot(1, 2, 2) + plt.imshow(img_canvas[..., ::-1]) - # In[3]: + f.savefig(name) + return dict( + img_points=img_points, + img_canvas=img_canvas, + ) - model = 'coco_pose_iter_440000.pth.tar' - if not os.path.exists(model): - url = 'https://drive.google.com/u/0/uc?export=download&confirm=f_Ix&id=0B1asvDK18cu_MmY1ZkpaOUhhRHM' - gdown.download( - url, - model, - quiet=False - ) + # In[5]: - - # In[4]: - - - state_dict = torch.load(model)['state_dict'] # getting the pre-trained model's parameters - # A state_dict is simply a Python dictionary object that maps each layer to its parameter tensor. - - model_pose = get_pose_model() # building the model (see fn. defn. above). To see the architecture, see below cell. - model_pose.load_state_dict(state_dict) # Loading the parameters (weights, biases) into the model. - - model_pose.float() # I'm not sure why this is used. No difference if you remove it. - - if use_gpu is None: - use_gpu = True - - if use_gpu: - model_pose.cuda() - model_pose = torch.nn.DataParallel(model_pose, device_ids=range(torch.cuda.device_count())) - cudnn.benchmark = True - - def estimate_pose( - img_ori, - name=None, - scale_param=None, - display=None, - ): - if display is None: - display = True - - if scale_param is None: - scale_param = [0.5, 1.0, 1.5, 2.0] - - if display: - if name is None: - name = tempfile.mktemp( - dir='/kaggle/working', - suffix='.png', - ) - pprint.pprint( - ['estimate_pose', dict(name=name)], - ) - - # People might be at different scales in the image, perform inference at multiple scales to boost results - - # Predict Heatmaps for approximate joint position - # Use Part Affinity Fields (PAF's) as guidance to link joints to form skeleton - # PAF's are just unit vectors along the limb encoding the direction of the limb - # A dot product of possible joint connection will be high if actual limb else low - - img_canvas = None - img_points = None - - try: - paf_info, heatmap_info = get_paf_and_heatmap(model_pose, img_ori, scale_param) - peaks = extract_heatmap_info(heatmap_info) - sp_k, con_all = extract_paf_info(img_ori, paf_info, peaks) - - subsets, candidates = get_subsets(con_all, sp_k, peaks) - - subsets, img_points = draw_key_point(subsets, peaks, img_ori) - img_canvas = link_key_point(img_points, candidates, subsets) - except ZeroDivisionError: - pprint.pprint('zero de') - - img_points = img_ori.copy() - img_canvas = img_ori.copy() - - # After predicting Heatmaps and PAF's, proceeed to link joints correctly - if display: - f = plt.figure(figsize=(15, 10)) - - plt.subplot(1, 2, 1) - plt.imshow(img_points[...,::-1]) - - plt.subplot(1, 2, 2) - plt.imshow(img_canvas[...,::-1]) - - f.savefig(name) - - return dict( - img_points=img_points, - img_canvas=img_canvas, - ) - - - # In[5]: - - return dict( - cv2=cv2, - estimate_pose=estimate_pose, - model_pose=model_pose, - ) + return dict( + cv2=cv2, + estimate_pose=estimate_pose, + model_pose=model_pose, + ) def kernel_8( - o_7, + o_7, ): - for i, o in enumerate([ - '../input/indonesian-traditional-dance/tgagrakanyar/tga_00%d0.jpg' % k - for k in range(6) - ]): - arch_image = o - img_ori = o_7['cv2'].imread(arch_image) - o_7['estimate_pose'](img_ori) + for i, o in enumerate(['../input/indonesian-traditional-dance/tgagrakanyar/tga_00%d0.jpg' % k for k in range(6)]): + arch_image = o + img_ori = o_7['cv2'].imread(arch_image) + o_7['estimate_pose'](img_ori) + def kernel_9_benchmark( - o_7, + o_7, ): - import datetime + import datetime + + t1 = o_7['cv2'].imread('../input/indonesian-traditional-dance/tgagrakanyar/tga_0000.jpg') + t5 = 10 + t2 = datetime.datetime.now() + for k in range(t5): + o_7['estimate_pose']( + img_ori=t1, + scale_param=[1.0], + display=False, + ) + t3 = datetime.datetime.now() + t4 = (t3 - t2).total_seconds() / t5 + pprint.pprint(['kernel_9_benchmark', dict(t4=t4, t5=t5)]) - t1 = o_7['cv2'].imread('../input/indonesian-traditional-dance/tgagrakanyar/tga_0000.jpg') - t5 = 10 - t2 = datetime.datetime.now() - for k in range(t5): - o_7['estimate_pose']( - img_ori=t1, - scale_param=[1.0], - display=False, - ) - t3 = datetime.datetime.now() - t4 = (t3 - t2).total_seconds() / t5 - pprint.pprint( - ['kernel_9_benchmark', dict(t4=t4, t5=t5)] - ) def kernel_10(): - import torch + import torch - # Model - model = torch.hub.load('ultralytics/yolov5', 'yolov5s') # or yolov5m, yolov5x, custom + # Model + model = torch.hub.load('ultralytics/yolov5', 'yolov5s') # or yolov5m, yolov5x, custom - # Images - img = 'https://ultralytics.com/images/zidane.jpg' # or file, PIL, OpenCV, numpy, multiple + # Images + img = 'https://ultralytics.com/images/zidane.jpg' # or file, PIL, OpenCV, numpy, multiple - # Inference - results = model(img) + # Inference + results = model(img) - # Results - results.print() # or .show(), .save(), .crop(), .pandas(), etc. + # Results + results.print() # or .show(), .save(), .crop(), .pandas(), etc. + + return dict( + model=model, + ) - return dict( - model=model, - ) def kernel_11_benchmark( - o_7, - o_10, + o_7, + o_10, ): - import datetime + import datetime - t1 = o_7['cv2'].imread('../input/indonesian-traditional-dance/tgagrakanyar/tga_0000.jpg') - t5 = 10 - t2 = datetime.datetime.now() - for k in range(t5): - t6 = o_10['model'](t1) - t7 = t6.pandas().xywhn + t1 = o_7['cv2'].imread('../input/indonesian-traditional-dance/tgagrakanyar/tga_0000.jpg') + t5 = 10 + t2 = datetime.datetime.now() + for k in range(t5): + t6 = o_10['model'](t1) + t7 = t6.pandas().xywhn + + t3 = datetime.datetime.now() + t4 = (t3 - t2).total_seconds() / t5 + pprint.pprint(['kernel_11_benchmark', dict(t4=t4, t5=t5)]) - t3 = datetime.datetime.now() - t4 = (t3 - t2).total_seconds() / t5 - pprint.pprint( - ['kernel_11_benchmark', dict(t4=t4, t5=t5)] - ) def kernel_13( - o_6=None, + o_6=None, ): - t2 = [ - '/kaggle/working', - '/kaggle/input/garbage' - ] + t2 = ['/kaggle/working', '/kaggle/input/garbage'] - t3 = [ - os.path.join( - o, - 'kernel_13-object-detection.nc', - ) - for o in t2 - ] + t3 = [ + os.path.join( + o, + 'kernel_13-object-detection.nc', + ) + for o in t2 + ] - t4 = [ - o - for o in t3 - if os.path.exists(o) - ] + t4 = [o for o in t3 if os.path.exists(o)] + if not len(t4) > 0 or not o_6 is None: + t1 = pandas.concat( + sum([[o2['t11'][0].assign(frame_id=k, video_path=o['video_path']) for k, o2 in enumerate(o['frames'])] for o in o_6['t8']], []) + ).to_xarray() + t5 = t3[0] + t1.to_netcdf(t5) + del t1 + elif len(t4) > 0: + t5 = t4[0] + else: + raise NotImplementedError - if not len(t4) > 0 or not o_6 is None: - t1 = pandas.concat( - sum( - [ - [ - o2['t11'][0].assign( - frame_id=k, - video_path=o['video_path'] - ) - for k, o2 in enumerate(o['frames']) - ] for o in o_6['t8'] - ], - [] - ) - ).to_xarray() - t5 = t3[0] - t1.to_netcdf(t5) - del t1 - elif len(t4) > 0: - t5 = t4[0] - else: - raise NotImplementedError + t1 = xarray.load_dataset(t5) - t1 = xarray.load_dataset(t5) + return dict( + t1=t1, + ) - return dict( - t1=t1, - ) def kernel_14( - skip_o_6=None, - skip_o_7_10_12_13=None, - run_benchmark=None, + skip_o_6=None, + skip_o_7_10_12_13=None, + run_benchmark=None, ): - if skip_o_6 is None: - skip_o_6 = True + if skip_o_6 is None: + skip_o_6 = True - if skip_o_7_10_12_13 is None: - skip_o_7_10_12_13 = True + if skip_o_7_10_12_13 is None: + skip_o_7_10_12_13 = True - if run_benchmark is None: - run_benchmark = False + if run_benchmark is None: + run_benchmark = False - o_3 = kernel_3(should_exist=True) - o_4 = kernel_4(o_3=o_3) - o_5 = kernel_5(o_4=o_4) + o_3 = kernel_3(should_exist=True) + o_4 = kernel_4(o_3=o_3) + o_5 = kernel_5(o_4=o_4) - if not skip_o_7_10_12_13: - o_7 = kernel_7() + if not skip_o_7_10_12_13: + o_7 = kernel_7() - o_10 = kernel_10() - o_12 = kernel_12() - else: - o_7 = None - o_10 = None - o_12 = None + o_10 = kernel_10() + o_12 = kernel_12() + else: + o_7 = None + o_10 = None + o_12 = None - if not skip_o_6: - o_6 = kernel_6( - o_7=None, - o_10=o_10, - o_12=None, - max_frames=10000 - ) - else: - o_6 = None + if not skip_o_6: + o_6 = kernel_6(o_7=None, o_10=o_10, o_12=None, max_frames=10000) + else: + o_6 = None - if not skip_o_7_10_12_13: - o_13 = kernel_13( - o_6=o_6, - ) - else: - o_13 = None + if not skip_o_7_10_12_13: + o_13 = kernel_13( + o_6=o_6, + ) + else: + o_13 = None - if run_benchmark: - o_11 = kernel_11_benchmark(o_7=o_7, o_10=o_10) - o_9 = kernel_9_benchmark(o_7=o_7) - o_8 = kernel_8(o_7=o_7) + if run_benchmark: + o_11 = kernel_11_benchmark(o_7=o_7, o_10=o_10) + o_9 = kernel_9_benchmark(o_7=o_7) + o_8 = kernel_8(o_7=o_7) + + return dict( + o_3=o_3, + o_13=o_13, + o_7=o_7, + ) - return dict( - o_3=o_3, - o_13=o_13, - o_7=o_7, - ) def kernel_15( - o_14, + o_14, ): - t1 = pandas.DataFrame( - numpy.unique( - o_14['o_13']['t1']['name'].data, - return_counts=True - ) - ).T - pprint.pprint( - dict( - t1=t1, - ) - ) + t1 = pandas.DataFrame(numpy.unique(o_14['o_13']['t1']['name'].data, return_counts=True)).T + pprint.pprint( + dict( + t1=t1, + ) + ) - t29 = 'output-gif' - if not os.path.exists(t29): - os.makedirs(t29, exist_ok=True) + t29 = 'output-gif' + if not os.path.exists(t29): + os.makedirs(t29, exist_ok=True) - for t2 in [ - 'baseball glove', - 'baseball bat', - 'sports ball', - 'person', - ]: - t28 = t2.replace(' ', '-') - t3 = o_14['o_13']['t1'] - t4 = numpy.where(t3.name.data == t2)[0] + for t2 in [ + 'baseball glove', + 'baseball bat', + 'sports ball', + 'person', + ]: + t28 = t2.replace(' ', '-') + t3 = o_14['o_13']['t1'] + t4 = numpy.where(t3.name.data == t2)[0] - t30 = 'output-png/%s' % t28 - if not os.path.exists(t30): - os.makedirs(t30, exist_ok=True) + t30 = 'output-png/%s' % t28 + if not os.path.exists(t30): + os.makedirs(t30, exist_ok=True) - numpy.random.seed(0) - t22 = numpy.random.choice(t4, 10) - pprint.pprint(t22) - import tqdm - t24 = [] - t27 = [] - for t5 in tqdm.tqdm(t22): - t6 = t3.video_path.data[t5] - t7 = t3.frame_id.data[t5] - t8 = t3.to_dataframe().iloc[t5] - #pprint.pprint([t6, t7]) - #pprint.pprint(t8) + numpy.random.seed(0) + t22 = numpy.random.choice(t4, 10) + pprint.pprint(t22) + import tqdm - import cv2 - import matplotlib.pyplot + t24 = [] + t27 = [] + for t5 in tqdm.tqdm(t22): + t6 = t3.video_path.data[t5] + t7 = t3.frame_id.data[t5] + t8 = t3.to_dataframe().iloc[t5] + # pprint.pprint([t6, t7]) + # pprint.pprint(t8) - t9 = cv2.VideoCapture(t6) - t9.set(cv2.CAP_PROP_POS_FRAMES, t7) - t10 = t9.read() - t9.release() - t11 = t10[1] - t12 = cv2.cvtColor(t11, cv2.COLOR_BGR2RGB) - t13 = t12.copy() - t15 = numpy.array([t8.xcenter, t8.ycenter, t8.width, t8.height]) - t16 = numpy.array([t13.shape[1], t13.shape[0], t13.shape[1], t13.shape[0]]) - t17 = t15 * t16 - t18 = t17[:2] - t17[2:] / 2 - t19 = t17[:2] + t17[2:] / 2 - t20 = numpy.array([ - t18[0], t18[1], - t19[0], t19[1], - ]) - t21 = numpy.round(t20).astype(numpy.int32) - t14 = cv2.rectangle( - t13, - tuple(t21[:2]), - tuple(t21[2:]), - (0, 255, 0), - 1, - ) + import cv2 + import matplotlib.pyplot - if False: - t32 = o_14['o_7']['estimate_pose']( - t12, - scale_param=[1.0], - display=False, - )['img_canvas'] - else: - t32 = kernel_16([t12])['t6'][0] + t9 = cv2.VideoCapture(t6) + t9.set(cv2.CAP_PROP_POS_FRAMES, t7) + t10 = t9.read() + t9.release() + t11 = t10[1] + t12 = cv2.cvtColor(t11, cv2.COLOR_BGR2RGB) + t13 = t12.copy() + t15 = numpy.array([t8.xcenter, t8.ycenter, t8.width, t8.height]) + t16 = numpy.array([t13.shape[1], t13.shape[0], t13.shape[1], t13.shape[0]]) + t17 = t15 * t16 + t18 = t17[:2] - t17[2:] / 2 + t19 = t17[:2] + t17[2:] / 2 + t20 = numpy.array( + [ + t18[0], + t18[1], + t19[0], + t19[1], + ] + ) + t21 = numpy.round(t20).astype(numpy.int32) + t14 = cv2.rectangle( + t13, + tuple(t21[:2]), + tuple(t21[2:]), + (0, 255, 0), + 1, + ) - f = matplotlib.pyplot.figure(figsize=(8, 12)) - f.suptitle( - 'name %s, frame_id %d\nvideo_path %s' % ( - t8['name'], - t8.frame_id, - t8.video_path, - ) - ) - matplotlib.pyplot.subplot(2, 1, 1) - matplotlib.pyplot.title( - 'score %s' % ( - t8.confidence, - ) - ) - matplotlib.pyplot.imshow(t14) - matplotlib.pyplot.subplot(2, 1, 2) - matplotlib.pyplot.imshow(t32) - t25 = os.path.join( - t30, - 'kernel_15-%s-%05d.jpg' % ( - t28, - t7, - ) - ) - f.savefig(t25) - t24.append(t25) - matplotlib.pyplot.close(f) + if False: + t32 = o_14['o_7']['estimate_pose']( + t12, + scale_param=[1.0], + display=False, + )['img_canvas'] + else: + t32 = kernel_16([t12])['t6'][0] - t27.append([t8, t21]) - pprint.pprint( - pandas.concat([ - o[0] - for o in t27 - ], axis=1).T - ) + f = matplotlib.pyplot.figure(figsize=(8, 12)) + f.suptitle( + 'name %s, frame_id %d\nvideo_path %s' + % ( + t8['name'], + t8.frame_id, + t8.video_path, + ) + ) + matplotlib.pyplot.subplot(2, 1, 1) + matplotlib.pyplot.title('score %s' % (t8.confidence,)) + matplotlib.pyplot.imshow(t14) + matplotlib.pyplot.subplot(2, 1, 2) + matplotlib.pyplot.imshow(t32) + t25 = os.path.join( + t30, + 'kernel_15-%s-%05d.jpg' + % ( + t28, + t7, + ), + ) + f.savefig(t25) + t24.append(t25) + matplotlib.pyplot.close(f) - t23 = os.path.join( - t29, - 'output-%s.gif' % t28 - ) - if os.path.exists(t23): - subprocess.check_call(['rm', t23]) + t27.append([t8, t21]) + pprint.pprint(pandas.concat([o[0] for o in t27], axis=1).T) + + t23 = os.path.join(t29, 'output-%s.gif' % t28) + if os.path.exists(t23): + subprocess.check_call(['rm', t23]) + + subprocess.check_call( + [ + 'convert', + '-delay', + '100', + '-loop', + '0', + *t24, + t23, + ] + ) - subprocess.check_call( - [ - 'convert', - '-delay', - '100', - '-loop', - '0', - *t24, - t23, - ] - ) def kernel_16(images): - assert isinstance(images, list) + assert isinstance(images, list) - import cv2 - import subprocess - import os + import cv2 + import subprocess + import os - t2 = '/kaggle/working/test-input' - subprocess.check_call([ - 'rm', - '-fr', - t2, - ]) + t2 = '/kaggle/working/test-input' + subprocess.check_call( + [ + 'rm', + '-fr', + t2, + ] + ) - subprocess.check_call([ - 'mkdir', - '-p', - t2, - ]) + subprocess.check_call( + [ + 'mkdir', + '-p', + t2, + ] + ) - t1 = [] - for i, o in enumerate(images): - t5 = cv2.cvtColor(o, cv2.COLOR_RGB2BGR) - t8 = 'image-%d.jpg' % i - t3 = os.path.join( - t2, - t8, - ) - cv2.imwrite(t3, t5) - t1.append( - dict( - image_name=t8, - image_path=t3, - image_canvas=o, - image_index=i, - ) - ) + t1 = [] + for i, o in enumerate(images): + t5 = cv2.cvtColor(o, cv2.COLOR_RGB2BGR) + t8 = 'image-%d.jpg' % i + t3 = os.path.join( + t2, + t8, + ) + cv2.imwrite(t3, t5) + t1.append( + dict( + image_name=t8, + image_path=t3, + image_canvas=o, + image_index=i, + ) + ) - t4 = '/kaggle/working/test-output' - subprocess.check_call([ - 'rm', - '-fr', - t4, - ]) + t4 = '/kaggle/working/test-output' + subprocess.check_call( + [ + 'rm', + '-fr', + t4, + ] + ) - subprocess.check_call([ - 'mkdir', - '-p', - t4, - ]) + subprocess.check_call( + [ + 'mkdir', + '-p', + t4, + ] + ) - with subprocess.Popen(''' + with subprocess.Popen( + """ cd /kaggle/working/AlphaPose && python3 \ scripts/demo_inference.py \ @@ -1236,398 +1223,414 @@ def kernel_16(images): --indir %s \ --outdir %s \ --save_img - ''' % (t2, t4), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + """ + % (t2, t4), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as p: + pprint.pprint(p.communicate()) + p.wait() + assert p.returncode == 0 - pprint.pprint(p.communicate()) - p.wait() - assert p.returncode == 0 + t6 = [] + with io.open(os.path.join(t4, 'alphapose-results.json'), 'r') as f: + t7 = json.load(f) + for o in t1: + t9 = os.path.join(t4, 'vis', o['image_name']) + assert os.path.exists(t9) + t10 = cv2.imread(t9, cv2.IMREAD_COLOR) + t11 = cv2.cvtColor(t10, cv2.COLOR_BGR2RGB) + t6.append(t11) + return dict( + images=images, + t1=t1, + t6=t6, + t7=t7, + ) - t6 = [] - with io.open( - os.path.join( - t4, - 'alphapose-results.json' - ), - 'r' - ) as f: - t7 = json.load(f) - for o in t1: - t9 = os.path.join( - t4, - 'vis', - o['image_name'] - ) - assert os.path.exists(t9) - t10 = cv2.imread(t9, cv2.IMREAD_COLOR) - t11 = cv2.cvtColor(t10, cv2.COLOR_BGR2RGB) - t6.append(t11) - return dict( - images=images, - t1=t1, - t6=t6, - t7=t7, - ) def kernel_17( - o_14, - max_images=None, + o_14, + max_images=None, ): - if max_images is None: - max_images = 10 + if max_images is None: + max_images = 10 - t50 = [] - for t2 in [ - 'baseball glove', - 'baseball bat', - 'sports ball', - 'person', - ]: - t28 = t2.replace(' ', '-') - t3 = o_14['o_13']['t1'] - t4 = numpy.where(t3.name.data == t2)[0] + t50 = [] + for t2 in [ + 'baseball glove', + 'baseball bat', + 'sports ball', + 'person', + ]: + t28 = t2.replace(' ', '-') + t3 = o_14['o_13']['t1'] + t4 = numpy.where(t3.name.data == t2)[0] - t30 = 'output-png/%s' % t28 - if not os.path.exists(t30): - os.makedirs(t30, exist_ok=True) + t30 = 'output-png/%s' % t28 + if not os.path.exists(t30): + os.makedirs(t30, exist_ok=True) - numpy.random.seed(0) - t22 = numpy.random.choice(t4, max_images) - pprint.pprint(t22) - import tqdm - t24 = [] - t27 = [] - for t5 in tqdm.tqdm(t22): - t6 = t3.video_path.data[t5] - t7 = t3.frame_id.data[t5] - t8 = t3.to_dataframe().iloc[t5] - #pprint.pprint([t6, t7]) - #pprint.pprint(t8) + numpy.random.seed(0) + t22 = numpy.random.choice(t4, max_images) + pprint.pprint(t22) + import tqdm - import cv2 - import matplotlib.pyplot + t24 = [] + t27 = [] + for t5 in tqdm.tqdm(t22): + t6 = t3.video_path.data[t5] + t7 = t3.frame_id.data[t5] + t8 = t3.to_dataframe().iloc[t5] + # pprint.pprint([t6, t7]) + # pprint.pprint(t8) - t9 = cv2.VideoCapture(t6) - t9.set(cv2.CAP_PROP_POS_FRAMES, t7) - t10 = t9.read() - t9.release() - t11 = t10[1] - t12 = cv2.cvtColor(t11, cv2.COLOR_BGR2RGB) - t13 = t12.copy() + import cv2 + import matplotlib.pyplot - t50.append( - dict( - t2=t2, - t28=t28, - t6=t6, - t5=t5, - t7=t7, - t12=t12, - ) - ) + t9 = cv2.VideoCapture(t6) + t9.set(cv2.CAP_PROP_POS_FRAMES, t7) + t10 = t9.read() + t9.release() + t11 = t10[1] + t12 = cv2.cvtColor(t11, cv2.COLOR_BGR2RGB) + t13 = t12.copy() + + t50.append( + dict( + t2=t2, + t28=t28, + t6=t6, + t5=t5, + t7=t7, + t12=t12, + ) + ) + + return dict( + t50=t50, + ) - return dict( - t50=t50, - ) def kernel_18(o_17): - t1 = [o['t12'] for o in o_17['t50']] - t2 = kernel_16(t1) + t1 = [o['t12'] for o in o_17['t50']] + t2 = kernel_16(t1) + + return dict( + t2=t2, + ) - return dict( - t2=t2, - ) def kernel_19(o_18): - kernel_25( - o_18['t2']['t6'] - ) + kernel_25(o_18['t2']['t6']) + def kernel_20( - o_18, - o_21=None, + o_18, + o_21=None, ): - if o_21 is None: - o_21 = kernel_21() + if o_21 is None: + o_21 = kernel_21() - import cv2 - import numpy - import os + import cv2 + import numpy + import os + t1 = numpy.array(o_18['t2']['t7'][0]['keypoints']).reshape(17, -1) + t2 = o_18['t2']['t6'][0] + t3 = o_18['t2']['t1'][0]['image_canvas'].copy() + assert o_18['t2']['t7'][0]['image_id'] == os.path.split(o_18['t2']['t1'][0]['image_name'])[1] - t1 = numpy.array(o_18['t2']['t7'][0]['keypoints']).reshape(17, -1) - t2 = o_18['t2']['t6'][0] - t3 = o_18['t2']['t1'][0]['image_canvas'].copy() - assert o_18['t2']['t7'][0]['image_id'] == os.path.split(o_18['t2']['t1'][0]['image_name'])[1] + for i, o2 in enumerate(o_21['p_color']): + if i >= 17: + print('fuck %d' % i) + continue + o1 = t1[i, :] + cv2.circle(t3, tuple(o1[:2].astype(numpy.int32)), 3, o2, -1) + cv2.imwrite('output.jpg', cv2.cvtColor(t3, cv2.COLOR_RGB2BGR)) + cv2.imwrite('output-v2.jpg', cv2.cvtColor(t2, cv2.COLOR_RGB2BGR)) - for i, o2 in enumerate(o_21['p_color']): - if i >= 17: - print('fuck %d' % i) - continue - o1 = t1[i, :] - cv2.circle(t3, tuple(o1[:2].astype(numpy.int32)), 3, o2, -1) - cv2.imwrite('output.jpg', cv2.cvtColor(t3, cv2.COLOR_RGB2BGR)) - cv2.imwrite('output-v2.jpg', cv2.cvtColor(t2, cv2.COLOR_RGB2BGR)) def kernel_21(): - l_pair = [ - (0, 1), (0, 2), (1, 3), (2, 4), # Head - (5, 6), (5, 7), (7, 9), (6, 8), (8, 10), - (17, 11), (17, 12), # Body - (11, 13), (12, 14), (13, 15), (14, 16) - ] - p_color = [ - # Nose, LEye, REye, LEar, REar - (0, 255, 255), (0, 191, 255), (0, 255, 102), (0, 77, 255), (0, 255, 0), - # LShoulder, RShoulder, LElbow, RElbow, LWrist, RWrist - (77, 255, 255), (77, 255, 204), (77, 204, 255), (191, 255, 77), (77, 191, 255), (191, 255, 77), - # LHip, RHip, LKnee, Rknee, LAnkle, RAnkle - (204, 77, 255), (77, 255, 204), (191, 77, 255), (77, 255, 191), (127, 77, 255), (77, 255, 127), - # Not Present - ## Neck - #(0, 255, 255), - ] + l_pair = [ + (0, 1), + (0, 2), + (1, 3), + (2, 4), # Head + (5, 6), + (5, 7), + (7, 9), + (6, 8), + (8, 10), + (17, 11), + (17, 12), # Body + (11, 13), + (12, 14), + (13, 15), + (14, 16), + ] + p_color = [ + # Nose, LEye, REye, LEar, REar + (0, 255, 255), + (0, 191, 255), + (0, 255, 102), + (0, 77, 255), + (0, 255, 0), + # LShoulder, RShoulder, LElbow, RElbow, LWrist, RWrist + (77, 255, 255), + (77, 255, 204), + (77, 204, 255), + (191, 255, 77), + (77, 191, 255), + (191, 255, 77), + # LHip, RHip, LKnee, Rknee, LAnkle, RAnkle + (204, 77, 255), + (77, 255, 204), + (191, 77, 255), + (77, 255, 191), + (127, 77, 255), + (77, 255, 127), + # Not Present + ## Neck + # (0, 255, 255), + ] - line_color = [(0, 215, 255), (0, 255, 204), (0, 134, 255), (0, 255, 50), - (77, 255, 222), (77, 196, 255), (77, 135, 255), (191, 255, 77), (77, 255, 77), - (77, 222, 255), (255, 156, 127), - (0, 127, 255), (255, 127, 77), (0, 77, 255), (255, 77, 36)] - #https://debuggercafe.com/wp-content/uploads/2020/10/keypoint_exmp.jpg - # Seems like No Neck actually - title = [ - 'Nose', - 'LEye', - 'REye', - 'LEar', - 'REar', - 'LShoulder', - 'RShoulder', - 'LElbow', - 'RElbow', - 'LWrist', - 'RWrist', - 'LHip', - 'RHip', - 'LKnee', - 'RKnee', - 'LAnkle', - 'RAnkle', - #'Neck', - ] + line_color = [ + (0, 215, 255), + (0, 255, 204), + (0, 134, 255), + (0, 255, 50), + (77, 255, 222), + (77, 196, 255), + (77, 135, 255), + (191, 255, 77), + (77, 255, 77), + (77, 222, 255), + (255, 156, 127), + (0, 127, 255), + (255, 127, 77), + (0, 77, 255), + (255, 77, 36), + ] + # https://debuggercafe.com/wp-content/uploads/2020/10/keypoint_exmp.jpg + # Seems like No Neck actually + title = [ + 'Nose', + 'LEye', + 'REye', + 'LEar', + 'REar', + 'LShoulder', + 'RShoulder', + 'LElbow', + 'RElbow', + 'LWrist', + 'RWrist', + 'LHip', + 'RHip', + 'LKnee', + 'RKnee', + 'LAnkle', + 'RAnkle', + #'Neck', + ] + + return dict( + l_pair=l_pair, + p_color=p_color, + line_color=line_color, + ) - return dict( - l_pair=l_pair, - p_color=p_color, - line_color=line_color, - ) def kernel_22(o_18): + t1 = o_18['t2']['t7'] + t2 = [numpy.array(o['keypoints']).reshape(17, 3) for o in t1] - t1 = o_18['t2']['t7'] - t2 = [ - numpy.array(o['keypoints']).reshape(17, 3) - for o in t1 - ] + o_31 = kernel_31( + image_id=[o['image_id'] for o in t1], + image_size=numpy.array([[list(o['image_canvas'].shape) for o in o_18['t2']['t1'] if o['image_name'] == t1[i]['image_id']][0] for i in range(len(t2))]), + keypoints=numpy.stack(t2, axis=0), + ) + t12 = o_31['t12'] - o_31 = kernel_31( - image_id=[ - o['image_id'] - for o in t1 - ], - image_size=numpy.array([ - [ - list(o['image_canvas'].shape) - for o in o_18['t2']['t1'] - if o['image_name'] == t1[i]['image_id'] - ][0] - for i in range(len(t2)) - ]), - keypoints=numpy.stack(t2, axis=0), - ) - t12 = o_31['t12'] + return dict( + t4=t12, + ) - return dict( - t4=t12, - ) def kernel_24( - img, - keypoints, - o_21=None, + img, + keypoints, + o_21=None, ): - t3 = img.copy() + t3 = img.copy() - if o_21 is None: - o_21 = kernel_21() + if o_21 is None: + o_21 = kernel_21() - import cv2 - import numpy - import os + import cv2 + import numpy + import os - for i, o2 in enumerate(o_21['p_color']): - if i >= 17: - print('fuck %d' % i) - continue - o1 = keypoints[i, :] - cv2.circle(t3, tuple(o1[:2].astype(numpy.int32)), 3, o2, -1) + for i, o2 in enumerate(o_21['p_color']): + if i >= 17: + print('fuck %d' % i) + continue + o1 = keypoints[i, :] + cv2.circle(t3, tuple(o1[:2].astype(numpy.int32)), 3, o2, -1) + + return dict( + t3=t3, + ) - return dict( - t3=t3, - ) def kernel_25(images, delay=None): - if delay is None: - delay = 100 + if delay is None: + delay = 100 - import tqdm - import os - import cv2 - import subprocess + import tqdm + import os + import cv2 + import subprocess - if len(images) == 0: - pprint.pprint(['kernel_25', 'no images']) - return + if len(images) == 0: + pprint.pprint(['kernel_25', 'no images']) + return - t3 = 'kernel_25-output.dir' - os.makedirs(t3, exist_ok=True) - t6 = [] - for i, o in tqdm.tqdm(enumerate(images)): - t4 = 'image-%03d.jpg' % i - t5 = os.path.join(t3, t4) - t8 = cv2.cvtColor(o, cv2.COLOR_RGB2BGR) - cv2.imwrite(t5, t8) - t6.append(t5) + t3 = 'kernel_25-output.dir' + os.makedirs(t3, exist_ok=True) + t6 = [] + for i, o in tqdm.tqdm(enumerate(images)): + t4 = 'image-%03d.jpg' % i + t5 = os.path.join(t3, t4) + t8 = cv2.cvtColor(o, cv2.COLOR_RGB2BGR) + cv2.imwrite(t5, t8) + t6.append(t5) - t7 = os.path.join(t3, 'output.gif') + t7 = os.path.join(t3, 'output.gif') - if False: - subprocess.check_call( - [ - 'convert', - '-delay', - '%d' % delay, - '-loop', - '0', - *t6, - t7, - ] - ) + if False: + subprocess.check_call( + [ + 'convert', + '-delay', + '%d' % delay, + '-loop', + '0', + *t6, + t7, + ] + ) - t8 = os.path.join( - t3, - 'output.mp4', - ) - if os.path.exists(t8): - os.unlink(t8) + t8 = os.path.join( + t3, + 'output.mp4', + ) + if os.path.exists(t8): + os.unlink(t8) - if False: - subprocess.check_call( - [ - 'ffmpeg', - '-i', - t7, - t8, - ] - ) - t9 = os.path.join(t3, 'input.txt') + if False: + subprocess.check_call( + [ + 'ffmpeg', + '-i', + t7, + t8, + ] + ) + t9 = os.path.join(t3, 'input.txt') + + with io.open(t9, 'w') as f: + f.write('\n'.join(['file %s' % os.path.split(o)[1] for o in t6])) + subprocess.check_call( + [ + 'ffmpeg', + '-r', + '%d' % int(100 / delay), + '-f', + 'concat', + '-i', + '%s' % t9, + t8, + ] + ) - with io.open(t9, 'w') as f: - f.write( - '\n'.join( - [ - 'file %s' % os.path.split(o)[1] - for o in t6 - ] - ) - ) - subprocess.check_call( - [ - 'ffmpeg', - '-r', - '%d' % int(100 / delay), - '-f', - 'concat', - '-i', - '%s' % t9, - t8, - ] - ) def kernel_26(o_18, image_name): - t1 = [ - i - for i, o in enumerate(o_18['t2']['t1']) - if o['image_name'] == image_name - ] - assert len(t1) == 1 - return t1[0] + t1 = [i for i, o in enumerate(o_18['t2']['t1']) if o['image_name'] == image_name] + assert len(t1) == 1 + return t1[0] + def kernel_23(o_18, o_22, ids=None): - import numpy - import tqdm + import numpy + import tqdm - if ids is None: - ids = numpy.s_[:] + if ids is None: + ids = numpy.s_[:] - t1 = numpy.arange(len(o_22['t4'])) - t2 = t1[ids] + t1 = numpy.arange(len(o_22['t4'])) + t2 = t1[ids] - pprint.pprint(t2[:5]) - t7 = [] - for o in tqdm.tqdm(t2): - t3 = o_22['t4'][o] - t9 = kernel_26(o_18=o_18, image_name=t3['image_name']) - t4 = o_18['t2']['t1'][t9]['image_canvas'] - t10 = o_18['t2']['t6'][t9] - t4 = [ - o['image_canvas'] - for o in o_18['t2']['t1'] - if o['image_name'] == t3['image_name'] - ] - assert len(t4) == 1 - t5 = t4[0] - t6 = kernel_24(t5, t3['keypoints']) - t7.append(t6['t3']) - t7.append(t10) + pprint.pprint(t2[:5]) + t7 = [] + for o in tqdm.tqdm(t2): + t3 = o_22['t4'][o] + t9 = kernel_26(o_18=o_18, image_name=t3['image_name']) + t4 = o_18['t2']['t1'][t9]['image_canvas'] + t10 = o_18['t2']['t6'][t9] + t4 = [o['image_canvas'] for o in o_18['t2']['t1'] if o['image_name'] == t3['image_name']] + assert len(t4) == 1 + t5 = t4[0] + t6 = kernel_24(t5, t3['keypoints']) + t7.append(t6['t3']) + t7.append(t10) + + kernel_25(t7) - kernel_25(t7) def kernel_27(): - import tqdm - import os - import subprocess - import pprint + import tqdm + import os + import subprocess + import pprint - t5 = '/kaggle/working/ATL AT TOR - April 19, 2015-T0MUK91ZWys.mp4' - t3 = '/kaggle/working/kernel_27-output.dir' - os.makedirs(t3, exist_ok=True) - #R = int(round(30 / 5)) - FRAMERATE = 4 - SLICE_LENGTH = 5 * 6 - for i in tqdm.tqdm(range(100)): - t1 = [SLICE_LENGTH * i, SLICE_LENGTH * (i + 1)] - t2 = os.path.join(t3, 'slice-%d' % i) - os.makedirs(t2, exist_ok=True) - t4 = os.path.join(t2, 'output.mp4') - if os.path.exists(t4): - os.unlink(t4) - with subprocess.Popen([ - 'ffmpeg', - '-i', - '%s' % t5, - '-filter:v', - 'fps=fps=%d' % FRAMERATE, - '-ss', - '%d' % t1[0], - '-t', - '%d' % (t1[1] - t1[0]), - '%s' % t4, - ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: - if False: - pprint.pprint(p.communicate()) - p.wait() - assert p.returncode == 0 + t5 = '/kaggle/working/ATL AT TOR - April 19, 2015-T0MUK91ZWys.mp4' + t3 = '/kaggle/working/kernel_27-output.dir' + os.makedirs(t3, exist_ok=True) + # R = int(round(30 / 5)) + FRAMERATE = 4 + SLICE_LENGTH = 5 * 6 + for i in tqdm.tqdm(range(100)): + t1 = [SLICE_LENGTH * i, SLICE_LENGTH * (i + 1)] + t2 = os.path.join(t3, 'slice-%d' % i) + os.makedirs(t2, exist_ok=True) + t4 = os.path.join(t2, 'output.mp4') + if os.path.exists(t4): + os.unlink(t4) + with subprocess.Popen( + [ + 'ffmpeg', + '-i', + '%s' % t5, + '-filter:v', + 'fps=fps=%d' % FRAMERATE, + '-ss', + '%d' % t1[0], + '-t', + '%d' % (t1[1] - t1[0]), + '%s' % t4, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as p: + if False: + pprint.pprint(p.communicate()) + p.wait() + assert p.returncode == 0 - t6 = ''' + t6 = """ cd /kaggle/working/AlphaPose && \ python3 \ scripts/demo_inference.py \ @@ -1635,90 +1638,91 @@ def kernel_27(): --checkpoint pretrained_models/fast_res50_256x192.pth \ --video %s \ --outdir %s - ''' % (t4, t2) - if False: - pprint.pprint([t4, t2, t6]) - with subprocess.Popen(t6, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: - if False: - pprint.pprint(p.communicate()) - p.wait() - assert p.returncode == 0 + """ % (t4, t2) + if False: + pprint.pprint([t4, t2, t6]) + with subprocess.Popen(t6, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + if False: + pprint.pprint(p.communicate()) + p.wait() + assert p.returncode == 0 + def kernel_28( - video_path=None, - framerate=None, - max_seconds=None, - video_id=None, + video_path=None, + framerate=None, + max_seconds=None, + video_id=None, ): - if video_id is None: - video_id = '' + if video_id is None: + video_id = '' - import cv2 - import tqdm - import os - import subprocess - import pprint + import cv2 + import tqdm + import os + import subprocess + import pprint - if framerate is None: - framerate = 4 - if max_seconds is None: - max_seconds = 999999 + if framerate is None: + framerate = 4 + if max_seconds is None: + max_seconds = 999999 - if video_path is None: - video_path = '/kaggle/working/ATL AT TOR - April 19, 2015-T0MUK91ZWys.mp4' - t5 = video_path - t3 = '/kaggle/working/kernel_28-output%s.dir' % video_id - t13 = '/root/kernel_28-output.dir/tmp-slice' - os.makedirs(t3, exist_ok=True) + if video_path is None: + video_path = '/kaggle/working/ATL AT TOR - April 19, 2015-T0MUK91ZWys.mp4' + t5 = video_path + t3 = '/kaggle/working/kernel_28-output%s.dir' % video_id + t13 = '/root/kernel_28-output.dir/tmp-slice' + os.makedirs(t3, exist_ok=True) - cap = None + cap = None - try: - cap = cv2.VideoCapture(t5) - fps = cap.get(cv2.CAP_PROP_FPS) # OpenCV2 version 2 used "CV_CAP_PROP_FPS" - frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - real_duration = frame_count/fps - duration = min(real_duration, max_seconds) - pprint.pprint( - dict( - fps=fps, - frame_count=frame_count, - duration=duration, - real_duration=real_duration, - ) - ) + try: + cap = cv2.VideoCapture(t5) + fps = cap.get(cv2.CAP_PROP_FPS) # OpenCV2 version 2 used "CV_CAP_PROP_FPS" + frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + real_duration = frame_count / fps + duration = min(real_duration, max_seconds) + pprint.pprint( + dict( + fps=fps, + frame_count=frame_count, + duration=duration, + real_duration=real_duration, + ) + ) - #R = int(round(30 / 5)) - FRAMERATE = framerate - SLICE_LENGTH = 5 * 6 - for i in tqdm.tqdm(range(int(duration / SLICE_LENGTH + 1e-8))): - t2 = os.path.join(t3, 'slice-%d' % i) - if os.path.exists(t2): - pprint.pprint(['t2', t2, 'exists', 'continue']) - continue + # R = int(round(30 / 5)) + FRAMERATE = framerate + SLICE_LENGTH = 5 * 6 + for i in tqdm.tqdm(range(int(duration / SLICE_LENGTH + 1e-8))): + t2 = os.path.join(t3, 'slice-%d' % i) + if os.path.exists(t2): + pprint.pprint(['t2', t2, 'exists', 'continue']) + continue - t1 = [SLICE_LENGTH * i, SLICE_LENGTH * (i + 1)] + t1 = [SLICE_LENGTH * i, SLICE_LENGTH * (i + 1)] - if os.path.exists(t13): - subprocess.check_call(['rm', '-fr', t13]) - os.makedirs(t13, exist_ok=True) + if os.path.exists(t13): + subprocess.check_call(['rm', '-fr', t13]) + os.makedirs(t13, exist_ok=True) - t8 = numpy.array([t1[0] * fps, t1[1] * fps]).astype(numpy.int32) - cap.set(cv2.CAP_PROP_POS_FRAMES, t8[0]) - t12 = numpy.arange(t8[0], t8[1], SLICE_LENGTH * FRAMERATE) - for k in range(t8[1] - t8[0]): - ret, frame = cap.read() - t11 = cap.get(cv2.CAP_PROP_POS_FRAMES) - if numpy.isin(t11, t12): - t10 = os.path.join( - t13, - 'frame-%d.jpg' % t11, - ) - cv2.imwrite(t10, frame) + t8 = numpy.array([t1[0] * fps, t1[1] * fps]).astype(numpy.int32) + cap.set(cv2.CAP_PROP_POS_FRAMES, t8[0]) + t12 = numpy.arange(t8[0], t8[1], SLICE_LENGTH * FRAMERATE) + for k in range(t8[1] - t8[0]): + ret, frame = cap.read() + t11 = cap.get(cv2.CAP_PROP_POS_FRAMES) + if numpy.isin(t11, t12): + t10 = os.path.join( + t13, + 'frame-%d.jpg' % t11, + ) + cv2.imwrite(t10, frame) - os.makedirs(t2, exist_ok=True) + os.makedirs(t2, exist_ok=True) - t6 = ''' + t6 = """ cd /kaggle/working/AlphaPose && \ python3 \ scripts/demo_inference.py \ @@ -1726,533 +1730,500 @@ def kernel_28( --checkpoint pretrained_models/fast_res50_256x192.pth \ --indir %s \ --outdir %s - ''' % (t13, t2) - if False: - pprint.pprint([ - # t4, - t2, t6, - ]) - with subprocess.Popen(t6, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: - if False: - pprint.pprint(p.communicate()) - p.wait() - assert p.returncode == 0 - finally: - if not cap is None: - cap.release() + """ % (t13, t2) + if False: + pprint.pprint( + [ + # t4, + t2, + t6, + ] + ) + with subprocess.Popen(t6, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + if False: + pprint.pprint(p.communicate()) + p.wait() + assert p.returncode == 0 + finally: + if not cap is None: + cap.release() -def kernel_29(video_path=None, video_id=None,): - if video_id is None: - video_id = '' - if video_path is None: - video_path = '/kaggle/working/ATL AT TOR - April 19, 2015-T0MUK91ZWys.mp4' +def kernel_29( + video_path=None, + video_id=None, +): + if video_id is None: + video_id = '' - assert os.path.exists(video_path) + if video_path is None: + video_path = '/kaggle/working/ATL AT TOR - April 19, 2015-T0MUK91ZWys.mp4' - t40 = 'kernel_29-poses%s.json' % video_id - t4 = os.path.join('/kaggle/working', t40) - t6 = [ - t4, - os.path.join('/kaggle/input/garbage', t40), - ] + assert os.path.exists(video_path) - t7 = [ - o - for o in t6 - if os.path.exists(o) - ] + t40 = 'kernel_29-poses%s.json' % video_id + t4 = os.path.join('/kaggle/working', t40) + t6 = [ + t4, + os.path.join('/kaggle/input/garbage', t40), + ] - if len(t7) == 0: + t7 = [o for o in t6 if os.path.exists(o)] - t1 = [ - dict( - data=json.load( - io.open( - o, - 'r' - ) - ), - input_path=o - ) - for o in glob.glob( - '/kaggle/working/kernel_28-output%s.dir/slice-*/*.json' % video_id - ) - ] + if len(t7) == 0: + t1 = [dict(data=json.load(io.open(o, 'r')), input_path=o) for o in glob.glob('/kaggle/working/kernel_28-output%s.dir/slice-*/*.json' % video_id)] - assert len(t1) > 0 + assert len(t1) > 0 - with io.open(t4, 'w') as f: - f.write(json.dumps(t1)) + with io.open(t4, 'w') as f: + f.write(json.dumps(t1)) - t7.append(t4) + t7.append(t4) - with io.open(t7[0], 'r') as f: - t1 = json.load(f) + with io.open(t7[0], 'r') as f: + t1 = json.load(f) - t8 = sum([ - o['data'] - for o in t1 - ], []) - t10 = re.compile(r'frame-(\d+)\.jpg') + t8 = sum([o['data'] for o in t1], []) + t10 = re.compile(r'frame-(\d+)\.jpg') - for i, o in enumerate(t8): - o['frame_id'] = int(t10.match(o['image_id'])[1]) + for i, o in enumerate(t8): + o['frame_id'] = int(t10.match(o['image_id'])[1]) - t9 = sorted( - t8, - key=lambda o: o['frame_id'] - ) + t9 = sorted(t8, key=lambda o: o['frame_id']) - t2 = pandas.DataFrame(t9) + t2 = pandas.DataFrame(t9) - t5 = t2.to_xarray() - t5['keypoints'] = xarray.DataArray( - [ - numpy.array(o).reshape(17, 3) - for o in t5['keypoints'].data - ], - dims=['index', 'joint', 'feature'], - ) - o_31 = kernel_31( - image_id=t5.image_id.data, - image_size=[ - kernel_32(video_path) - ] * t5.index.shape[0], - keypoints=t5.keypoints.data, - ) - for k, v in o_31['t13'].items(): - #pprint.pprint([k, v]) - t5[k] = v + t5 = t2.to_xarray() + t5['keypoints'] = xarray.DataArray( + [numpy.array(o).reshape(17, 3) for o in t5['keypoints'].data], + dims=['index', 'joint', 'feature'], + ) + o_31 = kernel_31( + image_id=t5.image_id.data, + image_size=[kernel_32(video_path)] * t5.index.shape[0], + keypoints=t5.keypoints.data, + ) + for k, v in o_31['t13'].items(): + # pprint.pprint([k, v]) + t5[k] = v + + return dict( + t5=t5, + t2=t2, + t9=t9, + ) - return dict( - t5=t5, - t2=t2, - t9=t9, - ) def kernel_30( - o_29, - ids=None, - delay=None, - prev_frames=None, - max_frames=None, - max_ids=None, - video_path=None, - low_portion=None, - low_mean_conf=None, - no_dots=None, + o_29, + ids=None, + delay=None, + prev_frames=None, + max_frames=None, + max_ids=None, + video_path=None, + low_portion=None, + low_mean_conf=None, + no_dots=None, ): - if no_dots is None: - no_dots = False + if no_dots is None: + no_dots = False - if low_portion is None: - low_portion = 0.1 - if low_mean_conf is None: - low_mean_conf = 0.6 + if low_portion is None: + low_portion = 0.1 + if low_mean_conf is None: + low_mean_conf = 0.6 - if video_path is None: - video_path = '/kaggle/working/ATL AT TOR - April 19, 2015-T0MUK91ZWys.mp4' + if video_path is None: + video_path = '/kaggle/working/ATL AT TOR - April 19, 2015-T0MUK91ZWys.mp4' - if max_frames is None: - max_frames = 9999 - if max_ids is None: - max_ids = 70 + if max_frames is None: + max_frames = 9999 + if max_ids is None: + max_ids = 70 - if prev_frames is None: - prev_frames = 0 + if prev_frames is None: + prev_frames = 0 - t5 = o_29['t5'] + t5 = o_29['t5'] - if delay is None: - delay = 200 + if delay is None: + delay = 200 - if ids is None: - numpy.random.seed(0) - ids = numpy.random.choice( - t5.index.shape[0], - 10 - ) - elif ids == 'v1': - t8 = numpy.where( - numpy.logical_and( - o_29['t5'].portion > low_portion, - o_29['t5'].mean_conf > low_mean_conf, - ) - )[0] - ids = numpy.random.choice( - t8, - min(len(t8), 70), - replace=False, - ) - elif ids == 'v2': - t8 = numpy.stack( - [ - o_29['t5'].portion > 0.1, - o_29['t5'].mean_conf > 0.6, - o_29['t5']['t11'].data, - ], - axis=0 - ).prod(axis=0) - pprint.pprint([t8.sum(), t8.mean()]) - ids = numpy.random.choice( - numpy.where(t8)[0], - min(70, t8.sum()), - ) - elif ids == 'v3': - t8 = numpy.stack( - [ - o_29['t5'].portion > 0.1, - o_29['t5'].mean_conf > 0.6, - o_29['t5']['t10'].data, - ], - axis=0 - ).prod(axis=0) - pprint.pprint([t8.sum(), t8.mean()]) - ids = numpy.random.choice( - numpy.where(t8)[0], - min(70, t8.sum()), - ) - elif ids == 'v4': - t8 = numpy.stack( - [ - o_29['t5'].portion > 0.1, - o_29['t5'].min_conf > 0.6, - o_29['t5']['t9'].data > 0, - ], - axis=0 - ).prod(axis=0) - pprint.pprint([t8.sum(), t8.mean()]) - ids = numpy.random.choice( - numpy.where(t8)[0], - min(70, t8.sum()), - ) - elif ids == 'v5': - t8 = numpy.stack( - [ - o_29['t5'].portion > 0.1, - o_29['t5'].min_conf > 0.5, - o_29['t5']['t8'].data > 0, - ], - axis=0 - ).prod(axis=0) - pprint.pprint([t8.sum(), t8.mean()]) - ids = numpy.random.choice( - numpy.where(t8)[0], - min(70, t8.sum()), - ) - elif ids == 'v6': - t8 = numpy.stack( - [ - o_29['t5'].portion > 0.02, - o_29['t5'].min_conf > 0.4, - o_29['t5']['t8'].data > 0, - ], - axis=0 - ).prod(axis=0) - ids = numpy.random.choice( - numpy.where(t8)[0], - min(70, t8.sum()), - replace=False, - ) - pprint.pprint([ - t8.sum(), - t8.mean(), - ids, - o_29['t5'].sel(index=o_29['t5'].index.data[ids[:5]]).to_dict(), - ]) - elif ids == 'v7': - t8 = numpy.stack( - [ - o_29['t5'].portion > 0.02, - o_29['t5'].min_conf > 0.4, - numpy.stack( - [ - o_29['t5']['t4'].data > 0, - o_29['t5']['t5'].data > 0, - ], - axis=0 - ).sum(axis=0) > 0, - ], - axis=0 - ).prod(axis=0) - ids = numpy.random.choice( - numpy.where(t8)[0], - min(70, t8.sum()), - replace=False, - ) - pprint.pprint([ - t8.sum(), - t8.mean(), - ids, - o_29['t5'].sel(index=o_29['t5'].index.data[ids[:5]]).to_dict(), - ]) - else: - assert isinstance(ids, numpy.ndarray) + if ids is None: + numpy.random.seed(0) + ids = numpy.random.choice(t5.index.shape[0], 10) + elif ids == 'v1': + t8 = numpy.where( + numpy.logical_and( + o_29['t5'].portion > low_portion, + o_29['t5'].mean_conf > low_mean_conf, + ) + )[0] + ids = numpy.random.choice( + t8, + min(len(t8), 70), + replace=False, + ) + elif ids == 'v2': + t8 = numpy.stack( + [ + o_29['t5'].portion > 0.1, + o_29['t5'].mean_conf > 0.6, + o_29['t5']['t11'].data, + ], + axis=0, + ).prod(axis=0) + pprint.pprint([t8.sum(), t8.mean()]) + ids = numpy.random.choice( + numpy.where(t8)[0], + min(70, t8.sum()), + ) + elif ids == 'v3': + t8 = numpy.stack( + [ + o_29['t5'].portion > 0.1, + o_29['t5'].mean_conf > 0.6, + o_29['t5']['t10'].data, + ], + axis=0, + ).prod(axis=0) + pprint.pprint([t8.sum(), t8.mean()]) + ids = numpy.random.choice( + numpy.where(t8)[0], + min(70, t8.sum()), + ) + elif ids == 'v4': + t8 = numpy.stack( + [ + o_29['t5'].portion > 0.1, + o_29['t5'].min_conf > 0.6, + o_29['t5']['t9'].data > 0, + ], + axis=0, + ).prod(axis=0) + pprint.pprint([t8.sum(), t8.mean()]) + ids = numpy.random.choice( + numpy.where(t8)[0], + min(70, t8.sum()), + ) + elif ids == 'v5': + t8 = numpy.stack( + [ + o_29['t5'].portion > 0.1, + o_29['t5'].min_conf > 0.5, + o_29['t5']['t8'].data > 0, + ], + axis=0, + ).prod(axis=0) + pprint.pprint([t8.sum(), t8.mean()]) + ids = numpy.random.choice( + numpy.where(t8)[0], + min(70, t8.sum()), + ) + elif ids == 'v6': + t8 = numpy.stack( + [ + o_29['t5'].portion > 0.02, + o_29['t5'].min_conf > 0.4, + o_29['t5']['t8'].data > 0, + ], + axis=0, + ).prod(axis=0) + ids = numpy.random.choice( + numpy.where(t8)[0], + min(70, t8.sum()), + replace=False, + ) + pprint.pprint( + [ + t8.sum(), + t8.mean(), + ids, + o_29['t5'].sel(index=o_29['t5'].index.data[ids[:5]]).to_dict(), + ] + ) + elif ids == 'v7': + t8 = numpy.stack( + [ + o_29['t5'].portion > 0.02, + o_29['t5'].min_conf > 0.4, + numpy.stack( + [ + o_29['t5']['t4'].data > 0, + o_29['t5']['t5'].data > 0, + ], + axis=0, + ).sum(axis=0) + > 0, + ], + axis=0, + ).prod(axis=0) + ids = numpy.random.choice( + numpy.where(t8)[0], + min(70, t8.sum()), + replace=False, + ) + pprint.pprint( + [ + t8.sum(), + t8.mean(), + ids, + o_29['t5'].sel(index=o_29['t5'].index.data[ids[:5]]).to_dict(), + ] + ) + else: + assert isinstance(ids, numpy.ndarray) - ids = ids[:max_ids] - pprint.pprint(['ids', ids]) + ids = ids[:max_ids] + pprint.pprint(['ids', ids]) - t7 = [] + t7 = [] - t16 = video_path - t15 = kernel_32(t16) + t16 = video_path + t15 = kernel_32(t16) - for o in tqdm.tqdm(ids): - t2 = t5.keypoints.data[o] - t3 = t5.frame_id.data[o] + for o in tqdm.tqdm(ids): + t2 = t5.keypoints.data[o] + t3 = t5.frame_id.data[o] - cap = None - try: - cap = cv2.VideoCapture(t16) - if prev_frames > 0: - t14 = kernel_24( - numpy.zeros(t15, dtype=numpy.uint8), - t2, - )['t3'] - t17 = (t14 == 0).astype(numpy.uint8) + cap = None + try: + cap = cv2.VideoCapture(t16) + if prev_frames > 0: + t14 = kernel_24( + numpy.zeros(t15, dtype=numpy.uint8), + t2, + )['t3'] + t17 = (t14 == 0).astype(numpy.uint8) - for k in range(prev_frames): - cap.set(cv2.CAP_PROP_POS_FRAMES, t3 - prev_frames + k) - ret, frame = cap.read() - t12 = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - #t13 = kernel_24(t12, t2)['t3'] - t13 = t12 * t17 + t14 - t7.append(t13) + for k in range(prev_frames): + cap.set(cv2.CAP_PROP_POS_FRAMES, t3 - prev_frames + k) + ret, frame = cap.read() + t12 = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + # t13 = kernel_24(t12, t2)['t3'] + t13 = t12 * t17 + t14 + t7.append(t13) + + cap.set(cv2.CAP_PROP_POS_FRAMES, t3) + ret, frame = cap.read() + t4 = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + if not no_dots: + t6 = kernel_24(t4, t2)['t3'] + else: + t6 = t4 + t7.append(t6) + finally: + if not cap is None: + cap.release() + kernel_25(t7[:max_frames], delay=delay) - cap.set(cv2.CAP_PROP_POS_FRAMES, t3) - ret, frame = cap.read() - t4 = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - if not no_dots: - t6 = kernel_24(t4, t2)['t3'] - else: - t6 = t4 - t7.append(t6) - finally: - if not cap is None: - cap.release() - kernel_25(t7[:max_frames], delay=delay) def kernel_31(image_id, image_size, keypoints): - def get_angle(a,b): - from math import sqrt, acos, degrees, atan - #print(a) - #print(b) - del_y = a[1]-b[1] - del_x = b[0]-a[0] - if del_x == 0: - del_x = 0.1 - #print("Del_X : "+str(del_x)+"-----Del_Y: "+str(del_y)) - angle = 0 + def get_angle(a, b): + from math import sqrt, acos, degrees, atan - if del_x > 0 and del_y > 0: - angle = degrees(atan(del_y / del_x)) - elif del_x < 0 and del_y > 0: - angle = degrees(atan(del_y / del_x)) + 180 + # print(a) + # print(b) + del_y = a[1] - b[1] + del_x = b[0] - a[0] + if del_x == 0: + del_x = 0.1 + # print("Del_X : "+str(del_x)+"-----Del_Y: "+str(del_y)) + angle = 0 - return angle + if del_x > 0 and del_y > 0: + angle = degrees(atan(del_y / del_x)) + elif del_x < 0 and del_y > 0: + angle = degrees(atan(del_y / del_x)) + 180 - def angle_gor(a,b,c,d): - import numpy as np - from math import sqrt, acos, degrees, atan - ab=[a[0]-b[0],a[1]-b[1]] - ab1=[c[0]-d[0],c[1]-d[1]] - cos = \ - abs( - ab[0] * ab1[0] + ab[1] * ab1[1] - ) / ( - sqrt( - ab[0] ** 2 + ab[1] ** 2 - ) * \ - sqrt( - ab1[0] ** 2 + ab1[1] ** 2 - ) + 1e-8 - ) - ang = acos(cos) - return ang*180/np.pi + return angle + def angle_gor(a, b, c, d): + import numpy as np + from math import sqrt, acos, degrees, atan - def sit_ang(a,b,c,d): - ang=angle_gor(a,b,c,d) - s1=0 - if not ang is None: - #print("Angle",ang) - if ang < 120 and ang>40: - s1=1 - return s1 + ab = [a[0] - b[0], a[1] - b[1]] + ab1 = [c[0] - d[0], c[1] - d[1]] + cos = abs(ab[0] * ab1[0] + ab[1] * ab1[1]) / (sqrt(ab[0] ** 2 + ab[1] ** 2) * sqrt(ab1[0] ** 2 + ab1[1] ** 2) + 1e-8) + ang = acos(cos) + return ang * 180 / np.pi - def sit_rec(a,b,c,d): - from math import sqrt, acos, degrees, atan + def sit_ang(a, b, c, d): + ang = angle_gor(a, b, c, d) + s1 = 0 + if not ang is None: + # print("Angle",ang) + if ang < 120 and ang > 40: + s1 = 1 + return s1 - ab = [a[0] - b[0], a[1] - b[1]] - ab1 = [c[0] - d[0], c[1] - d[1]] - l1=sqrt(ab[0]**2+ab[1]**2) - l2=sqrt(ab1[0]**2+ab1[1]**2) - s=0 - if l1!=0 and l2!=0: - #print(l1,l2, "---------->>>") - if l2/l1>=1.5: - s=1 - return s + def sit_rec(a, b, c, d): + from math import sqrt, acos, degrees, atan - t3 = [] - for i, o in enumerate(keypoints): - t4 = numpy.min(o[:, 0]) - t5 = numpy.max(o[:, 0]) - t6 = numpy.min(o[:, 1]) - t7 = numpy.max(o[:, 1]) + ab = [a[0] - b[0], a[1] - b[1]] + ab1 = [c[0] - d[0], c[1] - d[1]] + l1 = sqrt(ab[0] ** 2 + ab[1] ** 2) + l2 = sqrt(ab1[0] ** 2 + ab1[1] ** 2) + s = 0 + if l1 != 0 and l2 != 0: + # print(l1,l2, "---------->>>") + if l2 / l1 >= 1.5: + s = 1 + return s - t8 = (t5 - t4) * (t7 - t6) + t3 = [] + for i, o in enumerate(keypoints): + t4 = numpy.min(o[:, 0]) + t5 = numpy.max(o[:, 0]) + t6 = numpy.min(o[:, 1]) + t7 = numpy.max(o[:, 1]) - t9 = image_size[i] - t10 = t9[0] * t9[1] - t3.append( - dict( - t8=t8, - t9=t9, - t10=t10, - ) - ) - t12 = [ - dict( - square=t3[i]['t8'], - total=t3[i]['t10'], - portion=t3[i]['t8'] / (t3[i]['t10'] + 1e-8), - keypoints=keypoints[i], - image_name=image_id[i], - ) - for i in range(len(t3)) - ] + t8 = (t5 - t4) * (t7 - t6) - for i, o in enumerate(t12): - t1 = o['keypoints'] - t2 = get_angle( - t1[3, :2], - t1[4, :2], - ) - t3 = get_angle( - t1[6, :2], - t1[7, :2], - ) - t4 = 0 - if 30 < t2 and t2 < 150: - t4 = 1 - t5 = 0 - if 30 < t3 and t3 < 150: - t5 = 1 - t6 = t4 + t6 - t7 = 0 - if t6 == 1: - t7 = 1 - t8 = 0 - t8 += sit_rec( - t1[9, :2], - t1[10, :2], - t1[10, :2], - t1[11, :2], - ) - t8 += sit_rec( - t1[12, :2], - t1[13, :2], - t1[13, :2], - t1[14, :2], - ) - t9 = 0 - t9 += sit_ang( - t1[9, :2], - t1[10, :2], - t1[10, :2], - t1[11, :2], - ) - t9 += sit_ang( - t1[12, :2], - t1[13, :2], - t1[13, :2], - t1[14, :2], - ) - t10 = 0 - if t8 > 0 or t9 > 0: - t10 = 1 - t11 = 0 - if t8 == 0 and t9 == 0: - t11 = 1 - o.update( - dict( - t2=t2, - t3=t3, - t4=t4, - t5=t5, - t6=t6, - t7=t7, - t8=t8, - t9=t9, - t10=t10, - t11=t11, - mean_conf=t1[:, 2].mean(), - min_conf=t1[:, 2].min(), - ) - ) + t9 = image_size[i] + t10 = t9[0] * t9[1] + t3.append( + dict( + t8=t8, + t9=t9, + t10=t10, + ) + ) + t12 = [ + dict( + square=t3[i]['t8'], + total=t3[i]['t10'], + portion=t3[i]['t8'] / (t3[i]['t10'] + 1e-8), + keypoints=keypoints[i], + image_name=image_id[i], + ) + for i in range(len(t3)) + ] - t13 = pandas.DataFrame(t12).drop(columns=['keypoints']).to_xarray() + for i, o in enumerate(t12): + t1 = o['keypoints'] + t2 = get_angle( + t1[3, :2], + t1[4, :2], + ) + t3 = get_angle( + t1[6, :2], + t1[7, :2], + ) + t4 = 0 + if 30 < t2 and t2 < 150: + t4 = 1 + t5 = 0 + if 30 < t3 and t3 < 150: + t5 = 1 + t6 = t4 + t6 + t7 = 0 + if t6 == 1: + t7 = 1 + t8 = 0 + t8 += sit_rec( + t1[9, :2], + t1[10, :2], + t1[10, :2], + t1[11, :2], + ) + t8 += sit_rec( + t1[12, :2], + t1[13, :2], + t1[13, :2], + t1[14, :2], + ) + t9 = 0 + t9 += sit_ang( + t1[9, :2], + t1[10, :2], + t1[10, :2], + t1[11, :2], + ) + t9 += sit_ang( + t1[12, :2], + t1[13, :2], + t1[13, :2], + t1[14, :2], + ) + t10 = 0 + if t8 > 0 or t9 > 0: + t10 = 1 + t11 = 0 + if t8 == 0 and t9 == 0: + t11 = 1 + o.update( + dict( + t2=t2, + t3=t3, + t4=t4, + t5=t5, + t6=t6, + t7=t7, + t8=t8, + t9=t9, + t10=t10, + t11=t11, + mean_conf=t1[:, 2].mean(), + min_conf=t1[:, 2].min(), + ) + ) + + t13 = pandas.DataFrame(t12).drop(columns=['keypoints']).to_xarray() + + return dict( + t12=t12, + t13=t13, + ) - return dict( - t12=t12, - t13=t13, - ) def kernel_32(video_path): - assert os.path.exists(video_path) + assert os.path.exists(video_path) - cap = None - try: - cap = cv2.VideoCapture(video_path) - ret, frame = cap.read() - return frame.shape - finally: - if not cap is None: - cap.release() + cap = None + try: + cap = cv2.VideoCapture(video_path) + ret, frame = cap.read() + return frame.shape + finally: + if not cap is None: + cap.release() def kernel_33(): - o_14 = kernel_14() - o_15 = kernel_15(o_14=o_14) + o_14 = kernel_14() + o_15 = kernel_15(o_14=o_14) - assert os.system(r''' python repo/d1/mlb_player_v5.py ''') == 0 - o_17 = kernel_17(o_14=o_14, max_images=100) + assert os.system(r""" python repo/d1/mlb_player_v5.py """) == 0 + o_17 = kernel_17(o_14=o_14, max_images=100) + + o_18 = kernel_18(o_17=o_17) + o_19 = kernel_19(o_18=o_18) + o_20 = kernel_20(o_18=o_18) + o_22 = kernel_22(o_18=o_18) + import pandas + + o_23 = kernel_23(o_18=o_18, o_22=o_22, ids=pandas.DataFrame(o_22['t4']).query('portion > 0.1').index.values) + o_27 = kernel_27() + o_28 = kernel_28() + o_29 = kernel_29() + import numpy + + o_30 = kernel_30(o_29=o_29, ids='v1') - o_18 = kernel_18(o_17=o_17) - o_19 = kernel_19(o_18=o_18) - o_20 = kernel_20(o_18=o_18) - o_22 = kernel_22(o_18=o_18) - import pandas - o_23 = kernel_23( - o_18=o_18, - o_22=o_22, - ids=pandas.DataFrame( - o_22['t4'] - ).query('portion > 0.1').index.values - ) - o_27 = kernel_27() - o_28 = kernel_28() - o_29 = kernel_29() - import numpy - o_30 = kernel_30(o_29=o_29, ids='v1') def kernel_34(o_14): - t1 = numpy.unique( - o_14['o_3']['t5']['events']['event'], - return_counts=True, - ) - t2 = [ - [o, o2] - for o, o2 in zip(*t1) - ] - pprint.pprint( - t2 - ) + t1 = numpy.unique( + o_14['o_3']['t5']['events']['event'], + return_counts=True, + ) + t2 = [[o, o2] for o, o2 in zip(*t1)] + pprint.pprint(t2) + def kernel_35(): - return r''' + return r""" 1. https://posetrack.net/leaderboard.php 2. https://posetrack.net/users/view_entry_details.php?entry=CF_chl2 3. https://posetrack.net/users/view_entry_details.php?entry=DCPose_chl2 @@ -2262,23 +2233,27 @@ def kernel_35(): 7. https://github.com/leoxiaobin/deep-high-resolution-net.pytorch 8. https://github.com/CMU-Perceptual-Computing-Lab/openpose https://github.com/CMU-Perceptual-Computing-Lab/openpose/issues/1736#issuecomment-736846794 - ''' - -def kernel_36(): - # -*- coding: utf-8 -*- - """OpenPose.ipynb - - Automatically generated by Colaboratory. - - Original file is located at - https://colab.research.google.com/drive/1VDjRLKAu9KLQky0gv4RLJjeH6NRPiWXy """ - assert os.system(r''' - cd openpose && rm -rf build || true && mkdir build && cd build && cmake .. && make -j`nproc` - ''') == 0 - """# Pose Detection with OpenPose +def kernel_36(): + # -*- coding: utf-8 -*- + """OpenPose.ipynb + + Automatically generated by Colaboratory. + + Original file is located at + https://colab.research.google.com/drive/1VDjRLKAu9KLQky0gv4RLJjeH6NRPiWXy + """ + + assert ( + os.system(r""" + cd openpose && rm -rf build || true && mkdir build && cd build && cmake .. && make -j`nproc` + """) + == 0 + ) + + """# Pose Detection with OpenPose This notebook uses an open source project [CMU-Perceptual-Computing-Lab/openpose](https://github.com/CMU-Perceptual-Computing-Lab/openpose.git) to detect/track multi person poses on a video from your Google Drive @@ -2287,153 +2262,177 @@ def kernel_36(): ## Choose a video from your Google Drive """ - from google.colab import drive - drive.mount('/content/drive') + from google.colab import drive - """## Install OpenPose on Google Colab + drive.mount('/content/drive') + + """## Install OpenPose on Google Colab """ - # import os - from os.path import exists, join, basename, splitext + # import os + from os.path import exists, join, basename, splitext - git_repo_url = 'https://github.com/CMU-Perceptual-Computing-Lab/openpose.git' - project_name = splitext(basename(git_repo_url))[0] + git_repo_url = 'https://github.com/CMU-Perceptual-Computing-Lab/openpose.git' + project_name = splitext(basename(git_repo_url))[0] - if 1 or not exists(project_name): - assert os.system(r'''!rm -rf openpose''') == 0 - # see: https://github.com/CMU-Perceptual-Computing-Lab/openpose/issues/949 - print("install new CMake becaue of CUDA10") - cmake_version = 'cmake-3.20.2-linux-x86_64.tar.gz' - if not exists(cmake_version): - assert os.system(r'''!wget -q 'https://cmake.org/files/v3.20/{cmake_version}' ''') == 0 - assert os.system(r'''!tar xfz {cmake_version} --strip-components=1 -C /usr/local ''') == 0 + if 1 or not exists(project_name): + assert os.system(r"""!rm -rf openpose""") == 0 + # see: https://github.com/CMU-Perceptual-Computing-Lab/openpose/issues/949 + print('install new CMake becaue of CUDA10') + cmake_version = 'cmake-3.20.2-linux-x86_64.tar.gz' + if not exists(cmake_version): + assert os.system(r"""!wget -q 'https://cmake.org/files/v3.20/{cmake_version}' """) == 0 + assert os.system(r"""!tar xfz {cmake_version} --strip-components=1 -C /usr/local """) == 0 - print("clone openpose") - assert os.system(r'''!git clone -q --depth 1 $git_repo_url ''') == 0 - print("install system dependencies") - assert os.system(r'''!apt-get -qq install -y libatlas-base-dev libprotobuf-dev libleveldb-dev libsnappy-dev libhdf5-serial-dev protobuf-compiler libgflags-dev libgoogle-glog-dev liblmdb-dev opencl-headers ocl-icd-opencl-dev libviennacl-dev ''') == 0 - print("build openpose") - assert os.system(r'''!cd openpose && rm -rf build || true && mkdir build && cd build && cmake .. && make -j`nproc` ''') == 0 + print('clone openpose') + assert os.system(r"""!git clone -q --depth 1 $git_repo_url """) == 0 + print('install system dependencies') + assert ( + os.system( + r"""!apt-get -qq install -y libatlas-base-dev libprotobuf-dev libleveldb-dev libsnappy-dev libhdf5-serial-dev protobuf-compiler libgflags-dev libgoogle-glog-dev liblmdb-dev opencl-headers ocl-icd-opencl-dev libviennacl-dev """ + ) + == 0 + ) + print('build openpose') + assert os.system(r"""!cd openpose && rm -rf build || true && mkdir build && cd build && cmake .. && make -j`nproc` """) == 0 - """## From a Google Drive's folder""" + """## From a Google Drive's folder""" - # import os - # from os.path import exists, join, basename, splitext + # import os + # from os.path import exists, join, basename, splitext - folder_path = '/content/drive/My Drive/openpose/' - files = os.listdir(folder_path) - files.reverse() - for filename in files: - if filename.endswith('.mp4') and not filename.endswith('-openpose.mp4'): - print(filename) - colab_video_path = folder_path + filename - print(colab_video_path) - colab_openpose_video_path = colab_video_path.replace('.mp4', '') + '-openpose.mp4' - print(colab_openpose_video_path) - if not exists(colab_openpose_video_path): - assert os.system(r'''!cd openpose && ./build/examples/openpose/openpose.bin --hand --face --number_people_max 12 --video '{colab_video_path}' --display 0 --write_video_with_audio --write_video '{colab_openpose_video_path}' # --net_resolution "-1x736" --scale_number 4 --scale_gap 0.25 ''') == 0 + folder_path = '/content/drive/My Drive/openpose/' + files = os.listdir(folder_path) + files.reverse() + for filename in files: + if filename.endswith('.mp4') and not filename.endswith('-openpose.mp4'): + print(filename) + colab_video_path = folder_path + filename + print(colab_video_path) + colab_openpose_video_path = colab_video_path.replace('.mp4', '') + '-openpose.mp4' + print(colab_openpose_video_path) + if not exists(colab_openpose_video_path): + assert ( + os.system( + r"""!cd openpose && ./build/examples/openpose/openpose.bin --hand --face --number_people_max 12 --video '{colab_video_path}' --display 0 --write_video_with_audio --write_video '{colab_openpose_video_path}' # --net_resolution "-1x736" --scale_number 4 --scale_gap 0.25 """ + ) + == 0 + ) - """## From Youtube (Downloaded to your Drive)""" + """## From Youtube (Downloaded to your Drive)""" - assert os.system(r'''!pip install youtube-dl ''') == 0 + assert os.system(r"""!pip install youtube-dl """) == 0 - youtube_id = '2021-05-07_22-00-55_UTC' - assert os.system(r'''!youtube-dl -f mp4 -o '/content/drive/My Drive/openpose/%(id)s.mp4' {youtube_id} ''') == 0 - colab_video_path = '/content/drive/My Drive/openpose/' + youtube_id + '.mp4' - colab_openpose_video_path = colab_video_path.replace('.mp4', '') + '-openpose.mp4' + youtube_id = '2021-05-07_22-00-55_UTC' + assert os.system(r"""!youtube-dl -f mp4 -o '/content/drive/My Drive/openpose/%(id)s.mp4' {youtube_id} """) == 0 + colab_video_path = '/content/drive/My Drive/openpose/' + youtube_id + '.mp4' + colab_openpose_video_path = colab_video_path.replace('.mp4', '') + '-openpose.mp4' - assert os.system(r'''!cd openpose && ./build/examples/openpose/openpose.bin --number_people_max 12 --video '{colab_video_path}' --display 0 --write_video_with_audio --write_video '{colab_openpose_video_path}' # --net_resolution "-1x736" --scale_number 4 --scale_gap 0.25 ''') == 0 + assert ( + os.system( + r"""!cd openpose && ./build/examples/openpose/openpose.bin --number_people_max 12 --video '{colab_video_path}' --display 0 --write_video_with_audio --write_video '{colab_openpose_video_path}' # --net_resolution "-1x736" --scale_number 4 --scale_gap 0.25 """ + ) + == 0 + ) + """## Run OpenPose""" + assert ( + os.system( + r"""!wget 'https://cdn-eu-hz-4.ufile.io/get/z5yes0he?token=ZWUyZjMyNDBmNTRlMWY0NjRkZTUzYzQzNjg2OWY4ODgzMzA1ODFjNzY0YzYxYWFjNzM1ZTU2OWYwNzQyNzVkOWUwYjY3MjdiZmI1MDg0NmIyMzA3ODhiZmQ2YmI3ZWYyOTE3NDBlNDEwY2JlZTc5N2MwMmIxYjYzZGJjYTZjYzl3K3dtK25xdXE2M1ZUZXZMUU5CYzI0WjBtUEk4OTJQYzFvaTJXa1huRkdtNlhjZGxlTHkvTEJVNVlucnBOZFdHMXBiVkszeHBKWEFaTzRKK1h4Zy9BRDNvbkttNDhJVXRKZjJuSUIyTDVaR0UydjM3amtVdE93aFQ4ZXFGTlZYeVZiTXQwT3pkOEdLTklQRUhZNmR1MHRPeVdMb3ZHRWV3R3BYWStQcElkeUdDTkVvUU1IRTUxaE1acDlCM3htRDBaS2ZjWEpLMXlpNXRWUWdLWEoreld3N3VvLy9ocXE2RUxPS2pDdENtVjB0Zi9qWGh3WEIwK3RzU0xZekloK3k2' -O output.mp4 """ + ) + == 0 + ) - """## Run OpenPose""" + # video_folder = os.path.dirname(colab_video_path) + # video_base_name = os.path.basename(colab_video_path) + # print(video_base_name) + # import os + # from os.path import exists, join, basename, splitext + # colab_video_path = '/content/drive/My Drive/bachata.mp4' + colab_video_path = '/content/output.mp4' + colab_openpose_video_path = colab_video_path.replace('.mp4', '') + '-openpose.mp4' - assert os.system(r'''!wget 'https://cdn-eu-hz-4.ufile.io/get/z5yes0he?token=ZWUyZjMyNDBmNTRlMWY0NjRkZTUzYzQzNjg2OWY4ODgzMzA1ODFjNzY0YzYxYWFjNzM1ZTU2OWYwNzQyNzVkOWUwYjY3MjdiZmI1MDg0NmIyMzA3ODhiZmQ2YmI3ZWYyOTE3NDBlNDEwY2JlZTc5N2MwMmIxYjYzZGJjYTZjYzl3K3dtK25xdXE2M1ZUZXZMUU5CYzI0WjBtUEk4OTJQYzFvaTJXa1huRkdtNlhjZGxlTHkvTEJVNVlucnBOZFdHMXBiVkszeHBKWEFaTzRKK1h4Zy9BRDNvbkttNDhJVXRKZjJuSUIyTDVaR0UydjM3amtVdE93aFQ4ZXFGTlZYeVZiTXQwT3pkOEdLTklQRUhZNmR1MHRPeVdMb3ZHRWV3R3BYWStQcElkeUdDTkVvUU1IRTUxaE1acDlCM3htRDBaS2ZjWEpLMXlpNXRWUWdLWEoreld3N3VvLy9ocXE2RUxPS2pDdENtVjB0Zi9qWGh3WEIwK3RzU0xZekloK3k2' -O output.mp4 ''') == 0 + assert ( + os.system( + r"""!cd openpose && ./build/examples/openpose/openpose.bin --number_people_max 12 --video '{colab_video_path}' --display 0 --write_video_with_audio --write_video '{colab_openpose_video_path}' # --net_resolution "-1x736" --scale_number 4 --scale_gap 0.25 """ + ) + == 0 + ) - # video_folder = os.path.dirname(colab_video_path) - # video_base_name = os.path.basename(colab_video_path) - # print(video_base_name) - # import os - # from os.path import exists, join, basename, splitext - #colab_video_path = '/content/drive/My Drive/bachata.mp4' - colab_video_path = '/content/output.mp4' - colab_openpose_video_path = colab_video_path.replace('.mp4', '') + '-openpose.mp4' + assert os.system(r"""!cmake -version """) == 0 - assert os.system(r'''!cd openpose && ./build/examples/openpose/openpose.bin --number_people_max 12 --video '{colab_video_path}' --display 0 --write_video_with_audio --write_video '{colab_openpose_video_path}' # --net_resolution "-1x736" --scale_number 4 --scale_gap 0.25 ''') == 0 - - assert os.system(r'''!cmake -version ''') == 0 def kernel_37(): - return r''' + return r""" 1. https://www.youtube.com/channel/UCcTy8yBARva3WzrmIcaaWPA/about 2. https://www.play-cricket.com/website/results/4721802 - ''' + """ + def kernel_38( - video_path, - framerate=None, - max_seconds=None, - ids=None, - prev_frames=None, - delay=None, - low_mean_conf=None, - low_portion=None, - no_dots=None, + video_path, + framerate=None, + max_seconds=None, + ids=None, + prev_frames=None, + delay=None, + low_mean_conf=None, + low_portion=None, + no_dots=None, ): - if ids is None: - ids = 'v7' + if ids is None: + ids = 'v7' - if prev_frames is None: - prev_frames = 59 + if prev_frames is None: + prev_frames = 59 - if delay is None: - delay = 3 + if delay is None: + delay = 3 - t2 = hashlib.sha256( - video_path.encode('utf-8') - ).hexdigest() + t2 = hashlib.sha256(video_path.encode('utf-8')).hexdigest() - t1 = '/kaggle/working/video%s.mp4' % t2 + t1 = '/kaggle/working/video%s.mp4' % t2 - if not os.path.exists(t1): - subprocess.check_call( - [ - 'youtube-dl', - '-f', - '18', - video_path, - '-o', - t1, - ] - ) + if not os.path.exists(t1): + subprocess.check_call( + [ + 'youtube-dl', + '-f', + '18', + video_path, + '-o', + t1, + ] + ) - kernel_28( - t1, - framerate=framerate, - max_seconds=max_seconds, - video_id=t2, - ) + kernel_28( + t1, + framerate=framerate, + max_seconds=max_seconds, + video_id=t2, + ) - o_29 = kernel_29( - t1, - video_id=t2, - ) + o_29 = kernel_29( + t1, + video_id=t2, + ) - o_30 = kernel_30( - o_29=o_29, - ids=ids, - delay=delay, - prev_frames=prev_frames, - max_frames=9999, - max_ids=999, - video_path=t1, - low_mean_conf=low_mean_conf, - low_portion=low_portion, - no_dots=no_dots, - ) + o_30 = kernel_30( + o_29=o_29, + ids=ids, + delay=delay, + prev_frames=prev_frames, + max_frames=9999, + max_ids=999, + video_path=t1, + low_mean_conf=low_mean_conf, + low_portion=low_portion, + no_dots=no_dots, + ) - return dict( - o_29=o_29, - o_30=o_30, - ) + return dict( + o_29=o_29, + o_30=o_30, + ) diff --git a/python/online/fxreader/pr34/tests/test_crypto.py b/python/online/fxreader/pr34/tests/test_crypto.py index 1c39c70..f886ca0 100644 --- a/python/online/fxreader/pr34/tests/test_crypto.py +++ b/python/online/fxreader/pr34/tests/test_crypto.py @@ -3,34 +3,34 @@ import unittest class TestCrypto(unittest.TestCase): - def test_password_utils(self) -> None: - salt = b'asdfasdfasdf' + def test_password_utils(self) -> None: + salt = b'asdfasdfasdf' - secret = 'blah' + secret = 'blah' - hash_res = crypto.PasswordUtils.secret_hash( - secret, - mode='bytes', - salt=salt, - ) - self.assertEqual( - hash_res, - ( - salt, - b'\xdak\xd15\xfa\x8e\xc8\r\xc3\xd2c\xf1m\xb0\xbf\xe6\x98\x01$!j\xc8\xc0Hh\x84\xea,\x91\x8b\x08\xce', - ), - ) + hash_res = crypto.PasswordUtils.secret_hash( + secret, + mode='bytes', + salt=salt, + ) + self.assertEqual( + hash_res, + ( + salt, + b'\xdak\xd15\xfa\x8e\xc8\r\xc3\xd2c\xf1m\xb0\xbf\xe6\x98\x01$!j\xc8\xc0Hh\x84\xea,\x91\x8b\x08\xce', + ), + ) - check_res = crypto.PasswordUtils.secret_check( - secret, - *hash_res, - ) + check_res = crypto.PasswordUtils.secret_check( + secret, + *hash_res, + ) - self.assertTrue(check_res) + self.assertTrue(check_res) - self.assertFalse( - crypto.PasswordUtils.secret_check( - secret + 'asdfasdfsdf', - *hash_res, - ) - ) + self.assertFalse( + crypto.PasswordUtils.secret_check( + secret + 'asdfasdfsdf', + *hash_res, + ) + )