import subprocess import os import requests import sys import io import copy import traceback import datetime import pprint import logging import json import time logger = logging.getLogger(__name__) class Launcher: def run(self): logging.basicConfig(level=logging.INFO) with io.open( 'tmp/d1/cpanel.json', 'r' ) as f: t3 = json.load(f) t2 = copy.deepcopy(t3) ssh_known_hosts : list[str] = [] for k, v in t2.items(): if 'ssh_known_hosts' in v: ssh_known_hosts.append(v['ssh_known_hosts']) if len(ssh_known_hosts) > 0: subprocess.check_call( r''' mkdir -p ~/.ssh && \ cat $SSH_KNOWN_HOSTS > ~/.ssh/known_hosts ''', env=dict(list(os.environ.items())) | dict( SSH_KNOWN_HOSTS=' '.join(ssh_known_hosts), ), shell=True ) for k in t2: v = t2[k] v['task'] = lambda : subprocess.Popen( v['task_cmd'], stdin=subprocess.DEVNULL, ) def stop_task(task: subprocess.Popen[bytes]) -> None: task.terminate() try: task.wait(1) except: task.kill() t1 = dict() shutdown = False while True: try: for k, v in t2.items(): if not k in t1: logging.info(json.dumps(dict( task=k, status='starting', ))) t1[k] = v['task']() logging.info(json.dumps(dict( task=k, status='started', ))) continue o = t1[k] not_alive = None try: not_alive = not ( requests.get(v['url'], timeout=0.5).status_code == 200 ) except: logging.error(json.dumps(dict( error=traceback.format_exc(), time_iso=datetime.datetime.now().isoformat(), ))) not_alive = True if not_alive: logging.error(json.dumps( dict( args=o.args, k=k, #o=pprint.pformat(o.__dict__), status='not_alive', time_iso=datetime.datetime.now().isoformat(), ) )) #stop_task(o) #del t1[k] continue if not o.poll() is None: logging.error(json.dumps( dict( #o=pprint.pformat(o.__dict__), args=o.args, k=k, return_code=o.poll(), status='crashed', time_iso=datetime.datetime.now().isoformat(), ) )) del t1[k] continue if shutdown: break print('\r%s tasks %d' % ( datetime.datetime.now().isoformat(), len(t1), ), end='') sys.stdout.flush() except KeyboardInterrupt: print('\nshutting down') break finally: time.sleep(5 * 60) for o in t1: stop_task(o) if __name__ == '__main__': Launcher().run()