142 lines
4.0 KiB
Python
142 lines
4.0 KiB
Python
import subprocess
|
|
import os
|
|
import requests
|
|
import sys
|
|
import io
|
|
import copy
|
|
import traceback
|
|
import datetime
|
|
import pprint
|
|
import logging
|
|
import json
|
|
import time
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class Launcher:
|
|
def run(self):
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
with io.open(
|
|
'tmp/d1/cpanel.json', 'r'
|
|
) as f:
|
|
t3 = json.load(f)
|
|
|
|
t2 = copy.deepcopy(t3)
|
|
|
|
ssh_known_hosts : list[str] = []
|
|
|
|
for k, v in t2.items():
|
|
if 'ssh_known_hosts' in v:
|
|
ssh_known_hosts.append(v['ssh_known_hosts'])
|
|
|
|
if len(ssh_known_hosts) > 0:
|
|
subprocess.check_call(
|
|
r'''
|
|
mkdir -p ~/.ssh && \
|
|
cat $SSH_KNOWN_HOSTS > ~/.ssh/known_hosts
|
|
''', env=dict(list(os.environ.items())) | dict(
|
|
SSH_KNOWN_HOSTS=' '.join(ssh_known_hosts),
|
|
),
|
|
shell=True
|
|
)
|
|
|
|
for k in t2:
|
|
v = t2[k]
|
|
v['task'] = lambda : subprocess.Popen(
|
|
v['task_cmd'],
|
|
stdin=subprocess.DEVNULL,
|
|
)
|
|
|
|
def stop_task(task: subprocess.Popen[bytes]) -> None:
|
|
task.terminate()
|
|
try:
|
|
task.wait(1)
|
|
except:
|
|
task.kill()
|
|
|
|
t1 = dict()
|
|
|
|
shutdown = False
|
|
|
|
while True:
|
|
try:
|
|
for k, v in t2.items():
|
|
if not k in t1:
|
|
logging.info(json.dumps(dict(
|
|
task=k,
|
|
status='starting',
|
|
)))
|
|
t1[k] = v['task']()
|
|
logging.info(json.dumps(dict(
|
|
task=k,
|
|
status='started',
|
|
)))
|
|
continue
|
|
|
|
o = t1[k]
|
|
|
|
not_alive = None
|
|
|
|
try:
|
|
not_alive = not (
|
|
requests.get(v['url'], timeout=0.5).status_code
|
|
== 200
|
|
)
|
|
except:
|
|
logging.error(json.dumps(dict(
|
|
error=traceback.format_exc(),
|
|
time_iso=datetime.datetime.now().isoformat(),
|
|
)))
|
|
not_alive = True
|
|
|
|
if not_alive:
|
|
logging.error(json.dumps(
|
|
dict(
|
|
args=o.args,
|
|
k=k,
|
|
#o=pprint.pformat(o.__dict__),
|
|
status='not_alive',
|
|
time_iso=datetime.datetime.now().isoformat(),
|
|
)
|
|
))
|
|
|
|
#stop_task(o)
|
|
#del t1[k]
|
|
continue
|
|
|
|
if not o.poll() is None:
|
|
logging.error(json.dumps(
|
|
dict(
|
|
#o=pprint.pformat(o.__dict__),
|
|
args=o.args,
|
|
k=k,
|
|
return_code=o.poll(),
|
|
status='crashed',
|
|
time_iso=datetime.datetime.now().isoformat(),
|
|
)
|
|
))
|
|
del t1[k]
|
|
continue
|
|
|
|
if shutdown:
|
|
break
|
|
|
|
print('\r%s tasks %d' % (
|
|
datetime.datetime.now().isoformat(),
|
|
len(t1),
|
|
), end='')
|
|
sys.stdout.flush()
|
|
except KeyboardInterrupt:
|
|
print('\nshutting down')
|
|
break
|
|
finally:
|
|
time.sleep(5 * 60)
|
|
|
|
for o in t1:
|
|
stop_task(o)
|
|
|
|
if __name__ == '__main__':
|
|
Launcher().run()
|