[~] Refactor
This commit is contained in:
parent
58bbf5386e
commit
ee7be6a4b6
406
d1/f2.py
406
d1/f2.py
@ -1,4 +1,8 @@
|
||||
import os
|
||||
import select
|
||||
import socket
|
||||
import copy
|
||||
import re
|
||||
import shutil
|
||||
import datetime
|
||||
import tempfile
|
||||
@ -16,6 +20,10 @@ sys.path.insert(0, os.path.dirname(__file__))
|
||||
|
||||
|
||||
class Application:
|
||||
MAX_CHUNK = 512 * 1024 * 1024
|
||||
LOG_SIZE = 10 * 1024 * 1024
|
||||
MAX_TIME = 16
|
||||
|
||||
def __init__(self, environ, start_response):
|
||||
self.environ = environ
|
||||
self.start_response = start_response
|
||||
@ -33,7 +41,7 @@ class Application:
|
||||
except:
|
||||
return
|
||||
|
||||
if log_size > 10 * 1024 * 1024:
|
||||
if log_size > Application.LOG_SIZE:
|
||||
try:
|
||||
log_path2 = os.path.splitext(self.log_path)
|
||||
os.rename(
|
||||
@ -46,9 +54,12 @@ class Application:
|
||||
except:
|
||||
return
|
||||
|
||||
def op1(self, data=None):
|
||||
def op1(self, data=None, json_data=None,):
|
||||
if data is None:
|
||||
data = traceback.format_exc()
|
||||
if not json_data is None:
|
||||
data = json.dumps(json_data)
|
||||
else:
|
||||
data = traceback.format_exc()
|
||||
|
||||
self.trim_log()
|
||||
|
||||
@ -57,7 +68,8 @@ class Application:
|
||||
'a'
|
||||
) as f:
|
||||
f.write(
|
||||
'\n%s\n%s\n' % (
|
||||
'[%d] %s\n%s\n' % (
|
||||
os.getpid(),
|
||||
datetime.datetime.now().isoformat(),
|
||||
data,
|
||||
)
|
||||
@ -77,6 +89,73 @@ class Application:
|
||||
self.output_dat = tempfile.mktemp(suffix='.dat')
|
||||
self.input_dat = tempfile.mktemp(suffix='.dat')
|
||||
|
||||
def op10(
|
||||
self,
|
||||
headers_text=None,
|
||||
headers_lines=None,
|
||||
as_dict=None,
|
||||
detailed=None,
|
||||
):
|
||||
if detailed is None:
|
||||
detailed = True
|
||||
|
||||
if headers_text is None:
|
||||
headers_text = ''.join(
|
||||
[
|
||||
'%s\r\n' % o
|
||||
for o in headers_lines
|
||||
]
|
||||
)
|
||||
|
||||
if as_dict is None:
|
||||
as_dict = False
|
||||
|
||||
headers_list = [
|
||||
(o[:o.find(':')], o[o.find(':') + 2:])
|
||||
for o in headers_text.splitlines()[1:]
|
||||
if len(o) > 0
|
||||
]
|
||||
|
||||
if as_dict:
|
||||
headers = dict(headers_list)
|
||||
else:
|
||||
headers = headers_list
|
||||
|
||||
if detailed:
|
||||
self.op1(dict(headers_text=headers_text))
|
||||
return dict(
|
||||
first_line='%s\r\n' % headers_text.splitlines()[0],
|
||||
headers=headers,
|
||||
last_line='%s\r\n' % headers_text.splitlines()[-1]
|
||||
)
|
||||
else:
|
||||
return headers
|
||||
|
||||
def op11(
|
||||
self,
|
||||
headers_dict,
|
||||
old_headers_text,
|
||||
protocol=None,
|
||||
status_suffix=None,
|
||||
):
|
||||
first_line_split = lambda : \
|
||||
old_headers_text.splitlines()[0].split(' ', maxsplit=1)
|
||||
if protocol is None:
|
||||
protocol = first_line_split()[0]
|
||||
|
||||
if status_suffix is None:
|
||||
status_suffix = first_line_split()[1]
|
||||
|
||||
return ''.join(sum([
|
||||
[
|
||||
'%s %s\r\n' % (protocol, status_suffix),
|
||||
],
|
||||
[
|
||||
'%s: %s\r\n' % (k, v)
|
||||
for k, v in headers_dict.items()
|
||||
]
|
||||
], []))
|
||||
|
||||
def op5(
|
||||
self,
|
||||
status_code=None,
|
||||
@ -85,23 +164,28 @@ class Application:
|
||||
content=None,
|
||||
headers_text=None,
|
||||
):
|
||||
content_length = None
|
||||
if content is None:
|
||||
content = b''
|
||||
|
||||
if isinstance(content, bytes):
|
||||
content_length = len(content)
|
||||
|
||||
if not headers_text is None:
|
||||
if not any([o.startswith('Content-Length') for o in headers_text]):
|
||||
headers_text = \
|
||||
headers_text.replace(
|
||||
'Transfer-Encoding: chunked\r\n',
|
||||
''
|
||||
)
|
||||
headers_text += 'Content-Length: %d\r\n' % len(content)
|
||||
if False:
|
||||
if not any([o.startswith('Content-Length') for o in headers_text]):
|
||||
headers_text = \
|
||||
headers_text.replace(
|
||||
'Transfer-Encoding: chunked\r\n',
|
||||
''
|
||||
)
|
||||
headers_text += 'Content-Length: %d\r\n' % len(content)
|
||||
|
||||
t13 = headers_text.splitlines()[0]
|
||||
t14 = t13.find(' ')
|
||||
t15 = t13[t14 + 1:]
|
||||
status_suffix = t15
|
||||
headers = [
|
||||
(o[:o.find(':')], o[o.find(':') + 1:])
|
||||
for o in headers_text.splitlines()[1:]
|
||||
]
|
||||
headers = self.op10(headers_text)
|
||||
else:
|
||||
if status_code is None:
|
||||
status_code = 200
|
||||
@ -114,70 +198,219 @@ class Application:
|
||||
status_suffix = '%d %s' % (status_code, status_text)
|
||||
headers = [('Content-Type', content_type)]
|
||||
|
||||
t1 = self.start_response(
|
||||
status_suffix,
|
||||
headers,
|
||||
)
|
||||
assert isinstance(content, bytes)
|
||||
self.op1(json_data=dict(
|
||||
headers_text=headers_text,
|
||||
content_length=content_length,
|
||||
content_type=content_type,
|
||||
status_code=status_code,
|
||||
status_text=status_text,
|
||||
status_suffix=status_suffix,
|
||||
))
|
||||
|
||||
t1(content)
|
||||
if isinstance(content, bytes):
|
||||
t1 = self.start_response(
|
||||
status_suffix,
|
||||
headers,
|
||||
)
|
||||
t1(content)
|
||||
|
||||
return []
|
||||
return []
|
||||
else:
|
||||
headers_lines = []
|
||||
content_chunks = []
|
||||
returncode = None
|
||||
for chunk in content:
|
||||
current_headers = chunk[0]
|
||||
headers_lines.extend(current_headers)
|
||||
if len(chunk[1]) > 0:
|
||||
content_chunks.append(chunk[1])
|
||||
returncode = chunk[2]
|
||||
|
||||
def op6(self, cmd_args,):
|
||||
self.op1(json.dumps(cmd_args))
|
||||
if len(headers_lines) > 0 and headers_lines[-1] == '':
|
||||
break
|
||||
|
||||
if not returncode is None and returncode != 0:
|
||||
self.op1(
|
||||
json_data=dict(
|
||||
headers_lines=headers_lines,
|
||||
returncode=returncode,
|
||||
)
|
||||
)
|
||||
|
||||
output_stream = self.environ['passenger.hijack'](True)
|
||||
|
||||
headers_detailed = self.op10(
|
||||
headers_lines=headers_lines,
|
||||
detailed=True,
|
||||
as_dict=True,
|
||||
)
|
||||
|
||||
self.op1(
|
||||
json_data=dict(
|
||||
headers_detailed=headers_detailed,
|
||||
)
|
||||
)
|
||||
|
||||
get_output_length = lambda : sum([len(o) for o in content_chunks])
|
||||
finished_output = False
|
||||
first_output = False
|
||||
sent_bytes = 0
|
||||
|
||||
def dump_headers():
|
||||
self.op1(
|
||||
json_data=dict(
|
||||
aciton='dump_headers',
|
||||
sent_byte=sent_bytes,
|
||||
output_length=get_output_length()
|
||||
)
|
||||
)
|
||||
output_stream.sendall(
|
||||
headers_detailed['first_line'].encode('latin-1')
|
||||
)
|
||||
|
||||
if headers_detailed['headers'].get('Transfer-Encoding') == 'chunked':
|
||||
del headers_detailed['headers']['Transfer-Encoding']
|
||||
assert sent_bytes == 0
|
||||
|
||||
if finished_output:
|
||||
content_length = get_output_length()
|
||||
else:
|
||||
content_length = Application.MAX_CHUNK
|
||||
|
||||
headers_detailed['headers']['Content-Length'] = \
|
||||
'%d' % content_length
|
||||
self.op1(
|
||||
json_data=dict(
|
||||
headers_detailed=headers_detailed,
|
||||
)
|
||||
)
|
||||
headers_detailed['headers']['Connection'] = 'close'
|
||||
|
||||
for k, v in headers_detailed['headers'].items():
|
||||
output_stream.sendall(
|
||||
(
|
||||
'%s: %s\r\n' % (k, v)
|
||||
).encode('latin-1')
|
||||
)
|
||||
output_stream.sendall(
|
||||
headers_detailed['last_line'].encode('latin-1')
|
||||
)
|
||||
|
||||
while True:
|
||||
if not finished_output:
|
||||
try:
|
||||
self.op1(json_data=dict(action='fetch-chunk-started'))
|
||||
chunk_detailed = next(content)
|
||||
self.op1(
|
||||
json_data=dict(
|
||||
action='fetch-chunk-done',
|
||||
returncode=chunk_detailed[2],
|
||||
stderr=chunk_detailed[0],
|
||||
chunk_length=len(chunk_detailed[1]),
|
||||
)
|
||||
)
|
||||
if (len(chunk_detailed[1]) > 0):
|
||||
content_chunks.append(chunk_detailed[1])
|
||||
except StopIteration:
|
||||
finished_output = True
|
||||
|
||||
if (
|
||||
not finished_output and get_output_length() > 2048 or \
|
||||
finished_output
|
||||
) and not first_output:
|
||||
dump_headers()
|
||||
first_output = True
|
||||
|
||||
if first_output and len(content_chunks) > 0:
|
||||
chunk = content_chunks[0]
|
||||
del content_chunks[0]
|
||||
if len(chunk) > 0:
|
||||
output_stream.sendall(chunk)
|
||||
sent_bytes += len(chunk)
|
||||
self.op1(
|
||||
json_data=dict(sent_bytes=sent_bytes)
|
||||
)
|
||||
|
||||
if finished_output and first_output and len(content_chunks) == 0:
|
||||
break
|
||||
|
||||
try:
|
||||
output_stream.shutdown(socket.SHUT_WR)
|
||||
except:
|
||||
output_stream.close()
|
||||
|
||||
self.op1(json_data=dict(
|
||||
action='close_connection',
|
||||
extra=pprint.pformat([
|
||||
output_stream,
|
||||
output_stream.__class__,
|
||||
dir(output_stream),
|
||||
]),
|
||||
))
|
||||
return
|
||||
|
||||
def op9(
|
||||
self,
|
||||
cmd_args,
|
||||
headers=None,
|
||||
):
|
||||
if not headers is None:
|
||||
extra_cmd_args = sum([
|
||||
['--header', '%s: %s' % (k, v)]
|
||||
for k, v in headers.items()
|
||||
], [])
|
||||
else:
|
||||
extra_cmd_args = []
|
||||
|
||||
cmd_args2 = cmd_args + extra_cmd_args
|
||||
|
||||
self.op1(json.dumps(cmd_args2))
|
||||
|
||||
with subprocess.Popen(
|
||||
cmd_args,
|
||||
cmd_args2,
|
||||
stderr=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stdin=subprocess.PIPE
|
||||
) as p:
|
||||
stderr_lines = []
|
||||
returncode = None
|
||||
|
||||
try:
|
||||
p.wait(20)
|
||||
curl_stderr = p.stderr.read().decode('utf-8')
|
||||
response_headers = [
|
||||
o[2:]
|
||||
for o in curl_stderr.splitlines()
|
||||
if o.startswith('< ')
|
||||
]
|
||||
t9 = '\r\n'.join(response_headers)
|
||||
if not 'Content-Length: 0' in t9:
|
||||
for k in range(100):
|
||||
try:
|
||||
with io.open(self.output_dat, 'rb') as f:
|
||||
t10 = f.read()
|
||||
break
|
||||
except:
|
||||
time.sleep(0.05)
|
||||
else:
|
||||
t10 = b''
|
||||
|
||||
try:
|
||||
output_dat_stat = repr(
|
||||
os.stat(self.output_dat)
|
||||
)
|
||||
except:
|
||||
output_dat_stat = None
|
||||
stderr_poll = select.poll()
|
||||
stderr_poll.register(p.stderr, select.POLLIN)
|
||||
while True:
|
||||
if len(stderr_poll.poll(1)) > 0:
|
||||
stderr = p.stderr.readline(1024).decode('utf-8')
|
||||
stderr_lines.append(stderr)
|
||||
else:
|
||||
stderr = ''
|
||||
|
||||
self.op1(
|
||||
json.dumps(
|
||||
dict(
|
||||
curl_return_code=p.returncode,
|
||||
curl_stderr=curl_stderr,
|
||||
output_dat=self.output_dat,
|
||||
output_dat_stat=output_dat_stat,
|
||||
headers=t9,
|
||||
)
|
||||
)
|
||||
)
|
||||
stdout = p.stdout.read(1024)
|
||||
|
||||
response_headers = [
|
||||
o[2:]
|
||||
for o in stderr.splitlines()
|
||||
if o.startswith('< ')
|
||||
]
|
||||
|
||||
returncode = p.poll()
|
||||
yield (response_headers, stdout, returncode)
|
||||
|
||||
if len(stderr) == 0 and len(stdout) == 0 and not returncode is None:
|
||||
break
|
||||
|
||||
assert returncode == 0
|
||||
except Exception as exception:
|
||||
self.op1(json_data=dict(
|
||||
stderr_lines=stderr_lines,
|
||||
))
|
||||
raise exception
|
||||
finally:
|
||||
p.terminate()
|
||||
|
||||
return dict(
|
||||
headers_text=t9,
|
||||
content=t10
|
||||
)
|
||||
def op6(self, cmd_args, headers=None,):
|
||||
return self.op9(cmd_args, headers)
|
||||
|
||||
def op7(self):
|
||||
try:
|
||||
@ -208,28 +441,57 @@ class Application:
|
||||
|
||||
proxy_url = self.op7()
|
||||
|
||||
self.op1(json_data=headers)
|
||||
|
||||
t17 = [
|
||||
'curl',
|
||||
'http://%s%s' % (proxy_url, uri),
|
||||
*sum([
|
||||
['--header', '%s: %s' % (k, v)]
|
||||
for k, v in headers.items()
|
||||
], []),
|
||||
'-X', method,
|
||||
'-v',
|
||||
'--no-buffer',
|
||||
'--silent',
|
||||
'--data-binary', '@%s' % self.input_dat,
|
||||
'--max-filesize', '%d' % (60 * 1024 * 1024),
|
||||
'-o', self.output_dat,
|
||||
'--max-filesize', '%d' % Application.MAX_CHUNK,
|
||||
'--max-time', '%d' % Application.MAX_TIME,
|
||||
#'-o', self.output_dat,
|
||||
'-q',
|
||||
]
|
||||
|
||||
return self.op6(t17)
|
||||
return self.op6(
|
||||
t17,
|
||||
headers=headers,
|
||||
)
|
||||
finally:
|
||||
self.op3()
|
||||
|
||||
def run(self):
|
||||
self.op1(
|
||||
data=pprint.pformat(
|
||||
dict(
|
||||
environ=self.environ
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
if 'HTTP_FUCK' in self.environ:
|
||||
output_stream = self.environ['passenger.hijack'](True)
|
||||
|
||||
content = 'fuck off'
|
||||
output_stream.sendall(
|
||||
''.join([
|
||||
'HTTP/1.1 200\r\n',
|
||||
'Status: 200\r\n',
|
||||
'Connection-Length: %d\r\n' % len(content),
|
||||
'Connection: close\r\n',
|
||||
'\r\n',
|
||||
]).encode('latin-1')
|
||||
)
|
||||
output_stream.sendall(content.encode('utf-8'))
|
||||
output_stream.shutdown(socket.SHUT_WR)
|
||||
#self.start_response(200, [])
|
||||
#return []
|
||||
return None
|
||||
|
||||
try:
|
||||
t4 = int(self.environ.get('CONTENT_LENGTH', '0'))
|
||||
t3 = self.environ['wsgi.input'].read(t4)
|
||||
@ -258,7 +520,7 @@ class Application:
|
||||
)
|
||||
|
||||
return self.op5(
|
||||
**o_8,
|
||||
content=o_8,
|
||||
)
|
||||
except:
|
||||
self.op1()
|
||||
|
||||
@ -1,3 +1,17 @@
|
||||
|
||||
#PassengerResponseBufferHighWatermark 134217728
|
||||
#PassengerLogLevel 7
|
||||
#PassengerLogFile fuck.txt
|
||||
#PassengerHighPerformance on
|
||||
PassengerAppRoot "/home/productg/p1/p1"
|
||||
PassengerBaseURI "/"
|
||||
PassengerPython "/home/productg/p1/tmp/env3/bin/python"
|
||||
PassengerBufferResponse off
|
||||
PassengerBufferUpload off
|
||||
#PassengerMaxRequests 1
|
||||
#PassengerMaxRequestQueueSize 1
|
||||
#PassengerForceMaxConcurrentRequestsPerProcess 1
|
||||
|
||||
<IfModule mod_rewrite.c>
|
||||
RewriteEngine On
|
||||
RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization}]
|
||||
@ -5,7 +19,3 @@ RewriteBase /
|
||||
RewriteCond %{REQUEST_FILENAME} !-f
|
||||
RewriteCond %{REQUEST_FILENAME} !-d
|
||||
</IfModule>
|
||||
|
||||
PassengerAppRoot "/home/productg/p1/p1"
|
||||
PassengerBaseURI "/"
|
||||
PassengerPython "/home/productg/p1/tmp/env3/bin/python"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user