[~] Refactor

d1/f2.py

@@ -1,4 +1,8 @@
 import os
+import select
+import socket
+import copy
+import re
 import shutil
 import datetime
 import tempfile
@@ -16,6 +20,10 @@ sys.path.insert(0, os.path.dirname(__file__))
 
 
 class Application:
+    MAX_CHUNK = 512 * 1024 * 1024
+    LOG_SIZE = 10 * 1024 * 1024
+    MAX_TIME = 16
+
     def __init__(self, environ, start_response):
         self.environ = environ
         self.start_response = start_response
@@ -33,7 +41,7 @@ class Application:
         except:
             return
 
-        if log_size > 10 * 1024 * 1024:
+        if log_size > Application.LOG_SIZE:
             try:
                 log_path2 = os.path.splitext(self.log_path)
                 os.rename(
@@ -46,9 +54,12 @@ class Application:
             except:
                 return
 
-    def op1(self, data=None):
+    def op1(self, data=None, json_data=None,):
         if data is None:
-            data = traceback.format_exc()
+            if not json_data is None:
+                data = json.dumps(json_data)
+            else:
+                data = traceback.format_exc()
 
         self.trim_log()
 
@@ -57,7 +68,8 @@ class Application:
             'a'
         ) as f:
             f.write(
-                '\n%s\n%s\n' % (
+                '[%d] %s\n%s\n' % (
+                    os.getpid(),
                     datetime.datetime.now().isoformat(),
                     data,
                 )
@@ -77,6 +89,73 @@ class Application:
         self.output_dat = tempfile.mktemp(suffix='.dat')
         self.input_dat = tempfile.mktemp(suffix='.dat')
 
+    def op10(
+        self,
+        headers_text=None,
+        headers_lines=None,
+        as_dict=None,
+        detailed=None,
+    ):
+        if detailed is None:
+            detailed = True
+
+        if headers_text is None:
+            headers_text = ''.join(
+                [
+                    '%s\r\n' % o
+                    for o in headers_lines
+                ]
+            )
+
+        if as_dict is None:
+            as_dict = False
+
+        headers_list = [
+            (o[:o.find(':')], o[o.find(':') + 2:])
+            for o in headers_text.splitlines()[1:]
+            if len(o) > 0
+        ]
+
+        if as_dict:
+            headers = dict(headers_list)
+        else:
+            headers = headers_list
+
+        if detailed:
+            self.op1(dict(headers_text=headers_text))
+            return dict(
+                first_line='%s\r\n' % headers_text.splitlines()[0],
+                headers=headers,
+                last_line='%s\r\n' % headers_text.splitlines()[-1]
+            )
+        else:
+            return headers
+
+    def op11(
+        self,
+        headers_dict,
+        old_headers_text,
+        protocol=None,
+        status_suffix=None,
+    ):
+        first_line_split = lambda : \
+            old_headers_text.splitlines()[0].split(' ', maxsplit=1)
+        if protocol is None:
+            protocol = first_line_split()[0]
+
+        if status_suffix is None:
+            status_suffix = first_line_split()[1]
+
+        return ''.join(sum([
+            [
+                '%s %s\r\n' % (protocol, status_suffix),
+            ],
+            [
+                '%s: %s\r\n' % (k, v)
+                for k, v in headers_dict.items()
+            ]
+        ], []))
+
     def op5(
         self,
         status_code=None,
@@ -85,23 +164,28 @@ class Application:
         content=None,
         headers_text=None,
     ):
+        content_length = None
+        if content is None:
+            content = b''
+
+        if isinstance(content, bytes):
+            content_length = len(content)
+
         if not headers_text is None:
-            if not any([o.startswith('Content-Length') for o in headers_text]):
+            if False:
-                headers_text = \
+                if not any([o.startswith('Content-Length') for o in headers_text]):
-                    headers_text.replace(
+                    headers_text = \
-                        'Transfer-Encoding: chunked\r\n',
+                        headers_text.replace(
-                        ''
+                            'Transfer-Encoding: chunked\r\n',
-                    )
+                            ''
-                headers_text += 'Content-Length: %d\r\n' % len(content)
+                        )
+                    headers_text += 'Content-Length: %d\r\n' % len(content)
 
             t13 = headers_text.splitlines()[0]
             t14 = t13.find(' ')
             t15 = t13[t14 + 1:]
             status_suffix = t15
-            headers = [
+            headers = self.op10(headers_text)
-                (o[:o.find(':')], o[o.find(':') + 1:])
-                for o in headers_text.splitlines()[1:]
-            ]
         else:
             if status_code is None:
                 status_code = 200
@@ -114,70 +198,219 @@ class Application:
             status_suffix = '%d %s' % (status_code, status_text)
             headers = [('Content-Type', content_type)]
 
-        t1 = self.start_response(
+        self.op1(json_data=dict(
-            status_suffix,
+            headers_text=headers_text,
-            headers,
+            content_length=content_length,
-        )
+            content_type=content_type,
-        assert isinstance(content, bytes)
+            status_code=status_code,
+            status_text=status_text,
+            status_suffix=status_suffix,
+        ))
 
-        t1(content)
+        if isinstance(content, bytes):
+            t1 = self.start_response(
+                status_suffix,
+                headers,
+            )
+            t1(content)
 
-        return []
+            return []
+        else:
+            headers_lines = []
+            content_chunks = []
+            returncode = None
+            for chunk in content:
+                current_headers = chunk[0]
+                headers_lines.extend(current_headers)
+                if len(chunk[1]) > 0:
+                    content_chunks.append(chunk[1])
+                returncode = chunk[2]
 
-    def op6(self, cmd_args,):
+                if len(headers_lines) > 0 and headers_lines[-1] == '':
-        self.op1(json.dumps(cmd_args))
+                    break
+
+            if not returncode is None and returncode != 0:
+                self.op1(
+                    json_data=dict(
+                        headers_lines=headers_lines,
+                        returncode=returncode,
+                    )
+                )
+
+            output_stream = self.environ['passenger.hijack'](True)
+
+            headers_detailed = self.op10(
+                headers_lines=headers_lines,
+                detailed=True,
+                as_dict=True,
+            )
+
+            self.op1(
+                json_data=dict(
+                    headers_detailed=headers_detailed,
+                )
+            )
+
+            get_output_length = lambda : sum([len(o) for o in content_chunks])
+            finished_output = False
+            first_output = False
+            sent_bytes = 0
+
+            def dump_headers():
+                self.op1(
+                    json_data=dict(
+                        aciton='dump_headers',
+                        sent_byte=sent_bytes,
+                        output_length=get_output_length()
+                    )
+                )
+                output_stream.sendall(
+                    headers_detailed['first_line'].encode('latin-1')
+                )
+
+                if headers_detailed['headers'].get('Transfer-Encoding') == 'chunked':
+                    del headers_detailed['headers']['Transfer-Encoding']
+                    assert sent_bytes == 0
+
+                    if finished_output:
+                        content_length = get_output_length()
+                    else:
+                        content_length = Application.MAX_CHUNK
+
+                    headers_detailed['headers']['Content-Length'] = \
+                        '%d' % content_length 
+                    self.op1(
+                        json_data=dict(
+                            headers_detailed=headers_detailed,
+                        )
+                    )
+                headers_detailed['headers']['Connection'] = 'close'
+
+                for k, v in headers_detailed['headers'].items():
+                    output_stream.sendall(
+                        (
+                            '%s: %s\r\n' % (k, v)
+                        ).encode('latin-1')
+                    )
+                output_stream.sendall(
+                    headers_detailed['last_line'].encode('latin-1')
+                )
+
+            while True:
+                if not finished_output:
+                    try:
+                        self.op1(json_data=dict(action='fetch-chunk-started'))
+                        chunk_detailed = next(content)
+                        self.op1(
+                            json_data=dict(
+                                action='fetch-chunk-done',
+                                returncode=chunk_detailed[2],
+                                stderr=chunk_detailed[0],
+                                chunk_length=len(chunk_detailed[1]),
+                            )
+                        )
+                        if (len(chunk_detailed[1]) > 0):
+                            content_chunks.append(chunk_detailed[1])
+                    except StopIteration:
+                        finished_output = True
+
+                if (
+                    not finished_output and get_output_length() > 2048 or \
+                    finished_output
+                ) and not first_output:
+                    dump_headers()
+                    first_output = True
+
+                if first_output and len(content_chunks) > 0:
+                    chunk = content_chunks[0]
+                    del content_chunks[0]
+                    if len(chunk) > 0:
+                        output_stream.sendall(chunk)
+                        sent_bytes += len(chunk)
+                        self.op1(
+                            json_data=dict(sent_bytes=sent_bytes)
+                        )
+
+                if finished_output and first_output and len(content_chunks) == 0:
+                    break
+
+            try:
+                output_stream.shutdown(socket.SHUT_WR)
+            except:
+                output_stream.close()
+
+            self.op1(json_data=dict(
+                action='close_connection',
+                extra=pprint.pformat([
+                    output_stream,
+                    output_stream.__class__,
+                    dir(output_stream),
+                ]),
+            ))
+            return
+
+    def op9(
+        self,
+        cmd_args,
+        headers=None,
+    ):
+        if not headers is None:
+            extra_cmd_args = sum([
+                ['--header', '%s: %s' % (k, v)]
+                for k, v in headers.items()
+            ], [])
+        else:
+            extra_cmd_args = []
+
+        cmd_args2 = cmd_args + extra_cmd_args
+
+        self.op1(json.dumps(cmd_args2))
 
         with subprocess.Popen(
-            cmd_args,
+            cmd_args2,
             stderr=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stdin=subprocess.PIPE
         ) as p:
+            stderr_lines = []
+            returncode = None
+
             try:
-                p.wait(20)
-                curl_stderr = p.stderr.read().decode('utf-8')
-                response_headers = [
-                    o[2:]
-                    for o in curl_stderr.splitlines()
-                    if o.startswith('< ')
-                ]
-                t9 = '\r\n'.join(response_headers)
-                if not 'Content-Length: 0' in t9:
-                    for k in range(100):
-                        try:
-                            with io.open(self.output_dat, 'rb') as f:
-                                t10 = f.read()
-                            break
-                        except:
-                            time.sleep(0.05)
-                else:
-                    t10 = b''
 
-                try:
+                stderr_poll = select.poll()
-                    output_dat_stat = repr(
+                stderr_poll.register(p.stderr, select.POLLIN)
-                        os.stat(self.output_dat)
+                while True:
-                    )
+                    if len(stderr_poll.poll(1)) > 0:
-                except:
+                        stderr = p.stderr.readline(1024).decode('utf-8')
-                    output_dat_stat = None
+                        stderr_lines.append(stderr)
+                    else:
+                        stderr = ''
 
-                self.op1(
+                    stdout = p.stdout.read(1024)
-                    json.dumps(
+
-                        dict(
+                    response_headers = [
-                            curl_return_code=p.returncode,
+                        o[2:]
-                            curl_stderr=curl_stderr,
+                        for o in stderr.splitlines()
-                            output_dat=self.output_dat,
+                        if o.startswith('< ')
-                            output_dat_stat=output_dat_stat,
+                    ]
-                            headers=t9,
+
-                        )
+                    returncode = p.poll()
-                    )
+                    yield (response_headers, stdout, returncode)
-                )
+
+                    if len(stderr) == 0 and len(stdout) == 0 and not returncode is None:
+                        break
+
+                assert returncode == 0
+            except Exception as exception:
+                self.op1(json_data=dict(
+                    stderr_lines=stderr_lines,
+                ))
+                raise exception
             finally:
                 p.terminate()
 
-        return dict(
+    def op6(self, cmd_args, headers=None,):
-            headers_text=t9,
+        return self.op9(cmd_args, headers)
-            content=t10
-        )
 
     def op7(self):
         try:
@@ -208,28 +441,57 @@ class Application:
 
             proxy_url = self.op7()
 
+            self.op1(json_data=headers)
+
             t17 = [
                 'curl',
                 'http://%s%s' % (proxy_url, uri),
-                *sum([
-                    ['--header', '%s: %s' % (k, v)]
-                    for k, v in headers.items()
-                ], []),
                 '-X', method,
                 '-v',
                 '--no-buffer',
                 '--silent',
                 '--data-binary', '@%s' % self.input_dat,
-                '--max-filesize', '%d' % (60 * 1024 * 1024),
+                '--max-filesize', '%d' % Application.MAX_CHUNK,
-                '-o', self.output_dat,
+                '--max-time', '%d' % Application.MAX_TIME,
+                #'-o', self.output_dat,
                 '-q',
             ]
 
-            return self.op6(t17)
+            return self.op6(
+                t17,
+                headers=headers,
+            )
         finally:
             self.op3()
 
     def run(self):
+        self.op1(
+            data=pprint.pformat(
+                dict(
+                    environ=self.environ
+                )
+            )
+        )
+
+        if 'HTTP_FUCK' in self.environ:
+            output_stream = self.environ['passenger.hijack'](True)
+
+            content = 'fuck off'
+            output_stream.sendall(
+                ''.join([
+                    'HTTP/1.1 200\r\n',
+                    'Status: 200\r\n',
+                    'Connection-Length: %d\r\n' % len(content),
+                    'Connection: close\r\n',
+                    '\r\n',
+                ]).encode('latin-1')
+            )
+            output_stream.sendall(content.encode('utf-8'))
+            output_stream.shutdown(socket.SHUT_WR)
+            #self.start_response(200, [])
+            #return []
+            return None
+
         try:
             t4 = int(self.environ.get('CONTENT_LENGTH', '0'))
             t3 = self.environ['wsgi.input'].read(t4)
@@ -258,7 +520,7 @@ class Application:
             )
 
             return self.op5(
-                **o_8,
+                content=o_8,
             )
         except:
             self.op1()

d1/wsgi/.htaccess

@@ -1,3 +1,17 @@
+
+#PassengerResponseBufferHighWatermark 134217728
+#PassengerLogLevel 7
+#PassengerLogFile fuck.txt
+#PassengerHighPerformance on
+PassengerAppRoot "/home/productg/p1/p1"
+PassengerBaseURI "/"
+PassengerPython "/home/productg/p1/tmp/env3/bin/python"
+PassengerBufferResponse off
+PassengerBufferUpload off
+#PassengerMaxRequests 1
+#PassengerMaxRequestQueueSize 1
+#PassengerForceMaxConcurrentRequestsPerProcess 1
+
 <IfModule mod_rewrite.c>
 RewriteEngine On
 RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization}]
@@ -5,7 +19,3 @@ RewriteBase /
 RewriteCond %{REQUEST_FILENAME} !-f
 RewriteCond %{REQUEST_FILENAME} !-d
 </IfModule>
-
-PassengerAppRoot "/home/productg/p1/p1"
-PassengerBaseURI "/"
-PassengerPython "/home/productg/p1/tmp/env3/bin/python"