source: mod_gnutls/test/https-test-client.py @ 0a16644

asyncioproxy-ticket
Last change on this file since 0a16644 was 0a16644, checked in by Fiona Klute <fiona.klute@…>, 23 months ago

https-test-client.py: Stop filtering after the first read block of data

If the erroneous log line shows up it's the first piece of data we
receive. Just copy everything after.

  • Property mode set to 100755
File size: 12.3 KB
Line 
1#!/usr/bin/python3
2# PYTHON_ARGCOMPLETE_OK
3
4# Copyright 2019 Fiona Klute
5#
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10#     http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17
18import socket
19import subprocess
20import yaml
21
22from http.client import HTTPConnection
23from multiprocessing import Process
24from time import sleep
25
26class HTTPSubprocessConnection(HTTPConnection):
27    def __init__(self, command, host, port=None,
28                 output_filter=None,
29                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
30                 blocksize=8192):
31        super(HTTPSubprocessConnection, self).__init__(host, port, timeout,
32                                                       source_address=None,
33                                                       blocksize=blocksize)
34        # "command" must be a list containing binary and command line
35        # parameters
36        self.command = command
37        # This will be the subprocess reference when connected
38        self._sproc = None
39        # The subprocess return code is stored here on close()
40        self.returncode = None
41        # The set_tunnel method of the super class is not supported
42        # (see exception doc)
43        self.set_tunnel = None
44        # This method will be run in a separate process and filter the
45        # stdout of self._sproc. Its arguments are self._sproc.stdout
46        # and the socket back to the HTTP connection (write-only).
47        self._output_filter = output_filter
48        # output filter process
49        self._fproc = None
50
51    def connect(self):
52        s_local, s_remote = socket.socketpair(socket.AF_UNIX,
53                                              socket.SOCK_STREAM)
54        s_local.settimeout(self.timeout)
55
56        # TODO: Maybe capture stderr?
57        if self._output_filter:
58            self._sproc = subprocess.Popen(self.command, stdout=subprocess.PIPE,
59                                           stdin=s_remote, close_fds=True,
60                                           bufsize=0)
61            self._fproc = Process(target=self._output_filter,
62                                  args=(self._sproc.stdout, s_remote))
63            self._fproc.start()
64        else:
65            self._sproc = subprocess.Popen(self.command, stdout=s_remote,
66                                           stdin=s_remote, close_fds=True,
67                                           bufsize=0)
68        s_remote.close()
69        self.sock = s_local
70
71    def close(self):
72        # close socket to subprocess for writing
73        if self.sock:
74            self.sock.shutdown(socket.SHUT_WR)
75
76        # Wait for the process to stop, send SIGTERM/SIGKILL if
77        # necessary
78        try:
79            self.returncode = self._sproc.wait(self.timeout)
80        except subprocess.TimeoutExpired:
81            try:
82                self._sproc.terminate()
83                self.returncode = self._sproc.wait(self.timeout)
84            except subprocess.TimeoutExpired:
85                self._sproc.kill()
86                self.returncode = self._sproc.wait(self.timeout)
87
88        # filter process receives HUP on pipe when the subprocess
89        # terminates
90        if self._fproc:
91            self._fproc.join()
92
93        # close the connection in the super class, which also calls
94        # self.sock.close()
95        super().close()
96
97
98
99class TestRequest(yaml.YAMLObject):
100    yaml_tag = '!request'
101    def __init__(self, path, method='GET', headers=dict(),
102                 expect=dict(status=200)):
103        self.method = method
104        self.path = path
105        self.headers = headers
106        self.expect = expect
107
108    def __repr__(self):
109        return (f'{self.__class__.__name__!s}(path={self.path!r}, '
110                f'method={self.method!r}, headers={self.headers!r}, '
111                f'expect={self.expect!r})')
112
113    def _check_body(self, body):
114        """
115        >>> r1 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'exactly': 'test\\n'}})
116        >>> r1._check_body('test\\n')
117        >>> r1._check_body('xyz\\n')
118        Traceback (most recent call last):
119        ...
120        https-test-client.TestExpectationFailed: Unexpected body: 'xyz\\n' != 'test\\n'
121        >>> r2 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': ['tes', 'est']}})
122        >>> r2._check_body('test\\n')
123        >>> r2._check_body('est\\n')
124        Traceback (most recent call last):
125        ...
126        https-test-client.TestExpectationFailed: Unexpected body: 'est\\n' does not contain 'tes'
127        >>> r3 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': 'test'}})
128        >>> r3._check_body('test\\n')
129        """
130        if 'exactly' in self.expect['body'] \
131           and body != self.expect['body']['exactly']:
132            raise TestExpectationFailed(
133                f'Unexpected body: {body!r} != '
134                f'{self.expect["body"]["exactly"]!r}')
135        if 'contains' in self.expect['body']:
136            if type(self.expect['body']['contains']) is str:
137                self.expect['body']['contains'] = [
138                    self.expect['body']['contains']]
139            for s in self.expect['body']['contains']:
140                if not s in body:
141                    raise TestExpectationFailed(
142                        f'Unexpected body: {body!r} does not contain '
143                        f'{s!r}')
144
145    def check_response(self, response, body):
146        if self.expects_conn_reset():
147            raise TestExpectationFailed(
148                'Got a response, but connection should have failed!')
149        if response.status != self.expect['status']:
150            raise TestExpectationFailed(
151                f'Unexpected status: {response.status} != '
152                f'{self.expect["status"]}')
153        if 'body' in self.expect:
154            self._check_body(body)
155
156    def expects_conn_reset(self):
157        if 'reset' in self.expect:
158            return self.expect['reset']
159        return False
160
161    @classmethod
162    def _from_yaml(cls, loader, node):
163        fields = loader.construct_mapping(node)
164        req = TestRequest(**fields)
165        return req
166
167class TestConnection(yaml.YAMLObject):
168    yaml_tag = '!connection'
169
170    def __init__(self, actions, gnutls_params=[], protocol='https'):
171        self.gnutls_params = gnutls_params
172        self.actions = actions
173        self.protocol = protocol
174
175    def __repr__(self):
176        return (f'{self.__class__.__name__!s}'
177                f'(gnutls_params={self.gnutls_params!r}, '
178                f'actions={self.actions!r}, protocol={self.protocol!r})')
179
180    @classmethod
181    def _from_yaml(cls, loader, node):
182        fields = loader.construct_mapping(node)
183        conn = TestConnection(**fields)
184        return conn
185
186# Override the default constructors. Pyyaml ignores default parameters
187# otherwise.
188yaml.add_constructor('!request', TestRequest._from_yaml, yaml.Loader)
189yaml.add_constructor('!connection', TestConnection._from_yaml, yaml.Loader)
190
191
192
193class TestExpectationFailed(Exception):
194    """Raise if a test failed. The constructor should be called with a
195    string describing the problem."""
196    pass
197
198
199
200def filter_cert_log(in_stream, out_stream):
201    import fcntl
202    import os
203    import select
204    # This filters out a log line about loading client
205    # certificates that is mistakenly sent to stdout. My fix has
206    # been merged, but buggy binaries will probably be around for
207    # a while.
208    # https://gitlab.com/gnutls/gnutls/merge_requests/1125
209    cert_log = b'Processed 1 client X.509 certificates...\n'
210
211    # Set the input to non-blocking mode
212    fd = in_stream.fileno()
213    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
214    fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
215
216    # The poll object allows waiting for events on non-blocking IO
217    # channels.
218    poller = select.poll()
219    poller.register(fd)
220
221    init_done = False
222    run_loop = True
223    while run_loop:
224        # The returned tuples are file descriptor and event, but
225        # we're only listening on one stream anyway, so we don't
226        # need to check it here.
227        for x, event in poller.poll():
228            # Critical: "event" is a bitwise OR of the POLL* constants
229            if event & select.POLLIN or event & select.POLLPRI:
230                data = in_stream.read()
231                if not init_done:
232                    # If the erroneous log line shows up it's the
233                    # first piece of data we receive. Just copy
234                    # everything after.
235                    init_done = True
236                    if cert_log in data:
237                        data = data.replace(cert_log, b'')
238                out_stream.send(data)
239            if event & select.POLLHUP or event & select.POLLRDHUP:
240                # Stop the loop, but process any other events that
241                # might be in the list returned by poll() first.
242                run_loop = False
243
244    in_stream.close()
245    out_stream.close()
246
247
248
249def format_response(resp, body):
250    s = f'{resp.status} {resp.reason}\n'
251    s = s + '\n'.join(f'{name}: {value}' for name, value in resp.getheaders())
252    s = s + '\n\n' + body
253    return s
254
255
256
257if __name__ == "__main__":
258    import argparse
259    parser = argparse.ArgumentParser(
260        description='Send HTTP requests through gnutls-cli',
261        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
262    parser.add_argument('host', nargs='?', help='Access the specified host',
263                        default='localhost')
264    parser.add_argument('--insecure', action='store_true',
265                        help='do not validate the server certificate')
266    parser.add_argument('-p', '--port', type=int,
267                        help='Access the specified port', default='8000')
268    parser.add_argument('--x509cafile', type=str,
269                        help='Use the specified CA to validate the '
270                        'server certificate')
271    parser.add_argument('--test-config', type=argparse.FileType('r'),
272                        help='load YAML test configuration')
273
274    # enable bash completion if argcomplete is available
275    try:
276        import argcomplete
277        argcomplete.autocomplete(parser)
278    except ImportError:
279        pass
280
281    args = parser.parse_args()
282
283    test_conn = None
284    test_actions = None
285
286    if args.test_config:
287        config = yaml.load(args.test_config, Loader=yaml.Loader)
288        if type(config) is TestConnection:
289            test_conn = config
290            print(test_conn)
291            test_actions = test_conn.actions
292    else:
293        # simple default request
294        test_actions = [TestRequest(path='/test.txt',
295                                    expect={'status': 200, 'body': 'test\n'},
296                                    method='GET')]
297
298
299    # note: "--logfile" option requires GnuTLS version >= 3.6.7
300    command = ['gnutls-cli', '--logfile=/dev/stderr']
301    if args.insecure:
302        command.append('--insecure')
303    if args.x509cafile:
304        command.append('--x509cafile')
305        command.append(args.x509cafile)
306    if test_conn != None:
307        for s in test_conn.gnutls_params:
308            command.append('--' + s)
309    command = command + ['-p', str(args.port), args.host]
310
311    conn = HTTPSubprocessConnection(command, args.host, port=args.port,
312                                    output_filter=filter_cert_log,
313                                    timeout=6.0)
314
315    try:
316        for act in test_actions:
317            if type(act) is TestRequest:
318                try:
319                    conn.request(act.method, act.path, headers=act.headers)
320                    resp = conn.getresponse()
321                except ConnectionResetError as err:
322                    if act.expects_conn_reset():
323                        print('connection reset as expected.')
324                        break
325                    else:
326                        raise err
327                body = resp.read().decode()
328                print(format_response(resp, body))
329                act.check_response(resp, body)
330            else:
331                raise TypeError(f'Unsupported action requested: {act!r}')
332    finally:
333        conn.close()
Note: See TracBrowser for help on using the repository browser.