source: mod_gnutls/test/https-test-client.py @ 851173b

proxy-ticket
Last change on this file since 851173b was 851173b, checked in by Fiona Klute <fiona.klute@…>, 11 months ago

TestConnection?: Rename unused "protocol" parameter to "transport"

Fits better, in case support for different transports is ever
implemented.

  • Property mode set to 100755
File size: 14.8 KB
Line 
1#!/usr/bin/python3
2# PYTHON_ARGCOMPLETE_OK
3
4# Copyright 2019 Fiona Klute
5#
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10#     http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17
18import re
19import socket
20import subprocess
21import yaml
22
23from http.client import HTTPConnection
24from multiprocessing import Process
25from time import sleep
26
27class HTTPSubprocessConnection(HTTPConnection):
28    def __init__(self, command, host, port=None,
29                 output_filter=None,
30                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
31                 blocksize=8192):
32        super(HTTPSubprocessConnection, self).__init__(host, port, timeout,
33                                                       source_address=None,
34                                                       blocksize=blocksize)
35        # "command" must be a list containing binary and command line
36        # parameters
37        self.command = command
38        # This will be the subprocess reference when connected
39        self._sproc = None
40        # The subprocess return code is stored here on close()
41        self.returncode = None
42        # The set_tunnel method of the super class is not supported
43        # (see exception doc)
44        self.set_tunnel = None
45        # This method will be run in a separate process and filter the
46        # stdout of self._sproc. Its arguments are self._sproc.stdout
47        # and the socket back to the HTTP connection (write-only).
48        self._output_filter = output_filter
49        # output filter process
50        self._fproc = None
51
52    def connect(self):
53        s_local, s_remote = socket.socketpair(socket.AF_UNIX,
54                                              socket.SOCK_STREAM)
55        s_local.settimeout(self.timeout)
56
57        # TODO: Maybe capture stderr?
58        if self._output_filter:
59            self._sproc = subprocess.Popen(self.command, stdout=subprocess.PIPE,
60                                           stdin=s_remote, close_fds=True,
61                                           bufsize=0)
62            self._fproc = Process(target=self._output_filter,
63                                  args=(self._sproc.stdout, s_remote))
64            self._fproc.start()
65        else:
66            self._sproc = subprocess.Popen(self.command, stdout=s_remote,
67                                           stdin=s_remote, close_fds=True,
68                                           bufsize=0)
69        s_remote.close()
70        self.sock = s_local
71
72    def close(self):
73        # close socket to subprocess for writing
74        if self.sock:
75            self.sock.shutdown(socket.SHUT_WR)
76
77        # Wait for the process to stop, send SIGTERM/SIGKILL if
78        # necessary
79        if self._sproc:
80            try:
81                self.returncode = self._sproc.wait(self.timeout)
82            except subprocess.TimeoutExpired:
83                try:
84                    self._sproc.terminate()
85                    self.returncode = self._sproc.wait(self.timeout)
86                except subprocess.TimeoutExpired:
87                    self._sproc.kill()
88                    self.returncode = self._sproc.wait(self.timeout)
89
90        # filter process receives HUP on pipe when the subprocess
91        # terminates
92        if self._fproc:
93            self._fproc.join()
94
95        # close the connection in the super class, which also calls
96        # self.sock.close()
97        super().close()
98
99
100
101class TestRequest(yaml.YAMLObject):
102    yaml_tag = '!request'
103    def __init__(self, path, method='GET', headers=dict(),
104                 expect=dict(status=200)):
105        self.method = method
106        self.path = path
107        self.headers = headers
108        self.expect = expect
109
110    def __repr__(self):
111        return (f'{self.__class__.__name__!s}(path={self.path!r}, '
112                f'method={self.method!r}, headers={self.headers!r}, '
113                f'expect={self.expect!r})')
114
115    def _check_body(self, body):
116        """
117        >>> r1 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'exactly': 'test\\n'}})
118        >>> r1._check_body('test\\n')
119        >>> r1._check_body('xyz\\n')
120        Traceback (most recent call last):
121        ...
122        https-test-client.TestExpectationFailed: Unexpected body: 'xyz\\n' != 'test\\n'
123        >>> r2 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': ['tes', 'est']}})
124        >>> r2._check_body('test\\n')
125        >>> r2._check_body('est\\n')
126        Traceback (most recent call last):
127        ...
128        https-test-client.TestExpectationFailed: Unexpected body: 'est\\n' does not contain 'tes'
129        >>> r3 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': 'test'}})
130        >>> r3._check_body('test\\n')
131        """
132        if 'exactly' in self.expect['body'] \
133           and body != self.expect['body']['exactly']:
134            raise TestExpectationFailed(
135                f'Unexpected body: {body!r} != '
136                f'{self.expect["body"]["exactly"]!r}')
137        if 'contains' in self.expect['body']:
138            if type(self.expect['body']['contains']) is str:
139                self.expect['body']['contains'] = [
140                    self.expect['body']['contains']]
141            for s in self.expect['body']['contains']:
142                if not s in body:
143                    raise TestExpectationFailed(
144                        f'Unexpected body: {body!r} does not contain '
145                        f'{s!r}')
146
147    def check_response(self, response, body):
148        if self.expects_conn_reset():
149            raise TestExpectationFailed(
150                'Got a response, but connection should have failed!')
151        if response.status != self.expect['status']:
152            raise TestExpectationFailed(
153                f'Unexpected status: {response.status} != '
154                f'{self.expect["status"]}')
155        if 'body' in self.expect:
156            self._check_body(body)
157
158    def expects_conn_reset(self):
159        if 'reset' in self.expect:
160            return self.expect['reset']
161        return False
162
163    @classmethod
164    def _from_yaml(cls, loader, node):
165        fields = loader.construct_mapping(node)
166        req = TestRequest(**fields)
167        return req
168
169class TestConnection(yaml.YAMLObject):
170    yaml_tag = '!connection'
171
172    def __init__(self, actions, gnutls_params=[], transport='gnutls'):
173        self.gnutls_params = gnutls_params
174        self.actions = actions
175        self.transport = transport
176
177    def __repr__(self):
178        return (f'{self.__class__.__name__!s}'
179                f'(gnutls_params={self.gnutls_params!r}, '
180                f'actions={self.actions!r}, transport={self.transport!r})')
181
182    @classmethod
183    def _from_yaml(cls, loader, node):
184        fields = loader.construct_mapping(node)
185        conn = TestConnection(**fields)
186        return conn
187
188class TestRaw10(TestRequest):
189    """This is a minimal (and likely incomplete) HTTP/1.0 test client for
190    the one test case that strictly requires HTTP/1.0. All request
191    parameters (method, path, headers) MUST be specified in the config
192    file.
193
194    """
195    yaml_tag = '!raw10'
196    status_re = re.compile('^HTTP/([\d\.]+) (\d+) (.*)$')
197
198    def __init__(self, method, path, headers, expect):
199        self.method = method
200        self.path = path
201        self.headers = headers
202        self.expect = expect
203
204    def __repr__(self):
205        return (f'{self.__class__.__name__!s}'
206                f'(method={self.method!r}, path={self.path!r}, '
207                f'headers={self.headers!r}, expect={self.expect!r})')
208
209    def run(self, command, timeout=None):
210        req = f'{self.method} {self.path} HTTP/1.0\r\n'
211        for name, value in self.headers.items():
212            req = req + f'{name}: {value}\r\n'
213        req = req + f'\r\n'
214        proc = subprocess.Popen(command, stdout=subprocess.PIPE,
215                                stdin=subprocess.PIPE, close_fds=True,
216                                bufsize=0)
217        try:
218            # Note: errs will be empty because stderr is not captured
219            outs, errs = proc.communicate(input=req.encode(),
220                                          timeout=timeout)
221        except TimeoutExpired:
222            proc.kill()
223            outs, errs = proc.communicate()
224
225        # first line of the received data must be the status
226        status, rest = outs.decode().split('\r\n', maxsplit=1)
227        # headers and body are separated by double newline
228        headers, body = rest.split('\r\n\r\n', maxsplit=1)
229        # log response for debugging
230        print(f'{status}\n{headers}\n\n{body}')
231
232        m = self.status_re.match(status)
233        if m:
234            status_code = int(m.group(2))
235            status_expect = self.expect.get('status')
236            if status_expect and not status_code == status_expect:
237                raise TestExpectationFailed('Unexpected status code: '
238                                            f'{status}, expected '
239                                            f'{status_expect}')
240        else:
241            raise TestExpectationFailed(f'Invalid status line: "{status}"')
242
243        if 'body' in self.expect:
244            self._check_body(body)
245
246# Override the default constructors. Pyyaml ignores default parameters
247# otherwise.
248yaml.add_constructor('!request', TestRequest._from_yaml, yaml.Loader)
249yaml.add_constructor('!connection', TestConnection._from_yaml, yaml.Loader)
250
251
252
253class TestExpectationFailed(Exception):
254    """Raise if a test failed. The constructor should be called with a
255    string describing the problem."""
256    pass
257
258
259
260def filter_cert_log(in_stream, out_stream):
261    import fcntl
262    import os
263    import select
264    # This filters out a log line about loading client
265    # certificates that is mistakenly sent to stdout. My fix has
266    # been merged, but buggy binaries will probably be around for
267    # a while.
268    # https://gitlab.com/gnutls/gnutls/merge_requests/1125
269    cert_log = b'Processed 1 client X.509 certificates...\n'
270
271    # Set the input to non-blocking mode
272    fd = in_stream.fileno()
273    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
274    fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
275
276    # The poll object allows waiting for events on non-blocking IO
277    # channels.
278    poller = select.poll()
279    poller.register(fd)
280
281    init_done = False
282    run_loop = True
283    while run_loop:
284        # The returned tuples are file descriptor and event, but
285        # we're only listening on one stream anyway, so we don't
286        # need to check it here.
287        for x, event in poller.poll():
288            # Critical: "event" is a bitwise OR of the POLL* constants
289            if event & select.POLLIN or event & select.POLLPRI:
290                data = in_stream.read()
291                if not init_done:
292                    # If the erroneous log line shows up it's the
293                    # first piece of data we receive. Just copy
294                    # everything after.
295                    init_done = True
296                    if cert_log in data:
297                        data = data.replace(cert_log, b'')
298                out_stream.send(data)
299            if event & select.POLLHUP or event & select.POLLRDHUP:
300                # Stop the loop, but process any other events that
301                # might be in the list returned by poll() first.
302                run_loop = False
303
304    in_stream.close()
305    out_stream.close()
306
307
308
309def format_response(resp, body):
310    s = f'{resp.status} {resp.reason}\n'
311    s = s + '\n'.join(f'{name}: {value}' for name, value in resp.getheaders())
312    s = s + '\n\n' + body
313    return s
314
315
316
317if __name__ == "__main__":
318    import argparse
319    parser = argparse.ArgumentParser(
320        description='Send HTTP requests through gnutls-cli',
321        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
322    parser.add_argument('host', nargs='?', help='Access the specified host',
323                        default='localhost')
324    parser.add_argument('--insecure', action='store_true',
325                        help='do not validate the server certificate')
326    parser.add_argument('-p', '--port', type=int,
327                        help='Access the specified port', default='8000')
328    parser.add_argument('--x509cafile', type=str,
329                        help='Use the specified CA to validate the '
330                        'server certificate')
331    parser.add_argument('--test-config', type=argparse.FileType('r'),
332                        help='load YAML test configuration')
333
334    # enable bash completion if argcomplete is available
335    try:
336        import argcomplete
337        argcomplete.autocomplete(parser)
338    except ImportError:
339        pass
340
341    args = parser.parse_args()
342
343    test_conn = None
344    test_actions = None
345
346    if args.test_config:
347        config = yaml.load(args.test_config, Loader=yaml.Loader)
348        if type(config) is TestConnection:
349            test_conn = config
350            print(test_conn)
351            test_actions = test_conn.actions
352    else:
353        # simple default request
354        test_actions = [TestRequest(path='/test.txt',
355                                    expect={'status': 200, 'body': 'test\n'},
356                                    method='GET')]
357
358
359    # note: "--logfile" option requires GnuTLS version >= 3.6.7
360    command = ['gnutls-cli', '--logfile=/dev/stderr']
361    if args.insecure:
362        command.append('--insecure')
363    if args.x509cafile:
364        command.append('--x509cafile')
365        command.append(args.x509cafile)
366    if test_conn != None:
367        for s in test_conn.gnutls_params:
368            command.append('--' + s)
369    command = command + ['-p', str(args.port), args.host]
370
371    conn = HTTPSubprocessConnection(command, args.host, port=args.port,
372                                    output_filter=filter_cert_log,
373                                    timeout=6.0)
374
375    try:
376        for act in test_actions:
377            if type(act) is TestRequest:
378                try:
379                    conn.request(act.method, act.path, headers=act.headers)
380                    resp = conn.getresponse()
381                except ConnectionResetError as err:
382                    if act.expects_conn_reset():
383                        print('connection reset as expected.')
384                        break
385                    else:
386                        raise err
387                body = resp.read().decode()
388                print(format_response(resp, body))
389                act.check_response(resp, body)
390            elif type(act) is TestRaw10:
391                act.run(command, conn.timeout)
392            else:
393                raise TypeError(f'Unsupported action requested: {act!r}')
394    finally:
395        conn.close()
Note: See TracBrowser for help on using the repository browser.