source: mod_gnutls/test/mgstest/tests.py @ e3e0de1

proxy-ticket
Last change on this file since e3e0de1 was e3e0de1, checked in by Fiona Klute <fiona.klute@…>, 12 months ago

https-test-client.py: Make host and port configurable per connection

Both variables can use environment variables. If unset they default to
TEST_TARGET and TEST_PORT. This makes it possible to connect to
different servers in one test, for example in proxy tests to check
that the backend server is behaving as expected.

  • Property mode set to 100644
File size: 12.3 KB
Line 
1#!/usr/bin/python3
2
3# Copyright 2019 Fiona Klute
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Test objects and support functions for the mod_gnutls test
18suite. The classes defined in this module represent structures in the
19YAML test configuration files.
20
21"""
22
23import os
24import re
25import subprocess
26import yaml
27
28from string import Template
29
30from . import TestExpectationFailed
31from .http import HTTPSubprocessConnection
32
33class TestConnection(yaml.YAMLObject):
34    """An HTTP connection in a test. It includes parameters for the
35    transport (currently gnutls-cli only), and the actions
36    (e.g. sending requests) to take using this connection.
37
38    Note that running one TestConnection object may result in multiple
39    sequential network connections, if the transport gets closed in a
40    non-failure way (e.g. following a "Connection: close" request) and
41    there are more actions, or (rarely) if an action requires its own
42    transport.
43
44    """
45    yaml_tag = '!connection'
46
47    def __init__(self, actions, host=None, port=None, gnutls_params=[],
48                 transport='gnutls'):
49        self.gnutls_params = gnutls_params
50        self.actions = actions
51        self.transport = transport
52        if host:
53            self.host = subst_env(host)
54        else:
55            self.host = os.environ.get('TEST_TARGET', 'localhost')
56        if port:
57            self.port = int(subst_env(port))
58        else:
59            self.port = int(os.environ.get('TEST_PORT', 8000))
60
61    def __repr__(self):
62        return (f'{self.__class__.__name__!s}'
63                f'(host={self.host!r}, port={self.port!r}, '
64                f'gnutls_params={self.gnutls_params!r}, '
65                f'actions={self.actions!r}, transport={self.transport!r})')
66
67    def run(self, timeout=5.0):
68        # note: "--logfile" option requires GnuTLS version >= 3.6.7
69        command = ['gnutls-cli', '--logfile=/dev/stderr']
70        for s in self.gnutls_params:
71            command.append('--' + s)
72        command = command + ['-p', str(self.port), self.host]
73
74        conn = HTTPSubprocessConnection(command, self.host, self.port,
75                                        output_filter=filter_cert_log,
76                                        timeout=timeout)
77
78        try:
79            for act in self.actions:
80                if type(act) is TestRequest:
81                    act.run(conn)
82                elif type(act) is TestRaw10:
83                    act.run(command, timeout)
84                else:
85                    raise TypeError(f'Unsupported action requested: {act!r}')
86        finally:
87            conn.close()
88
89    @classmethod
90    def _from_yaml(cls, loader, node):
91        fields = loader.construct_mapping(node)
92        conn = TestConnection(**fields)
93        return conn
94
95
96
97class TestRequest(yaml.YAMLObject):
98    """Test action that sends an HTTP/1.1 request.
99
100    The path must be specified in the configuration file, all other
101    parameters (method, headers, expected response) have
102    defaults.
103
104    Options for checking the response currently are:
105    * require a specific response status
106    * require the body to exactly match a specific string
107    * require the body to contain all of a list of strings
108
109    """
110    yaml_tag = '!request'
111    def __init__(self, path, method='GET', headers=dict(),
112                 expect=dict(status=200)):
113        self.method = method
114        self.path = path
115        self.headers = headers
116        self.expect = expect
117
118    def __repr__(self):
119        return (f'{self.__class__.__name__!s}(path={self.path!r}, '
120                f'method={self.method!r}, headers={self.headers!r}, '
121                f'expect={self.expect!r})')
122
123    def run(self, conn):
124        try:
125            conn.request(self.method, self.path, headers=self.headers)
126            resp = conn.getresponse()
127        except ConnectionResetError as err:
128            if self.expects_conn_reset():
129                print('connection reset as expected.')
130                return
131            else:
132                raise err
133        body = resp.read().decode()
134        print(format_response(resp, body))
135        self.check_response(resp, body)
136
137    def check_body(self, body):
138        """
139        >>> r1 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'exactly': 'test\\n'}})
140        >>> r1.check_body('test\\n')
141        >>> r1.check_body('xyz\\n')
142        Traceback (most recent call last):
143        ...
144        mgstest.TestExpectationFailed: Unexpected body: 'xyz\\n' != 'test\\n'
145        >>> r2 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': ['tes', 'est']}})
146        >>> r2.check_body('test\\n')
147        >>> r2.check_body('est\\n')
148        Traceback (most recent call last):
149        ...
150        mgstest.TestExpectationFailed: Unexpected body: 'est\\n' does not contain 'tes'
151        >>> r3 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': 'test'}})
152        >>> r3.check_body('test\\n')
153        """
154        if 'exactly' in self.expect['body'] \
155           and body != self.expect['body']['exactly']:
156            raise TestExpectationFailed(
157                f'Unexpected body: {body!r} != '
158                f'{self.expect["body"]["exactly"]!r}')
159        if 'contains' in self.expect['body']:
160            if type(self.expect['body']['contains']) is str:
161                self.expect['body']['contains'] = [
162                    self.expect['body']['contains']]
163            for s in self.expect['body']['contains']:
164                if not s in body:
165                    raise TestExpectationFailed(
166                        f'Unexpected body: {body!r} does not contain '
167                        f'{s!r}')
168
169    def check_response(self, response, body):
170        if self.expects_conn_reset():
171            raise TestExpectationFailed(
172                'Got a response, but connection should have failed!')
173        if response.status != self.expect['status']:
174            raise TestExpectationFailed(
175                f'Unexpected status: {response.status} != '
176                f'{self.expect["status"]}')
177        if 'body' in self.expect:
178            self.check_body(body)
179
180    def expects_conn_reset(self):
181        """Returns True if running this request is expected to fail due to the
182        connection being reset. That usually means the underlying TLS
183        connection failed.
184
185        >>> r1 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': 'test'}})
186        >>> r1.expects_conn_reset()
187        False
188        >>> r2 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'reset': True})
189        >>> r2.expects_conn_reset()
190        True
191        """
192        if 'reset' in self.expect:
193            return self.expect['reset']
194        return False
195
196    @classmethod
197    def _from_yaml(cls, loader, node):
198        fields = loader.construct_mapping(node)
199        req = TestRequest(**fields)
200        return req
201
202
203
204class TestRaw10(TestRequest):
205    """Test action that sends a request using a minimal (and likely
206    incomplete) HTTP/1.0 test client for the one test case that
207    strictly requires HTTP/1.0.
208
209    All request parameters (method, path, headers) MUST be specified
210    in the config file. Checks on status and body work the same as for
211    TestRequest.
212
213    """
214    yaml_tag = '!raw10'
215    status_re = re.compile('^HTTP/([\d\.]+) (\d+) (.*)$')
216
217    def __init__(self, method, path, headers, expect):
218        self.method = method
219        self.path = path
220        self.headers = headers
221        self.expect = expect
222
223    def __repr__(self):
224        return (f'{self.__class__.__name__!s}'
225                f'(method={self.method!r}, path={self.path!r}, '
226                f'headers={self.headers!r}, expect={self.expect!r})')
227
228    def run(self, command, timeout=None):
229        req = f'{self.method} {self.path} HTTP/1.0\r\n'
230        for name, value in self.headers.items():
231            req = req + f'{name}: {value}\r\n'
232        req = req + f'\r\n'
233        proc = subprocess.Popen(command, stdout=subprocess.PIPE,
234                                stdin=subprocess.PIPE, close_fds=True,
235                                bufsize=0)
236        try:
237            # Note: errs will be empty because stderr is not captured
238            outs, errs = proc.communicate(input=req.encode(),
239                                          timeout=timeout)
240        except TimeoutExpired:
241            proc.kill()
242            outs, errs = proc.communicate()
243
244        # first line of the received data must be the status
245        status, rest = outs.decode().split('\r\n', maxsplit=1)
246        # headers and body are separated by double newline
247        headers, body = rest.split('\r\n\r\n', maxsplit=1)
248        # log response for debugging
249        print(f'{status}\n{headers}\n\n{body}')
250
251        m = self.status_re.match(status)
252        if m:
253            status_code = int(m.group(2))
254            status_expect = self.expect.get('status')
255            if status_expect and not status_code == status_expect:
256                raise TestExpectationFailed('Unexpected status code: '
257                                            f'{status}, expected '
258                                            f'{status_expect}')
259        else:
260            raise TestExpectationFailed(f'Invalid status line: "{status}"')
261
262        if 'body' in self.expect:
263            self.check_body(body)
264
265
266
267# Override the default constructors. Pyyaml ignores default parameters
268# otherwise.
269yaml.add_constructor('!request', TestRequest._from_yaml, yaml.Loader)
270yaml.add_constructor('!connection', TestConnection._from_yaml, yaml.Loader)
271
272
273
274def filter_cert_log(in_stream, out_stream):
275    """Filter to stop an erroneous gnutls-cli log message.
276
277    This function filters out a log line about loading client
278    certificates that is mistakenly sent to stdout from gnutls-cli. My
279    fix (https://gitlab.com/gnutls/gnutls/merge_requests/1125) has
280    been merged, but buggy binaries will probably be around for a
281    while.
282
283    The filter is meant to run in a multiprocessing.Process or
284    threading.Thread that receives the stdout of gnutls-cli as
285    in_stream, and a connection for further processing as out_stream.
286
287    """
288    import fcntl
289    import os
290    import select
291    # message to filter
292    cert_log = b'Processed 1 client X.509 certificates...\n'
293
294    # Set the input to non-blocking mode
295    fd = in_stream.fileno()
296    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
297    fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
298
299    # The poll object allows waiting for events on non-blocking IO
300    # channels.
301    poller = select.poll()
302    poller.register(fd)
303
304    init_done = False
305    run_loop = True
306    while run_loop:
307        # The returned tuples are file descriptor and event, but
308        # we're only listening on one stream anyway, so we don't
309        # need to check it here.
310        for x, event in poller.poll():
311            # Critical: "event" is a bitwise OR of the POLL* constants
312            if event & select.POLLIN or event & select.POLLPRI:
313                data = in_stream.read()
314                if not init_done:
315                    # If the erroneous log line shows up it's the
316                    # first piece of data we receive. Just copy
317                    # everything after.
318                    init_done = True
319                    if cert_log in data:
320                        data = data.replace(cert_log, b'')
321                out_stream.send(data)
322            if event & select.POLLHUP or event & select.POLLRDHUP:
323                # Stop the loop, but process any other events that
324                # might be in the list returned by poll() first.
325                run_loop = False
326
327    in_stream.close()
328    out_stream.close()
329
330
331
332def format_response(resp, body):
333    s = f'{resp.status} {resp.reason}\n'
334    s = s + '\n'.join(f'{name}: {value}' for name, value in resp.getheaders())
335    s = s + '\n\n' + body
336    return s
337
338
339
340def subst_env(text):
341    t = Template(text)
342    return t.substitute(os.environ)
Note: See TracBrowser for help on using the repository browser.