source: mod_gnutls/test/mgstest/tests.py @ 09774e2

proxy-ticket
Last change on this file since 09774e2 was 09774e2, checked in by Fiona Klute <fiona.klute@…>, 11 months ago

Optionally log HTTP responses to another stream/file

  • Property mode set to 100644
File size: 14.7 KB
Line 
1#!/usr/bin/python3
2
3# Copyright 2019 Fiona Klute
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Test objects and support functions for the mod_gnutls test
18suite. The classes defined in this module represent structures in the
19YAML test configuration files.
20
21"""
22
23import os
24import re
25import subprocess
26import sys
27import yaml
28
29from enum import Enum, auto
30from http.client import HTTPConnection
31from string import Template
32
33from . import TestExpectationFailed
34from .http import HTTPSubprocessConnection
35
36class Transports(Enum):
37    GNUTLS = auto()
38    PLAIN = auto()
39
40    def __repr__(self):
41        return f'{self.__class__.__name__!s}.{self.name}'
42
43class TestConnection(yaml.YAMLObject):
44    """An HTTP connection in a test. It includes parameters for the
45    transport (currently gnutls-cli only), and the actions
46    (e.g. sending requests) to take using this connection.
47
48    Note that running one TestConnection object may result in multiple
49    sequential network connections, if the transport gets closed in a
50    non-failure way (e.g. following a "Connection: close" request) and
51    there are more actions, or (rarely) if an action requires its own
52    transport.
53
54    """
55    yaml_tag = '!connection'
56
57    def __init__(self, actions, host=None, port=None, gnutls_params=[],
58                 transport='gnutls', description=None):
59        self.gnutls_params = gnutls_params
60        self.actions = actions
61        self.transport = Transports[transport.upper()]
62        self.description = description
63        if host:
64            self.host = subst_env(host)
65        else:
66            self.host = os.environ.get('TEST_TARGET', 'localhost')
67        if port:
68            self.port = int(subst_env(port))
69        else:
70            self.port = int(os.environ.get('TEST_PORT', 8000))
71
72    def __repr__(self):
73        return (f'{self.__class__.__name__!s}'
74                f'(host={self.host!r}, port={self.port!r}, '
75                f'gnutls_params={self.gnutls_params!r}, '
76                f'actions={self.actions!r}, transport={self.transport!r}, '
77                f'description={self.description!r})')
78
79    def run(self, timeout=5.0, conn_log=None, response_log=None):
80        # note: "--logfile" option requires GnuTLS version >= 3.6.7
81        command = ['gnutls-cli', '--logfile=/dev/stderr']
82        for s in self.gnutls_params:
83            command.append('--' + s)
84        command = command + ['-p', str(self.port), self.host]
85
86        if self.transport == Transports.GNUTLS:
87            conn = HTTPSubprocessConnection(command, self.host, self.port,
88                                            output_filter=filter_cert_log,
89                                            stderr_log=conn_log,
90                                            timeout=timeout)
91        elif self.transport == Transports.PLAIN:
92            conn = HTTPConnection(self.host, port=self.port,
93                                  timeout=timeout)
94
95        try:
96            for act in self.actions:
97                if type(act) is TestRequest:
98                    act.run(conn, response_log)
99                elif type(act) is TestRaw10:
100                    act.run(command, timeout, conn_log, response_log)
101                else:
102                    raise TypeError(f'Unsupported action requested: {act!r}')
103        finally:
104            conn.close()
105            sys.stdout.flush()
106
107    @classmethod
108    def _from_yaml(cls, loader, node):
109        fields = loader.construct_mapping(node)
110        conn = TestConnection(**fields)
111        return conn
112
113
114
115class TestRequest(yaml.YAMLObject):
116    """Test action that sends an HTTP/1.1 request.
117
118    The path must be specified in the configuration file, all other
119    parameters (method, headers, expected response) have
120    defaults.
121
122    Options for checking the response currently are:
123    * require a specific response status
124    * require the body to exactly match a specific string
125    * require the body to contain all of a list of strings
126
127    """
128    yaml_tag = '!request'
129    def __init__(self, path, method='GET', headers=dict(),
130                 expect=dict(status=200)):
131        self.method = method
132        self.path = path
133        self.headers = headers
134        self.expect = expect
135
136    def __repr__(self):
137        return (f'{self.__class__.__name__!s}(path={self.path!r}, '
138                f'method={self.method!r}, headers={self.headers!r}, '
139                f'expect={self.expect!r})')
140
141    def run(self, conn, response_log=None):
142        try:
143            conn.request(self.method, self.path, headers=self.headers)
144            resp = conn.getresponse()
145            if self.expects_conn_reset():
146                raise TestExpectationFailed(
147                    'Expected connection reset did not occur!')
148        except (BrokenPipeError, ConnectionResetError) as err:
149            if self.expects_conn_reset():
150                print('connection reset as expected.')
151                return
152            else:
153                raise err
154        body = resp.read().decode()
155        log_str = format_response(resp, body)
156        print(log_str)
157        if response_log:
158            print(log_str, file=response_log)
159        self.check_response(resp, body)
160
161    def check_headers(self, headers):
162        for name, expected in self.expect['headers'].items():
163            value = headers.get(name)
164            expected = subst_env(expected)
165            if value != expected:
166                raise TestExpectationFailed(
167                    f'Unexpected value in header {name}: "{value}", '
168                    f'expected "{expected}"')
169
170    def check_body(self, body):
171        """
172        >>> r1 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'exactly': 'test\\n'}})
173        >>> r1.check_body('test\\n')
174        >>> r1.check_body('xyz\\n')
175        Traceback (most recent call last):
176        ...
177        mgstest.TestExpectationFailed: Unexpected body: 'xyz\\n' != 'test\\n'
178        >>> r2 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': ['tes', 'est']}})
179        >>> r2.check_body('test\\n')
180        >>> r2.check_body('est\\n')
181        Traceback (most recent call last):
182        ...
183        mgstest.TestExpectationFailed: Unexpected body: 'est\\n' does not contain 'tes'
184        >>> r3 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': 'test'}})
185        >>> r3.check_body('test\\n')
186        """
187        if 'exactly' in self.expect['body'] \
188           and body != self.expect['body']['exactly']:
189            raise TestExpectationFailed(
190                f'Unexpected body: {body!r} != '
191                f'{self.expect["body"]["exactly"]!r}')
192        if 'contains' in self.expect['body']:
193            if type(self.expect['body']['contains']) is str:
194                self.expect['body']['contains'] = [
195                    self.expect['body']['contains']]
196            for s in self.expect['body']['contains']:
197                if not s in body:
198                    raise TestExpectationFailed(
199                        f'Unexpected body: {body!r} does not contain '
200                        f'{s!r}')
201
202    def check_response(self, response, body):
203        if self.expects_conn_reset():
204            raise TestExpectationFailed(
205                'Got a response, but connection should have failed!')
206        if response.status != self.expect['status']:
207            raise TestExpectationFailed(
208                f'Unexpected status: {response.status} != '
209                f'{self.expect["status"]}')
210        if 'headers' in self.expect:
211            self.check_headers(dict(response.getheaders()))
212        if 'body' in self.expect:
213            self.check_body(body)
214
215    def expects_conn_reset(self):
216        """Returns True if running this request is expected to fail due to the
217        connection being reset. That usually means the underlying TLS
218        connection failed.
219
220        >>> r1 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': 'test'}})
221        >>> r1.expects_conn_reset()
222        False
223        >>> r2 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'reset': True})
224        >>> r2.expects_conn_reset()
225        True
226        """
227        if 'reset' in self.expect:
228            return self.expect['reset']
229        return False
230
231    @classmethod
232    def _from_yaml(cls, loader, node):
233        fields = loader.construct_mapping(node)
234        req = TestRequest(**fields)
235        return req
236
237
238
239class TestRaw10(TestRequest):
240    """Test action that sends a request using a minimal (and likely
241    incomplete) HTTP/1.0 test client for the one test case that
242    strictly requires HTTP/1.0.
243
244    All request parameters (method, path, headers) MUST be specified
245    in the config file. Checks on status and body work the same as for
246    TestRequest.
247
248    """
249    yaml_tag = '!raw10'
250    status_re = re.compile('^HTTP/([\d\.]+) (\d+) (.*)$')
251
252    def __init__(self, method, path, headers, expect):
253        self.method = method
254        self.path = path
255        self.headers = headers
256        self.expect = expect
257
258    def __repr__(self):
259        return (f'{self.__class__.__name__!s}'
260                f'(method={self.method!r}, path={self.path!r}, '
261                f'headers={self.headers!r}, expect={self.expect!r})')
262
263    def run(self, command, timeout=None, conn_log=None, response_log=None):
264        req = f'{self.method} {self.path} HTTP/1.0\r\n'
265        for name, value in self.headers.items():
266            req = req + f'{name}: {value}\r\n'
267        req = req + f'\r\n'
268        proc = subprocess.Popen(command,
269                                stdout=subprocess.PIPE,
270                                stderr=subprocess.PIPE,
271                                stdin=subprocess.PIPE,
272                                close_fds=True,
273                                bufsize=0)
274        try:
275            outs, errs = proc.communicate(input=req.encode(),
276                                          timeout=timeout)
277        except TimeoutExpired:
278            proc.kill()
279            outs, errs = proc.communicate()
280
281        if conn_log:
282            print(errs.decode(), file=conn_log)
283
284        # first line of the received data must be the status
285        status, rest = outs.decode().split('\r\n', maxsplit=1)
286        # headers and body are separated by double newline
287        headers, body = rest.split('\r\n\r\n', maxsplit=1)
288        # log response for debugging
289        print(f'{status}\n{headers}\n\n{body}')
290        if response_log:
291            print(f'{status}\n{headers}\n\n{body}', file=response_log)
292
293        m = self.status_re.match(status)
294        if m:
295            status_code = int(m.group(2))
296            status_expect = self.expect.get('status')
297            if status_expect and not status_code == status_expect:
298                raise TestExpectationFailed('Unexpected status code: '
299                                            f'{status}, expected '
300                                            f'{status_expect}')
301        else:
302            raise TestExpectationFailed(f'Invalid status line: "{status}"')
303
304        if 'body' in self.expect:
305            self.check_body(body)
306
307
308
309# Override the default constructors. Pyyaml ignores default parameters
310# otherwise.
311yaml.add_constructor('!request', TestRequest._from_yaml, yaml.Loader)
312yaml.add_constructor('!connection', TestConnection._from_yaml, yaml.Loader)
313
314
315
316def filter_cert_log(in_stream, out_stream):
317    """Filter to stop an erroneous gnutls-cli log message.
318
319    This function filters out a log line about loading client
320    certificates that is mistakenly sent to stdout from gnutls-cli. My
321    fix (https://gitlab.com/gnutls/gnutls/merge_requests/1125) has
322    been merged, but buggy binaries will probably be around for a
323    while.
324
325    The filter is meant to run in a multiprocessing.Process or
326    threading.Thread that receives the stdout of gnutls-cli as
327    in_stream, and a connection for further processing as out_stream.
328
329    """
330    import os
331    import select
332    # message to filter
333    cert_log = b'Processed 1 client X.509 certificates...\n'
334
335    # Set the input to non-blocking mode
336    fd = in_stream.fileno()
337    os.set_blocking(fd, False)
338
339    # The poll object allows waiting for events on non-blocking IO
340    # channels.
341    poller = select.poll()
342    poller.register(fd)
343
344    init_done = False
345    run_loop = True
346    while run_loop:
347        # The returned tuples are file descriptor and event, but
348        # we're only listening on one stream anyway, so we don't
349        # need to check it here.
350        for x, event in poller.poll():
351            # Critical: "event" is a bitwise OR of the POLL* constants
352            if event & select.POLLIN or event & select.POLLPRI:
353                data = in_stream.read()
354                if not init_done:
355                    # If the erroneous log line shows up it's the
356                    # first piece of data we receive. Just copy
357                    # everything after.
358                    init_done = True
359                    if cert_log in data:
360                        data = data.replace(cert_log, b'')
361                out_stream.send(data)
362            if event & select.POLLHUP or event & select.POLLRDHUP:
363                # Stop the loop, but process any other events that
364                # might be in the list returned by poll() first.
365                run_loop = False
366
367    in_stream.close()
368    out_stream.close()
369
370
371
372def format_response(resp, body):
373    s = f'{resp.status} {resp.reason}\n'
374    s = s + '\n'.join(f'{name}: {value}' for name, value in resp.getheaders())
375    s = s + '\n\n' + body
376    return s
377
378
379
380def subst_env(text):
381    t = Template(text)
382    return t.substitute(os.environ)
383
384
385
386def run_test_conf(test_config, timeout=5.0, conn_log=None, response_log=None):
387    conns = None
388
389    config = yaml.load(test_config, Loader=yaml.Loader)
390    if type(config) is TestConnection:
391        conns = [config]
392    elif type(config) is list:
393        # assume list elements are connections
394        conns = config
395    else:
396        raise TypeError(f'Unsupported configuration: {config!r}')
397    print(conns)
398    sys.stdout.flush()
399
400    for i, test_conn in enumerate(conns):
401        if test_conn.description:
402            print(f'Running test connection {i}: {test_conn.description}')
403        else:
404            print(f'Running test connection {i}.')
405        sys.stdout.flush()
406        test_conn.run(timeout=timeout, conn_log=conn_log,
407                      response_log=response_log)
Note: See TracBrowser for help on using the repository browser.