source: mod_gnutls/test/mgstest/tests.py @ 3deb86e

proxy-ticket
Last change on this file since 3deb86e was 3deb86e, checked in by Fiona Klute <fiona.klute@…>, 9 months ago

Reorganize imports, remove already done TODO

  • Property mode set to 100644
File size: 14.4 KB
Line 
1#!/usr/bin/python3
2
3# Copyright 2019-2020 Fiona Klute
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Test objects and support functions for the mod_gnutls test
18suite. The classes defined in this module represent structures in the
19YAML test configuration files.
20
21"""
22
23import os
24import re
25import select
26import subprocess
27import sys
28import yaml
29
30from enum import Enum, auto
31from http.client import HTTPConnection
32from string import Template
33
34from . import TestExpectationFailed
35from .http import HTTPSubprocessConnection
36
37class Transports(Enum):
38    GNUTLS = auto()
39    PLAIN = auto()
40
41    def __repr__(self):
42        return f'{self.__class__.__name__!s}.{self.name}'
43
44class TestConnection(yaml.YAMLObject):
45    """An HTTP connection in a test. It includes parameters for the
46    transport (currently gnutls-cli only), and the actions
47    (e.g. sending requests) to take using this connection.
48
49    Note that running one TestConnection object may result in multiple
50    sequential network connections, if the transport gets closed in a
51    non-failure way (e.g. following a "Connection: close" request) and
52    there are more actions, or (rarely) if an action requires its own
53    transport.
54
55    """
56    yaml_tag = '!connection'
57
58    def __init__(self, actions, host=None, port=None, gnutls_params=[],
59                 transport='gnutls', description=None):
60        self.gnutls_params = gnutls_params
61        self.actions = actions
62        self.transport = Transports[transport.upper()]
63        self.description = description
64        if host:
65            self.host = subst_env(host)
66        else:
67            self.host = os.environ.get('TEST_TARGET', 'localhost')
68        if port:
69            self.port = int(subst_env(port))
70        else:
71            self.port = int(os.environ.get('TEST_PORT', 8000))
72
73    def __repr__(self):
74        return (f'{self.__class__.__name__!s}'
75                f'(host={self.host!r}, port={self.port!r}, '
76                f'gnutls_params={self.gnutls_params!r}, '
77                f'actions={self.actions!r}, transport={self.transport!r}, '
78                f'description={self.description!r})')
79
80    def run(self, timeout=5.0, conn_log=None, response_log=None):
81        # note: "--logfile" option requires GnuTLS version >= 3.6.7
82        command = ['gnutls-cli', '--logfile=/dev/stderr']
83        for s in self.gnutls_params:
84            command.append('--' + s)
85        command = command + ['-p', str(self.port), self.host]
86
87        if self.transport == Transports.GNUTLS:
88            conn = HTTPSubprocessConnection(command, self.host, self.port,
89                                            output_filter=filter_cert_log,
90                                            stderr_log=conn_log,
91                                            timeout=timeout)
92        elif self.transport == Transports.PLAIN:
93            conn = HTTPConnection(self.host, port=self.port,
94                                  timeout=timeout)
95
96        try:
97            for act in self.actions:
98                if type(act) is TestRequest:
99                    act.run(conn, response_log)
100                elif type(act) is TestReq10:
101                    act.run(command, timeout, conn_log, response_log)
102                else:
103                    raise TypeError(f'Unsupported action requested: {act!r}')
104        finally:
105            conn.close()
106            sys.stdout.flush()
107
108    @classmethod
109    def from_yaml(cls, loader, node):
110        fields = loader.construct_mapping(node)
111        conn = cls(**fields)
112        return conn
113
114
115
116class TestRequest(yaml.YAMLObject):
117    """Test action that sends an HTTP/1.1 request.
118
119    The path must be specified in the configuration file, all other
120    parameters (method, headers, expected response) have
121    defaults.
122
123    Options for checking the response currently are:
124    * require a specific response status
125    * require the body to exactly match a specific string
126    * require the body to contain all of a list of strings
127
128    """
129    yaml_tag = '!request'
130    def __init__(self, path, method='GET', headers=dict(),
131                 expect=dict(status=200)):
132        self.method = method
133        self.path = path
134        self.headers = headers
135        self.expect = expect
136
137    def __repr__(self):
138        return (f'{self.__class__.__name__!s}(path={self.path!r}, '
139                f'method={self.method!r}, headers={self.headers!r}, '
140                f'expect={self.expect!r})')
141
142    def run(self, conn, response_log=None):
143        try:
144            conn.request(self.method, self.path, headers=self.headers)
145            resp = conn.getresponse()
146            if self.expects_conn_reset():
147                raise TestExpectationFailed(
148                    'Expected connection reset did not occur!')
149        except (BrokenPipeError, ConnectionResetError) as err:
150            if self.expects_conn_reset():
151                print('connection reset as expected.')
152                return
153            else:
154                raise err
155        body = resp.read().decode()
156        log_str = format_response(resp, body)
157        print(log_str)
158        if response_log:
159            print(log_str, file=response_log)
160        self.check_response(resp, body)
161
162    def check_headers(self, headers):
163        for name, expected in self.expect['headers'].items():
164            value = headers.get(name)
165            expected = subst_env(expected)
166            if value != expected:
167                raise TestExpectationFailed(
168                    f'Unexpected value in header {name}: "{value}", '
169                    f'expected "{expected}"')
170
171    def check_body(self, body):
172        """
173        >>> r1 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'exactly': 'test\\n'}})
174        >>> r1.check_body('test\\n')
175        >>> r1.check_body('xyz\\n')
176        Traceback (most recent call last):
177        ...
178        mgstest.TestExpectationFailed: Unexpected body: 'xyz\\n' != 'test\\n'
179        >>> r2 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': ['tes', 'est']}})
180        >>> r2.check_body('test\\n')
181        >>> r2.check_body('est\\n')
182        Traceback (most recent call last):
183        ...
184        mgstest.TestExpectationFailed: Unexpected body: 'est\\n' does not contain 'tes'
185        >>> r3 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': 'test'}})
186        >>> r3.check_body('test\\n')
187        """
188        if 'exactly' in self.expect['body'] \
189           and body != self.expect['body']['exactly']:
190            raise TestExpectationFailed(
191                f'Unexpected body: {body!r} != '
192                f'{self.expect["body"]["exactly"]!r}')
193        if 'contains' in self.expect['body']:
194            if type(self.expect['body']['contains']) is str:
195                self.expect['body']['contains'] = [
196                    self.expect['body']['contains']]
197            for s in self.expect['body']['contains']:
198                if not s in body:
199                    raise TestExpectationFailed(
200                        f'Unexpected body: {body!r} does not contain '
201                        f'{s!r}')
202
203    def check_response(self, response, body):
204        if self.expects_conn_reset():
205            raise TestExpectationFailed(
206                'Got a response, but connection should have failed!')
207        if response.status != self.expect['status']:
208            raise TestExpectationFailed(
209                f'Unexpected status: {response.status} != '
210                f'{self.expect["status"]}')
211        if 'headers' in self.expect:
212            self.check_headers(dict(response.getheaders()))
213        if 'body' in self.expect:
214            self.check_body(body)
215
216    def expects_conn_reset(self):
217        """Returns True if running this request is expected to fail due to the
218        connection being reset. That usually means the underlying TLS
219        connection failed.
220
221        >>> r1 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': 'test'}})
222        >>> r1.expects_conn_reset()
223        False
224        >>> r2 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'reset': True})
225        >>> r2.expects_conn_reset()
226        True
227        """
228        if 'reset' in self.expect:
229            return self.expect['reset']
230        return False
231
232    @classmethod
233    def from_yaml(cls, loader, node):
234        fields = loader.construct_mapping(node)
235        req = cls(**fields)
236        return req
237
238
239
240class TestReq10(TestRequest):
241    """Test action that sends a request using a minimal (and likely
242    incomplete) HTTP/1.0 test client for the one test case that
243    strictly requires HTTP/1.0.
244
245    Objects use the same default parameters as TestRequest, but note
246    that an empty "headers" parameter means that not even a "Host:"
247    header will be sent. All headers must be specified in the test
248    configuration file.
249
250    """
251    yaml_tag = '!request10'
252    status_re = re.compile('^HTTP/([\d\.]+) (\d+) (.*)$')
253
254    def __init__(self, **kwargs):
255        super().__init__(**kwargs)
256
257    def __repr__(self):
258        return (f'{self.__class__.__name__!s}'
259                f'(method={self.method!r}, path={self.path!r}, '
260                f'headers={self.headers!r}, expect={self.expect!r})')
261
262    def run(self, command, timeout=None, conn_log=None, response_log=None):
263        req = f'{self.method} {self.path} HTTP/1.0\r\n'
264        for name, value in self.headers.items():
265            req = req + f'{name}: {value}\r\n'
266        req = req + f'\r\n'
267        proc = subprocess.Popen(command,
268                                stdout=subprocess.PIPE,
269                                stderr=subprocess.PIPE,
270                                stdin=subprocess.PIPE,
271                                close_fds=True,
272                                bufsize=0)
273        try:
274            outs, errs = proc.communicate(input=req.encode(),
275                                          timeout=timeout)
276        except TimeoutExpired:
277            proc.kill()
278            outs, errs = proc.communicate()
279
280        print(errs.decode())
281        if conn_log:
282            print(errs.decode(), file=conn_log)
283
284        # first line of the received data must be the status
285        status, rest = outs.decode().split('\r\n', maxsplit=1)
286        # headers and body are separated by double newline
287        headers, body = rest.split('\r\n\r\n', maxsplit=1)
288        # log response for debugging
289        print(f'{status}\n{headers}\n\n{body}')
290        if response_log:
291            print(f'{status}\n{headers}\n\n{body}', file=response_log)
292
293        m = self.status_re.match(status)
294        if m:
295            status_code = int(m.group(2))
296            status_expect = self.expect.get('status')
297            if status_expect and not status_code == status_expect:
298                raise TestExpectationFailed('Unexpected status code: '
299                                            f'{status}, expected '
300                                            f'{status_expect}')
301        else:
302            raise TestExpectationFailed(f'Invalid status line: "{status}"')
303
304        if 'body' in self.expect:
305            self.check_body(body)
306
307
308
309def filter_cert_log(in_stream, out_stream):
310    """Filter to stop an erroneous gnutls-cli log message.
311
312    This function filters out a log line about loading client
313    certificates that is mistakenly sent to stdout from gnutls-cli. My
314    fix (https://gitlab.com/gnutls/gnutls/merge_requests/1125) has
315    been merged, but buggy binaries will probably be around for a
316    while.
317
318    The filter is meant to run in a multiprocessing.Process or
319    threading.Thread that receives the stdout of gnutls-cli as
320    in_stream, and a connection for further processing as out_stream.
321
322    """
323    # message to filter
324    cert_log = b'Processed 1 client X.509 certificates...\n'
325
326    # Set the input to non-blocking mode
327    fd = in_stream.fileno()
328    os.set_blocking(fd, False)
329
330    # The poll object allows waiting for events on non-blocking IO
331    # channels.
332    poller = select.poll()
333    poller.register(fd)
334
335    init_done = False
336    run_loop = True
337    while run_loop:
338        # The returned tuples are file descriptor and event, but
339        # we're only listening on one stream anyway, so we don't
340        # need to check it here.
341        for x, event in poller.poll():
342            # Critical: "event" is a bitwise OR of the POLL* constants
343            if event & select.POLLIN or event & select.POLLPRI:
344                data = in_stream.read()
345                if not init_done:
346                    # If the erroneous log line shows up it's the
347                    # first piece of data we receive. Just copy
348                    # everything after.
349                    init_done = True
350                    if cert_log in data:
351                        data = data.replace(cert_log, b'')
352                out_stream.send(data)
353            if event & select.POLLHUP or event & select.POLLRDHUP:
354                # Stop the loop, but process any other events that
355                # might be in the list returned by poll() first.
356                run_loop = False
357
358    in_stream.close()
359    out_stream.close()
360
361
362
363def format_response(resp, body):
364    s = f'{resp.status} {resp.reason}\n'
365    s = s + '\n'.join(f'{name}: {value}' for name, value in resp.getheaders())
366    s = s + '\n\n' + body
367    return s
368
369
370
371def subst_env(text):
372    t = Template(text)
373    return t.substitute(os.environ)
374
375
376
377def run_test_conf(test_config, timeout=5.0, conn_log=None, response_log=None):
378    conns = None
379
380    config = yaml.load(test_config, Loader=yaml.Loader)
381    if type(config) is TestConnection:
382        conns = [config]
383    elif type(config) is list:
384        # assume list elements are connections
385        conns = config
386    else:
387        raise TypeError(f'Unsupported configuration: {config!r}')
388    print(conns)
389    sys.stdout.flush()
390
391    for i, test_conn in enumerate(conns):
392        if test_conn.description:
393            print(f'Running test connection {i}: {test_conn.description}')
394        else:
395            print(f'Running test connection {i}.')
396        sys.stdout.flush()
397        test_conn.run(timeout=timeout, conn_log=conn_log,
398                      response_log=response_log)
Note: See TracBrowser for help on using the repository browser.