source: mod_gnutls/test/mgstest/tests.py @ 0e069b6

proxy-ticket
Last change on this file since 0e069b6 was 0e069b6, checked in by Fiona Klute <fiona.klute@…>, 12 months ago

Support plain HTTP in the Python test framework

This makes it possible to run test 26 "redirect HTTP to HTTPS" without
an external HTTP client, the only thing still different from most
other tests is the TEST_HTTP_PORT environment variable.

  • Property mode set to 100644
File size: 13.2 KB
Line 
1#!/usr/bin/python3
2
3# Copyright 2019 Fiona Klute
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Test objects and support functions for the mod_gnutls test
18suite. The classes defined in this module represent structures in the
19YAML test configuration files.
20
21"""
22
23import os
24import re
25import subprocess
26import sys
27import yaml
28
29from enum import Enum, auto
30from http.client import HTTPConnection
31from string import Template
32
33from . import TestExpectationFailed
34from .http import HTTPSubprocessConnection
35
36class Transports(Enum):
37    GNUTLS = auto()
38    PLAIN = auto()
39
40    def __repr__(self):
41        return f'{self.__class__.__name__!s}.{self.name}'
42
43class TestConnection(yaml.YAMLObject):
44    """An HTTP connection in a test. It includes parameters for the
45    transport (currently gnutls-cli only), and the actions
46    (e.g. sending requests) to take using this connection.
47
48    Note that running one TestConnection object may result in multiple
49    sequential network connections, if the transport gets closed in a
50    non-failure way (e.g. following a "Connection: close" request) and
51    there are more actions, or (rarely) if an action requires its own
52    transport.
53
54    """
55    yaml_tag = '!connection'
56
57    def __init__(self, actions, host=None, port=None, gnutls_params=[],
58                 transport='gnutls'):
59        self.gnutls_params = gnutls_params
60        self.actions = actions
61        self.transport = Transports[transport.upper()]
62        if host:
63            self.host = subst_env(host)
64        else:
65            self.host = os.environ.get('TEST_TARGET', 'localhost')
66        if port:
67            self.port = int(subst_env(port))
68        else:
69            self.port = int(os.environ.get('TEST_PORT', 8000))
70
71    def __repr__(self):
72        return (f'{self.__class__.__name__!s}'
73                f'(host={self.host!r}, port={self.port!r}, '
74                f'gnutls_params={self.gnutls_params!r}, '
75                f'actions={self.actions!r}, transport={self.transport!r})')
76
77    def run(self, timeout=5.0):
78        # note: "--logfile" option requires GnuTLS version >= 3.6.7
79        command = ['gnutls-cli', '--logfile=/dev/stderr']
80        for s in self.gnutls_params:
81            command.append('--' + s)
82        command = command + ['-p', str(self.port), self.host]
83
84        if self.transport == Transports.GNUTLS:
85            conn = HTTPSubprocessConnection(command, self.host, self.port,
86                                            output_filter=filter_cert_log,
87                                            timeout=timeout)
88        elif self.transport == Transports.PLAIN:
89            conn = HTTPConnection(self.host, port=self.port,
90                                  timeout=timeout)
91
92        try:
93            for act in self.actions:
94                if type(act) is TestRequest:
95                    act.run(conn)
96                elif type(act) is TestRaw10:
97                    act.run(command, timeout)
98                else:
99                    raise TypeError(f'Unsupported action requested: {act!r}')
100        finally:
101            conn.close()
102            sys.stdout.flush()
103
104    @classmethod
105    def _from_yaml(cls, loader, node):
106        fields = loader.construct_mapping(node)
107        conn = TestConnection(**fields)
108        return conn
109
110
111
112class TestRequest(yaml.YAMLObject):
113    """Test action that sends an HTTP/1.1 request.
114
115    The path must be specified in the configuration file, all other
116    parameters (method, headers, expected response) have
117    defaults.
118
119    Options for checking the response currently are:
120    * require a specific response status
121    * require the body to exactly match a specific string
122    * require the body to contain all of a list of strings
123
124    """
125    yaml_tag = '!request'
126    def __init__(self, path, method='GET', headers=dict(),
127                 expect=dict(status=200)):
128        self.method = method
129        self.path = path
130        self.headers = headers
131        self.expect = expect
132
133    def __repr__(self):
134        return (f'{self.__class__.__name__!s}(path={self.path!r}, '
135                f'method={self.method!r}, headers={self.headers!r}, '
136                f'expect={self.expect!r})')
137
138    def run(self, conn):
139        try:
140            conn.request(self.method, self.path, headers=self.headers)
141            resp = conn.getresponse()
142        except ConnectionResetError as err:
143            if self.expects_conn_reset():
144                print('connection reset as expected.')
145                return
146            else:
147                raise err
148        body = resp.read().decode()
149        print(format_response(resp, body))
150        self.check_response(resp, body)
151
152    def check_headers(self, headers):
153        for name, expected in self.expect['headers'].items():
154            value = headers.get(name)
155            expected = subst_env(expected)
156            if value != expected:
157                raise TestExpectationFailed(
158                    f'Unexpected value in header {name}: "{value}", '
159                    f'expected "{expected}"')
160
161    def check_body(self, body):
162        """
163        >>> r1 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'exactly': 'test\\n'}})
164        >>> r1.check_body('test\\n')
165        >>> r1.check_body('xyz\\n')
166        Traceback (most recent call last):
167        ...
168        mgstest.TestExpectationFailed: Unexpected body: 'xyz\\n' != 'test\\n'
169        >>> r2 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': ['tes', 'est']}})
170        >>> r2.check_body('test\\n')
171        >>> r2.check_body('est\\n')
172        Traceback (most recent call last):
173        ...
174        mgstest.TestExpectationFailed: Unexpected body: 'est\\n' does not contain 'tes'
175        >>> r3 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': 'test'}})
176        >>> r3.check_body('test\\n')
177        """
178        if 'exactly' in self.expect['body'] \
179           and body != self.expect['body']['exactly']:
180            raise TestExpectationFailed(
181                f'Unexpected body: {body!r} != '
182                f'{self.expect["body"]["exactly"]!r}')
183        if 'contains' in self.expect['body']:
184            if type(self.expect['body']['contains']) is str:
185                self.expect['body']['contains'] = [
186                    self.expect['body']['contains']]
187            for s in self.expect['body']['contains']:
188                if not s in body:
189                    raise TestExpectationFailed(
190                        f'Unexpected body: {body!r} does not contain '
191                        f'{s!r}')
192
193    def check_response(self, response, body):
194        if self.expects_conn_reset():
195            raise TestExpectationFailed(
196                'Got a response, but connection should have failed!')
197        if response.status != self.expect['status']:
198            raise TestExpectationFailed(
199                f'Unexpected status: {response.status} != '
200                f'{self.expect["status"]}')
201        if 'headers' in self.expect:
202            self.check_headers(dict(response.getheaders()))
203        if 'body' in self.expect:
204            self.check_body(body)
205
206    def expects_conn_reset(self):
207        """Returns True if running this request is expected to fail due to the
208        connection being reset. That usually means the underlying TLS
209        connection failed.
210
211        >>> r1 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': 'test'}})
212        >>> r1.expects_conn_reset()
213        False
214        >>> r2 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'reset': True})
215        >>> r2.expects_conn_reset()
216        True
217        """
218        if 'reset' in self.expect:
219            return self.expect['reset']
220        return False
221
222    @classmethod
223    def _from_yaml(cls, loader, node):
224        fields = loader.construct_mapping(node)
225        req = TestRequest(**fields)
226        return req
227
228
229
230class TestRaw10(TestRequest):
231    """Test action that sends a request using a minimal (and likely
232    incomplete) HTTP/1.0 test client for the one test case that
233    strictly requires HTTP/1.0.
234
235    All request parameters (method, path, headers) MUST be specified
236    in the config file. Checks on status and body work the same as for
237    TestRequest.
238
239    """
240    yaml_tag = '!raw10'
241    status_re = re.compile('^HTTP/([\d\.]+) (\d+) (.*)$')
242
243    def __init__(self, method, path, headers, expect):
244        self.method = method
245        self.path = path
246        self.headers = headers
247        self.expect = expect
248
249    def __repr__(self):
250        return (f'{self.__class__.__name__!s}'
251                f'(method={self.method!r}, path={self.path!r}, '
252                f'headers={self.headers!r}, expect={self.expect!r})')
253
254    def run(self, command, timeout=None):
255        req = f'{self.method} {self.path} HTTP/1.0\r\n'
256        for name, value in self.headers.items():
257            req = req + f'{name}: {value}\r\n'
258        req = req + f'\r\n'
259        proc = subprocess.Popen(command, stdout=subprocess.PIPE,
260                                stdin=subprocess.PIPE, close_fds=True,
261                                bufsize=0)
262        try:
263            # Note: errs will be empty because stderr is not captured
264            outs, errs = proc.communicate(input=req.encode(),
265                                          timeout=timeout)
266        except TimeoutExpired:
267            proc.kill()
268            outs, errs = proc.communicate()
269
270        # first line of the received data must be the status
271        status, rest = outs.decode().split('\r\n', maxsplit=1)
272        # headers and body are separated by double newline
273        headers, body = rest.split('\r\n\r\n', maxsplit=1)
274        # log response for debugging
275        print(f'{status}\n{headers}\n\n{body}')
276
277        m = self.status_re.match(status)
278        if m:
279            status_code = int(m.group(2))
280            status_expect = self.expect.get('status')
281            if status_expect and not status_code == status_expect:
282                raise TestExpectationFailed('Unexpected status code: '
283                                            f'{status}, expected '
284                                            f'{status_expect}')
285        else:
286            raise TestExpectationFailed(f'Invalid status line: "{status}"')
287
288        if 'body' in self.expect:
289            self.check_body(body)
290
291
292
293# Override the default constructors. Pyyaml ignores default parameters
294# otherwise.
295yaml.add_constructor('!request', TestRequest._from_yaml, yaml.Loader)
296yaml.add_constructor('!connection', TestConnection._from_yaml, yaml.Loader)
297
298
299
300def filter_cert_log(in_stream, out_stream):
301    """Filter to stop an erroneous gnutls-cli log message.
302
303    This function filters out a log line about loading client
304    certificates that is mistakenly sent to stdout from gnutls-cli. My
305    fix (https://gitlab.com/gnutls/gnutls/merge_requests/1125) has
306    been merged, but buggy binaries will probably be around for a
307    while.
308
309    The filter is meant to run in a multiprocessing.Process or
310    threading.Thread that receives the stdout of gnutls-cli as
311    in_stream, and a connection for further processing as out_stream.
312
313    """
314    import fcntl
315    import os
316    import select
317    # message to filter
318    cert_log = b'Processed 1 client X.509 certificates...\n'
319
320    # Set the input to non-blocking mode
321    fd = in_stream.fileno()
322    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
323    fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
324
325    # The poll object allows waiting for events on non-blocking IO
326    # channels.
327    poller = select.poll()
328    poller.register(fd)
329
330    init_done = False
331    run_loop = True
332    while run_loop:
333        # The returned tuples are file descriptor and event, but
334        # we're only listening on one stream anyway, so we don't
335        # need to check it here.
336        for x, event in poller.poll():
337            # Critical: "event" is a bitwise OR of the POLL* constants
338            if event & select.POLLIN or event & select.POLLPRI:
339                data = in_stream.read()
340                if not init_done:
341                    # If the erroneous log line shows up it's the
342                    # first piece of data we receive. Just copy
343                    # everything after.
344                    init_done = True
345                    if cert_log in data:
346                        data = data.replace(cert_log, b'')
347                out_stream.send(data)
348            if event & select.POLLHUP or event & select.POLLRDHUP:
349                # Stop the loop, but process any other events that
350                # might be in the list returned by poll() first.
351                run_loop = False
352
353    in_stream.close()
354    out_stream.close()
355
356
357
358def format_response(resp, body):
359    s = f'{resp.status} {resp.reason}\n'
360    s = s + '\n'.join(f'{name}: {value}' for name, value in resp.getheaders())
361    s = s + '\n\n' + body
362    return s
363
364
365
366def subst_env(text):
367    t = Template(text)
368    return t.substitute(os.environ)
Note: See TracBrowser for help on using the repository browser.