diff options
| author | Piotr Ożarowski <piotr@debian.org> | 2016-12-21 13:30:27 +0100 |
|---|---|---|
| committer | git-ubuntu importer <ubuntu-devel-discuss@lists.ubuntu.com> | 2016-12-21 16:24:43 +0000 |
| commit | e192dcec26c7ef90f24321b299aac5d10fcde41b (patch) | |
| tree | 2567903b7afe28e225cf91de0f9467ff89122d82 | |
| parent | 53beee81bdab2521ea635f34b4dd3e53e2aad2fa (diff) | |
1.2.0-1 (patches unapplied)import/1.2.0-1ubuntu/zesty-proposedubuntu/zesty-develubuntu/zestydebian/stretch
Imported using git-ubuntu import.
Notes
Notes:
* New upstream release
* debian/compat changed to 10 (this change also closes: 845178 thanks to
dh_autoreconf)
72 files changed, 2981 insertions, 964 deletions
diff --git a/CHANGES.rst b/CHANGES.rst index 5496dd0..39b6bd7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,138 +1,60 @@ CHANGES ======= -1.1.5 (2016-11-16) +1.2.0 (2016-12-17) ------------------ -- Fix static file serving in fallback mode #1401 - -1.1.4 (2016-11-14) ------------------- +- Extract `BaseRequest` from `web.Request`, introduce `web.Server` + (former `RequestHandlerFactory`), introduce new low-level web server + which is not coupled with `web.Application` and routing #1362 - Make `TestServer.make_url` compatible with `yarl.URL` #1389 -- Generate informative exception on redirects from server which - doesn't provide redirection headers #1396 - - -1.1.3 (2016-11-10) ------------------- - -- Support *root* resources for sub-applications #1379 - - -1.1.2 (2016-11-08) ------------------- - -- Allow starting variables with an underscore #1379 - -- Properly process UNIX sockets by gunicorn worker #1375 - -- Fix ordering for `FrozenList` - -- Don't propagate pre and post signals to sub-application #1377 - -1.1.1 (2016-11-04) ------------------- - -- Fix documentation generation #1120 - -1.1.0 (2016-11-03) ------------------- - -- Drop deprecated `WSClientDisconnectedError` (BACKWARD INCOMPATIBLE) - -- Use `yarl.URL` in client API. The change is 99% backward compatible - but `ClientResponse.url` is an `yarl.URL` instance now. #1217 - -- Close idle keep-alive connections on shutdown #1222 - -- Modify regex in AccessLogger to accept underscore and numbers #1225 - -- Use `yarl.URL` in web server API. `web.Request.rel_url` and - `web.Request.url` are added. URLs and templates are percent-encoded - now. #1224 - -- Accept `yarl.URL` by server redirections #1278 - -- Return `yarl.URL` by `.make_url()` testing utility #1279 - -- Properly format IPv6 addresses by `aiohttp.web.run_app` #1139 - -- Use `yarl.URL` by server API #1288 - - * Introduce `resource.url_for()`, deprecate `resource.url()`. - - * Implement `StaticResource`. - - * Inherit `SystemRoute` from `AbstractRoute` - - * Drop old-style routes: `Route`, `PlainRoute`, `DynamicRoute`, - `StaticRoute`, `ResourceAdapter`. - -- Revert `resp.url` back to `str`, introduce `resp.url_obj` #1292 - -- Raise ValueError if BasicAuth login has a ":" character #1307 - -- Fix bug when ClientRequest send payload file with opened as - open('filename', 'r+b') #1306 - -- Enhancement to AccessLogger (pass *extra* dict) #1303 - -- Show more verbose message on import errors #1319 - -- Added save and load functionality for `CookieJar` #1219 - -- Added option on `StaticRoute` to follow symlinks #1299 - -- Force encoding of `application/json` content type to utf-8 #1339 +- Implement range requests for static files #1382 -- Fix invalid invocations of `errors.LineTooLong` #1335 +- Support task attribute for StreamResponse #1410 -- Websockets: Stop `async for` iteration when connection is closed #1144 +- Drop `TestClient.app` property, use `TestClient.server.app` instead + (BACKWARD INCOMPATIBLE) -- Ensure TestClient HTTP methods return a context manager #1318 +- Drop `TestClient.handler` property, use `TestClient.server.handler` instead + (BACKWARD INCOMPATIBLE) -- Raise `ClientDisconnectedError` to `FlowControlStreamReader` read function - if `ClientSession` object is closed by client when reading data. #1323 +- `TestClient.server` property returns a test server instance, was + `asyncio.AbstractServer` (BACKWARD INCOMPATIBLE) -- Document deployment without `Gunicorn` #1120 +- Follow gunicorn's signal semantics in `Gunicorn[UVLoop]WebWorker` #1201 -- Add deprecation warning for MD5 and SHA1 digests when used for fingerprint - of site certs in TCPConnector. #1186 +- Call worker_int and worker_abort callbacks in + `Gunicorn[UVLoop]WebWorker` #1202 -- Implement sub-applications #1301 +- Has functional tests for client proxy #1218 -- Don't inherit `web.Request` from `dict` but implement - `MutableMapping` protocol. +- Fix bugs with client proxy target path and proxy host with port #1413 -- Implement frozen signals +- Fix bugs related to the use of unicode hostnames #1444 -- Don't inherit `web.Application` from `dict` but implement - `MutableMapping` protocol. +- Preserve cookie quoting/escaping #1453 -- Support freezing for web applications +- FileSender will send gzipped response if gzip version available #1426 -- Accept access_log parameter in `web.run_app`, use `None` to disable logging +- Don't override `Content-Length` header in `web.Response` if no body + was set #1400 -- Don't flap `tcp_cork` and `tcp_nodelay` in regular request handling. - `tcp_nodelay` is still enabled by default. +- Introduce `router.post_init()` for solving #1373 -- Improve performance of web server by removing premature computing of - Content-Type if the value was set by `web.Response` constructor. +- Fix raise error in case of multiple calls of `TimeServive.stop()` - While the patch boosts speed of trivial `web.Response(text='OK', - content_type='text/plain)` very well please don't expect significant - boost of your application -- a couple DB requests and business logic - is still the main bottleneck. +- Allow to raise web exceptions on router resolving stage #1460 -- Boost performance by adding a custom time service #1350 +- Add a warning for session creation outside of coroutine #1468 -- Extend `ClientResponse` with `content_type` and `charset` - properties like in `web.Request`. #1349 +- Avoid a race when application might start accepting incoming requests + but startup signals are not processed yet e98e8c6 -- Disable aiodns by default #559 +- Raise a `RuntimeError` when trying to change the status of the HTTP response + after the headers have been sent #1480 -- Don't flap `tcp_cork` in client code, use TCP_NODELAY mode by default. +- Fix bug with https proxy acquired cleanup #1340 -- Implement `web.Request.clone()` #1361 +- Use UTF-8 as the default encoding for multipart text parts #1484 diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 0a5a9a7..2d1bac1 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -10,6 +10,7 @@ Alexander Bayandin Alexander Karpinsky Alexander Koshevoy Alexander Malev +Alexander Mohr Alexander Shorin Alexander Travov Alexandru Mihai @@ -21,6 +22,7 @@ Alex Lisovoy Amy Boyle Andrei Ursulenko Andrej Antonov +Andrew Leech Andrew Svetlov Andrii Soldatenko Anton Kasyanov @@ -104,10 +106,13 @@ Mun Gwan-gyeong Nicolas Braem Nikolay Novik Olaf Conradi +Pahaz Blinov <pahaz.blinov@gmail.com> Pankaj Pandey Pau Freixes Paul Colomiets +Paulus Schoutsen Philipp A. +Rafael Viotti Raúl Cumplido "Required Field" <requiredfield256@gmail.com> Robert Lu @@ -117,6 +122,7 @@ Sebastian Hüther SeongSoo Cho Sergey Ninua Sergey Skripnick +Serhii Kostel Simon Kennedy Sin-Woo Bang Stanislas Plum @@ -48,8 +48,11 @@ cov-dev: .develop @echo "open file://`pwd`/coverage/index.html" cov-dev-full: .develop + @echo "Run without extensions" @AIOHTTP_NO_EXTENSIONS=1 py.test --cov=aiohttp tests + @echo "Run in debug mode" @PYTHONASYNCIODEBUG=1 py.test --cov=aiohttp --cov-append tests + @echo "Regular run" @py.test --cov=aiohttp --cov-report=term --cov-report=html --cov-append tests @echo "open file://`pwd`/coverage/index.html" @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: aiohttp -Version: 1.1.5 +Version: 1.2.0 Summary: http client/server for asyncio Home-page: https://github.com/KeepSafe/aiohttp/ Author: Andrew Svetlov @@ -161,141 +161,63 @@ Description: http client/server for asyncio CHANGES ======= - 1.1.5 (2016-11-16) + 1.2.0 (2016-12-17) ------------------ - - Fix static file serving in fallback mode #1401 - - 1.1.4 (2016-11-14) - ------------------ + - Extract `BaseRequest` from `web.Request`, introduce `web.Server` + (former `RequestHandlerFactory`), introduce new low-level web server + which is not coupled with `web.Application` and routing #1362 - Make `TestServer.make_url` compatible with `yarl.URL` #1389 - - Generate informative exception on redirects from server which - doesn't provide redirection headers #1396 - - - 1.1.3 (2016-11-10) - ------------------ - - - Support *root* resources for sub-applications #1379 - - - 1.1.2 (2016-11-08) - ------------------ - - - Allow starting variables with an underscore #1379 - - - Properly process UNIX sockets by gunicorn worker #1375 - - - Fix ordering for `FrozenList` - - - Don't propagate pre and post signals to sub-application #1377 - - 1.1.1 (2016-11-04) - ------------------ - - - Fix documentation generation #1120 - - 1.1.0 (2016-11-03) - ------------------ - - - Drop deprecated `WSClientDisconnectedError` (BACKWARD INCOMPATIBLE) - - - Use `yarl.URL` in client API. The change is 99% backward compatible - but `ClientResponse.url` is an `yarl.URL` instance now. #1217 - - - Close idle keep-alive connections on shutdown #1222 - - - Modify regex in AccessLogger to accept underscore and numbers #1225 - - - Use `yarl.URL` in web server API. `web.Request.rel_url` and - `web.Request.url` are added. URLs and templates are percent-encoded - now. #1224 - - - Accept `yarl.URL` by server redirections #1278 - - - Return `yarl.URL` by `.make_url()` testing utility #1279 - - - Properly format IPv6 addresses by `aiohttp.web.run_app` #1139 - - - Use `yarl.URL` by server API #1288 - - * Introduce `resource.url_for()`, deprecate `resource.url()`. - - * Implement `StaticResource`. - - * Inherit `SystemRoute` from `AbstractRoute` - - * Drop old-style routes: `Route`, `PlainRoute`, `DynamicRoute`, - `StaticRoute`, `ResourceAdapter`. - - - Revert `resp.url` back to `str`, introduce `resp.url_obj` #1292 - - - Raise ValueError if BasicAuth login has a ":" character #1307 - - - Fix bug when ClientRequest send payload file with opened as - open('filename', 'r+b') #1306 - - - Enhancement to AccessLogger (pass *extra* dict) #1303 - - - Show more verbose message on import errors #1319 - - - Added save and load functionality for `CookieJar` #1219 - - - Added option on `StaticRoute` to follow symlinks #1299 - - - Force encoding of `application/json` content type to utf-8 #1339 + - Implement range requests for static files #1382 - - Fix invalid invocations of `errors.LineTooLong` #1335 + - Support task attribute for StreamResponse #1410 - - Websockets: Stop `async for` iteration when connection is closed #1144 + - Drop `TestClient.app` property, use `TestClient.server.app` instead + (BACKWARD INCOMPATIBLE) - - Ensure TestClient HTTP methods return a context manager #1318 + - Drop `TestClient.handler` property, use `TestClient.server.handler` instead + (BACKWARD INCOMPATIBLE) - - Raise `ClientDisconnectedError` to `FlowControlStreamReader` read function - if `ClientSession` object is closed by client when reading data. #1323 + - `TestClient.server` property returns a test server instance, was + `asyncio.AbstractServer` (BACKWARD INCOMPATIBLE) - - Document deployment without `Gunicorn` #1120 + - Follow gunicorn's signal semantics in `Gunicorn[UVLoop]WebWorker` #1201 - - Add deprecation warning for MD5 and SHA1 digests when used for fingerprint - of site certs in TCPConnector. #1186 + - Call worker_int and worker_abort callbacks in + `Gunicorn[UVLoop]WebWorker` #1202 - - Implement sub-applications #1301 + - Has functional tests for client proxy #1218 - - Don't inherit `web.Request` from `dict` but implement - `MutableMapping` protocol. + - Fix bugs with client proxy target path and proxy host with port #1413 - - Implement frozen signals + - Fix bugs related to the use of unicode hostnames #1444 - - Don't inherit `web.Application` from `dict` but implement - `MutableMapping` protocol. + - Preserve cookie quoting/escaping #1453 - - Support freezing for web applications + - FileSender will send gzipped response if gzip version available #1426 - - Accept access_log parameter in `web.run_app`, use `None` to disable logging + - Don't override `Content-Length` header in `web.Response` if no body + was set #1400 - - Don't flap `tcp_cork` and `tcp_nodelay` in regular request handling. - `tcp_nodelay` is still enabled by default. + - Introduce `router.post_init()` for solving #1373 - - Improve performance of web server by removing premature computing of - Content-Type if the value was set by `web.Response` constructor. + - Fix raise error in case of multiple calls of `TimeServive.stop()` - While the patch boosts speed of trivial `web.Response(text='OK', - content_type='text/plain)` very well please don't expect significant - boost of your application -- a couple DB requests and business logic - is still the main bottleneck. + - Allow to raise web exceptions on router resolving stage #1460 - - Boost performance by adding a custom time service #1350 + - Add a warning for session creation outside of coroutine #1468 - - Extend `ClientResponse` with `content_type` and `charset` - properties like in `web.Request`. #1349 + - Avoid a race when application might start accepting incoming requests + but startup signals are not processed yet e98e8c6 - - Disable aiodns by default #559 + - Raise a `RuntimeError` when trying to change the status of the HTTP response + after the headers have been sent #1480 - - Don't flap `tcp_cork` in client code, use TCP_NODELAY mode by default. + - Fix bug with https proxy acquired cleanup #1340 - - Implement `web.Request.clone()` #1361 + - Use UTF-8 as the default encoding for multipart text parts #1484 Platform: UNKNOWN Classifier: License :: OSI Approved :: Apache Software License Classifier: Intended Audience :: Developers diff --git a/aiohttp.egg-info/PKG-INFO b/aiohttp.egg-info/PKG-INFO index 589e88b..3a79382 100644 --- a/aiohttp.egg-info/PKG-INFO +++ b/aiohttp.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: aiohttp -Version: 1.1.5 +Version: 1.2.0 Summary: http client/server for asyncio Home-page: https://github.com/KeepSafe/aiohttp/ Author: Andrew Svetlov @@ -161,141 +161,63 @@ Description: http client/server for asyncio CHANGES ======= - 1.1.5 (2016-11-16) + 1.2.0 (2016-12-17) ------------------ - - Fix static file serving in fallback mode #1401 - - 1.1.4 (2016-11-14) - ------------------ + - Extract `BaseRequest` from `web.Request`, introduce `web.Server` + (former `RequestHandlerFactory`), introduce new low-level web server + which is not coupled with `web.Application` and routing #1362 - Make `TestServer.make_url` compatible with `yarl.URL` #1389 - - Generate informative exception on redirects from server which - doesn't provide redirection headers #1396 - - - 1.1.3 (2016-11-10) - ------------------ - - - Support *root* resources for sub-applications #1379 - - - 1.1.2 (2016-11-08) - ------------------ - - - Allow starting variables with an underscore #1379 - - - Properly process UNIX sockets by gunicorn worker #1375 - - - Fix ordering for `FrozenList` - - - Don't propagate pre and post signals to sub-application #1377 - - 1.1.1 (2016-11-04) - ------------------ - - - Fix documentation generation #1120 - - 1.1.0 (2016-11-03) - ------------------ - - - Drop deprecated `WSClientDisconnectedError` (BACKWARD INCOMPATIBLE) - - - Use `yarl.URL` in client API. The change is 99% backward compatible - but `ClientResponse.url` is an `yarl.URL` instance now. #1217 - - - Close idle keep-alive connections on shutdown #1222 - - - Modify regex in AccessLogger to accept underscore and numbers #1225 - - - Use `yarl.URL` in web server API. `web.Request.rel_url` and - `web.Request.url` are added. URLs and templates are percent-encoded - now. #1224 - - - Accept `yarl.URL` by server redirections #1278 - - - Return `yarl.URL` by `.make_url()` testing utility #1279 - - - Properly format IPv6 addresses by `aiohttp.web.run_app` #1139 - - - Use `yarl.URL` by server API #1288 - - * Introduce `resource.url_for()`, deprecate `resource.url()`. - - * Implement `StaticResource`. - - * Inherit `SystemRoute` from `AbstractRoute` - - * Drop old-style routes: `Route`, `PlainRoute`, `DynamicRoute`, - `StaticRoute`, `ResourceAdapter`. - - - Revert `resp.url` back to `str`, introduce `resp.url_obj` #1292 - - - Raise ValueError if BasicAuth login has a ":" character #1307 - - - Fix bug when ClientRequest send payload file with opened as - open('filename', 'r+b') #1306 - - - Enhancement to AccessLogger (pass *extra* dict) #1303 - - - Show more verbose message on import errors #1319 - - - Added save and load functionality for `CookieJar` #1219 - - - Added option on `StaticRoute` to follow symlinks #1299 - - - Force encoding of `application/json` content type to utf-8 #1339 + - Implement range requests for static files #1382 - - Fix invalid invocations of `errors.LineTooLong` #1335 + - Support task attribute for StreamResponse #1410 - - Websockets: Stop `async for` iteration when connection is closed #1144 + - Drop `TestClient.app` property, use `TestClient.server.app` instead + (BACKWARD INCOMPATIBLE) - - Ensure TestClient HTTP methods return a context manager #1318 + - Drop `TestClient.handler` property, use `TestClient.server.handler` instead + (BACKWARD INCOMPATIBLE) - - Raise `ClientDisconnectedError` to `FlowControlStreamReader` read function - if `ClientSession` object is closed by client when reading data. #1323 + - `TestClient.server` property returns a test server instance, was + `asyncio.AbstractServer` (BACKWARD INCOMPATIBLE) - - Document deployment without `Gunicorn` #1120 + - Follow gunicorn's signal semantics in `Gunicorn[UVLoop]WebWorker` #1201 - - Add deprecation warning for MD5 and SHA1 digests when used for fingerprint - of site certs in TCPConnector. #1186 + - Call worker_int and worker_abort callbacks in + `Gunicorn[UVLoop]WebWorker` #1202 - - Implement sub-applications #1301 + - Has functional tests for client proxy #1218 - - Don't inherit `web.Request` from `dict` but implement - `MutableMapping` protocol. + - Fix bugs with client proxy target path and proxy host with port #1413 - - Implement frozen signals + - Fix bugs related to the use of unicode hostnames #1444 - - Don't inherit `web.Application` from `dict` but implement - `MutableMapping` protocol. + - Preserve cookie quoting/escaping #1453 - - Support freezing for web applications + - FileSender will send gzipped response if gzip version available #1426 - - Accept access_log parameter in `web.run_app`, use `None` to disable logging + - Don't override `Content-Length` header in `web.Response` if no body + was set #1400 - - Don't flap `tcp_cork` and `tcp_nodelay` in regular request handling. - `tcp_nodelay` is still enabled by default. + - Introduce `router.post_init()` for solving #1373 - - Improve performance of web server by removing premature computing of - Content-Type if the value was set by `web.Response` constructor. + - Fix raise error in case of multiple calls of `TimeServive.stop()` - While the patch boosts speed of trivial `web.Response(text='OK', - content_type='text/plain)` very well please don't expect significant - boost of your application -- a couple DB requests and business logic - is still the main bottleneck. + - Allow to raise web exceptions on router resolving stage #1460 - - Boost performance by adding a custom time service #1350 + - Add a warning for session creation outside of coroutine #1468 - - Extend `ClientResponse` with `content_type` and `charset` - properties like in `web.Request`. #1349 + - Avoid a race when application might start accepting incoming requests + but startup signals are not processed yet e98e8c6 - - Disable aiodns by default #559 + - Raise a `RuntimeError` when trying to change the status of the HTTP response + after the headers have been sent #1480 - - Don't flap `tcp_cork` in client code, use TCP_NODELAY mode by default. + - Fix bug with https proxy acquired cleanup #1340 - - Implement `web.Request.clone()` #1361 + - Use UTF-8 as the default encoding for multipart text parts #1484 Platform: UNKNOWN Classifier: License :: OSI Approved :: Apache Software License Classifier: Intended Audience :: Developers diff --git a/aiohttp.egg-info/SOURCES.txt b/aiohttp.egg-info/SOURCES.txt index 5ae2b03..8f69f42 100644 --- a/aiohttp.egg-info/SOURCES.txt +++ b/aiohttp.egg-info/SOURCES.txt @@ -33,6 +33,7 @@ aiohttp/test_utils.py aiohttp/web.py aiohttp/web_exceptions.py aiohttp/web_reqrep.py +aiohttp/web_server.py aiohttp/web_urldispatcher.py aiohttp/web_ws.py aiohttp/worker.py @@ -68,6 +69,7 @@ docs/testing.rst docs/third_party.rst docs/tutorial.rst docs/web.rst +docs/web_lowlevel.rst docs/web_reference.rst docs/whats_new_1_1.rst docs/_static/aiohttp-icon-128x128.png @@ -82,6 +84,7 @@ examples/client_json.py examples/client_ws.py examples/curl.py examples/fake_server.py +examples/lowlevel_srv.py examples/server.crt examples/server.csr examples/server.key @@ -121,6 +124,7 @@ tests/test_multipart.py tests/test_parser_buffer.py tests/test_protocol.py tests/test_proxy.py +tests/test_proxy_functional.py tests/test_pytest_plugin.py tests/test_resolver.py tests/test_run_app.py @@ -142,6 +146,7 @@ tests/test_web_request_handler.py tests/test_web_response.py tests/test_web_sendfile.py tests/test_web_sendfile_functional.py +tests/test_web_server.py tests/test_web_urldispatcher.py tests/test_web_websocket.py tests/test_web_websocket_functional.py diff --git a/aiohttp.egg-info/requires.txt b/aiohttp.egg-info/requires.txt index 2075b89..1b49143 100644 --- a/aiohttp.egg-info/requires.txt +++ b/aiohttp.egg-info/requires.txt @@ -1,4 +1,4 @@ chardet -multidict>=2.0 +multidict>=2.1.4 async_timeout>=1.1.0 -yarl>=0.5.0 +yarl>=0.8.1 diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index 057e9e8..21265cb 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -1,4 +1,4 @@ -__version__ = '1.1.5' +__version__ = '1.2.0' # Deprecated, keep it here for a while for backward compatibility. import multidict # noqa diff --git a/aiohttp/_websocket.c b/aiohttp/_websocket.c index 0500ae7..fdea978 100644 --- a/aiohttp/_websocket.c +++ b/aiohttp/_websocket.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.25.1 */ +/* Generated by Cython 0.25.2 */ /* BEGIN: Cython Metadata { @@ -16,7 +16,7 @@ END: Cython Metadata */ #elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) #error Cython requires Python 2.6+ or Python 3.2+. #else -#define CYTHON_ABI "0_25_1" +#define CYTHON_ABI "0_25_2" #include <stddef.h> #ifndef offsetof #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) @@ -199,7 +199,7 @@ END: Cython Metadata */ #endif #if CYTHON_FAST_PYCCALL #define __Pyx_PyFastCFunction_Check(func)\ - ((PyCFunction_Check(func) && METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))) + ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))))) #else #define __Pyx_PyFastCFunction_Check(func) 0 #endif @@ -352,10 +352,39 @@ END: Cython Metadata */ #define CYTHON_RESTRICT #endif #endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR +# if defined(__cplusplus) + template<class T> void CYTHON_MAYBE_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_MAYBE_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif #define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) #ifndef CYTHON_INLINE - #if defined(__GNUC__) + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #elif defined(__GNUC__) #define CYTHON_INLINE __inline__ #elif defined(_MSC_VER) #define CYTHON_INLINE __inline @@ -421,26 +450,6 @@ static CYTHON_INLINE float __PYX_NAN() { #define CYTHON_WITHOUT_ASSERTIONS #endif -#ifndef CYTHON_UNUSED -# if defined(__GNUC__) -# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -#endif -#ifndef CYTHON_NCP_UNUSED -# if CYTHON_COMPILING_IN_CPYTHON -# define CYTHON_NCP_UNUSED -# else -# define CYTHON_NCP_UNUSED CYTHON_UNUSED -# endif -#endif typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; diff --git a/aiohttp/abc.py b/aiohttp/abc.py index 90ed599..fb426f1 100644 --- a/aiohttp/abc.py +++ b/aiohttp/abc.py @@ -11,6 +11,15 @@ class AbstractRouter(ABC): def __init__(self): self._frozen = False + def post_init(self, app): + """Post init stage. + + It's not an abstract method for sake of backward compatibility + but if router wans to be aware about application it should + override it. + + """ + @property def frozen(self): return self._frozen diff --git a/aiohttp/client.py b/aiohttp/client.py index 6c5d029..4b49874 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -54,6 +54,17 @@ class ClientSession: if loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) + if not loop.is_running(): + warnings.warn("Creating a client session outside of coroutine is " + "a very dangerous idea", ResourceWarning, + stacklevel=2) + context = {'client_session': self, + 'message': 'Creating a client session outside ' + 'of coroutine'} + if self._source_traceback is not None: + context['source_traceback'] = self._source_traceback + loop.call_exception_handler(context) + if cookie_jar is None: cookie_jar = CookieJar(loop=loop) self._cookie_jar = cookie_jar diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 929f083..3b5b6c0 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -165,7 +165,7 @@ class ClientRequest: # add host if hdrs.HOST not in used_headers: - netloc = self.url.host + netloc = self.url.raw_host if not self.url.is_default_port(): netloc += ':' + str(self.url.port) self.headers[hdrs.HOST] = netloc @@ -185,7 +185,10 @@ class ClientRequest: for name, value in cookies.items(): if isinstance(value, http.cookies.Morsel): - c[value.key] = value.value + # Preserve coded_value + mrsl_val = value.get(value.key, http.cookies.Morsel()) + mrsl_val.set(value.key, value.value, value.coded_value) + c[name] = mrsl_val else: c[name] = value @@ -432,9 +435,19 @@ class ClientRequest: self._writer = None def send(self, writer, reader): - path = self.url.raw_path - if self.url.raw_query_string: - path += '?' + self.url.raw_query_string + # Specify request target: + # - CONNECT request must send authority form URI + # - not CONNECT proxy must send absolute form URI + # - most common is origin form URI + if self.method == hdrs.METH_CONNECT: + path = '{}:{}'.format(self.url.raw_host, self.url.port) + elif self.proxy and not self.ssl: + path = str(self.url) + else: + path = self.url.raw_path + if self.url.raw_query_string: + path += '?' + self.url.raw_query_string + request = aiohttp.Request(writer, self.method, path, self.version) @@ -522,6 +535,9 @@ class ClientResponse(HeadersMixin): @property def url(self): + warnings.warn("Deprecated, use .url_obj", + DeprecationWarning, + stacklevel=2) return str(self._url_obj) @property diff --git a/aiohttp/connector.py b/aiohttp/connector.py index f24354f..16403ad 100644 --- a/aiohttp/connector.py +++ b/aiohttp/connector.py @@ -91,6 +91,8 @@ class Connection: self._transport = None def detach(self): + if self._transport is not None: + self._connector._release_acquired(self._key, self._transport) self._transport = None @property @@ -351,7 +353,7 @@ class BaseConnector(object): waiter.set_result(None) break - def _release(self, key, req, transport, protocol, *, should_close=False): + def _release_acquired(self, key, transport): if self._closed: # acquired connection is already released on connector closing return @@ -362,9 +364,19 @@ class BaseConnector(object): except KeyError: # pragma: no cover # this may be result of undetermenistic order of objects # finalization due garbage collection. - pass - else: - if self._limit is not None and len(acquired) < self._limit: + return None + + return acquired + + def _release(self, key, req, transport, protocol, *, should_close=False): + if self._closed: + # acquired connection is already released on connector closing + return + + acquired = self._release_acquired(key, transport) + + if self._limit is not None and acquired is not None: + if len(acquired) < self._limit: self._release_waiter(key) resp = req.response @@ -634,7 +646,7 @@ class TCPConnector(BaseConnector): def _create_proxy_connection(self, req): proxy_req = ClientRequest( hdrs.METH_GET, req.proxy, - headers={hdrs.HOST: req.host}, + headers={hdrs.HOST: req.headers[hdrs.HOST]}, auth=req.proxy_auth, loop=self._loop) try: @@ -644,8 +656,6 @@ class TCPConnector(BaseConnector): except OSError as exc: raise ProxyConnectionError(*exc.args) from exc - if not req.ssl: - req.path = str(req.url) if hdrs.AUTHORIZATION in proxy_req.headers: auth = proxy_req.headers[hdrs.AUTHORIZATION] del proxy_req.headers[hdrs.AUTHORIZATION] @@ -665,7 +675,7 @@ class TCPConnector(BaseConnector): # to do this we must wrap raw socket into secure one # asyncio handles this perfectly proxy_req.method = hdrs.METH_CONNECT - proxy_req.path = '{}:{}'.format(req.host, req.port) + proxy_req.url = req.url key = (req.host, req.port, req.ssl) conn = Connection(self, key, proxy_req, transport, proto, self._loop) @@ -679,13 +689,19 @@ class TCPConnector(BaseConnector): raise else: conn.detach() - if resp.status != 200: - raise HttpProxyError(code=resp.status, message=resp.reason) - rawsock = transport.get_extra_info('socket', default=None) - if rawsock is None: - raise RuntimeError( - "Transport does not expose socket instance") - transport.pause_reading() + try: + if resp.status != 200: + raise HttpProxyError(code=resp.status, + message=resp.reason) + rawsock = transport.get_extra_info('socket', default=None) + if rawsock is None: + raise RuntimeError( + "Transport does not expose socket instance") + # Duplicate the socket, so now we can close proxy transport + rawsock = rawsock.dup() + finally: + transport.close() + transport, proto = yield from self._loop.create_connection( self._factory, ssl=self.ssl_context, sock=rawsock, server_hostname=req.host) diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py index c3c7b91..78b666f 100644 --- a/aiohttp/cookiejar.py +++ b/aiohttp/cookiejar.py @@ -91,7 +91,7 @@ class CookieJar(AbstractCookieJar): def update_cookies(self, cookies, response_url=URL()): """Update cookies.""" - hostname = response_url.host + hostname = response_url.raw_host if not self._unsafe and is_ip_address(hostname): # Don't accept cookies from IPs @@ -168,7 +168,7 @@ class CookieJar(AbstractCookieJar): """Returns this jar's cookies filtered by their attributes.""" self._do_expiration() filtered = SimpleCookie() - hostname = request_url.host or "" + hostname = request_url.raw_host or "" is_not_secure = request_url.scheme not in ("https", "wss") for cookie in self: @@ -195,7 +195,11 @@ class CookieJar(AbstractCookieJar): if is_not_secure and cookie["secure"]: continue - filtered[name] = cookie.value + # It's critical we use the Morsel so the coded_value + # (based on cookie version) is preserved + mrsl_val = cookie.get(cookie.key, Morsel()) + mrsl_val.set(cookie.key, cookie.value, cookie.coded_value) + filtered[name] = mrsl_val return filtered diff --git a/aiohttp/file_sender.py b/aiohttp/file_sender.py index 0717e0e..b2005a6 100644 --- a/aiohttp/file_sender.py +++ b/aiohttp/file_sender.py @@ -4,6 +4,8 @@ import os from . import hdrs from .helpers import create_future +from .web_exceptions import (HTTPNotModified, HTTPOk, HTTPPartialContent, + HTTPRequestRangeNotSatisfiable) from .web_reqrep import StreamResponse @@ -81,8 +83,8 @@ class FileSender: # See https://github.com/KeepSafe/aiohttp/issues/958 for details # send headers - headers = ['HTTP/{0.major}.{0.minor} 200 OK\r\n'.format( - request.version)] + headers = ['HTTP/{0.major}.{0.minor} {1} OK\r\n'.format( + request.version, resp.status)] for hdr, val in resp.headers.items(): headers.append('{}: {}\r\n'.format(hdr, val)) headers.append('\r\n') @@ -91,6 +93,7 @@ class FileSender: out_socket.setblocking(False) out_fd = out_socket.fileno() in_fd = fobj.fileno() + offset = fobj.tell() bheaders = ''.join(headers).encode('utf-8') headers_length = len(bheaders) @@ -100,7 +103,7 @@ class FileSender: try: yield from loop.sock_sendall(out_socket, bheaders) fut = create_future(loop) - self._sendfile_cb(fut, out_fd, in_fd, 0, count, loop, False) + self._sendfile_cb(fut, out_fd, in_fd, offset, count, loop, False) yield from fut finally: @@ -141,27 +144,60 @@ class FileSender: @asyncio.coroutine def send(self, request, filepath): """Send filepath to client using request.""" + gzip = False + if 'gzip' in request.headers.get(hdrs.ACCEPT_ENCODING, ''): + gzip_path = filepath.with_name(filepath.name + '.gz') + + if gzip_path.is_file(): + filepath = gzip_path + gzip = True + st = filepath.stat() modsince = request.if_modified_since if modsince is not None and st.st_mtime <= modsince.timestamp(): - from .web_exceptions import HTTPNotModified raise HTTPNotModified() ct, encoding = mimetypes.guess_type(str(filepath)) if not ct: ct = 'application/octet-stream' - resp = self._response_factory() + status = HTTPOk.status_code + file_size = st.st_size + count = file_size + + try: + rng = request.http_range + start = rng.start + end = rng.stop + except ValueError: + raise HTTPRequestRangeNotSatisfiable + + # If a range request has been made, convert start, end slice notation + # into file pointer offset and count + if start is not None or end is not None: + status = HTTPPartialContent.status_code + if start is None and end < 0: # return tail of file + start = file_size + end + count = -end + else: + count = (end or file_size) - start + + if start + count > file_size: + raise HTTPRequestRangeNotSatisfiable + + resp = self._response_factory(status=status) resp.content_type = ct if encoding: resp.headers[hdrs.CONTENT_ENCODING] = encoding + if gzip: + resp.headers[hdrs.VARY] = hdrs.ACCEPT_ENCODING resp.last_modified = st.st_mtime - file_size = st.st_size - - resp.content_length = file_size + resp.content_length = count with filepath.open('rb') as f: - yield from self._sendfile(request, resp, f, file_size) + if start: + f.seek(start) + yield from self._sendfile(request, resp, f, count) return resp diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 2f8bac3..7a3eba0 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -592,7 +592,8 @@ class TimeService: self._cb = loop.call_later(1, self._on_cb) def stop(self): - self._cb.cancel() + if self._cb: + self._cb.cancel() self._cb = None self._loop = None diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 093c856..af7f19b 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -332,10 +332,6 @@ class BodyPartReader(object): chunk = window[len(self._prev_chunk):idx] if not chunk: self._at_eof = True - if 0 < len(chunk) < len(sub) and not self._content_eof: - self._prev_chunk += chunk - self._at_eof = False - return b'' result = self._prev_chunk self._prev_chunk = chunk return result @@ -398,7 +394,9 @@ class BodyPartReader(object): :rtype: str """ data = yield from self.read(decode=True) - encoding = encoding or self.get_charset(default='latin1') + # see https://www.w3.org/TR/html5/forms.html#multipart/form-data-encoding-algorithm # NOQA + # and https://dvcs.w3.org/hg/xhr/raw-file/tip/Overview.html#dom-xmlhttprequest-send # NOQA + encoding = encoding or self.get_charset(default='utf-8') return data.decode(encoding) @asyncio.coroutine diff --git a/aiohttp/parsers.py b/aiohttp/parsers.py index a9922d4..7797958 100644 --- a/aiohttp/parsers.py +++ b/aiohttp/parsers.py @@ -58,7 +58,6 @@ _SocketSocketTransport -> import asyncio import asyncio.streams -import inspect import socket from . import errors @@ -183,7 +182,6 @@ class StreamParser: # init parser p = parser(output, self._buffer) - assert inspect.isgenerator(p), 'Generator is required' try: # initialize parser with data and parser buffers diff --git a/aiohttp/protocol.py b/aiohttp/protocol.py index 54d0570..4ff1266 100644 --- a/aiohttp/protocol.py +++ b/aiohttp/protocol.py @@ -15,7 +15,6 @@ from multidict import CIMultiDict, istr import aiohttp from . import errors, hdrs -from .helpers import reify from .log import internal_logger __all__ = ('HttpMessage', 'Request', 'Response', @@ -713,8 +712,6 @@ class HttpMessage(ABC): if self._send_headers and not self.headers_sent: self.send_headers() - assert self.writer is not None, 'send_headers() is not called.' - if self.filter: chunk = self.filter.send(chunk) while chunk not in (EOF_MARKER, EOL_MARKER): @@ -867,7 +864,7 @@ class Response(HttpMessage): def reason(self): return self._reason - @reify + @property def status_line(self): version = self.version return 'HTTP/{}.{} {} {}\r\n'.format( @@ -875,7 +872,7 @@ class Response(HttpMessage): def autochunked(self): return (self.length is None and - self.version >= HttpVersion11) + self._version >= HttpVersion11) def _add_default_headers(self): super()._add_default_headers() @@ -916,12 +913,12 @@ class Request(HttpMessage): def path(self): return self._path - @reify + @property def status_line(self): return '{0} {1} HTTP/{2[0]}.{2[1]}\r\n'.format( self.method, self.path, self.version) def autochunked(self): return (self.length is None and - self.version >= HttpVersion11 and + self._version >= HttpVersion11 and self.status not in (304, 204)) diff --git a/aiohttp/pytest_plugin.py b/aiohttp/pytest_plugin.py index d0c23f3..d0dbbdf 100644 --- a/aiohttp/pytest_plugin.py +++ b/aiohttp/pytest_plugin.py @@ -6,8 +6,8 @@ import pytest from aiohttp.web import Application from .test_utils import unused_port as _unused_port -from .test_utils import (TestClient, TestServer, loop_context, setup_test_loop, - teardown_test_loop) +from .test_utils import (RawTestServer, TestClient, TestServer, loop_context, + setup_test_loop, teardown_test_loop) @contextlib.contextmanager @@ -82,6 +82,27 @@ def test_server(loop): @pytest.yield_fixture +def raw_test_server(loop): + servers = [] + + @asyncio.coroutine + def go(handler, **kwargs): + server = RawTestServer(handler, loop=loop) + yield from server.start_server(**kwargs) + servers.append(server) + return server + + yield go + + @asyncio.coroutine + def finalize(): + while servers: + yield from servers.pop().close() + + loop.run_until_complete(finalize()) + + +@pytest.yield_fixture def test_client(loop): clients = [] @@ -97,6 +118,11 @@ def test_client(loop): assert __param.app.loop is loop, \ "TestServer is attached to other event loop" client = TestClient(__param, **kwargs) + elif isinstance(__param, RawTestServer): + assert not args, "args should be empty" + assert __param._loop is loop, \ + "TestServer is attached to other event loop" + client = TestClient(__param, **kwargs) else: __param = __param(loop, *args, **kwargs) client = TestClient(__param) diff --git a/aiohttp/server.py b/aiohttp/server.py index 719cf74..3024089 100644 --- a/aiohttp/server.py +++ b/aiohttp/server.py @@ -94,6 +94,8 @@ class ServerHttpProtocol(aiohttp.StreamProtocol): max_line_size=8190, max_headers=32768, max_field_size=8190, + lingering_time=30, + lingering_timeout=5, **kwargs): # process deprecated params @@ -116,9 +118,10 @@ class ServerHttpProtocol(aiohttp.StreamProtocol): self._tcp_keepalive = tcp_keepalive self._keepalive_timeout = keepalive_timeout self._slow_request_timeout = slow_request_timeout + self._lingering_time = float(lingering_time) + self._lingering_timeout = float(lingering_timeout) self._loop = loop if loop is not None else asyncio.get_event_loop() - self._request_prefix = aiohttp.HttpPrefixParser() self._request_parser = aiohttp.HttpRequestParser( max_line_size=max_line_size, max_field_size=max_field_size, @@ -184,9 +187,9 @@ class ServerHttpProtocol(aiohttp.StreamProtocol): def data_received(self, data): super().data_received(data) - # reading request - if not self._reading_request: - self._reading_request = True + # we can not gracefully shutdown handler + # if we in process of reading request + self._reading_request = True def keep_alive(self, val): """Set keep-alive connection mode. @@ -228,17 +231,10 @@ class ServerHttpProtocol(aiohttp.StreamProtocol): self._reading_request = False payload = None - with Timeout(max(self._slow_request_timeout, - self._keepalive_timeout), - loop=self._loop): - # read HTTP request method - prefix = reader.set_parser(self._request_prefix) - yield from prefix.read() - - # start reading request - self._reading_request = True - # start slow request timer + # slow request timer + with Timeout(max(self._slow_request_timeout, + self._keepalive_timeout), loop=self._loop): # read request headers httpstream = reader.set_parser(self._request_parser) message = yield from httpstream.read() @@ -251,7 +247,7 @@ class ServerHttpProtocol(aiohttp.StreamProtocol): raise errors.InvalidHeader(hdrs.CONTENT_LENGTH) from None if (content_length > 0 or - message.method == 'CONNECT' or + message.method == hdrs.METH_CONNECT or hdrs.SEC_WEBSOCKET_KEY1 in message.headers or 'chunked' in message.headers.get( hdrs.TRANSFER_ENCODING, '')): @@ -264,9 +260,25 @@ class ServerHttpProtocol(aiohttp.StreamProtocol): yield from self.handle_request(message, payload) - if payload and not payload.is_eof(): + if not payload.is_eof(): self.log_debug('Uncompleted request.') self._closing = True + + if self._lingering_time: + self.transport.write_eof() + self.log_debug( + 'Start lingering close timer for %s sec.', + self._lingering_time) + + end_time = self._loop.time() + self._lingering_time + + with suppress(asyncio.TimeoutError, + errors.ClientDisconnectedError): + while self._loop.time() < end_time: + with Timeout(self._lingering_timeout, + loop=self._loop): + # read and ignore + yield from payload.readany() else: reader.unset_parser() if not self._keepalive or not self._keepalive_timeout: diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index e17ae5c..48a391b 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -7,6 +7,7 @@ import gc import socket import sys import unittest +from abc import ABC, abstractmethod from unittest import mock from multidict import CIMultiDict @@ -19,7 +20,7 @@ from . import ClientSession, hdrs from .helpers import sentinel from .protocol import HttpVersion, RawRequestMessage from .signals import Signal -from .web import Application, Request, UrlMappingMatchInfo +from .web import Application, Request, Server, UrlMappingMatchInfo PY_35 = sys.version_info >= (3, 5) @@ -39,10 +40,8 @@ def unused_port(): return s.getsockname()[1] -class TestServer: - def __init__(self, app, *, scheme=sentinel, host='127.0.0.1'): - self.app = app - self._loop = app.loop +class BaseTestServer(ABC): + def __init__(self, *, scheme=sentinel, host='127.0.0.1'): self.port = None self.server = None self.handler = None @@ -66,18 +65,31 @@ class TestServer: self._root = URL('{}://{}:{}'.format(self.scheme, self.host, self.port)) - yield from self.app.startup() - self.handler = self.app.make_handler(**kwargs) - self.server = yield from self._loop.create_server(self.handler, + + handler = yield from self._make_factory(**kwargs) + self.server = yield from self._loop.create_server(handler, self.host, self.port, ssl=self._ssl) + @abstractmethod # pragma: no cover + @asyncio.coroutine + def _make_factory(self, **kwargs): + pass + def make_url(self, path): url = URL(path) assert not url.is_absolute() return self._root.join(url) + @property + def started(self): + return self.server is not None + + @property + def closed(self): + return self._closed + @asyncio.coroutine def close(self): """Close all fixtures created by the test client. @@ -91,16 +103,19 @@ class TestServer: exit when used as a context manager. """ - if self.server is not None and not self._closed: + if self.started and not self.closed: self.server.close() yield from self.server.wait_closed() - yield from self.app.shutdown() - yield from self.handler.finish_connections() - yield from self.app.cleanup() self._root = None self.port = None + yield from self._close_hook() self._closed = True + @abstractmethod + @asyncio.coroutine + def _close_hook(self): + pass # pragma: no cover + def __enter__(self): self._loop.run_until_complete(self.start_server()) return self @@ -119,6 +134,44 @@ class TestServer: yield from self.close() +class TestServer(BaseTestServer): + def __init__(self, app, *, scheme=sentinel, host='127.0.0.1'): + self.app = app + self._loop = app.loop + super().__init__(scheme=scheme, host=host) + + @asyncio.coroutine + def _make_factory(self, **kwargs): + yield from self.app.startup() + self.handler = self.app.make_handler(**kwargs) + return self.handler + + @asyncio.coroutine + def _close_hook(self): + yield from self.app.shutdown() + yield from self.handler.shutdown() + yield from self.app.cleanup() + + +class RawTestServer(BaseTestServer): + def __init__(self, handler, + *, loop=None, scheme=sentinel, host='127.0.0.1'): + if loop is None: + loop = asyncio.get_event_loop() + self._loop = loop + self._handler = handler + super().__init__(scheme=scheme, host=host) + + @asyncio.coroutine + def _make_factory(self, **kwargs): + self.handler = Server(self._handler, loop=self._loop, **kwargs) + return self.handler + + @asyncio.coroutine + def _close_hook(self): + return + + class TestClient: """ A test client implementation, for a aiohttp.web.Application. @@ -137,7 +190,7 @@ class TestClient: def __init__(self, app_or_server, *, scheme=sentinel, host=sentinel, cookie_jar=None, **kwargs): - if isinstance(app_or_server, TestServer): + if isinstance(app_or_server, BaseTestServer): if scheme is not sentinel or host is not sentinel: raise ValueError("scheme and host are mutable exclusive " "with TestServer parameter") @@ -150,7 +203,7 @@ class TestClient: else: raise TypeError("app_or_server should be either web.Application " "or TestServer instance") - self._loop = self._server.app.loop + self._loop = self._server._loop if cookie_jar is None: cookie_jar = aiohttp.CookieJar(unsafe=True, loop=self._loop) @@ -166,10 +219,6 @@ class TestClient: yield from self._server.start_server() @property - def app(self): - return self._server.app - - @property def host(self): return self._server.host @@ -178,16 +227,12 @@ class TestClient: return self._server.port @property - def handler(self): - return self._server.handler - - @property def server(self): - return self._server.server + return self._server @property def session(self): - """A raw handler to the aiohttp.ClientSession. + """An internal aiohttp.ClientSession. Unlike the methods on the TestClient, client session requests do not automatically include the host in the url queried, and @@ -201,11 +246,11 @@ class TestClient: @asyncio.coroutine def request(self, method, path, *args, **kwargs): - """Routes a request to the http server. + """Routes a request to tested http server. The interface is identical to asyncio.ClientSession.request, except the loop kwarg is overridden by the instance used by the - application. + test server. """ resp = yield from self._session.request( @@ -261,7 +306,7 @@ class TestClient: def ws_connect(self, path, *args, **kwargs): """Initiate websocket connection. - The api is identical to aiohttp.ClientSession.ws_connect. + The api corresponds to aiohttp.ClientSession.ws_connect. """ ws = yield from self._session.ws_connect( @@ -435,45 +480,6 @@ def make_mocked_request(method, path, headers=None, *, Useful in unit tests, when spinning full web server is overkill or specific conditions and errors are hard to trigger. - :param method: str, that represents HTTP method, like; GET, POST. - :type method: str - - :param path: str, The URL including *PATH INFO* without the host or scheme - :type path: str - - :param headers: mapping containing the headers. Can be anything accepted - by the multidict.CIMultiDict constructor. - :type headers: dict, multidict.CIMultiDict, list of pairs - - :param version: namedtuple with encoded HTTP version - :type version: aiohttp.protocol.HttpVersion - - :param closing: flag indicates that connection should be closed after - response. - :type closing: bool - - :param app: the aiohttp.web application attached for fake request - :type app: aiohttp.web.Application - - :param reader: object for storing and managing incoming data - :type reader: aiohttp.parsers.StreamParser - - :param writer: object for managing outcoming data - :type wirter: aiohttp.parsers.StreamWriter - - :param transport: asyncio transport instance - :type transport: asyncio.transports.Transport - - :param payload: raw payload reader object - :type payload: aiohttp.streams.FlowControlStreamReader - - :param sslcontext: ssl.SSLContext object, for HTTPS connection - :type sslcontext: ssl.SSLContext - - :param secure_proxy_ssl_header: A tuple representing a HTTP header/value - combination that signifies a request is secure. - :type secure_proxy_ssl_header: tuple - """ if version < HttpVersion(1, 1): @@ -508,9 +514,11 @@ def make_mocked_request(method, path, headers=None, *, time_service.time.return_value = 12345 time_service.strtime.return_value = "Tue, 15 Nov 1994 08:12:31 GMT" + task = mock.Mock() + req = Request(message, payload, transport, reader, writer, - time_service, + time_service, task, secure_proxy_ssl_header=secure_proxy_ssl_header) match_info = UrlMappingMatchInfo({}, mock.Mock()) diff --git a/aiohttp/web.py b/aiohttp/web.py index 2e838ce..d2457a6 100644 --- a/aiohttp/web.py +++ b/aiohttp/web.py @@ -7,15 +7,16 @@ from importlib import import_module from yarl import URL -from . import hdrs, web_exceptions, web_reqrep, web_urldispatcher, web_ws +from . import (hdrs, web_exceptions, web_reqrep, web_server, web_urldispatcher, + web_ws) from .abc import AbstractMatchInfo, AbstractRouter -from .helpers import FrozenList, TimeService, sentinel +from .helpers import FrozenList, sentinel from .log import access_logger, web_logger from .protocol import HttpVersion # noqa -from .server import ServerHttpProtocol from .signals import PostSignal, PreSignal, Signal from .web_exceptions import * # noqa from .web_reqrep import * # noqa +from .web_server import Server from .web_urldispatcher import * # noqa from .web_ws import * # noqa @@ -23,168 +24,29 @@ __all__ = (web_reqrep.__all__ + web_exceptions.__all__ + web_urldispatcher.__all__ + web_ws.__all__ + - ('Application', 'RequestHandler', - 'RequestHandlerFactory', 'HttpVersion', - 'MsgType')) - - -class RequestHandler(ServerHttpProtocol): - - _request = None - - def __init__(self, manager, app, router, time_service, *, - secure_proxy_ssl_header=None, **kwargs): - super().__init__(**kwargs) - - self._manager = manager - self._app = app - self._router = router - self._secure_proxy_ssl_header = secure_proxy_ssl_header - self._time_service = time_service - - def __repr__(self): - if self._request is None: - meth = 'none' - path = 'none' - else: - meth = self._request.method - path = self._request.rel_url.raw_path - return "<{} {}:{} {}>".format( - self.__class__.__name__, meth, path, - 'connected' if self.transport is not None else 'disconnected') - - def connection_made(self, transport): - super().connection_made(transport) - - self._manager.connection_made(self, transport) - - def connection_lost(self, exc): - self._manager.connection_lost(self, exc) - - super().connection_lost(exc) - - @asyncio.coroutine - def handle_request(self, message, payload): - self._manager._requests_count += 1 - if self.access_log: - now = self._loop.time() - - request = web_reqrep.Request( - message, payload, - self.transport, self.reader, self.writer, - self._time_service, - secure_proxy_ssl_header=self._secure_proxy_ssl_header) - self._request = request - try: - match_info = yield from self._router.resolve(request) - assert isinstance(match_info, AbstractMatchInfo), match_info - match_info.add_app(self._app) - match_info.freeze() - - resp = None - request._match_info = match_info - expect = request.headers.get(hdrs.EXPECT) - if expect: - resp = ( - yield from match_info.expect_handler(request)) - - if resp is None: - handler = match_info.handler - for app in match_info.apps: - for factory in reversed(app.middlewares): - handler = yield from factory(app, handler) - resp = yield from handler(request) - - assert isinstance(resp, web_reqrep.StreamResponse), \ - ("Handler {!r} should return response instance, " - "got {!r} [middlewares {!r}]").format( - match_info.handler, type(resp), self._middlewares) - except web_exceptions.HTTPException as exc: - resp = exc - - resp_msg = yield from resp.prepare(request) - yield from resp.write_eof() - - # notify server about keep-alive - self.keep_alive(resp.keep_alive) - - # Restore default state. - # Should be no-op if server code didn't touch these attributes. - self.writer.set_tcp_cork(False) - self.writer.set_tcp_nodelay(True) - - # log access - if self.access_log: - self.log_access(message, None, resp_msg, self._loop.time() - now) - - # for repr - self._request = None - - -class RequestHandlerFactory: - - def __init__(self, app, router, *, - handler=RequestHandler, loop=None, - secure_proxy_ssl_header=None, **kwargs): - self._app = app - self._router = router - self._handler = handler - self._loop = loop - self._connections = {} - self._secure_proxy_ssl_header = secure_proxy_ssl_header - self._kwargs = kwargs - self._kwargs.setdefault('logger', app.logger) - self._requests_count = 0 - self._time_service = TimeService(self._loop) - - @property - def requests_count(self): - """Number of processed requests.""" - return self._requests_count - - @property - def secure_proxy_ssl_header(self): - return self._secure_proxy_ssl_header - - @property - def connections(self): - return list(self._connections.keys()) - - def connection_made(self, handler, transport): - self._connections[handler] = transport - - def connection_lost(self, handler, exc=None): - if handler in self._connections: - del self._connections[handler] - - @asyncio.coroutine - def finish_connections(self, timeout=None): - coros = [conn.shutdown(timeout) for conn in self._connections] - yield from asyncio.gather(*coros, loop=self._loop) - self._connections.clear() - self._time_service.stop() - - def __call__(self): - return self._handler( - self, self._app, self._router, self._time_service, loop=self._loop, - secure_proxy_ssl_header=self._secure_proxy_ssl_header, - **self._kwargs) + web_server.__all__ + + ('Application', 'HttpVersion', 'MsgType')) class Application(MutableMapping): def __init__(self, *, logger=web_logger, loop=None, - router=None, handler_factory=RequestHandlerFactory, - middlewares=(), debug=False): + router=None, + middlewares=(), debug=...): if loop is None: loop = asyncio.get_event_loop() if router is None: - router = web_urldispatcher.UrlDispatcher(self) + router = web_urldispatcher.UrlDispatcher() assert isinstance(router, AbstractRouter), router + router.post_init(self) + + if debug is ...: + debug = loop.get_debug() + self._debug = debug self._router = router - self._handler_factory = handler_factory + self._secure_proxy_ssl_header = None self._loop = loop self.logger = logger @@ -299,7 +161,7 @@ class Application(MutableMapping): def middlewares(self): return self._middlewares - def make_handler(self, **kwargs): + def make_handler(self, *, secure_proxy_ssl_header=None, **kwargs): debug = kwargs.pop('debug', sentinel) if debug is not sentinel: warnings.warn( @@ -316,8 +178,9 @@ class Application(MutableMapping): "web_reference.html#aiohttp.web.Application" ) self.freeze() - return self._handler_factory(self, self.router, debug=self.debug, - loop=self.loop, **kwargs) + self._secure_proxy_ssl_header = secure_proxy_ssl_header + return Server(self._handle, request_factory=self._make_request, + debug=self.debug, loop=self.loop, **kwargs) @asyncio.coroutine def startup(self): @@ -356,8 +219,42 @@ class Application(MutableMapping): warnings.warn("Use .on_cleanup.append() instead", DeprecationWarning) self.on_cleanup.append(lambda app: func(app, *args, **kwargs)) - def copy(self): - raise NotImplementedError + def _make_request(self, message, payload, protocol, + _cls=web_reqrep.Request): + return _cls( + message, payload, + protocol.transport, protocol.reader, protocol.writer, + protocol.time_service, protocol._request_handler, + secure_proxy_ssl_header=self._secure_proxy_ssl_header) + + @asyncio.coroutine + def _handle(self, request): + match_info = yield from self._router.resolve(request) + assert isinstance(match_info, AbstractMatchInfo), match_info + match_info.add_app(self) + match_info.freeze() + + resp = None + request._match_info = match_info + expect = request.headers.get(hdrs.EXPECT) + if expect: + resp = ( + yield from match_info.expect_handler(request)) + + if resp is None: + handler = match_info.handler + for app in match_info.apps: + for factory in reversed(app.middlewares): + handler = yield from factory(app, handler) + resp = yield from handler(request) + + assert isinstance(resp, web_reqrep.StreamResponse), \ + ("Handler {!r} should return response instance, " + "got {!r} [middlewares {!r}]").format( + match_info.handler, type(resp), + [middleware for middleware in app.middlewares + for app in match_info.apps]) + return resp def __call__(self): """gunicorn compatibility""" @@ -385,11 +282,11 @@ def run_app(app, *, host='0.0.0.0', port=None, make_handler_kwargs['access_log_format'] = access_log_format handler = app.make_handler(access_log=access_log, **make_handler_kwargs) - server = loop.create_server(handler, host, port, ssl=ssl_context, - backlog=backlog) - srv, startup_res = loop.run_until_complete(asyncio.gather(server, - app.startup(), - loop=loop)) + + loop.run_until_complete(app.startup()) + srv = loop.run_until_complete(loop.create_server(handler, host, + port, ssl=ssl_context, + backlog=backlog)) scheme = 'https' if ssl_context else 'http' url = URL('{}://localhost'.format(scheme)) @@ -405,7 +302,7 @@ def run_app(app, *, host='0.0.0.0', port=None, srv.close() loop.run_until_complete(srv.wait_closed()) loop.run_until_complete(app.shutdown()) - loop.run_until_complete(handler.finish_connections(shutdown_timeout)) + loop.run_until_complete(handler.shutdown(shutdown_timeout)) loop.run_until_complete(app.cleanup()) loop.close() diff --git a/aiohttp/web_reqrep.py b/aiohttp/web_reqrep.py index d5c6a37..d9fce3e 100644 --- a/aiohttp/web_reqrep.py +++ b/aiohttp/web_reqrep.py @@ -4,13 +4,14 @@ import cgi import collections import datetime import enum -import http.cookies import io import json import math +import re import time import warnings from email.utils import parsedate +from http.cookies import SimpleCookie from types import MappingProxyType from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy @@ -22,7 +23,7 @@ from .protocol import WebResponse as ResponseImpl from .protocol import HttpVersion10, HttpVersion11 __all__ = ( - 'ContentCoding', 'Request', 'StreamResponse', 'Response', + 'ContentCoding', 'BaseRequest', 'Request', 'StreamResponse', 'Response', 'json_response' ) @@ -44,13 +45,13 @@ class ContentCoding(enum.Enum): ############################################################ -class Request(collections.MutableMapping, HeadersMixin): +class BaseRequest(collections.MutableMapping, HeadersMixin): POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT, hdrs.METH_TRACE, hdrs.METH_DELETE} def __init__(self, message, payload, transport, reader, writer, - time_service, *, + time_service, task, *, secure_proxy_ssl_header=None): self._message = message self._transport = transport @@ -59,10 +60,6 @@ class Request(collections.MutableMapping, HeadersMixin): self._post = None self._post_files_cache = None - # matchdict, route_name, handler - # or information about traversal lookup - self._match_info = None # initialized after route resolving - self._payload = payload self._read_bytes = None @@ -72,6 +69,7 @@ class Request(collections.MutableMapping, HeadersMixin): self._time_service = time_service self._state = {} self._cache = {} + self._task = task def clone(self, *, method=sentinel, rel_url=sentinel, headers=sentinel): @@ -99,13 +97,14 @@ class Request(collections.MutableMapping, HeadersMixin): message = self._message._replace(**dct) - return Request( + return self.__class__( message, self._payload, self._transport, self._reader, self._writer, self._time_service, + self._task, secure_proxy_ssl_header=self._secure_proxy_ssl_header) # MutableMapping API @@ -276,23 +275,10 @@ class Request(collections.MutableMapping, HeadersMixin): tzinfo=datetime.timezone.utc) return None - @reify + @property def keep_alive(self): """Is keepalive enabled by client?""" - if self.version < HttpVersion10: - return False - else: - return not self._message.should_close - - @property - def match_info(self): - """Result of route resolving.""" - return self._match_info - - @reify - def app(self): - """Application instance.""" - return self._match_info.apps[-1] + return not self._message.should_close @property def transport(self): @@ -306,11 +292,45 @@ class Request(collections.MutableMapping, HeadersMixin): A read-only dictionary-like object. """ raw = self.headers.get(hdrs.COOKIE, '') - parsed = http.cookies.SimpleCookie(raw) + parsed = SimpleCookie(raw) return MappingProxyType( {key: val.value for key, val in parsed.items()}) @property + def http_range(self, *, _RANGE=hdrs.RANGE): + """The content of Range HTTP header. + + Return a slice instance. + + """ + rng = self.headers.get(_RANGE) + start, end = None, None + if rng is not None: + try: + pattern = r'^bytes=(\d*)-(\d*)$' + start, end = re.findall(pattern, rng)[0] + except IndexError: # pattern was not found in header + raise ValueError("range not in acceptible format") + + end = int(end) if end else None + start = int(start) if start else None + + if start is None and end is not None: + # end with no start is to return tail of content + end = -end + + if start is not None and end is not None: + # end is inclusive in range header, exclusive for slice + end += 1 + + if start >= end: + raise ValueError('start cannot be after end') + + if start is end is None: # No valid range supplied + raise ValueError('No start or end of range specified') + return slice(start, end, 1) + + @property def content(self): """Return raw payload stream.""" return self._payload @@ -439,6 +459,39 @@ class Request(collections.MutableMapping, HeadersMixin): return "<{} {} {} >".format(self.__class__.__name__, self.method, ascii_encodable_path) + @asyncio.coroutine + def _prepare_hook(self, response): + return + yield # pragma: no cover + + +class Request(BaseRequest): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # matchdict, route_name, handler + # or information about traversal lookup + self._match_info = None # initialized after route resolving + + @property + def match_info(self): + """Result of route resolving.""" + return self._match_info + + @reify + def app(self): + """Application instance.""" + return self._match_info.apps[-1] + + @asyncio.coroutine + def _prepare_hook(self, response): + match_info = self._match_info + if match_info is None: + return + for app in match_info.apps: + yield from app.on_response_prepare.send(self, response) + ############################################################ # HTTP Response classes @@ -455,22 +508,20 @@ class StreamResponse(HeadersMixin): self._compression = False self._compression_force = False self._headers = CIMultiDict() - self._cookies = http.cookies.SimpleCookie() - self.set_status(status, reason) + self._cookies = SimpleCookie() self._req = None self._resp_impl = None self._eof_sent = False + self._task = None + if headers is not None: + # TODO: optimize CIMultiDict extending self._headers.extend(headers) - if hdrs.CONTENT_TYPE not in self._headers: - self._headers[hdrs.CONTENT_TYPE] = 'application/octet-stream' + self._headers.setdefault(hdrs.CONTENT_TYPE, 'application/octet-stream') - def _copy_cookies(self): - for cookie in self._cookies.values(): - value = cookie.output(header='')[1:] - self.headers.add(hdrs.SET_COOKIE, value) + self.set_status(status, reason) @property def prepared(self): @@ -482,6 +533,10 @@ class StreamResponse(HeadersMixin): return self.prepared @property + def task(self): + return self._task + + @property def status(self): return self._status @@ -498,6 +553,9 @@ class StreamResponse(HeadersMixin): return self._reason def set_status(self, status, reason=None): + if self.prepared: + raise RuntimeError("Cannot change the response status code after " + "the headers have been sent") self._status = int(status) if reason is None: reason = ResponseImpl.calc_reason(status) @@ -510,6 +568,14 @@ class StreamResponse(HeadersMixin): def force_close(self): self._keep_alive = False + @property + def body_length(self): + return self._resp_impl.body_length + + @property + def output_length(self): + return self._resp.impl.output_length + def enable_chunked_encoding(self, chunk_size=None): """Enables automatic chunked transfer encoding.""" self._chunked = True @@ -733,12 +799,18 @@ class StreamResponse(HeadersMixin): resp_impl = self._start_pre_check(request) if resp_impl is not None: return resp_impl - for app in request.match_info.apps: - yield from app.on_response_prepare.send(request, self) + yield from request._prepare_hook(self) return self._start(request) - def _start(self, request): + def _start(self, request, + HttpVersion10=HttpVersion10, + HttpVersion11=HttpVersion11, + CONNECTION=hdrs.CONNECTION, + DATE=hdrs.DATE, + SERVER=hdrs.SERVER, + SET_COOKIE=hdrs.SET_COOKIE, + TRANSFER_ENCODING=hdrs.TRANSFER_ENCODING): self._req = request keep_alive = self._keep_alive if keep_alive is None: @@ -753,9 +825,11 @@ class StreamResponse(HeadersMixin): not keep_alive, self._reason) - self._copy_cookies() - headers = self.headers + for cookie in self._cookies.values(): + value = cookie.output(header='')[1:] + headers.add(SET_COOKIE, value) + if self._compression: self._start_compression(request) @@ -764,27 +838,27 @@ class StreamResponse(HeadersMixin): raise RuntimeError("Using chunked encoding is forbidden " "for HTTP/{0.major}.{0.minor}".format( request.version)) - resp_impl.enable_chunked_encoding() + resp_impl.chunked = True if self._chunk_size: resp_impl.add_chunking_filter(self._chunk_size) - headers[hdrs.TRANSFER_ENCODING] = 'chunked' + headers[TRANSFER_ENCODING] = 'chunked' else: resp_impl.length = self.content_length - if hdrs.DATE not in headers: - headers[hdrs.DATE] = request._time_service.strtime() - headers.setdefault(hdrs.SERVER, resp_impl.SERVER_SOFTWARE) - if hdrs.CONNECTION not in headers: + headers.setdefault(DATE, request._time_service.strtime()) + headers.setdefault(SERVER, resp_impl.SERVER_SOFTWARE) + if CONNECTION not in headers: if keep_alive: if version == HttpVersion10: - headers[hdrs.CONNECTION] = 'keep-alive' + headers[CONNECTION] = 'keep-alive' else: if version == HttpVersion11: - headers[hdrs.CONNECTION] = 'close' + headers[CONNECTION] = 'close' resp_impl.headers = headers self._send_headers(resp_impl) + self._task = request._task return resp_impl def _send_headers(self, resp_impl): @@ -883,6 +957,8 @@ class Response(StreamResponse): super().__init__(status=status, reason=reason, headers=headers) if text is not None: self.text = text + elif body is None and hdrs.CONTENT_LENGTH in headers: + self._body = None else: self.body = body diff --git a/aiohttp/web_server.py b/aiohttp/web_server.py new file mode 100644 index 0000000..0813693 --- /dev/null +++ b/aiohttp/web_server.py @@ -0,0 +1,166 @@ +"""Low level HTTP server.""" + +import asyncio +import traceback +from html import escape as html_escape + +from .helpers import TimeService +from .server import ServerHttpProtocol +from .web_exceptions import HTTPException, HTTPInternalServerError +from .web_reqrep import BaseRequest + +__all__ = ('RequestHandler', 'Server') + + +class RequestHandler(ServerHttpProtocol): + _request = None + + def __init__(self, manager, **kwargs): + super().__init__(**kwargs) + self._manager = manager + self._request_factory = manager.request_factory + self._handler = manager.handler + self.time_service = manager.time_service + + def __repr__(self): + if self._request is None: + meth = 'none' + path = 'none' + else: + meth = self._request.method + path = self._request.rel_url.raw_path + return "<{} {}:{} {}>".format( + self.__class__.__name__, meth, path, + 'connected' if self.transport is not None else 'disconnected') + + def connection_made(self, transport): + super().connection_made(transport) + + self._manager.connection_made(self, transport) + + def connection_lost(self, exc): + self._manager.connection_lost(self, exc) + + super().connection_lost(exc) + self._request_factory = None + self._manager = None + self.time_service = None + self._handler = None + + @asyncio.coroutine + def handle_request(self, message, payload): + self._manager._requests_count += 1 + if self.access_log: + now = self._loop.time() + + request = self._request_factory(message, payload, self) + self._request = request + + try: + try: + resp = yield from self._handler(request) + except HTTPException as exc: + resp = exc + except Exception as exc: + msg = "<h1>500 Internal Server Error</h1>" + if self.debug: + try: + tb = traceback.format_exc() + tb = html_escape(tb) + msg += '<br><h2>Traceback:</h2>\n<pre>' + msg += tb + msg += '</pre>' + except: # pragma: no cover + pass + else: + msg += "Server got itself in trouble" + msg = ("<html><head><title>500 Internal Server Error</title>" + "</head><body>" + msg + "</body></html>") + resp = HTTPInternalServerError(text=msg, + content_type='text/html') + self.logger.exception( + "Error handling request", + exc_info=exc) + + yield from resp.prepare(request) + yield from resp.write_eof() + finally: + resp._task = None + + # notify server about keep-alive + # assign to parent class attr + self._keepalive = resp._keep_alive + + # Restore default state. + # Should be no-op if server code didn't touch these attributes. + self.writer.set_tcp_cork(False) + self.writer.set_tcp_nodelay(True) + + # log access + if self.access_log: + self.log_access(message, None, resp, self._loop.time() - now) + + # for repr + self._request = None + + +class Server: + + def __init__(self, handler, *, request_factory=None, loop=None, **kwargs): + if loop is None: + loop = asyncio.get_event_loop() + self._handler = handler + self._request_factory = request_factory or self._make_request + self._loop = loop + self._connections = {} + self._kwargs = kwargs + self._requests_count = 0 + self._time_service = TimeService(self._loop) + + @property + def requests_count(self): + """Number of processed requests.""" + return self._requests_count + + @property + def handler(self): + return self._handler + + @property + def request_factory(self): + return self._request_factory + + @property + def time_service(self): + return self._time_service + + @property + def connections(self): + return list(self._connections.keys()) + + def connection_made(self, handler, transport): + self._connections[handler] = transport + + def connection_lost(self, handler, exc=None): + if handler in self._connections: + del self._connections[handler] + + def _make_request(self, message, payload, protocol): + return BaseRequest( + message, payload, + protocol.transport, protocol.reader, protocol.writer, + protocol.time_service, protocol._request_handler) + + @asyncio.coroutine + def shutdown(self, timeout=None): + coros = [conn.shutdown(timeout) for conn in self._connections] + yield from asyncio.gather(*coros, loop=self._loop) + self._connections.clear() + self._time_service.stop() + + finish_connections = shutdown + + def __call__(self): + return RequestHandler( + self, loop=self._loop, + **self._kwargs) diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index 4cf8853..017452d 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -710,10 +710,14 @@ class UrlDispatcher(AbstractRouter, collections.abc.Mapping): ROUTE_RE = re.compile(r'(\{[_a-zA-Z][^{}]*(?:\{[^{}]*\}[^{}]*)*\})') NAME_SPLIT_RE = re.compile(r'[.:-]') - def __init__(self, app): + def __init__(self): super().__init__() self._resources = [] self._named_resources = {} + self._app = None + + def post_init(self, app): + assert app is not None self._app = app @asyncio.coroutine @@ -759,6 +763,9 @@ class UrlDispatcher(AbstractRouter, collections.abc.Mapping): assert isinstance(resource, AbstractResource), \ 'Instance of AbstractResource class is required, got {!r}'.format( resource) + if self._app is None: + raise RuntimeError(".post_init() should be called before " + "first resource registering") if self.frozen: raise RuntimeError("Cannot register a resource into " "frozen router.") diff --git a/aiohttp/worker.py b/aiohttp/worker.py index 94a9c44..1d70366 100644 --- a/aiohttp/worker.py +++ b/aiohttp/worker.py @@ -1,6 +1,7 @@ """Async gunicorn worker for aiohttp.web""" import asyncio +import logging import os import re import signal @@ -12,6 +13,8 @@ import gunicorn.workers.base as base from gunicorn.config import AccessLogFormat as GunicornAccessLogFormat from aiohttp.helpers import AccessLogger, ensure_future +from aiohttp.web_server import Server +from aiohttp.wsgi import WSGIServerHttpProtocol __all__ = ('GunicornWebWorker', 'GunicornUVLoopWebWorker') @@ -26,6 +29,7 @@ class GunicornWebWorker(base.Worker): self.servers = {} self.exit_code = 0 + self._notify_waiter = None def init_process(self): # create new event_loop after fork @@ -37,7 +41,8 @@ class GunicornWebWorker(base.Worker): super().init_process() def run(self): - self.loop.run_until_complete(self.wsgi.startup()) + if hasattr(self.wsgi, 'startup'): + self.loop.run_until_complete(self.wsgi.startup()) self._runner = ensure_future(self._run(), loop=self.loop) try: @@ -48,13 +53,16 @@ class GunicornWebWorker(base.Worker): sys.exit(self.exit_code) def make_handler(self, app): - return app.make_handler( - logger=self.log, - slow_request_timeout=self.cfg.timeout, - keepalive_timeout=self.cfg.keepalive, - access_log=self.log.access_log, - access_log_format=self._get_valid_log_format( - self.cfg.access_log_format)) + if hasattr(self.wsgi, 'make_handler'): + return app.make_handler( + logger=self.log, + slow_request_timeout=self.cfg.timeout, + keepalive_timeout=self.cfg.keepalive, + access_log=self.log.access_log, + access_log_format=self._get_valid_log_format( + self.cfg.access_log_format)) + else: + return WSGIServer(self.wsgi, self) @asyncio.coroutine def close(self): @@ -70,17 +78,19 @@ class GunicornWebWorker(base.Worker): yield from server.wait_closed() # send on_shutdown event - yield from self.wsgi.shutdown() + if hasattr(self.wsgi, 'shutdown'): + yield from self.wsgi.shutdown() # stop alive connections tasks = [ - handler.finish_connections( + handler.shutdown( timeout=self.cfg.graceful_timeout / 100 * 95) for handler in servers.values()] yield from asyncio.gather(*tasks, loop=self.loop) # cleanup application - yield from self.wsgi.cleanup() + if hasattr(self.wsgi, 'cleanup'): + yield from self.wsgi.cleanup() @asyncio.coroutine def _run(self): @@ -114,13 +124,28 @@ class GunicornWebWorker(base.Worker): self.alive = False self.log.info("Parent changed, shutting down: %s", self) else: - yield from asyncio.sleep(1.0, loop=self.loop) + yield from self._wait_next_notify() except BaseException: pass yield from self.close() + def _wait_next_notify(self): + self._notify_waiter_done() + + self._notify_waiter = waiter = asyncio.Future(loop=self.loop) + self.loop.call_later(1.0, self._notify_waiter_done) + + return waiter + + def _notify_waiter_done(self): + waiter = self._notify_waiter + if waiter is not None and not waiter.done(): + waiter.set_result(True) + + self._notify_waiter = None + def init_signals(self): # Set up signals through the event loop API. @@ -150,9 +175,20 @@ class GunicornWebWorker(base.Worker): def handle_quit(self, sig, frame): self.alive = False + # worker_int callback + self.cfg.worker_int(self) + + # init closing process + self._closing = ensure_future(self.close(), loop=self.loop) + + # close loop + self.loop.call_later(0.1, self._notify_waiter_done) + def handle_abort(self, sig, frame): self.alive = False self.exit_code = 1 + self.cfg.worker_abort(self) + sys.exit(1) @staticmethod def _create_ssl_context(cfg): @@ -184,6 +220,26 @@ class GunicornWebWorker(base.Worker): return source_format +class WSGIServer(Server): + + def __init__(self, app, worker): + super().__init__(app, loop=worker.loop) + + self.worker = worker + self.access_log_format = worker._get_valid_log_format( + worker.cfg.access_log_format) + + def __call__(self): + return WSGIServerHttpProtocol( + self.handler, readpayload=True, + loop=self._loop, + logger=self.worker.log, + debug=self.worker.log.loglevel == logging.DEBUG, + keep_alive=self.worker.cfg.keepalive, + access_log=self.worker.log.access_log, + access_log_format=self.access_log_format) + + class GunicornUVLoopWebWorker(GunicornWebWorker): def init_process(self): diff --git a/debian/.git-dpm b/debian/.git-dpm index b27907b..f88407e 100644 --- a/debian/.git-dpm +++ b/debian/.git-dpm @@ -1,11 +1,11 @@ # see git-dpm(1) from git-dpm package -98fe54bd7567469b73bb93caa7ea443afd70e309 -98fe54bd7567469b73bb93caa7ea443afd70e309 -98fe54bd7567469b73bb93caa7ea443afd70e309 -98fe54bd7567469b73bb93caa7ea443afd70e309 -python-aiohttp_1.1.5.orig.tar.gz -8d89d26cf656196de9cc35307a47654727a79fb0 -510247 +4a74b0a6d3f82f187d7c364e8662f23ceaba395e +4a74b0a6d3f82f187d7c364e8662f23ceaba395e +4a74b0a6d3f82f187d7c364e8662f23ceaba395e +4a74b0a6d3f82f187d7c364e8662f23ceaba395e +python-aiohttp_1.2.0.orig.tar.gz +e84ce3866174b472a7647d07b8bed67db6f02716 +519187 debianTag="debian/%e%v" patchedTag="patched/%e%v" upstreamTag="upstream/%e%u" diff --git a/debian/changelog b/debian/changelog index 2b1fe54..3870f65 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +python-aiohttp (1.2.0-1) unstable; urgency=medium + + * New upstream release + * debian/compat changed to 10 (this change also closes: 845178 thanks to + dh_autoreconf) + + -- Piotr Ożarowski <piotr@debian.org> Wed, 21 Dec 2016 13:30:27 +0100 + python-aiohttp (1.1.5-1) unstable; urgency=medium * New upstream release diff --git a/debian/compat b/debian/compat index ec63514..f599e28 100644 --- a/debian/compat +++ b/debian/compat @@ -1 +1 @@ -9 +10 diff --git a/debian/control b/debian/control index e60a743..6e540c4 100644 --- a/debian/control +++ b/debian/control @@ -5,7 +5,7 @@ Maintainer: Debian Python Modules Team <python-modules-team@lists.alioth.debian. Uploaders: Tianon Gravi <admwiggin@gmail.com>, Paul Tagliamonte <paultag@debian.org>, Piotr Ożarowski <piotr@debian.org> -Build-Depends: debhelper (>= 9), +Build-Depends: debhelper (>= 10), dh-python, python3-all-dev (>= 3.4~), python3-all-dbg (>= 3.4~), diff --git a/docs/changes.rst b/docs/changes.rst index 50ad8f4..005bd1f 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,3 +1,5 @@ +.. _aiohttp_changes: + .. include:: ../CHANGES.rst .. include:: ../HISTORY.rst diff --git a/docs/client.rst b/docs/client.rst index 2db791a..f5239f8 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -1,7 +1,7 @@ .. _aiohttp-client: -HTTP Client -=========== +Client +====== .. module:: aiohttp diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 6514740..65770c1 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1,7 +1,7 @@ .. _aiohttp-client-reference: -HTTP Client Reference -===================== +Client Reference +================ .. module:: aiohttp .. currentmodule:: aiohttp @@ -1204,6 +1204,8 @@ Response object URL of request (:class:`str`). + .. deprecated:: 1.1 + .. attribute:: url_obj URL of request (:class:`~yarl.URL`). diff --git a/docs/deployment.rst b/docs/deployment.rst index 4a287d0..a0add2f 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -206,14 +206,14 @@ Prepare environment ------------------- You firstly need to setup your deployment environment. This example is -based on Ubuntu 14.04. +based on `Ubuntu` 14.04. Create a directory for your application:: >> mkdir myapp >> cd myapp -Ubuntu has a bug in pyenv, so to create virtualenv you need to do some +`Ubuntu` has a bug in pyenv, so to create virtualenv you need to do some extra manipulation:: >> pyvenv-3.4 --without-pip venv diff --git a/docs/faq.rst b/docs/faq.rst index 653e96f..8f8f4a3 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -15,6 +15,18 @@ For this reason feature will not be implemented. But if you really want to use decorators just derive from web.Application and add desired method. +Has aiohttp the Flask Blueprint or Django App concept? +------------------------------------------------------ + +If you're planing to write big applications, maybe you must consider +use nested applications. They acts as a Flask Blueprint or like the +Django application concept. + +Using nested application you can add sub-applications to the main application. + +see: :ref:`aiohttp-web-nested-applications`. + + How to create route that catches urls with given prefix? --------------------------------------------------------- Try something like:: @@ -61,7 +73,7 @@ resource closing. The last Python 3.3, 3.4.0 compatible version of aiohttp is **v0.17.4**. -This should not be an issue for most aiohttp users (for example Ubuntu +This should not be an issue for most aiohttp users (for example `Ubuntu` 14.04.3 LTS provides python upgraded to 3.4.3), however libraries depending on aiohttp should consider this and either freeze aiohttp version or drop Python 3.3 support as well. @@ -234,7 +246,9 @@ example would be the following:: async def test_get(self, test_client, loop): with patch("main.AioESService", MagicMock( - side_effect=lambda *args, **kwargs: AioESService(*args, **kwargs, loop=loop))): + side_effect=lambda *args, **kwargs: AioESService(*args, + **kwargs, + loop=loop))): client = await test_client(create_app) resp = await client.get("/") assert resp.status == 200 @@ -306,7 +320,9 @@ And the full tests file:: async def test_get(self, test_client, loop): with patch("main.AioESService", MagicMock( - side_effect=lambda *args, **kwargs: AioESService(*args, **kwargs, loop=loop))): + side_effect=lambda *args, **kwargs: AioESService(*args, + **kwargs, + loop=loop))): client = await test_client(create_app) resp = await client.get("/") assert resp.status == 200 @@ -315,5 +331,37 @@ Note how we are using the ``side_effect`` feature for injecting the loop to the ``AioESService.__init__`` call. The use of ``**args, **kwargs`` is mandatory in order to propagate the arguments being used by the caller. + +API stability and deprecation policy +------------------------------------ + +aiohttp tries to not break existing users code. + +Obsolete attributes and methods are marked as *deprecated* in +documentation and raises :class:`DeprecationWarning` on usage. + +Deprecation period is usually a year and half. + +After the period is passed out deprecated code is be removed. + +Unfortunately we should break own rules if new functionality or bug +fixing forces us to do it (for example proper cookies support on +client side forced us to break backward compatibility twice). + +All *backward incompatible* changes are explicitly marked in +:ref:`CHANGES <aiohttp_changes>` chapter. + + +How to enable gzip compression globally for the whole application? +------------------------------------------------------------------ + +It's impossible. Choosing what to compress and where don't apply such +time consuming operation is very tricky matter. + +If you need global compression -- write own custom middleware. Or +enable compression in NGINX (you are deploying aiohttp behind reverse +proxy, isn't it). + + .. disqus:: :title: aiohttp FAQ diff --git a/docs/index.rst b/docs/index.rst index 786bf82..7006d62 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -198,8 +198,8 @@ Contents tutorial web web_reference + web_lowlevel abc - server multipart streams api @@ -208,6 +208,7 @@ Contents deployment faq third_party + server essays contributing changes diff --git a/docs/server.rst b/docs/server.rst index 8f325c0..5657832 100644 --- a/docs/server.rst +++ b/docs/server.rst @@ -1,7 +1,7 @@ .. _aiohttp-server: -Low-level HTTP Server -===================== +Deprecated Low-level Server +=========================== .. currentmodule:: aiohttp.server @@ -10,6 +10,13 @@ Low-level HTTP Server This topic describes the low-level HTTP support. For high-level interface please take a look on :mod:`aiohttp.web`. +.. deprecated:: 1.2 + + The module has been deprecated and will be eventually removed. Use + :ref:`aiohttp-web-lowlevel` instead. + + The new solution provides much more user friendly API. + Run a basic server ------------------ diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 91f43b1..f0eb758 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -77,6 +77,7 @@ hostnames HTTPException HttpProcessingError https +incapsulates impl Indices infos @@ -202,6 +203,7 @@ tp tuples UI un +unicode unittest Unittest unix diff --git a/docs/testing.rst b/docs/testing.rst index 92a5d75..96b20be 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -46,8 +46,30 @@ Moreover we may break *backward compatibility* without *deprecation peroid* for some very strong reason. -Pytest Example -~~~~~~~~~~~~~~ +The Test Client and Servers +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*aiohttp* test utils provides a scaffolding for testing aiohttp-based +web servers. + +They are consist of two parts: running test server and making HTTP +requests to this server. + +:class:`~aiohttp.test_utils.TestServer` runs :class:`aiohttp.web.Application` +based server, :class:`~aiohttp.test_utils.RawTestServer` starts +:class:`aiohttp.web.WebServer` low level server. + +For performing HTTP requests to these servers you have to create a +test client: :class:`~aiohttp.test_utils.TestClient` instance. + +The client incapsulates :class:`aiohttp.ClientSession` by providing +proxy methods to the client for common operations such as +*ws_connect*, *get*, *post*, etc. + + + +Pytest +~~~~~~ The :data:`test_client` fixture available from pytest-aiohttp_ plugin allows you to create a client to make requests to test your app. @@ -103,73 +125,75 @@ app test client:: assert await resp.text() == 'value: bar' -.. _aiohttp-testing-framework-agnostic-utilities: +Pytest tooling has the following fixtures: -Framework Agnostic Utilities -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. data:: test_server(app, **kwargs) -High level test creation:: + A fixture factory that creates + :class:`~aiohttp.test_utils.TestServer`:: - from aiohttp.test_utils import TestClient, loop_context - from aiohttp import request + async def test_f(loop, test_server): + app = web.Application(loop=loop) + # fill route table - # loop_context is provided as a utility. You can use any - # asyncio.BaseEventLoop class in it's place. - with loop_context() as loop: - app = _create_example_app(loop) - with TestClient(app) as client: + server = await test_server(app) - async def test_get_route(): - nonlocal client - resp = await client.get("/") - assert resp.status == 200 - text = await resp.text() - assert "Hello, world" in text + The server will be destroyed on exit from test function. - loop.run_until_complete(test_get_route()) + *app* is the :class:`aiohttp.web.Application` used + to start server. + *kwargs* are parameters passed to + :meth:`aiohttp.web.Application.make_handler` -If it's preferred to handle the creation / teardown on a more granular -basis, the TestClient object can be used directly:: - from aiohttp.test_utils import TestClient +.. data:: test_client(app, **kwargs) + test_client(server, **kwargs) + test_client(raw_server, **kwargs) - with loop_context() as loop: - app = _create_example_app(loop) - client = TestClient(app) - loop.run_until_complete(client.start_server()) - root = "http://127.0.0.1:{}".format(port) + A fixture factory that creates + :class:`~aiohttp.test_utils.TestClient` for access to tested server:: - async def test_get_route(): - resp = await client.get("/") - assert resp.status == 200 - text = await resp.text() - assert "Hello, world" in text + async def test_f(loop, test_client): + app = web.Application(loop=loop) + # fill route table - loop.run_until_complete(test_get_route()) - loop.run_until_complete(client.close()) + client = await test_client(app) + resp = await client.get('/') + *client* and responses are cleaned up after test function finishing. -A full list of the utilities provided can be found at the -:data:`api reference <aiohttp.test_utils>` + The fixture accepts :class:`aiohttp.web.Application`, + :class:`aiohttp.test_utils.TestServer` or + :class:`aiohttp.test_utils.RawTestServer` instance. + + *kwargs* are parameters passed to + :class:`aiohttp.test_utils.TestClient` constructor. + +.. data:: raw_test_server(handler, **kwargs) -The Test Client -~~~~~~~~~~~~~~~ + A fixture factory that creates + :class:`~aiohttp.test_utils.RawTestServer` instance from given web + handler. -The :class:`aiohttp.test_utils.TestClient` creates an asyncio server -for the web.Application object, as well as a -:class:`aiohttp.ClientSession` to perform requests. In addition, -*TestClient* provides proxy methods to the client for common -operations such as *ws_connect*, *get*, *post*, etc. + *handler* should be a coroutine which accepts a request and returns + response, e.g.:: -Please see the full api at the -:class:`TestClass api reference <aiohttp.test_utils.TestClient>` + async def test_f(raw_test_server, test_client): + async def handler(request): + return web.Response(text="OK") + + raw_server = await raw_test_server(handler) + client = await test_client(raw_server) + resp = await client.get('/') .. _aiohttp-testing-unittest-example: -Unittest Example -~~~~~~~~~~~~~~~~ +.. _aiohttp-testing-unittest-style: + +Unittest +~~~~~~~~ To test applications with the standard library's unittest or unittest-based functionality, the AioHTTPTestCase is provided:: @@ -206,6 +230,71 @@ functionality, the AioHTTPTestCase is provided:: self.loop.run_until_complete(test_get_route()) +.. class:: AioHTTPTestCase + + A base class to allow for unittest web applications using aiohttp. + + Derived from :class:`unittest.TestCase` + + Provides the following: + + .. attribute:: client + + an aiohttp test client, :class:`TestClient` instance. + + .. attribute:: loop + + The event loop in which the application and server are running. + + .. attribute:: app + + The application returned by :meth:`get_app` + (:class:`aiohttp.web.Application` instance). + + .. method:: get_app(loop) + + This method should be overridden + to return the :class:`aiohttp.web.Application` + object to test. + + :param loop: the event_loop to use + :type loop: asyncio.AbstractEventLoop + + :return: :class:`aiohttp.web.Application` instance. + + .. method:: setUp() + + Standard test initialization method. + + .. method:: tearDown() + + Standard test finalization method. + + + .. note:: + + The ``TestClient``'s methods are asynchronous: you have to + execute function on the test client using asynchronous methods. + + A basic test class wraps every test method by + :func:`unittest_run_loop` decorator:: + + class TestA(AioHTTPTestCase): + + @unittest_run_loop + async def test_f(self): + resp = await self.client.get('/') + + +.. decorator:: unittest_run_loop: + + A decorator dedicated to use with asynchronous methods of an + :class:`AioHTTPTestCase`. + + Handles executing an asynchronous function, using + the :attr:`AioHTTPTestCase.loop` of the :class:`AioHTTPTestCase`. + + Faking request object --------------------- @@ -240,8 +329,116 @@ conditions that hard to reproduce on real server:: way. +.. function:: make_mocked_request(method, path, headers=None, *, \ + version=HttpVersion(1, 1), \ + closing=False, \ + app=None, \ + reader=sentinel, \ + writer=sentinel, \ + transport=sentinel, \ + payload=sentinel, \ + sslcontext=None, \ + secure_proxy_ssl_header=None) + + Creates mocked web.Request testing purposes. + + Useful in unit tests, when spinning full web server is overkill or + specific conditions and errors are hard to trigger. + + :param method: str, that represents HTTP method, like; GET, POST. + :type method: str + + :param path: str, The URL including *PATH INFO* without the host or scheme + :type path: str + + :param headers: mapping containing the headers. Can be anything accepted + by the multidict.CIMultiDict constructor. + :type headers: dict, multidict.CIMultiDict, list of pairs + + :param version: namedtuple with encoded HTTP version + :type version: aiohttp.protocol.HttpVersion + + :param closing: flag indicates that connection should be closed after + response. + :type closing: bool + + :param app: the aiohttp.web application attached for fake request + :type app: aiohttp.web.Application + + :param reader: object for storing and managing incoming data + :type reader: aiohttp.parsers.StreamParser + + :param writer: object for managing outcoming data + :type wirter: aiohttp.parsers.StreamWriter + + :param transport: asyncio transport instance + :type transport: asyncio.transports.Transport + + :param payload: raw payload reader object + :type payload: aiohttp.streams.FlowControlStreamReader + + :param sslcontext: ssl.SSLContext object, for HTTPS connection + :type sslcontext: ssl.SSLContext + + :param secure_proxy_ssl_header: A tuple representing a HTTP header/value + combination that signifies a request is secure. + :type secure_proxy_ssl_header: tuple + + :return: :class:`aiohttp.web.Request` object. + + .. _aiohttp-testing-writing-testable-services: +.. _aiohttp-testing-framework-agnostic-utilities: + +Framework Agnostic Utilities +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +High level test creation:: + + from aiohttp.test_utils import TestClient, loop_context + from aiohttp import request + + # loop_context is provided as a utility. You can use any + # asyncio.BaseEventLoop class in it's place. + with loop_context() as loop: + app = _create_example_app(loop) + with TestClient(app) as client: + + async def test_get_route(): + nonlocal client + resp = await client.get("/") + assert resp.status == 200 + text = await resp.text() + assert "Hello, world" in text + + loop.run_until_complete(test_get_route()) + + +If it's preferred to handle the creation / teardown on a more granular +basis, the TestClient object can be used directly:: + + from aiohttp.test_utils import TestClient + + with loop_context() as loop: + app = _create_example_app(loop) + client = TestClient(app) + loop.run_until_complete(client.start_server()) + root = "http://127.0.0.1:{}".format(port) + + async def test_get_route(): + resp = await client.get("/") + assert resp.status == 200 + text = await resp.text() + assert "Hello, world" in text + + loop.run_until_complete(test_get_route()) + loop.run_until_complete(client.close()) + + +A full list of the utilities provided can be found at the +:data:`api reference <aiohttp.test_utils>` + Writing testable services ------------------------- @@ -327,9 +524,76 @@ Test server usually works in conjunction with :class:`aiohttp.test_utils.TestClient` which provides handy client methods for accessing to the server. +.. class:: BaseTestServer(*, scheme='http', host='127.0.0.1') + + Base class for test servers. + + :param str scheme: HTTP scheme, non-protected ``"http"`` by default. + + :param str host: a host for TCP socket, IPv4 *local host* + (``'127.0.0.1'``) by default. + + + .. attribute:: scheme + + A *scheme* for tested application, ``'http'`` for non-protected + run and ``'htttps'`` for TLS encrypted server. + + .. attribute:: host + + *host* used to start a test server. + + .. attribute:: port + + A random *port* used to start a server. + + .. attribute:: handler + + :class:`aiohttp.web.WebServer` used for HTTP requests serving. + + .. attribute:: server + + :class:`asyncio.AbstractServer` used for managing accepted connections. + + .. comethod:: start_server(**kwargs) + + Start a test server. + + .. comethod:: close() + + Stop and finish executed test server. + + .. method:: make_url(path) + + Return an *absolute* :class:`~yarl.URL` for given *path*. + + +.. class:: RawTestServer(handler, *, \ + loop=None, scheme="http", host='127.0.0.1') + + Low-level test server (derived from :class:`BaseTestServer`). + + :param handler: a coroutine for handling web requests. The + handler should accept + :class:`aiohttp.web.BaseRequest` and return a + response instance, + e.g. :class:`~aiohttp.web.StreamResponse` or + :class:`~aiohttp.web.Response`. + + The handler could raise + :class:`~aiohttp.web.HTTPException` as a signal for + non-200 HTTP response. + + :param str scheme: HTTP scheme, non-protected ``"http"`` by default. + + :param str host: a host for TCP socket, IPv4 *local host* + (``'127.0.0.1'``) by default. + + .. class:: TestServer(app, *, scheme="http", host='127.0.0.1') - Test server (not started yet after constructor call). + Test server (derived from :class:`BaseTestServer`) for starting + :class:`~aiohttp.web.Application`. :param app: :class:`aiohttp.web.Application` instance to run. @@ -343,6 +607,33 @@ for accessing to the server. :class:`aiohttp.web.Application` instance to run. + +Test Client +~~~~~~~~~~~ + +.. class:: TestClient(app_or_server, *, \ + scheme='http', host='127.0.0.1', \ + cookie_jar=None, **kwargs) + + A test client used for making calls to tested server. + + :param app_or_server: :class:`BaseTestServer` instance for making + client requests to it. + + If the parameter is + :class:`aiohttp.web.Application` the tool + creates :class:`TestServer` implicitly for + serving the application. + + :param cookie_jar: an optional :class:`aiohttp.CookieJar` instance, + may be useful with ``CookieJar(unsafe=True)`` + option. + + :param str scheme: HTTP scheme, non-protected ``"http"`` by default. + + :param str host: a host for TCP socket, IPv4 *local host* + (``'127.0.0.1'``) by default. + .. attribute:: scheme A *scheme* for tested application, ``'http'`` for non-protected @@ -356,14 +647,18 @@ for accessing to the server. A random *port* used to start a server. - .. attribute:: handler + .. attribute:: server - :class:`aiohttp.web.RequestHandlerFactory` returned by - ``self.app.make_handler()``. + :class:`BaseTestServer` test server instance used in conjunction + with client. - .. attribute:: server + .. attribute:: session - :class:`asyncio.AbstractServer` used for running :attr:`app`. + An internal :class:`aiohttp.ClientSession`. + + Unlike the methods on the :class:`TestClient`, client session + requests do not automatically include the host in the url + queried, and will require an absolute path to the resource. .. comethod:: start_server(**kwargs) @@ -375,14 +670,53 @@ for accessing to the server. .. method:: make_url(path) - Return :class:`~yarl.URL` for given *path*. + Return an *absolute* :class:`~yarl.URL` for given *path*. + .. comethod:: request(method, path, *args, **kwargs) -Test Client -~~~~~~~~~~~ + Routes a request to tested http server. + + The interface is identical to + :meth:`asyncio.ClientSession.request`, except the loop kwarg is + overridden by the instance used by the test server. + + .. comethod:: get(path, *args, **kwargs) + + Perform an HTTP GET request. + + .. comethod:: post(path, *args, **kwargs) + + Perform an HTTP POST request. + + .. comethod:: options(path, *args, **kwargs) + + Perform an HTTP OPTIONS request. + + .. comethod:: head(path, *args, **kwargs) + + Perform an HTTP HEAD request. + + .. comethod:: put(path, *args, **kwargs) -.. class:: TestClient + Perform an HTTP PUT request. + .. comethod:: patch(path, *args, **kwargs) + + Perform an HTTP PATCH request. + + .. comethod:: delete(path, *args, **kwargs) + + Perform an HTTP DELETE request. + + .. comethod:: delete(path, *args, **kwargs) + + Initiate websocket connection. + + The api corresponds to :meth:`aiohttp.ClientSession.ws_connect`. + + +Utilities +~~~~~~~~~ .. function:: make_mocked_coro(return_value) @@ -407,15 +741,27 @@ Test Client Return an unused port number for IPv4 TCP protocol. + :return int: ephemeral port number which could be reused by test server. + +.. function:: loop_context(loop_factory=<function asyncio.new_event_loop>) + + A contextmanager that creates an event_loop, for test purposes. + + Handles the creation and cleanup of a test loop. + +.. function:: setup_test_loop(loop_factory=<function asyncio.new_event_loop>) + + Create and return an :class:`asyncio.AbstractEventLoop` instance. + + The caller should also call teardown_test_loop, once they are done + with the loop. +.. function:: teardown_test_loop(loop) -.. automodule:: aiohttp.test_utils - :members: TestClient, AioHTTPTestCase, unittest_run_loop, - loop_context, setup_test_loop, teardown_test_loop, - make_mocked_request - :undoc-members: - :show-inheritance: + Teardown and cleanup an event_loop created by setup_test_loop. + :param loop: the loop to teardown + :type loop: asyncio.AbstractEventLoop diff --git a/docs/third_party.rst b/docs/third_party.rst index fadf6f2..bd16391 100644 --- a/docs/third_party.rst +++ b/docs/third_party.rst @@ -80,3 +80,5 @@ period ask to raise he status. - `aiohttp-cache <https://github.com/cr0hn/aiohttp-cache>`_ A cache system for aiohttp server. +- `aiocache <https://github.com/argaen/aiocache>`_ Caching for asyncio + with multiple backends (framework agnostic) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 1a51a29..ff25538 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1,7 +1,7 @@ .. _aiohttp-tutorial: -HTTP Server Tutorial -==================== +Server Tutorial +=============== Are you going to learn *aiohttp* but don't where to start? We have example for you. Polls application is a great example for getting @@ -35,7 +35,7 @@ command: .. code-block:: shell $ python -c 'import aiohttp; print(aiohttp.__version__)' - 0.22.0 + 1.1.5 Project structure looks very similar to other python based web projects: @@ -89,7 +89,83 @@ The following code creates an application:: loop = asyncio.get_event_loop() app = web.Application(loop=loop) + web.run_app(app, host='127.0.0.1', port=8080) +Save it under ``aiohttpdemo_polls/main.py`` and start the server: + +.. code-block:: shell + + $ python main.py + +You'll see the following output on the command line: + +.. code-block:: shell + + ======== Running on http://127.0.0.1:8080 ======== + (Press CTRL+C to quit) + +Open ``http://127.0.0.1:8080`` in browser or do + +.. code-block:: shell + + $ curl -X GET localhost:8080 + +Alas, for now both return only ``404: Not Found``. +To show something more meaningful let's create a route and a view. + +.. _aiohttp-tutorial-views: + +Views +----- + +Let's start from first views. Create the file ``aiohttpdemo_polls/views.py`` with the following:: + + from aiohttp import web + + + async def index(request): + return web.Response(text='Hello Aiohttp!') + +This is the simplest view possible in Aiohttp. +Now we should create a route for this ``index`` view. Put this into ``aiohttpdemo_polls/routes.py`` (it is a good practice to separate views, routes, models etc. You'll have more of each, and it is nice to have them in different places):: + + from views import index + + + def setup_routes(app): + app.router.add_get('/', index) + + +Also, we should call ``setup_routes`` function somewhere, and the best place is in the ``main.py`` :: + + import asyncio + from aiohttp import web + from routes import setup_routes + + + loop = asyncio.get_event_loop() + app = web.Application(loop=loop) + setup_routes(app) + web.run_app(app, host='127.0.0.1', port=8080) + +Start server again. Now if we open browser we can see: + +.. code-block:: shell + + $ curl -X GET localhost:8080 + Hello Aiohttp! + +Success! For now your working directory should look like this: + +.. code-block:: none + + . + ├── .. + └── polls + ├── aiohttpdemo_polls + │ ├── main.py + │ ├── routes.py + │ └── views.py .. _aiohttp-tutorial-config: @@ -255,36 +331,6 @@ Let's close DB connection in :attr:`~aiohtp.web.Application.on_cleanup` signal:: app.on_cleanup.append(close_pg) -.. _aiohttp-tutorial-views: - -Views ------ - -Let's start from first views. Open polls/aiohttpdemo_polls/views.py and put -next Python code inside file (``polls/aiohttpdemo_polls/views.py``):: - - from aiohttp import web - - - async def index(self, request): - return web.Response(text='Hello Aiohttp!') - -This is the simplest view possible in Aiohttp. Now we should add ``index`` view -to ``polls/aiohttpdemo_polls/routes.py``:: - - from .views import index - - - def setup_routes(app, project_root): - app.router.add_get('/', index) - -Now if we open browser we can see: - -.. code-block:: shell - - $ curl -X GET localhost:8080 - Hello Aiohttp! - .. _aiohttp-tutorial-templates: diff --git a/docs/web.rst b/docs/web.rst index 6f2cdb2..43d123b 100644 --- a/docs/web.rst +++ b/docs/web.rst @@ -1,7 +1,7 @@ .. _aiohttp-web: -HTTP Server Usage -================= +Server Usage +============ .. currentmodule:: aiohttp.web @@ -159,6 +159,25 @@ You can also specify a custom regex in the form ``{identifier:regex}``:: resource = app.router.add_resource(r'/{name:\d+}') +.. note:: + + Regex should match against *percent encoded* URL + (``request.rel_url_raw_path``). E.g. *space character* is encoded + as ``%20``. + + According to + `RFC 3986 <https://tools.ietf.org/html/rfc3986.html#appendix-A>`_ + allowed in path symbols are:: + + allowed = unreserved / pct-encoded / sub-delims + / ":" / "@" / "/" + + pct-encoded = "%" HEXDIG HEXDIG + + unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + + sub-delims = "!" / "$" / "&" / "'" / "(" / ")" + / "*" / "+" / "," / ";" / "=" .. _aiohttp-web-named-routes: @@ -172,8 +191,7 @@ Routes can also be given a *name*:: Which can then be used to access and build a *URL* for that resource later (e.g. in a :ref:`request handler <aiohttp-web-handler>`):: - >>> request.app.router.named_resources()['root'].url_for() - ... .with_query({"a": "b", "c": "d"}) + >>> request.app.router['root'].url_for().with_query({"a": "b", "c": "d"}) URL('/root?a=b&c=d') A more interesting example is building *URLs* for :ref:`variable @@ -591,7 +609,7 @@ should use :meth:`Request.multipart` which returns :ref:`multipart reader filename = mp3.filename - # You cannot relay on Content-Length if transfer is chunked. + # You cannot rely on Content-Length if transfer is chunked. size = 0 with open(os.path.join('/spool/yarrr-media/mp3/', filename), 'wb') as f: while True: @@ -636,10 +654,12 @@ with the peer:: .. _aiohttp-web-websocket-read-same-task: -Reading from the *WebSocket* (``await ws.receive()``) and closing it (``await ws.close()``) -**must only** be done inside the request handler *task*; however, writing -(``ws.send_str(...)``) to the *WebSocket* and canceling the handler task -may be delegated to other tasks. See also :ref:`FAQ section <aiohttp_faq_terminating_websockets>`. +Reading from the *WebSocket* (``await ws.receive()``) and closing it +(``await ws.close()``) **must only** be done inside the request +handler *task*; however, writing (``ws.send_str(...)``) to the +*WebSocket* and canceling the handler task may be delegated to other +tasks. See also :ref:`FAQ section +<aiohttp_faq_terminating_websockets>`. *aiohttp.web* creates an implicit :class:`asyncio.Task` for handling every incoming request. @@ -936,6 +956,7 @@ parameters. object creation is subject to change. As long as you are not creating new signals, but simply reusing existing ones, you will not be affected. +.. _aiohttp-web-nested-applications: Nested applications ------------------- @@ -1095,7 +1116,7 @@ Proper finalization procedure has three steps: 2. Fire :meth:`Application.shutdown` event. 3. Close accepted connections from clients by - :meth:`RequestHandlerFactory.finish_connections` call with + :meth:`Server.shutdown` call with reasonable small delay. 4. Call registered application finalizers by :meth:`Application.cleanup`. @@ -1116,7 +1137,7 @@ finalizing. It's pretty close to :func:`run_app` utility function:: srv.close() loop.run_until_complete(srv.wait_closed()) loop.run_until_complete(app.shutdown()) - loop.run_until_complete(handler.finish_connections(60.0)) + loop.run_until_complete(handler.shutdown(60.0)) loop.run_until_complete(app.cleanup()) loop.close() diff --git a/docs/web_lowlevel.rst b/docs/web_lowlevel.rst new file mode 100644 index 0000000..695d6d0 --- /dev/null +++ b/docs/web_lowlevel.rst @@ -0,0 +1,93 @@ +.. _aiohttp-web-lowlevel: + +Low Level Server +================ + +.. currentmodule:: aiohttp.web + + +This topic describes :mod:`aiohttp.web` based *low level* API. + +Abstract +-------- + +Sometimes user don't need high-level concepts introduced in +:ref:`aiohttp-web`: applications, routers, middlewares and signals. + +All what is needed is supporting asynchronous callable which accepts a +request and returns a response object. + +This is done by introducing :class:`aiohttp.web.WebServer` class which +serves a *protocol factory* role for +:meth:`asyncio.AbstractEventLoop.create_server` and bridges data +stream to *web handler* and sends result back. + + +Low level *web handler* should accept the single :class:`BaseRequest` +parameter and performs one of the following actions: + + 1. Return a :class:`Response` with the whole HTTP body stored in memory. + + 2. Create a :class:`StreamResponse`, send headers by + :meth:`StreamResponse.prepare` call, send data chunks by + :meth:`StreamResponse.write` / :meth:`StreamResponse.drain`, + return finished response. + + 3. Raise :class:`HTTPException` derived exception (see + :ref:`aiohttp-web-exceptions` section). + + All other exceptions not derived from :class:`HTTPException` + leads to *500 Internal Server Error* response. + + 4. Initiate and process Web-Socket connection by + :class:`WebSocketResponse` using (see :ref:`aiohttp-web-websockets`). + + +Run a Basic Low-Level Server +---------------------------- + +The following code demonstrates very trivial usage example:: + + import asyncio + from aiohttp import web + + + async def handler(request): + return web.Response(text="OK") + + + async def main(loop): + server = web.Server(handler) + await loop.create_server(server, "127.0.0.1", 8080) + print("======= Serving on http://127.0.0.1:8080/ ======") + + # pause here for very long time by serving HTTP requests and + # waiting for keyboard interruption + await asyncio.sleep(100*3600) + + + loop = asyncio.get_event_loop() + + try: + loop.run_until_complete(main(loop)) + except KeyboardInterrupt: + pass + loop.close() + + +In the snippet we have ``handler`` which returns a regular +:class:`Response` with ``"OK"`` in BODY. + +This *handler* is processed by ``server`` (:class:`Server` which acts +as *protocol factory*). Network communication is created by +``loop.create_server`` call to serve ``http://127.0.0.1:8080/``. + +The handler should process every request: ``GET``, ``POST``, +Web-Socket for every *path*. + +The example is very basic: it always return ``200 OK`` response, real +life code should be much more complex. + + +.. disqus:: + :title: aiohttp.web low-level server diff --git a/docs/web_reference.rst b/docs/web_reference.rst index f5283d4..b55bfd8 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -1,7 +1,7 @@ .. _aiohttp-web-reference: -HTTP Server Reference -===================== +Server Reference +================ .. module:: aiohttp.web @@ -10,27 +10,22 @@ HTTP Server Reference .. _aiohttp-web-request: -Request -------- +Request and Base Request +------------------------ The Request object contains all the information about an incoming HTTP request. -Every :ref:`handler<aiohttp-web-handler>` accepts a request instance as the -first positional parameter. +:class:`BaseRequest` is used for :ref:`Low-Level +Servers<aiohttp-web-lowlevel>` (which have no applications, routers, signals +and middlewares) and :class:`Request` has an *application* and *match +info* attributes. -A :class:`Request` is a :obj:`dict`-like object, allowing it to be used for -:ref:`sharing data<aiohttp-web-data-sharing>` among -:ref:`aiohttp-web-middlewares` and :ref:`aiohttp-web-signals` handlers. +A :class:`BaseRequest`/:class:`Request` are :obj:`dict`-like objects, +allowing them to be used for :ref:`sharing +data<aiohttp-web-data-sharing>` among :ref:`aiohttp-web-middlewares` +and :ref:`aiohttp-web-signals` handlers. -Although :class:`Request` is :obj:`dict`-like object, it can't be duplicated -like one using :meth:`Request.copy`. - -.. note:: - - You should never create the :class:`Request` instance manually -- - :mod:`aiohttp.web` does it for you. - -.. class:: Request +.. class:: BaseRequest .. attribute:: version @@ -190,22 +185,6 @@ like one using :meth:`Request.copy`. Read-only :class:`bool` property. - .. attribute:: match_info - - Read-only property with :class:`~aiohttp.abc.AbstractMatchInfo` - instance for result of route resolving. - - .. note:: - - Exact type of property depends on used router. If - ``app.router`` is :class:`UrlDispatcher` the property contains - :class:`UrlMappingMatchInfo` instance. - - .. attribute:: app - - An :class:`Application` instance used to call :ref:`request handler - <aiohttp-web-handler>`, Read-only property. - .. attribute:: transport An :ref:`transport<asyncio-transport>` used to process request, @@ -268,6 +247,31 @@ like one using :meth:`Request.copy`. Returns :class:`int` or ``None`` if *Content-Length* is absent. + .. attribute:: http_range + + Read-only property that returns information about *Range* HTTP header. + + Returns a :class:`slice` where ``.start`` is *left inclusive + bound*, ``.stop`` is *right exclusive bound* and ``.step`` is + ``1``. + + The property might be used in two manners: + + 1. Attribute-access style (example assumes that both left and + right borders are set, the real logic for case of open bounds + is more complex):: + + rng = request.http_rangea + with open(filename, 'rb') as f: + f.seek(rng.start) + return f.read(rng.stop-rng.start) + + 2. Slice-style:: + + return buffer[request.http_range] + + .. versionadded:: 1.2 + .. attribute:: if_modified_since Read-only property that returns the date specified in the @@ -389,6 +393,41 @@ like one using :meth:`Request.copy`. internal machinery. +.. class:: Request + + An request used for receiving request's information by *web handler*. + + Every :ref:`handler<aiohttp-web-handler>` accepts a request + instance as the first positional parameter. + + The class in derived from :class:`BaseRequest`, shares all parent's + attributes and methods but has a couple of additional properties: + + .. attribute:: match_info + + Read-only property with :class:`~aiohttp.abc.AbstractMatchInfo` + instance for result of route resolving. + + .. note:: + + Exact type of property depends on used router. If + ``app.router`` is :class:`UrlDispatcher` the property contains + :class:`UrlMappingMatchInfo` instance. + + .. attribute:: app + + An :class:`Application` instance used to call :ref:`request handler + <aiohttp-web-handler>`, Read-only property. + + .. note:: + + You should never create the :class:`Request` instance manually + -- :mod:`aiohttp.web` does it for you. But + :meth:`~BaseRequest.clone` may be used for cloning *modified* + request copy with changed *path*, *method* etc. + + + .. _aiohttp-web-response: @@ -460,6 +499,15 @@ StreamResponse .. deprecated:: 0.18 + .. attribute:: task + + A task that serves HTTP request handling. + + May be useful for graceful shutdown of long-running requests + (streaming, long polling or web-socket). + + .. versionadded:: 1.2 + .. attribute:: status Read-only property for *HTTP response status code*, :class:`int`. @@ -809,6 +857,17 @@ WebSocketResponse communicate with websocket client by :meth:`send_str`, :meth:`receive` and others. + :param bool autoping: Automatically send + :const:`~aiohttp.WSMsgType.PONG` on + :const:`~aiohttp.WSMsgType.PING` + message from client, and handle + :const:`~aiohttp.WSMsgType.PONG` + responses from client. + Note that server doesn't send + :const:`~aiohttp.WSMsgType.PING` + requests, you need to do this explicitly + using :meth:`ping` method. + .. versionadded:: 0.19 The class supports ``async for`` statement for iterating over @@ -1102,8 +1161,8 @@ Application is a synonym for web-server. To get fully working example, you have to make *application*, register supported urls in *router* and create a *server socket* with -:class:`~aiohttp.web.RequestHandlerFactory` as a *protocol -factory*. *RequestHandlerFactory* could be constructed with +:class:`~aiohttp.web.Server` as a *protocol +factory*. *Server* could be constructed with :meth:`Application.make_handler`. *Application* contains a *router* instance and a list of callbacks that @@ -1261,6 +1320,14 @@ duplicated like one using :meth:`Application.copy`. :param int max_field_size: Optional maximum header field size. Default: ``8190``. + :param float lingering_time: maximum time during which the server + reads and ignore additional data coming from the client when + lingering close is on. Use ``0`` for disabling lingering on + server channel closing. + + :param float lingering_timeout: maximum waiting time for more + client data to arrive when lingering close is in effect + You should pass result of the method as *protocol_factory* to :meth:`~asyncio.AbstractEventLoop.create_server`, e.g.:: @@ -1339,31 +1406,44 @@ duplicated like one using :meth:`Application.copy`. router for your application). -RequestHandlerFactory -^^^^^^^^^^^^^^^^^^^^^ +Server +^^^^^^ + +A protocol factory compatible with +:meth:`~asyncio.AbstreactEventLoop.create_server`. + +.. class:: Server + + The class is responsible for creating HTTP protocol + objects that can handle HTTP connections. - A protocol factory compatible with - :meth:`~asyncio.AbstreactEventLoop.create_server`. + .. attribute:: Server.connections - .. class:: RequestHandlerFactory + List of all currently opened connections. - RequestHandlerFactory is responsible for creating HTTP protocol - objects that can handle HTTP connections. + .. attribute:: requests_count - .. attribute:: RequestHandlerFactory.connections + Amount of processed requests. + + .. versionadded:: 1.0 - List of all currently opened connections. + .. coroutinemethod:: Server.shutdown(timeout) - .. attribute:: requests_count + A :ref:`coroutine<coroutine>` that should be called to close all opened + connections. - Amount of processed requests. + .. coroutinemethod:: Server.finish_connections(timeout) - .. versionadded:: 1.0 + .. deprecated:: 1.2 - .. coroutinemethod:: RequestHandlerFactory.finish_connections(timeout) + A deprecated alias for :meth:`shutdown`. - A :ref:`coroutine<coroutine>` that should be called to close all opened - connections. + .. versionchanged:: 1.2 + + ``Server`` was called ``RequestHandlerFactory`` before ``aiohttp==1.2``. + + The rename has no deprecation period but it's safe: no user + should instantiate the class by hands. Router @@ -1497,6 +1577,9 @@ Router is any object that implements :class:`AbstractRouter` interface. system call even if the platform supports it. This can be accomplished by by setting environment variable ``AIOHTTP_NOSENDFILE=1``. + If a gzip version of the static content exists at file path + ``.gz``, it + will be used for the response. + .. warning:: Use :meth:`add_static` for development only. In production, @@ -1511,6 +1594,9 @@ Router is any object that implements :class:`AbstractRouter` interface. Disable ``sendfile`` by setting environment variable ``AIOHTTP_NOSENDFILE=1`` + .. versionchanged:: 1.2.0 + Send gzip version if file path + ``.gz`` exists. + :param str prefix: URL path prefix for handled static files :param path: path to the folder in file system that contains diff --git a/examples/fake_server.py b/examples/fake_server.py index fdd9f66..bde9c96 100755 --- a/examples/fake_server.py +++ b/examples/fake_server.py @@ -101,7 +101,7 @@ class FakeFacebook: self.server.close() await self.server.wait_closed() await self.app.shutdown() - await self.handler.finish_connections() + await self.handler.shutdown() await self.app.cleanup() @get('/v2.7/me') diff --git a/examples/lowlevel_srv.py b/examples/lowlevel_srv.py new file mode 100644 index 0000000..6699e08 --- /dev/null +++ b/examples/lowlevel_srv.py @@ -0,0 +1,25 @@ +import asyncio +from aiohttp import web + + +async def handler(request): + return web.Response(text="OK") + + +async def main(loop): + server = web.Server(handler) + await loop.create_server(server, "127.0.0.1", 8080) + print("======= Serving on http://127.0.0.1:8080/ ======") + + # pause here for very long time by serving HTTP requests and + # waiting for keyboard interruption + await asyncio.sleep(100*3600) + + +loop = asyncio.get_event_loop() + +try: + loop.run_until_complete(main(loop)) +except KeyboardInterrupt: + pass +loop.close() diff --git a/examples/web_rewrite_headers_middleware.py b/examples/web_rewrite_headers_middleware.py index b85db12..4e8c644 100755 --- a/examples/web_rewrite_headers_middleware.py +++ b/examples/web_rewrite_headers_middleware.py @@ -5,7 +5,7 @@ Example for rewriting response headers by middleware. import asyncio -from aiohttp.web import Application, HTTPException, Response +from aiohttp.web import Application, HTTPException, Response, run_app @asyncio.coroutine @@ -22,27 +22,19 @@ def middleware_factory(app, next_handler): response = yield from next_handler(request) except HTTPException as exc: response = exc - if not response.started: + if not response.prepared: response.headers['SERVER'] = "Secured Server Software" return response return middleware -@asyncio.coroutine def init(loop): app = Application(loop=loop, middlewares=[middleware_factory]) app.router.add_get('/', handler) - - requests_handler = app.make_handler() - srv = yield from loop.create_server(requests_handler, '127.0.0.1', 8080) - print("Server started at http://127.0.0.1:8080") - return srv, requests_handler + return app loop = asyncio.get_event_loop() -srv, requests_handler = loop.run_until_complete(init(loop)) -try: - loop.run_forever() -except KeyboardInterrupt: - loop.run_until_complete(requests_handler.finish_connections()) +app = init(loop) +run_app(app) @@ -54,8 +54,8 @@ with codecs.open(os.path.join(os.path.abspath(os.path.dirname( raise RuntimeError('Unable to determine version.') -install_requires = ['chardet', 'multidict>=2.0', - 'async_timeout>=1.1.0', 'yarl>=0.5.0'] +install_requires = ['chardet', 'multidict>=2.1.4', + 'async_timeout>=1.1.0', 'yarl>=0.8.1'] if sys.version_info < (3, 4, 2): raise RuntimeError("aiohttp requires Python 3.4.2+") diff --git a/tests/test_client_connection.py b/tests/test_client_connection.py index 8aa5e3b..59991d4 100644 --- a/tests/test_client_connection.py +++ b/tests/test_client_connection.py @@ -95,5 +95,16 @@ def test_detach(connector, key, request, transport, protocol, loop): assert not conn.closed conn.detach() assert conn._transport is None + assert connector._release_acquired.called assert not connector._release.called assert conn.closed + + +def test_detach_closed(connector, key, request, transport, protocol, loop): + conn = Connection(connector, key, request, + transport, protocol, loop) + conn.release() + conn.detach() + + assert not connector._release_acquired.called + assert conn._transport is None diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index a515b43..a011f89 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -337,7 +337,7 @@ def test_format_task_get(test_server, loop): def test_str_params(loop, test_client): @asyncio.coroutine def handler(request): - assert 'q=t+est' in request.query_string + assert 'q=t est' in request.query_string return web.Response() app = web.Application(loop=loop) diff --git a/tests/test_client_request.py b/tests/test_client_request.py index 6595022..d057250 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -143,6 +143,16 @@ def test_host_header_host_with_nondefault_port(make_request): assert req.headers['HOST'] == 'python.org:99' +def test_host_header_host_idna_encode(make_request): + req = make_request('get', 'http://xn--9caa.com') + assert req.headers['HOST'] == 'xn--9caa.com' + + +def test_host_header_host_unicode(make_request): + req = make_request('get', 'http://éé.com') + assert req.headers['HOST'] == 'xn--9caa.com' + + def test_host_header_explicit_host(make_request): req = make_request('get', 'http://python.org/', headers={'host': 'example.com'}) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index 74e0c07..c97fdaa 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -371,6 +371,7 @@ def test_request_ctx_manager_props(loop): assert isinstance(ctx_mgr.gi_frame, types.FrameType) assert not ctx_mgr.gi_running assert isinstance(ctx_mgr.gi_code, types.CodeType) + yield from asyncio.sleep(0.1, loop=loop) @asyncio.coroutine @@ -435,3 +436,9 @@ def test_proxy_str(session, params): allow_redirects=True, proxy='http://proxy.com', **params)] + + +def test_create_session_outside_of_coroutine(loop): + with pytest.warns(ResourceWarning): + sess = ClientSession(loop=loop) + sess.close() diff --git a/tests/test_connector.py b/tests/test_connector.py index 5c2aa35..73e3d69 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -149,11 +149,43 @@ def test_get_expired(loop): conn.close() +def test_release_acquired(loop): + conn = aiohttp.BaseConnector(loop=loop, limit=5) + conn._release_waiter = unittest.mock.Mock() + + key, tr = 1, unittest.mock.Mock() + conn._acquired[key].add(tr) + acquired = conn._release_acquired(key, tr) + assert 0 == len(conn._acquired[key]) + assert acquired == conn._acquired[key] + assert not conn._release_waiter.called + + acquired = conn._release_acquired(key, tr) + assert 0 == len(conn._acquired[key]) + assert acquired is None + + conn.close() + + +def test_release_acquired_closed(loop): + conn = aiohttp.BaseConnector(loop=loop, limit=5) + conn._release_waiter = unittest.mock.Mock() + + key, tr = 1, unittest.mock.Mock() + conn._acquired[key].add(tr) + conn._closed = True + conn._release_acquired(key, tr) + assert 1 == len(conn._acquired[key]) + assert not conn._release_waiter.called + conn.close() + + def test_release(loop): loop.time = mock.Mock(return_value=10) conn = aiohttp.BaseConnector(loop=loop) conn._start_cleanup_task = unittest.mock.Mock() + conn._release_waiter = unittest.mock.Mock() req = unittest.mock.Mock() resp = req.response = unittest.mock.Mock() resp._should_close = False @@ -162,11 +194,63 @@ def test_release(loop): key = 1 conn._acquired[key].add(tr) conn._release(key, req, tr, proto) + assert conn._release_waiter.called assert conn._conns[1][0] == (tr, proto, 10) assert conn._start_cleanup_task.called conn.close() +def test_release_already_closed(loop): + conn = aiohttp.BaseConnector(loop=loop) + + tr, proto = unittest.mock.Mock(), unittest.mock.Mock() + key = 1 + conn._acquired[key].add(tr) + conn.close() + + conn._start_cleanup_task = unittest.mock.Mock() + conn._release_waiter = unittest.mock.Mock() + conn._release_acquired = unittest.mock.Mock() + req = unittest.mock.Mock() + + conn._release(key, req, tr, proto) + assert not conn._release_waiter.called + assert not conn._start_cleanup_task.called + assert not conn._release_acquired.called + + +def test_release_do_not_call_release_waiter(loop): + req = unittest.mock.Mock() + resp = req.response = unittest.mock.Mock() + resp._should_close = False + tr, proto = unittest.mock.Mock(), unittest.mock.Mock() + key = 1 + + # limit is None + conn = aiohttp.BaseConnector(limit=None, loop=loop) + conn._release_waiter = unittest.mock.Mock() + conn._acquired[key].add(tr) + conn._release(key, req, tr, proto) + assert not conn._release_waiter.called + conn.close() + + # acquired key error + conn = aiohttp.BaseConnector(loop=loop) + conn._release_waiter = unittest.mock.Mock() + conn._release(key, req, tr, proto) + assert not conn._release_waiter.called + conn.close() + + # acquired len >= limit + conn = aiohttp.BaseConnector(limit=1, loop=loop) + conn._release_waiter = unittest.mock.Mock() + conn._acquired[key].add(unittest.mock.Mock()) + conn._acquired[key].add(tr) + conn._release(key, req, tr, proto) + assert not conn._release_waiter.called + conn.close() + + def test_release_close(loop): conn = aiohttp.BaseConnector(loop=loop) req = unittest.mock.Mock() diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py index 54290f4..9d487f8 100644 --- a/tests/test_cookiejar.py +++ b/tests/test_cookiejar.py @@ -162,6 +162,32 @@ def test_save_load(loop, cookies_to_send, cookies_to_receive): assert jar_test == cookies_to_receive +def test_update_cookie_with_unicode_domain(loop): + cookies = ( + "idna-domain-first=first; Domain=xn--9caa.com; Path=/;", + "idna-domain-second=second; Domain=xn--9caa.com; Path=/;", + ) + + jar = CookieJar(loop=loop) + jar.update_cookies(SimpleCookie(cookies[0]), URL("http://éé.com/")) + jar.update_cookies(SimpleCookie(cookies[1]), URL("http://xn--9caa.com/")) + + jar_test = SimpleCookie() + for cookie in jar: + jar_test[cookie.key] = cookie + + assert jar_test == SimpleCookie(" ".join(cookies)) + + +def test_filter_cookie_with_unicode_domain(loop): + jar = CookieJar(loop=loop) + jar.update_cookies(SimpleCookie( + "idna-domain-first=first; Domain=xn--9caa.com; Path=/; " + )) + assert len(jar.filter_cookies(URL("http://éé.com"))) == 1 + assert len(jar.filter_cookies(URL("http://xn--9caa.com"))) == 1 + + def test_ctor_ith_default_loop(loop): asyncio.set_event_loop(loop) jar = CookieJar() @@ -218,6 +244,16 @@ def test_preserving_ip_domain_cookies(loop): 'Cookie: shared-cookie=first') +def test_preserving_quoted_cookies(loop): + jar = CookieJar(loop=loop, unsafe=True) + jar.update_cookies(SimpleCookie( + "ip-cookie=\"second\"; Domain=127.0.0.1;" + )) + cookies_sent = jar.filter_cookies(URL("http://127.0.0.1/")).output( + header='Cookie:') + assert cookies_sent == 'Cookie: ip-cookie=\"second\"' + + def test_ignore_domain_ending_with_dot(loop): jar = CookieJar(loop=loop, unsafe=True) jar.update_cookies(SimpleCookie("cookie=val; Domain=example.com.;"), @@ -234,7 +270,7 @@ class TestCookieJarBase(unittest.TestCase): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) - # N.B. those need to be overriden in child test cases + # N.B. those need to be overridden in child test cases self.jar = CookieJar(loop=self.loop) def tearDown(self): diff --git a/tests/test_helpers.py b/tests/test_helpers.py index a68cd90..4a1c083 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -404,6 +404,12 @@ class TestTimeService: assert time_service._cb is None assert time_service._loop is None + def test_double_stopping(self, time_service): + time_service.stop() + time_service.stop() + assert time_service._cb is None + assert time_service._loop is None + def test_time(self, time_service): t = time_service._time assert t == time_service.time() diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 37397c2..e82e28e 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -361,6 +361,13 @@ class PartReaderTestCase(TestCase): result = yield from obj.text() self.assertEqual('Hello, world!', result) + def test_read_text_default_encoding(self): + obj = aiohttp.multipart.BodyPartReader( + self.boundary, {}, + Stream('Привет, Мир!\r\n--:--'.encode('utf-8'))) + result = yield from obj.text() + self.assertEqual('Привет, Мир!', result) + def test_read_text_encoding(self): obj = aiohttp.multipart.BodyPartReader( self.boundary, {}, diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 7b27d42..5f0a7d4 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -40,7 +40,7 @@ class TestProxy(unittest.TestCase): tr, proto = mock.Mock(), mock.Mock() self.loop.create_connection = make_mocked_coro((tr, proto)) conn = self.loop.run_until_complete(connector.connect(req)) - self.assertEqual(req.path, 'http://www.python.org') + self.assertEqual(req.url, URL('http://www.python.org')) self.assertIs(conn._transport, tr) self.assertIs(conn._protocol, proto) @@ -62,6 +62,21 @@ class TestProxy(unittest.TestCase): "proxy_auth must be None or BasicAuth() tuple", ) + @mock.patch('aiohttp.Request') + def test_connect_request_with_unicode_host(self, Request_mock): + loop = mock.Mock() + request = ClientRequest("CONNECT", URL("http://éé.com/"), + loop=loop) + + request.response_class = mock.Mock() + request.write_bytes = mock.Mock() + request.write_bytes.return_value = asyncio.Future(loop=loop) + request.write_bytes.return_value.set_result(None) + request.send(mock.Mock(), mock.Mock()) + + Request_mock.assert_called_with(mock.ANY, mock.ANY, "xn--9caa.com:80", + mock.ANY) + def test_proxy_connection_error(self): connector = aiohttp.TCPConnector(loop=self.loop) connector._resolve_host = make_mocked_coro( @@ -105,7 +120,7 @@ class TestProxy(unittest.TestCase): self.assertNotIn('PROXY-AUTHORIZATION', req.headers) conn = self.loop.run_until_complete(connector.connect(req)) - self.assertEqual(req.path, 'http://www.python.org') + self.assertEqual(req.url, URL('http://www.python.org')) self.assertNotIn('AUTHORIZATION', req.headers) self.assertIn('PROXY-AUTHORIZATION', req.headers) self.assertNotIn('AUTHORIZATION', proxy_req.headers) @@ -148,7 +163,7 @@ class TestProxy(unittest.TestCase): self.assertNotIn('PROXY-AUTHORIZATION', req.headers) conn = self.loop.run_until_complete(connector.connect(req)) - self.assertEqual(req.path, 'http://www.python.org') + self.assertEqual(req.url, URL('http://www.python.org')) self.assertNotIn('AUTHORIZATION', req.headers) self.assertIn('PROXY-AUTHORIZATION', req.headers) self.assertNotIn('AUTHORIZATION', proxy_req.headers) @@ -212,8 +227,8 @@ class TestProxy(unittest.TestCase): self.assertEqual(req.url.path, '/') self.assertEqual(proxy_req.method, 'CONNECT') - self.assertEqual(proxy_req.path, 'www.python.org:443') - tr.pause_reading.assert_called_once_with() + self.assertEqual(proxy_req.url, URL('https://www.python.org')) + tr.close.assert_called_once_with() tr.get_extra_info.assert_called_with('socket', default=None) self.loop.run_until_complete(proxy_req.close()) @@ -341,7 +356,7 @@ class TestProxy(unittest.TestCase): loop=self.loop, ) self.loop.run_until_complete(connector._create_connection(req)) - self.assertEqual(req.path, 'http://localhost:1234/path') + self.assertEqual(req.url, URL('http://localhost:1234/path')) def test_proxy_auth_property(self): req = aiohttp.ClientRequest( @@ -393,8 +408,8 @@ class TestProxy(unittest.TestCase): self.assertEqual(req.url.path, '/') self.assertEqual(proxy_req.method, 'CONNECT') - self.assertEqual(proxy_req.path, 'www.python.org:443') - tr.pause_reading.assert_called_once_with() + self.assertEqual(proxy_req.url, URL('https://www.python.org')) + tr.close.assert_called_once_with() tr.get_extra_info.assert_called_with('socket', default=None) self.loop.run_until_complete(proxy_req.close()) @@ -494,7 +509,7 @@ class TestProxyConnector(unittest.TestCase): tr, proto = mock.Mock(), mock.Mock() self.loop.create_connection = make_mocked_coro((tr, proto)) conn = self.loop.run_until_complete(connector.connect(req)) - self.assertEqual(req.path, 'http://www.python.org') + self.assertEqual(req.url, URL('http://www.python.org')) self.assertIs(conn._transport, tr) self.assertIs(conn._protocol, proto) diff --git a/tests/test_proxy_functional.py b/tests/test_proxy_functional.py new file mode 100644 index 0000000..9fd30d2 --- /dev/null +++ b/tests/test_proxy_functional.py @@ -0,0 +1,500 @@ +import asyncio +from functools import partial +from unittest import mock + +import pytest +from yarl import URL + +import aiohttp +import aiohttp.helpers +import aiohttp.web + + +@pytest.fixture +def proxy_test_server(raw_test_server, loop, monkeypatch): + """Handle all proxy requests and imitate remote server response.""" + + _patch_ssl_transport(monkeypatch) + + default_response = dict( + status=200, + headers=None, + body=None) + + @asyncio.coroutine + def proxy_handler(request, proxy_mock): + proxy_mock.request = request + proxy_mock.requests_list.append(request) + + response = default_response.copy() + if isinstance(proxy_mock.return_value, dict): + response.update(proxy_mock.return_value) + + if request.method == 'CONNECT': + response['body'] = None + + return aiohttp.web.Response(**response) + + @asyncio.coroutine + def proxy_server(): + proxy_mock = mock.Mock() + proxy_mock.request = None + proxy_mock.requests_list = [] + + handler = partial(proxy_handler, proxy_mock=proxy_mock) + server = yield from raw_test_server(handler) + + proxy_mock.server = server + proxy_mock.url = server.make_url('/') + + return proxy_mock + + return proxy_server + + +@asyncio.coroutine +def _request(method, url, loop=None, **kwargs): + with aiohttp.ClientSession(loop=loop) as client: + resp = yield from client.request(method, url, **kwargs) + yield from resp.release() + return resp + + +@pytest.fixture() +def get_request(loop): + return partial(_request, method='GET', loop=loop) + + +@asyncio.coroutine +def test_proxy_http_absolute_path(proxy_test_server, get_request): + url = 'http://aiohttp.io/path?query=yes' + proxy = yield from proxy_test_server() + + yield from get_request(url=url, proxy=proxy.url) + + assert len(proxy.requests_list) == 1 + assert proxy.request.method == 'GET' + assert proxy.request.host == 'aiohttp.io' + assert proxy.request.path_qs == 'http://aiohttp.io/path?query=yes' + + +@asyncio.coroutine +def test_proxy_http_raw_path(proxy_test_server, get_request): + url = 'http://aiohttp.io:2561/space sheep?q=can:fly' + raw_url = 'http://aiohttp.io:2561/space%20sheep?q=can:fly' + proxy = yield from proxy_test_server() + + yield from get_request(url=url, proxy=proxy.url) + + assert proxy.request.host == 'aiohttp.io:2561' + assert proxy.request.path_qs == raw_url + + +@asyncio.coroutine +def test_proxy_http_idna_support(proxy_test_server, get_request): + url = 'http://éé.com/' + raw_url = 'http://xn--9caa.com/' + proxy = yield from proxy_test_server() + + yield from get_request(url=url, proxy=proxy.url) + + assert proxy.request.host == 'xn--9caa.com' + assert proxy.request.path_qs == raw_url + + +@asyncio.coroutine +def test_proxy_http_connection_error(get_request): + url = 'http://aiohttp.io/path' + proxy_url = 'http://localhost:2242/' + + with pytest.raises(aiohttp.ProxyConnectionError): + yield from get_request(url=url, proxy=proxy_url) + + +@asyncio.coroutine +def test_proxy_http_bad_response(proxy_test_server, get_request): + url = 'http://aiohttp.io/path' + proxy = yield from proxy_test_server() + proxy.return_value = dict( + status=502, + headers={'Proxy-Agent': 'TestProxy'}) + + resp = yield from get_request(url=url, proxy=proxy.url) + + assert resp.status == 502 + assert resp.headers['Proxy-Agent'] == 'TestProxy' + + +@asyncio.coroutine +def test_proxy_http_auth(proxy_test_server, get_request): + url = 'http://aiohttp.io/path' + proxy = yield from proxy_test_server() + + yield from get_request(url=url, proxy=proxy.url) + + assert 'Authorization' not in proxy.request.headers + assert 'Proxy-Authorization' not in proxy.request.headers + + auth = aiohttp.helpers.BasicAuth('user', 'pass') + yield from get_request(url=url, auth=auth, proxy=proxy.url) + + assert 'Authorization' in proxy.request.headers + assert 'Proxy-Authorization' not in proxy.request.headers + + yield from get_request(url=url, proxy_auth=auth, proxy=proxy.url) + + assert 'Authorization' not in proxy.request.headers + assert 'Proxy-Authorization' in proxy.request.headers + + yield from get_request(url=url, auth=auth, + proxy_auth=auth, proxy=proxy.url) + + assert 'Authorization' in proxy.request.headers + assert 'Proxy-Authorization' in proxy.request.headers + + +@asyncio.coroutine +def test_proxy_http_auth_utf8(proxy_test_server, get_request): + url = 'http://aiohttp.io/path' + auth = aiohttp.helpers.BasicAuth('юзер', 'пасс', 'utf-8') + proxy = yield from proxy_test_server() + + yield from get_request(url=url, auth=auth, proxy=proxy.url) + + assert 'Authorization' in proxy.request.headers + assert 'Proxy-Authorization' not in proxy.request.headers + + +@asyncio.coroutine +def test_proxy_http_auth_from_url(proxy_test_server, get_request): + url = 'http://aiohttp.io/path' + proxy = yield from proxy_test_server() + + auth_url = URL(url).with_user('user').with_password('pass') + yield from get_request(url=auth_url, proxy=proxy.url) + + assert 'Authorization' in proxy.request.headers + assert 'Proxy-Authorization' not in proxy.request.headers + + proxy_url = URL(proxy.url).with_user('user').with_password('pass') + yield from get_request(url=url, proxy=proxy_url) + + assert 'Authorization' not in proxy.request.headers + assert 'Proxy-Authorization' in proxy.request.headers + + +@asyncio.coroutine +def test_proxy_http_acquired_cleanup(proxy_test_server, loop): + url = 'http://aiohttp.io/path' + key = ('aiohttp.io', 80, False) + + conn = aiohttp.TCPConnector(loop=loop) + sess = aiohttp.ClientSession(connector=conn, loop=loop) + proxy = yield from proxy_test_server() + + assert 0 == len(conn._acquired.keys()) + + @asyncio.coroutine + def request(): + resp = yield from sess.get(url, proxy=proxy.url) + + assert 1 == len(conn._acquired.keys()) + assert 1 == len(conn._acquired[key]) + + yield from resp.release() + + yield from request() + + assert 0 == len(conn._acquired[key]) + + yield from sess.close() + + +@asyncio.coroutine +def test_proxy_http_acquired_cleanup_force(proxy_test_server, loop): + url = 'http://aiohttp.io/path' + key = ('aiohttp.io', 80, False) + + conn = aiohttp.TCPConnector(force_close=True, loop=loop) + sess = aiohttp.ClientSession(connector=conn, loop=loop) + proxy = yield from proxy_test_server() + + assert 0 == len(conn._acquired.keys()) + + @asyncio.coroutine + def request(): + resp = yield from sess.get(url, proxy=proxy.url) + + assert 1 == len(conn._acquired.keys()) + assert 1 == len(conn._acquired[key]) + + yield from resp.release() + + yield from request() + + assert 0 == len(conn._acquired[key]) + + yield from sess.close() + + +@asyncio.coroutine +def test_proxy_http_multi_conn_limit(proxy_test_server, loop): + url = 'http://aiohttp.io/path' + limit, multi_conn_num = 1, 5 + + conn = aiohttp.TCPConnector(limit=limit, loop=loop) + sess = aiohttp.ClientSession(connector=conn, loop=loop) + proxy = yield from proxy_test_server() + + current_pid = None + + @asyncio.coroutine + def request(pid): + # process requests only one by one + nonlocal current_pid + + resp = yield from sess.get(url, proxy=proxy.url) + + current_pid = pid + yield from asyncio.sleep(0.2, loop=loop) + assert current_pid == pid + + yield from resp.release() + return resp + + requests = [request(pid) for pid in range(multi_conn_num)] + responses = yield from asyncio.gather(*requests, loop=loop) + + assert len(responses) == multi_conn_num + assert set(resp.status for resp in responses) == {200} + + yield from sess.close() + + +@asyncio.coroutine +def test_proxy_https_connect(proxy_test_server, get_request): + proxy = yield from proxy_test_server() + url = 'https://www.google.com.ua/search?q=aiohttp proxy' + + yield from get_request(url=url, proxy=proxy.url) + + connect = proxy.requests_list[0] + assert connect.method == 'CONNECT' + assert connect.path == 'www.google.com.ua:443' + assert connect.host == 'www.google.com.ua' + + assert proxy.request.host == 'www.google.com.ua' + assert proxy.request.path_qs == '/search?q=aiohttp+proxy' + + +@asyncio.coroutine +def test_proxy_https_connect_with_port(proxy_test_server, get_request): + proxy = yield from proxy_test_server() + url = 'https://secure.aiohttp.io:2242/path' + + yield from get_request(url=url, proxy=proxy.url) + + connect = proxy.requests_list[0] + assert connect.method == 'CONNECT' + assert connect.path == 'secure.aiohttp.io:2242' + assert connect.host == 'secure.aiohttp.io:2242' + + assert proxy.request.host == 'secure.aiohttp.io:2242' + assert proxy.request.path_qs == '/path' + + +@asyncio.coroutine +def test_proxy_https_send_body(proxy_test_server, loop): + sess = aiohttp.ClientSession(loop=loop) + proxy = yield from proxy_test_server() + proxy.return_value = {'status': 200, 'body': b'1'*(2**20)} + url = 'https://www.google.com.ua/search?q=aiohttp proxy' + + resp = yield from sess.get(url, proxy=proxy.url) + body = yield from resp.read() + yield from resp.release() + yield from sess.close() + + assert body == b'1'*(2**20) + + +@asyncio.coroutine +def test_proxy_https_idna_support(proxy_test_server, get_request): + url = 'https://éé.com/' + proxy = yield from proxy_test_server() + + yield from get_request(url=url, proxy=proxy.url) + + connect = proxy.requests_list[0] + assert connect.method == 'CONNECT' + assert connect.path == 'xn--9caa.com:443' + assert connect.host == 'xn--9caa.com' + + +@asyncio.coroutine +def test_proxy_https_connection_error(get_request): + url = 'https://secure.aiohttp.io/path' + proxy_url = 'http://localhost:2242/' + + with pytest.raises(aiohttp.ProxyConnectionError): + yield from get_request(url=url, proxy=proxy_url) + + +@asyncio.coroutine +def test_proxy_https_bad_response(proxy_test_server, get_request): + url = 'https://secure.aiohttp.io/path' + proxy = yield from proxy_test_server() + proxy.return_value = dict( + status=502, + headers={'Proxy-Agent': 'TestProxy'}) + + with pytest.raises(aiohttp.HttpProxyError): + yield from get_request(url=url, proxy=proxy.url) + + assert len(proxy.requests_list) == 1 + assert proxy.request.method == 'CONNECT' + assert proxy.request.path == 'secure.aiohttp.io:443' + + +@asyncio.coroutine +def test_proxy_https_auth(proxy_test_server, get_request): + url = 'https://secure.aiohttp.io/path' + auth = aiohttp.helpers.BasicAuth('user', 'pass') + + proxy = yield from proxy_test_server() + yield from get_request(url=url, proxy=proxy.url) + + connect = proxy.requests_list[0] + assert 'Authorization' not in connect.headers + assert 'Proxy-Authorization' not in connect.headers + assert 'Authorization' not in proxy.request.headers + assert 'Proxy-Authorization' not in proxy.request.headers + + proxy = yield from proxy_test_server() + yield from get_request(url=url, auth=auth, proxy=proxy.url) + + connect = proxy.requests_list[0] + assert 'Authorization' not in connect.headers + assert 'Proxy-Authorization' not in connect.headers + assert 'Authorization' in proxy.request.headers + assert 'Proxy-Authorization' not in proxy.request.headers + + proxy = yield from proxy_test_server() + yield from get_request(url=url, proxy_auth=auth, proxy=proxy.url) + + connect = proxy.requests_list[0] + assert 'Authorization' not in connect.headers + assert 'Proxy-Authorization' in connect.headers + assert 'Authorization' not in proxy.request.headers + assert 'Proxy-Authorization' not in proxy.request.headers + + proxy = yield from proxy_test_server() + yield from get_request(url=url, auth=auth, + proxy_auth=auth, proxy=proxy.url) + + connect = proxy.requests_list[0] + assert 'Authorization' not in connect.headers + assert 'Proxy-Authorization' in connect.headers + assert 'Authorization' in proxy.request.headers + assert 'Proxy-Authorization' not in proxy.request.headers + + +@asyncio.coroutine +def test_proxy_https_acquired_cleanup(proxy_test_server, loop): + url = 'https://secure.aiohttp.io/path' + key = ('secure.aiohttp.io', 443, True) + + conn = aiohttp.TCPConnector(loop=loop) + sess = aiohttp.ClientSession(connector=conn, loop=loop) + proxy = yield from proxy_test_server() + + assert 0 == len(conn._acquired.keys()) + + @asyncio.coroutine + def request(): + resp = yield from sess.get(url, proxy=proxy.url) + + assert 1 == len(conn._acquired.keys()) + assert 1 == len(conn._acquired[key]) + + yield from resp.release() + + yield from request() + + assert 0 == len(conn._acquired[key]) + + yield from sess.close() + + +@asyncio.coroutine +def test_proxy_https_acquired_cleanup_force(proxy_test_server, loop): + url = 'https://secure.aiohttp.io/path' + key = ('secure.aiohttp.io', 443, True) + + conn = aiohttp.TCPConnector(force_close=True, loop=loop) + sess = aiohttp.ClientSession(connector=conn, loop=loop) + proxy = yield from proxy_test_server() + + assert 0 == len(conn._acquired.keys()) + + @asyncio.coroutine + def request(): + resp = yield from sess.get(url, proxy=proxy.url) + + assert 1 == len(conn._acquired.keys()) + assert 1 == len(conn._acquired[key]) + + yield from resp.release() + + yield from request() + + assert 0 == len(conn._acquired[key]) + + yield from sess.close() + + +@asyncio.coroutine +def test_proxy_https_multi_conn_limit(proxy_test_server, loop): + url = 'https://secure.aiohttp.io/path' + limit, multi_conn_num = 1, 5 + + conn = aiohttp.TCPConnector(limit=limit, loop=loop) + sess = aiohttp.ClientSession(connector=conn, loop=loop) + proxy = yield from proxy_test_server() + + current_pid = None + + @asyncio.coroutine + def request(pid): + # process requests only one by one + nonlocal current_pid + + resp = yield from sess.get(url, proxy=proxy.url) + + current_pid = pid + yield from asyncio.sleep(0.2, loop=loop) + assert current_pid == pid + + yield from resp.release() + return resp + + requests = [request(pid) for pid in range(multi_conn_num)] + responses = yield from asyncio.gather(*requests, loop=loop) + + assert len(responses) == multi_conn_num + assert set(resp.status for resp in responses) == {200} + + yield from sess.close() + + +def _patch_ssl_transport(monkeypatch): + """Make ssl transport substitution to prevent ssl handshake.""" + def _make_ssl_transport_dummy(self, rawsock, protocol, sslcontext, + waiter=None, **kwargs): + return self._make_socket_transport(rawsock, protocol, waiter, + extra=kwargs.get('extra'), + server=kwargs.get('server')) + + monkeypatch.setattr( + "asyncio.selector_events.BaseSelectorEventLoop._make_ssl_transport", + _make_ssl_transport_dummy) diff --git a/tests/test_pytest_plugin.py b/tests/test_pytest_plugin.py index 860c916..c2738d1 100644 --- a/tests/test_pytest_plugin.py +++ b/tests/test_pytest_plugin.py @@ -114,7 +114,7 @@ def test_set_value(cli): assert resp.status == 200 text = yield from resp.text() assert text == 'thanks for the data' - assert cli.app['value'] == 'foo' + assert cli.server.app['value'] == 'foo' @asyncio.coroutine @@ -123,7 +123,7 @@ def test_get_value(cli): assert resp.status == 200 text = yield from resp.text() assert text == 'value: unknown' - cli.app['value'] = 'bar' + cli.server.app['value'] = 'bar' resp = yield from cli.get('/') assert resp.status == 200 text = yield from resp.text() diff --git a/tests/test_server.py b/tests/test_server.py index 3377616..76cc212 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -199,8 +199,7 @@ def test_bad_method(srv, loop): def test_line_too_long(srv, loop): transport = mock.Mock() srv.connection_made(transport) - - srv.reader.feed_data(b''.join([b'a' for _ in range(10000)])) + srv.data_received(b''.join([b'a' for _ in range(10000)]) + b'\r\n\r\n') loop.run_until_complete(srv._request_handler) assert transport.write.mock_calls[0][1][0].startswith( @@ -330,8 +329,18 @@ def test_handle(srv, loop): assert transport.close.called -def test_handle_uncompleted(srv, loop): +def test_handle_uncompleted(make_srv, loop): transport = mock.Mock() + closed = False + + def close(): + nonlocal closed + closed = True + + transport.close = close + + srv = make_srv(lingering_timeout=0) + srv.connection_made(transport) srv.logger.exception = mock.Mock() @@ -345,10 +354,84 @@ def test_handle_uncompleted(srv, loop): loop.run_until_complete(srv._request_handler) assert handle.called - assert transport.close.called + assert closed srv.logger.exception.assert_called_with("Error handling request") +@asyncio.coroutine +def test_lingering(srv, loop): + + transport = mock.Mock() + srv.connection_made(transport) + + yield from asyncio.sleep(0, loop=loop) + assert not transport.close.called + + srv.reader.feed_data( + b'GET / HTTP/1.0\r\n' + b'Host: example.com\r\n' + b'Content-Length: 0\r\n\r\n') + + srv.reader.feed_data(b'123') + + yield from asyncio.sleep(0, loop=loop) + assert not transport.close.called + srv.reader.feed_eof() + + yield from asyncio.sleep(0, loop=loop) + transport.close.assert_called_with() + + +@asyncio.coroutine +def test_lingering_disabled(make_srv, loop): + srv = make_srv(lingering_time=0) + + transport = mock.Mock() + srv.connection_made(transport) + + yield from asyncio.sleep(0, loop=loop) + assert not transport.close.called + + srv.reader.feed_data( + b'GET / HTTP/1.0\r\n' + b'Host: example.com\r\n' + b'Content-Length: 50\r\n\r\n') + + srv.reader.feed_data(b'123') + + yield from asyncio.sleep(0, loop=loop) + assert not transport.close.called + srv.reader.feed_eof() + + yield from asyncio.sleep(0, loop=loop) + transport.close.assert_called_with() + + +@asyncio.coroutine +def test_lingering_zero_timeout(make_srv, loop): + srv = make_srv(lingering_time=1e-30) + + transport = mock.Mock() + srv.connection_made(transport) + + yield from asyncio.sleep(0, loop=loop) + assert not transport.close.called + + srv.reader.feed_data( + b'GET / HTTP/1.0\r\n' + b'Host: example.com\r\n' + b'Content-Length: 50\r\n\r\n') + + srv.reader.feed_data(b'123') + + yield from asyncio.sleep(0, loop=loop) + assert not transport.close.called + srv.reader.feed_eof() + + yield from asyncio.sleep(0, loop=loop) + transport.close.assert_called_with() + + def test_handle_coro(srv, loop): transport = mock.Mock() diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index d4327aa..c9eca16 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -9,7 +9,7 @@ import aiohttp from aiohttp import web, web_reqrep from aiohttp.test_utils import TestClient as _TestClient from aiohttp.test_utils import TestServer as _TestServer -from aiohttp.test_utils import (AioHTTPTestCase, loop_context, +from aiohttp.test_utils import (AioHTTPTestCase, RawTestServer, loop_context, make_mocked_request, setup_test_loop, teardown_test_loop, unittest_run_loop) @@ -226,12 +226,10 @@ def test_make_mocked_request_transport(): def test_test_client_props(loop): app = _create_example_app(loop) client = _TestClient(app, host='localhost') - assert client.app == app assert client.host == 'localhost' assert client.port is None with client: assert isinstance(client.port, int) - assert client.handler is not None assert client.server is not None assert client.port is None @@ -278,3 +276,12 @@ def test_server_make_url_yarl_compatibility(loop): make_url('http://foo.com') with pytest.raises(AssertionError): make_url(URL('http://foo.com')) + + +def test_raw_server_implicit_loop(loop): + @asyncio.coroutine + def handler(request): + pass + asyncio.set_event_loop(loop) + srv = RawTestServer(handler) + assert srv._loop is loop diff --git a/tests/test_urldispatch.py b/tests/test_urldispatch.py index a3819d0..996a8c2 100644 --- a/tests/test_urldispatch.py +++ b/tests/test_urldispatch.py @@ -13,7 +13,7 @@ from aiohttp import hdrs, web from aiohttp.test_utils import make_mocked_request from aiohttp.web import HTTPMethodNotAllowed, HTTPNotFound, Response from aiohttp.web_urldispatcher import (AbstractResource, ResourceRoute, - SystemRoute, View, + SystemRoute, UrlDispatcher, View, _defaultExpectHandler) @@ -1011,3 +1011,9 @@ def test_convert_empty_path_to_slash_on_freezing(router): assert resource.get_info() == {'path': ''} router.freeze() assert resource.get_info() == {'path': '/'} + + +def test_add_to_non_initialized_router(): + router = UrlDispatcher() + with pytest.raises(RuntimeError): + router.add_get('/', make_handler()) diff --git a/tests/test_web_application.py b/tests/test_web_application.py index 1a74bcf..ec000be 100644 --- a/tests/test_web_application.py +++ b/tests/test_web_application.py @@ -18,12 +18,6 @@ def test_app_call(loop): assert app is app() -def test_app_copy(loop): - app = web.Application(loop=loop) - with pytest.raises(NotImplementedError): - app.copy() - - def test_app_default_loop(loop): asyncio.set_event_loop(loop) app = web.Application() @@ -34,16 +28,18 @@ def test_app_default_loop(loop): def test_app_make_handler_debug_exc(loop, mocker, debug): app = web.Application(loop=loop, debug=debug) - mocker.spy(app, '_handler_factory') + srv = mocker.patch('aiohttp.web.Server') app.make_handler() with pytest.warns(DeprecationWarning) as exc: app.make_handler(debug=debug) assert 'parameter is deprecated' in exc[0].message.args[0] - assert app._handler_factory.call_count == 2 - app._handler_factory.assert_called_with(app, app.router, loop=loop, - debug=debug) + assert srv.call_count == 2 + srv.assert_called_with(app._handle, + request_factory=app._make_request, + loop=loop, + debug=debug) with pytest.raises(ValueError) as exc: app.make_handler(debug=not debug) @@ -175,3 +171,16 @@ def test_app_delitem(loop): assert len(app) == 1 del app['key'] assert len(app) == 0 + + +def test_secure_proxy_ssl_header_default(loop): + app = web.Application(loop=loop) + assert app._secure_proxy_ssl_header is None + + +@asyncio.coroutine +def test_secure_proxy_ssl_header_non_default(loop): + app = web.Application(loop=loop) + hdr = ('X-Forwarded-Proto', 'https') + app.make_handler(secure_proxy_ssl_header=hdr) + assert app._secure_proxy_ssl_header is hdr diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 0bee655..7101e21 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -52,7 +52,7 @@ def test_handler_returns_not_response(loop, test_server, test_client): resp = yield from client.get('/') assert 500 == resp.status - logger.exception.assert_called_with("Error handling request") + assert logger.exception.called @asyncio.coroutine @@ -825,19 +825,19 @@ def test_requests_count(loop, test_client): app = web.Application(loop=loop) app.router.add_get('/', handler) client = yield from test_client(app) - assert client.handler.requests_count == 0 + assert client.server.handler.requests_count == 0 resp = yield from client.get('/') assert 200 == resp.status - assert client.handler.requests_count == 1 + assert client.server.handler.requests_count == 1 resp = yield from client.get('/') assert 200 == resp.status - assert client.handler.requests_count == 2 + assert client.server.handler.requests_count == 2 resp = yield from client.get('/') assert 200 == resp.status - assert client.handler.requests_count == 3 + assert client.server.handler.requests_count == 3 @asyncio.coroutine @@ -1185,3 +1185,26 @@ def test_custom_date_header(loop, test_client): resp = yield from client.get('/') assert 200 == resp.status assert resp.headers['Date'] == 'Sun, 30 Oct 2016 03:13:52 GMT' + + +@asyncio.coroutine +def test_response_task(loop, test_client): + + srv_resp = None + + @asyncio.coroutine + def handler(request): + nonlocal srv_resp + srv_resp = web.StreamResponse() + assert srv_resp.task is None + yield from srv_resp.prepare(request) + assert srv_resp.task is not None + return srv_resp + + app = web.Application(loop=loop) + app.router.add_get('/', handler) + client = yield from test_client(app) + + resp = yield from client.get('/') + assert 200 == resp.status + assert srv_resp.task is None diff --git a/tests/test_web_request_handler.py b/tests/test_web_request_handler.py index 6c0c7ce..b724e47 100644 --- a/tests/test_web_request_handler.py +++ b/tests/test_web_request_handler.py @@ -64,9 +64,3 @@ def test_finish_connection_timeout(loop): manager.connection_lost(handler, None) assert manager.connections == [] handler.shutdown.assert_called_with(0.1) - - -def test_secure_proxy_ssl_header_default(loop): - app = web.Application(loop=loop) - manager = app.make_handler() - assert manager.secure_proxy_ssl_header is None diff --git a/tests/test_web_response.py b/tests/test_web_response.py index ed766a3..b044e31 100644 --- a/tests/test_web_response.py +++ b/tests/test_web_response.py @@ -905,6 +905,14 @@ def test_drain_before_start(): yield from resp.drain() +@asyncio.coroutine +def test_changing_status_after_prepare_raises(): + resp = StreamResponse() + yield from resp.prepare(make_request('GET', '/')) + with pytest.raises(RuntimeError): + resp.set_status(400) + + def test_nonstr_text_in_ctor(): with pytest.raises(TypeError): Response(text=b'data') @@ -948,6 +956,11 @@ def test_text_with_empty_payload(): assert resp.text is None +def test_response_with_content_length_header_without_body(): + resp = Response(headers={'Content-Length': 123}) + assert resp.content_length == 123 + + class TestJSONResponse: def test_content_type_is_application_json_by_default(self): diff --git a/tests/test_web_sendfile.py b/tests/test_web_sendfile.py index ccc24d8..420c8e9 100644 --- a/tests/test_web_sendfile.py +++ b/tests/test_web_sendfile.py @@ -1,8 +1,11 @@ import os from unittest import mock -from aiohttp import helpers +from yarl import URL + +from aiohttp import hdrs, helpers from aiohttp.file_sender import FileSender +from aiohttp.test_utils import make_mocked_coro, make_mocked_request def test_env_nosendfile(): @@ -75,3 +78,109 @@ def test__sendfile_cb_return_on_cancelling(loop): assert not fake_loop.add_writer.called assert not fake_loop.remove_writer.called assert not m_os.sendfile.called + + +def test_using_gzip_if_header_present_and_file_available(loop): + request = make_mocked_request( + 'GET', URL('http://python.org/logo.png'), headers={ + hdrs.ACCEPT_ENCODING: 'gzip' + } + ) + + gz_filepath = mock.Mock() + gz_filepath.open = mock.mock_open() + gz_filepath.is_file.return_value = True + gz_filepath.stat.return_value = mock.MagicMock() + gz_filepath.stat.st_size = 1024 + + filepath = mock.Mock() + filepath.name = 'logo.png' + filepath.open = mock.mock_open() + filepath.with_name.return_value = gz_filepath + + file_sender = FileSender() + file_sender._sendfile = make_mocked_coro(None) + + loop.run_until_complete(file_sender.send(request, filepath)) + + assert not filepath.open.called + assert gz_filepath.open.called + + +def test_gzip_if_header_not_present_and_file_available(loop): + request = make_mocked_request( + 'GET', URL('http://python.org/logo.png'), headers={ + } + ) + + gz_filepath = mock.Mock() + gz_filepath.open = mock.mock_open() + gz_filepath.is_file.return_value = True + + filepath = mock.Mock() + filepath.name = 'logo.png' + filepath.open = mock.mock_open() + filepath.with_name.return_value = gz_filepath + filepath.stat.return_value = mock.MagicMock() + filepath.stat.st_size = 1024 + + file_sender = FileSender() + file_sender._sendfile = make_mocked_coro(None) + + loop.run_until_complete(file_sender.send(request, filepath)) + + assert filepath.open.called + assert not gz_filepath.open.called + + +def test_gzip_if_header_not_present_and_file_not_available(loop): + request = make_mocked_request( + 'GET', URL('http://python.org/logo.png'), headers={ + } + ) + + gz_filepath = mock.Mock() + gz_filepath.open = mock.mock_open() + gz_filepath.is_file.return_value = False + + filepath = mock.Mock() + filepath.name = 'logo.png' + filepath.open = mock.mock_open() + filepath.with_name.return_value = gz_filepath + filepath.stat.return_value = mock.MagicMock() + filepath.stat.st_size = 1024 + + file_sender = FileSender() + file_sender._sendfile = make_mocked_coro(None) + + loop.run_until_complete(file_sender.send(request, filepath)) + + assert filepath.open.called + assert not gz_filepath.open.called + + +def test_gzip_if_header_present_and_file_not_available(loop): + request = make_mocked_request( + 'GET', URL('http://python.org/logo.png'), headers={ + hdrs.ACCEPT_ENCODING: 'gzip' + } + ) + + gz_filepath = mock.Mock() + gz_filepath.open = mock.mock_open() + gz_filepath.is_file.return_value = False + + filepath = mock.Mock() + filepath.name = 'logo.png' + filepath.open = mock.mock_open() + filepath.with_name.return_value = gz_filepath + filepath.stat.return_value = mock.MagicMock() + filepath.stat.st_size = 1024 + + file_sender = FileSender() + file_sender._sendfile = make_mocked_coro(None) + + loop.run_until_complete(file_sender.send(request, filepath)) + + assert filepath.open.called + assert not gz_filepath.open.called diff --git a/tests/test_web_sendfile_functional.py b/tests/test_web_sendfile_functional.py index b8a3d9d..09917e1 100644 --- a/tests/test_web_sendfile_functional.py +++ b/tests/test_web_sendfile_functional.py @@ -323,3 +323,128 @@ def test_static_file_huge(loop, test_client, tmpdir): off += len(chunk) cnt += 1 f.close() + + +@asyncio.coroutine +def test_static_file_range(loop, test_client, sender): + filepath = (pathlib.Path(__file__).parent / + 'software_development_in_picture.jpg') + + @asyncio.coroutine + def handler(request): + resp = yield from sender(chunk_size=16).send(request, filepath) + return resp + + app = web.Application(loop=loop) + app.router.add_get('/', handler) + client = yield from test_client(lambda loop: app) + + with filepath.open('rb') as f: + content = f.read() + + # Ensure the whole file requested in parts is correct + responses = yield from asyncio.gather( + client.get('/', headers={'Range': 'bytes=0-999'}), + client.get('/', headers={'Range': 'bytes=1000-1999'}), + client.get('/', headers={'Range': 'bytes=2000-'}), + loop=loop + ) + assert len(responses) == 3 + assert responses[0].status == 206, \ + "failed 'bytes=0-999': %s" % responses[0].reason + assert responses[1].status == 206, \ + "failed 'bytes=1000-1999': %s" % responses[1].reason + assert responses[2].status == 206, \ + "failed 'bytes=2000-': %s" % responses[2].reason + + body = yield from asyncio.gather( + *(resp.read() for resp in responses), + loop=loop + ) + + assert len(body[0]) == 1000, \ + "failed 'bytes=0-999', received %d bytes" % len(body[0]) + assert len(body[1]) == 1000, \ + "failed 'bytes=1000-1999', received %d bytes" % len(body[1]) + responses[0].close() + responses[1].close() + responses[2].close() + + assert content == b"".join(body) + + +@asyncio.coroutine +def test_static_file_range_tail(loop, test_client, sender): + filepath = (pathlib.Path(__file__).parent / + 'software_development_in_picture.jpg') + + @asyncio.coroutine + def handler(request): + resp = yield from sender(chunk_size=16).send(request, filepath) + return resp + + app = web.Application(loop=loop) + app.router.add_get('/', handler) + client = yield from test_client(lambda loop: app) + + with filepath.open('rb') as f: + content = f.read() + + # Ensure the tail of the file is correct + resp = yield from client.get('/', headers={'Range': 'bytes=-500'}) + assert resp.status == 206, resp.reason + body4 = yield from resp.read() + resp.close() + assert content[-500:] == body4 + + +@asyncio.coroutine +def test_static_file_invalid_range(loop, test_client, sender): + filepath = (pathlib.Path(__file__).parent / + 'software_development_in_picture.jpg') + + @asyncio.coroutine + def handler(request): + resp = yield from sender(chunk_size=16).send(request, filepath) + return resp + + app = web.Application(loop=loop) + app.router.add_get('/', handler) + client = yield from test_client(lambda loop: app) + + flen = filepath.stat().st_size + + # range must be in bytes + resp = yield from client.get('/', headers={'Range': 'blocks=0-10'}) + assert resp.status == 416, 'Range must be in bytes' + resp.close() + + # Range end is inclusive + resp = yield from client.get('/', headers={'Range': 'bytes=0-%d' % flen}) + assert resp.status == 416, 'Range end must be inclusive' + resp.close() + + # start > end + resp = yield from client.get('/', headers={'Range': 'bytes=100-0'}) + assert resp.status == 416, "Range start can't be greater than end" + resp.close() + + # start > end + resp = yield from client.get('/', headers={'Range': 'bytes=10-9'}) + assert resp.status == 416, "Range start can't be greater than end" + resp.close() + + # non-number range + resp = yield from client.get('/', headers={'Range': 'bytes=a-f'}) + assert resp.status == 416, 'Range must be integers' + resp.close() + + # double dash range + resp = yield from client.get('/', headers={'Range': 'bytes=0--10'}) + assert resp.status == 416, 'double dash in range' + resp.close() + + # no range + resp = yield from client.get('/', headers={'Range': 'bytes=-'}) + assert resp.status == 416, 'no range given' + resp.close() diff --git a/tests/test_web_server.py b/tests/test_web_server.py new file mode 100644 index 0000000..8feeb54 --- /dev/null +++ b/tests/test_web_server.py @@ -0,0 +1,73 @@ +import asyncio +from unittest import mock + +from aiohttp import web + + +@asyncio.coroutine +def test_simple_server(raw_test_server, test_client): + @asyncio.coroutine + def handler(request): + return web.Response(text=str(request.rel_url)) + + server = yield from raw_test_server(handler) + client = yield from test_client(server) + resp = yield from client.get('/path/to') + assert resp.status == 200 + txt = yield from resp.text() + assert txt == '/path/to' + + +@asyncio.coroutine +def test_raw_server_not_http_exception(raw_test_server, test_client): + exc = RuntimeError("custom runtime error") + + @asyncio.coroutine + def handler(request): + raise exc + + logger = mock.Mock() + server = yield from raw_test_server(handler, logger=logger) + client = yield from test_client(server) + resp = yield from client.get('/path/to') + assert resp.status == 500 + + txt = yield from resp.text() + assert "<h1>500 Internal Server Error</h1>" in txt + + logger.exception.assert_called_with( + "Error handling request", + exc_info=exc) + + +@asyncio.coroutine +def test_raw_server_not_http_exception_debug(raw_test_server, test_client): + exc = RuntimeError("custom runtime error") + + @asyncio.coroutine + def handler(request): + raise exc + + logger = mock.Mock() + server = yield from raw_test_server(handler, logger=logger, debug=True) + client = yield from test_client(server) + resp = yield from client.get('/path/to') + assert resp.status == 500 + + txt = yield from resp.text() + assert "<h2>Traceback:</h2>" in txt + + logger.exception.assert_called_with( + "Error handling request", + exc_info=exc) + + +def test_create_web_server_with_implicit_loop(loop): + asyncio.set_event_loop(loop) + + @asyncio.coroutine + def handler(request): + return web.Response() # pragma: no cover + + srv = web.Server(handler) + assert srv._loop is loop diff --git a/tests/test_web_urldispatcher.py b/tests/test_web_urldispatcher.py index e29f387..2d8733c 100644 --- a/tests/test_web_urldispatcher.py +++ b/tests/test_web_urldispatcher.py @@ -8,7 +8,7 @@ from unittest.mock import MagicMock import pytest -from aiohttp import web +from aiohttp import abc, web from aiohttp.web_urldispatcher import SystemRoute @@ -270,3 +270,21 @@ def test_system_route(): assert "<SystemRoute 201: test>" == repr(route) assert 201 == route.status assert 'test' == route.reason + + +@asyncio.coroutine +def test_412_is_returned(loop, test_client): + + class MyRouter(abc.AbstractRouter): + + @asyncio.coroutine + def resolve(self, request): + raise web.HTTPPreconditionFailed() + + app = web.Application(router=MyRouter(), loop=loop) + + client = yield from test_client(app) + + resp = yield from client.get('/') + + assert resp.status == 412 diff --git a/tests/test_worker.py b/tests/test_worker.py index 315f314..0abd867 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -77,16 +77,57 @@ def test_run(worker, loop): assert loop.is_closed() +def test_run_wsgi(worker, loop): + worker.wsgi = lambda env, start_resp: start_resp() + + worker.loop = loop + worker._run = mock.Mock( + wraps=asyncio.coroutine(lambda: None)) + with pytest.raises(SystemExit): + worker.run() + assert worker._run.called + assert loop.is_closed() + + def test_handle_quit(worker): - worker.handle_quit(object(), object()) - assert not worker.alive - assert worker.exit_code == 0 + with mock.patch('aiohttp.worker.ensure_future') as m_ensure_future: + worker.loop = mock.Mock() + worker.handle_quit(object(), object()) + assert not worker.alive + assert worker.exit_code == 0 + assert m_ensure_future.called + worker.loop.call_later.asset_called_with( + 0.1, worker._notify_waiter_done) def test_handle_abort(worker): - worker.handle_abort(object(), object()) - assert not worker.alive - assert worker.exit_code == 1 + with mock.patch('aiohttp.worker.sys') as m_sys: + worker.handle_abort(object(), object()) + assert not worker.alive + assert worker.exit_code == 1 + m_sys.exit.assert_called_with(1) + + +def test__wait_next_notify(worker): + worker.loop = mock.Mock() + worker._notify_waiter_done = mock.Mock() + fut = worker._wait_next_notify() + + assert worker._notify_waiter == fut + worker.loop.call_later.assert_called_with(1.0, worker._notify_waiter_done) + + +def test__notify_waiter_done(worker): + worker._notify_waiter = None + worker._notify_waiter_done() + assert worker._notify_waiter is None + + waiter = worker._notify_waiter = mock.Mock() + worker._notify_waiter.done.return_value = False + worker._notify_waiter_done() + + assert worker._notify_waiter is None + waiter.set_result.assert_called_with(True) def test_init_signals(worker): @@ -108,6 +149,20 @@ def test_make_handler(worker, mocker): assert worker._get_valid_log_format.called +def test_make_handler_wsgi(worker, mocker): + worker.wsgi = lambda env, start_resp: start_resp() + worker.loop = mock.Mock() + worker.log = mock.Mock() + worker.cfg = mock.Mock() + worker.cfg.access_log_format = ACCEPTABLE_LOG_FORMAT + mocker.spy(worker, '_get_valid_log_format') + + f = worker.make_handler(worker.wsgi) + assert isinstance(f, base_worker.WSGIServer) + assert isinstance(f(), base_worker.WSGIServerHttpProtocol) + assert worker._get_valid_log_format.called + + @pytest.mark.parametrize('source,result', [ (ACCEPTABLE_LOG_FORMAT, ACCEPTABLE_LOG_FORMAT), (AsyncioWorker.DEFAULT_GUNICORN_LOG_FORMAT, @@ -205,6 +260,7 @@ def test__run_exc(worker, loop): handler = mock.Mock() handler.requests_count = 0 worker.servers = {mock.Mock(): handler} + worker._wait_next_notify = mock.Mock() worker.ppid = 1 worker.alive = True worker.sockets = [] @@ -223,7 +279,7 @@ def test__run_exc(worker, loop): yield from worker._run() - m_sleep.assert_called_with(1.0, loop=loop) + assert worker._wait_next_notify.called worker.close.assert_called_with() @@ -238,8 +294,8 @@ def test_close(worker, loop): app = worker.wsgi = mock.Mock() app.cleanup = make_mocked_coro(None) handler.connections = [object()] - handler.finish_connections.return_value = helpers.create_future(loop) - handler.finish_connections.return_value.set_result(1) + handler.shutdown.return_value = helpers.create_future(loop) + handler.shutdown.return_value.set_result(1) app.shutdown.return_value = helpers.create_future(loop) app.shutdown.return_value.set_result(None) @@ -247,7 +303,28 @@ def test_close(worker, loop): yield from worker.close() app.shutdown.assert_called_with() app.cleanup.assert_called_with() - handler.finish_connections.assert_called_with(timeout=95.0) + handler.shutdown.assert_called_with(timeout=95.0) + srv.close.assert_called_with() + assert worker.servers is None + + yield from worker.close() + + +@asyncio.coroutine +def test_close_wsgi(worker, loop): + srv = mock.Mock() + srv.wait_closed = make_mocked_coro(None) + handler = mock.Mock() + worker.servers = {srv: handler} + worker.log = mock.Mock() + worker.loop = loop + worker.wsgi = lambda env, start_resp: start_resp() + handler.connections = [object()] + handler.shutdown.return_value = helpers.create_future(loop) + handler.shutdown.return_value.set_result(1) + + yield from worker.close() + handler.shutdown.assert_called_with(timeout=95.0) srv.close.assert_called_with() assert worker.servers is None |
