diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac73ec2b..ba7cb9cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -93,6 +93,7 @@ repos: rev: v0.45.0 hooks: - id: markdownlint + exclude: docs/source/include-toplevel-* - repo: local diff --git a/README-Unicode.md b/README-Unicode.md index 3f21dff9..3e52c910 100644 --- a/README-Unicode.md +++ b/README-Unicode.md @@ -95,7 +95,7 @@ for i in 2.7 3.{6,7};do echo "$i:"; LC_ALL=C python$i -c 'open("/usr/share/hwdata/pci.ids").read()';done ``` -``` +```text 2.7: 3.6: Traceback (most recent call last): @@ -106,65 +106,31 @@ UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position 97850: ordi 3.7: ``` -This error means that the `'ascii' codec` cannot handle input ord() >= 128, and as some Video cards use `²` to reference their power, the `ascii` codec chokes on them. - -It means `xcp.pci.PCIIds()` cannot use `open("/usr/share/hwdata/pci.ids").read()`. +The `'ascii'` codec fails on all bytes >128. +For example, it cannot decode the bytes representing `²` (UTF-8: power of two) in the PCI IDs database. +To read `/usr/share/hwdata/pci.ids`, we must use `encoding="utf-8"`. While Python 3.7 and newer use UTF-8 mode by default, it does not set up an error handler for `UnicodeDecodeError`. -As it happens, some older tools output ISO-8859-1 characters hard-coded and these aren't valid UTF-8 sequences, and even newer Python versions need error handlers to not fail: +Also, some older tools output ISO-8859-1 characters +These aren't valid UTF-8 sequences. +For all Python versions, we need to use error handlers to handle them: ```sh echo -e "\0262" # ISO-8859-1 for: "²" python3 -c 'open(".text").read()' ``` -``` +```text Traceback (most recent call last): File "", line 1, in File "", line 322, in decode UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb2 in position 0: invalid start byte ``` -Of course, `xcp/net/ifrename` won't be affected but it would be good to fix the -warning for them as well in an intelligent way. See the proposal for that below. - -There are a couple of possibilities and Python because 2.7 does not support the -arguments we need to pass to ensure that all users of open() will work, we need -to make passing the arguments conditional on Python >= 3. - -1. Overriding `open()`, while technically working would not only affect xcp.python but the entire program: - - ```py - if sys.version_info >= (3, 0): - original_open = __builtins__["open"] - def uopen(*args, **kwargs): - if "b" not in (args[1] \ - if len(args) >= 2 else kwargs.get("mode", "")): - kwargs.setdefault("encoding", "UTF-8") - kwargs.setdefault("errors", "replace") - return original_open(*args, **kwargs) - __builtins__["open"] = uopen - ``` - -2. This is sufficient but is not very nice: - - ```py - # xcp/utf8mode.py - if sys.version_info >= (3, 0): - open_utf8args = {"encoding": "utf-8", "errors": "replace"} - else: - open_utf8args = {} - # xcp/{cmd,pci,environ?,logger?}.py tests/test_{pci,biodevname?,...?}.py - + from .utf8mode import open_utf8args - ... - - open(filename) - + open(filename, **open_utf8args) - ``` - - But, `pylint` will still warn about these lines, so I propose: - -3. Instead, use a wrapper function, which will also silence the `pylint` warnings at the locations which have been changed to use it: +To fix these issues, `xcp.compat`, provides a wrapper for `open()`. +It adds `encoding="utf-8", errors="replace"` +to enable UTF-8 conversion and handle encoding errors: ```py # xcp/utf8mode.py @@ -182,9 +148,3 @@ to make passing the arguments conditional on Python >= 3. + utf8open(filename) ``` -Using the 3rd option, the `pylint` warnings for the changed locations -`unspecified-encoding` and `consider-using-with` don't appear without -explicitly disabling them. - -PS: Since utf8open() still returns a context-manager, `with open(...) as f:` -would still work. diff --git a/docs/source/conf.py b/docs/source/conf.py index 97d79512..5407c091 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -48,7 +48,7 @@ myst_heading_anchors = 2 templates_path = ["_templates"] -exclude_patterns = [] +exclude_patterns: list[str] = [] # -- Options for HTML output ------------------------------------------------- diff --git a/stubs/pytest_httpserver.pyi b/stubs/pytest_httpserver.pyi deleted file mode 100644 index 0ee27850..00000000 --- a/stubs/pytest_httpserver.pyi +++ /dev/null @@ -1,134 +0,0 @@ -import abc -from enum import Enum -from ssl import SSLContext -from typing import Any, Callable, Iterable, Mapping, MutableMapping, Optional, Pattern, Tuple, Union - -# pylint: disable=import-error, no-name-in-module, super-init-not-called, multiple-statements, too-few-public-methods, invalid-name, line-too-long -from _typeshed import Incomplete - -from werkzeug.wrappers import Request, Response - -URI_DEFAULT: str -METHOD_ALL: str -HEADERS_T = Union[Mapping[str, Union[str, Iterable[str]]], Iterable[Tuple[str, str]]] -HVMATCHER_T = Callable[[str, Optional[str], str], bool] - -class Error(Exception): ... -class NoHandlerError(Error): ... -class HTTPServerError(Error): ... -class NoMethodFoundForMatchingHeaderValueError(Error): ... - -class WaitingSettings: - raise_assertions: Incomplete - stop_on_nohandler: Incomplete - timeout: Incomplete - def __init__(self, raise_assertions: bool = ..., stop_on_nohandler: bool = ..., timeout: float = ...) -> None: ... - -class Waiting: - def __init__(self) -> None: ... - def complete(self, result: bool): ... - @property - def result(self) -> bool: ... - @property - def elapsed_time(self) -> float: ... - -class HeaderValueMatcher: - DEFAULT_MATCHERS: MutableMapping[str, Callable[[Optional[str], str], bool]] - matchers: Incomplete - def __init__(self, matchers: Optional[Mapping[str, Callable[[Optional[str], str], bool]]] = ...) -> None: ... - @staticmethod - def authorization_header_value_matcher(actual: Optional[str], expected: str) -> bool: ... - @staticmethod - def default_header_value_matcher(actual: Optional[str], expected: str) -> bool: ... - def __call__(self, header_name: str, actual: Optional[str], expected: str) -> bool: ... - -class URIPattern(abc.ABC, metaclass=abc.ABCMeta): - @abc.abstractmethod - def match(self, uri: str) -> bool: ... - -class RequestMatcher: - uri: Incomplete - method: Incomplete - query_string: Incomplete - query_matcher: Incomplete - json: Incomplete - headers: Incomplete - data: Incomplete - data_encoding: Incomplete - header_value_matcher: Incomplete - # def __init__(self) - def match_data(self, request: Request) -> bool: ... - def match_uri(self, request: Request) -> bool: ... - def match_json(self, request: Request) -> bool: ... - def match(self, request: Request) -> bool: ... - -class RequestHandlerBase(abc.ABC, metaclass=abc.ABCMeta): - def respond_with_json(self, response_json, status: int = ..., headers: Optional[Mapping[str, str]] = ..., content_type: str = ...): ... - def respond_with_data(self, response_data: Union[str, bytes] = ..., status: int = ..., headers: Optional[HEADERS_T] = ..., mimetype: Optional[str] = ..., content_type: Optional[str] = ...): ... - @abc.abstractmethod - def respond_with_response(self, response: Response): ... - -class RequestHandler(RequestHandlerBase): - matcher: Incomplete - request_handler: Incomplete - def __init__(self, matcher: RequestMatcher) -> None: ... - def respond(self, request: Request) -> Response: ... - def respond_with_handler(self, func: Callable[[Request], Response]): ... - def respond_with_response(self, response: Response): ... - -class HandlerType(Enum): - PERMANENT: str - ONESHOT: str - ORDERED: str - -class HTTPServerBase(abc.ABC, metaclass=abc.ABCMeta): - host: Incomplete - port: Incomplete - server: Incomplete - server_thread: Incomplete - assertions: Incomplete - handler_errors: Incomplete - log: Incomplete - ssl_context: Incomplete - no_handler_status_code: int - def __init__(self, host: str, port: int, ssl_context: Optional[SSLContext] = ...) -> None: ... - def clear(self) -> None: ... - def clear_assertions(self) -> None: ... - def clear_handler_errors(self) -> None: ... - def clear_log(self) -> None: ... - def url_for(self, suffix: str): ... - def create_matcher(self, *args, **kwargs) -> RequestMatcher: ... - def thread_target(self) -> None: ... - def is_running(self) -> bool: ... - def start(self) -> None: ... - def stop(self) -> None: ... - def add_assertion(self, obj) -> None: ... - def check(self) -> None: ... - def check_assertions(self) -> None: ... - def check_handler_errors(self) -> None: ... - def respond_nohandler(self, request: Request, extra_message: str = ...): ... - @abc.abstractmethod - def dispatch(self, request: Request) -> Response: ... - def application(self, request: Request): ... - def __enter__(self): ... - def __exit__(self, *args, **kwargs) -> None: ... - @staticmethod - def format_host(host): ... - -class HTTPServer(HTTPServerBase): - DEFAULT_LISTEN_HOST: str - DEFAULT_LISTEN_PORT: int - ordered_handlers: Incomplete - oneshot_handlers: Incomplete - handlers: Incomplete - permanently_failed: bool - default_waiting_settings: Incomplete - def __init__(self, host=..., port=..., ssl_context: Optional[SSLContext] = ..., default_waiting_settings: Optional[WaitingSettings] = ...) -> None: ... - def clear(self) -> None: ... - def clear_all_handlers(self) -> None: ... - def expect_request(self, uri: Union[str, URIPattern, Pattern[str]], method: str = ..., data: Union[str, bytes, None] = ..., data_encoding: str = ..., header_value_matcher: Optional[HVMATCHER_T] = ..., handler_type: HandlerType = ..., json: Any = ...) -> RequestHandler: ... - def format_matchers(self) -> str: ... - def respond_nohandler(self, request: Request, extra_message: str = ...): ... - def respond_permanent_failure(self): ... - def dispatch(self, request: Request) -> Response: ... - def wait(self, raise_assertions: Optional[bool] = ..., stop_on_nohandler: Optional[bool] = ..., timeout: Optional[float] = ...): ... diff --git a/tox.ini b/tox.ini index 82d52514..bcfdeab4 100644 --- a/tox.ini +++ b/tox.ini @@ -119,9 +119,9 @@ commands = coverage xml -o {envlogdir}/coverage.xml --fail-under {env:XCP_COVERAGE_MIN:78} coverage lcov -o {envlogdir}/coverage.lcov coverage html -d {envlogdir}/htmlcov - coverage html -d {envlogdir}/htmlcov-tests --fail-under {env:TESTS_COVERAGE_MIN:96} \ + coverage html -d {envlogdir}/htmlcov-tests --fail-under {env:TESTS_COVERAGE_MIN:95} \ --include="tests/*" - diff-cover --compare-branch=origin/master --exclude xcp/dmv.py \ + diff-cover --compare-branch=origin/master \ {env:PY3_DIFFCOVER_OPTIONS} --fail-under {env:DIFF_COVERAGE_MIN:92} \ --html-report {envlogdir}/coverage-diff.html \ {envlogdir}/coverage.xml