From 119f0ea9b6c6a37808a40702371ed8e7256929f4 Mon Sep 17 00:00:00 2001 From: Rohan McGovern Date: Wed, 29 Jun 2022 16:19:05 +1000 Subject: [PATCH] Make it all pass mypy --- poetry.lock | 69 +++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + repo_autoindex/__init__.py | 2 ++ repo_autoindex/api.py | 3 +- repo_autoindex/base.py | 2 +- repo_autoindex/cmd.py | 3 +- repo_autoindex/template.py | 2 +- repo_autoindex/tree.py | 7 ++-- repo_autoindex/yum.py | 50 ++++++++++++++++++--------- tox.ini | 2 +- 10 files changed, 116 insertions(+), 25 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0a1dc0c..d7f9cc4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -140,6 +140,32 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "mypy" +version = "0.961" +description = "Optional static typing for Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +mypy-extensions = ">=0.4.3" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=3.10" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "packaging" version = "21.3" @@ -211,6 +237,14 @@ category = "dev" optional = false python-versions = ">=3.7" +[[package]] +name = "typing-extensions" +version = "4.2.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "dev" +optional = false +python-versions = ">=3.7" + [[package]] name = "yarl" version = "1.7.2" @@ -226,7 +260,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = ">=3.9,<4" -content-hash = "de28df6c78aea6f3baa2be842fd72f68fff8bd1e3949a7de27fddc90e28e7a20" +content-hash = "ec537cb22f7488971d12450524aec30918e6d42c6eac78fcd4022fe8293d935a" [metadata.files] aiohttp = [ @@ -507,6 +541,35 @@ multidict = [ {file = "multidict-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:4bae31803d708f6f15fd98be6a6ac0b6958fcf68fda3c77a048a4f9073704aae"}, {file = "multidict-6.0.2.tar.gz", hash = "sha256:5ff3bd75f38e4c43f1f470f2df7a4d430b821c4ce22be384e1459cb57d6bb013"}, ] +mypy = [ + {file = "mypy-0.961-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:697540876638ce349b01b6786bc6094ccdaba88af446a9abb967293ce6eaa2b0"}, + {file = "mypy-0.961-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b117650592e1782819829605a193360a08aa99f1fc23d1d71e1a75a142dc7e15"}, + {file = "mypy-0.961-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bdd5ca340beffb8c44cb9dc26697628d1b88c6bddf5c2f6eb308c46f269bb6f3"}, + {file = "mypy-0.961-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3e09f1f983a71d0672bbc97ae33ee3709d10c779beb613febc36805a6e28bb4e"}, + {file = "mypy-0.961-cp310-cp310-win_amd64.whl", hash = "sha256:e999229b9f3198c0c880d5e269f9f8129c8862451ce53a011326cad38b9ccd24"}, + {file = "mypy-0.961-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b24be97351084b11582fef18d79004b3e4db572219deee0212078f7cf6352723"}, + {file = "mypy-0.961-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f4a21d01fc0ba4e31d82f0fff195682e29f9401a8bdb7173891070eb260aeb3b"}, + {file = "mypy-0.961-cp36-cp36m-win_amd64.whl", hash = "sha256:439c726a3b3da7ca84a0199a8ab444cd8896d95012c4a6c4a0d808e3147abf5d"}, + {file = "mypy-0.961-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5a0b53747f713f490affdceef835d8f0cb7285187a6a44c33821b6d1f46ed813"}, + {file = "mypy-0.961-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0e9f70df36405c25cc530a86eeda1e0867863d9471fe76d1273c783df3d35c2e"}, + {file = "mypy-0.961-cp37-cp37m-win_amd64.whl", hash = "sha256:b88f784e9e35dcaa075519096dc947a388319cb86811b6af621e3523980f1c8a"}, + {file = "mypy-0.961-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d5aaf1edaa7692490f72bdb9fbd941fbf2e201713523bdb3f4038be0af8846c6"}, + {file = "mypy-0.961-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9f5f5a74085d9a81a1f9c78081d60a0040c3efb3f28e5c9912b900adf59a16e6"}, + {file = "mypy-0.961-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f4b794db44168a4fc886e3450201365c9526a522c46ba089b55e1f11c163750d"}, + {file = "mypy-0.961-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:64759a273d590040a592e0f4186539858c948302c653c2eac840c7a3cd29e51b"}, + {file = "mypy-0.961-cp38-cp38-win_amd64.whl", hash = "sha256:63e85a03770ebf403291ec50097954cc5caf2a9205c888ce3a61bd3f82e17569"}, + {file = "mypy-0.961-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5f1332964963d4832a94bebc10f13d3279be3ce8f6c64da563d6ee6e2eeda932"}, + {file = "mypy-0.961-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:006be38474216b833eca29ff6b73e143386f352e10e9c2fbe76aa8549e5554f5"}, + {file = "mypy-0.961-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9940e6916ed9371809b35b2154baf1f684acba935cd09928952310fbddaba648"}, + {file = "mypy-0.961-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a5ea0875a049de1b63b972456542f04643daf320d27dc592d7c3d9cd5d9bf950"}, + {file = "mypy-0.961-cp39-cp39-win_amd64.whl", hash = "sha256:1ece702f29270ec6af25db8cf6185c04c02311c6bb21a69f423d40e527b75c56"}, + {file = "mypy-0.961-py3-none-any.whl", hash = "sha256:03c6cc893e7563e7b2949b969e63f02c000b32502a1b4d1314cabe391aa87d66"}, + {file = "mypy-0.961.tar.gz", hash = "sha256:f730d56cb924d371c26b8eaddeea3cc07d78ff51c521c6d04899ac6904b75492"}, +] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, @@ -531,6 +594,10 @@ tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +typing-extensions = [ + {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, + {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, +] yarl = [ {file = "yarl-1.7.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f2a8508f7350512434e41065684076f640ecce176d262a7d54f0da41d99c5a95"}, {file = "yarl-1.7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da6df107b9ccfe52d3a48165e48d72db0eca3e3029b5b8cb4fe6ee3cb870ba8b"}, diff --git a/pyproject.toml b/pyproject.toml index 96278ca..aa07567 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ Jinja2 = ">=3.1.2" [tool.poetry.dev-dependencies] pytest = ">=7.1.2" +mypy = ">=0.961" [tool.poetry.scripts] "repo-autoindex" = "repo_autoindex.cmd:entrypoint" diff --git a/repo_autoindex/__init__.py b/repo_autoindex/__init__.py index c8410c8..e4e075a 100644 --- a/repo_autoindex/__init__.py +++ b/repo_autoindex/__init__.py @@ -1,2 +1,4 @@ from .api import autoindex from .base import Fetcher, GeneratedIndex + +__all__ = ["autoindex", "Fetcher", "GeneratedIndex"] diff --git a/repo_autoindex/api.py b/repo_autoindex/api.py index 95504ee..b555811 100644 --- a/repo_autoindex/api.py +++ b/repo_autoindex/api.py @@ -1,6 +1,7 @@ import gzip import logging from collections.abc import AsyncGenerator +from typing import Optional import aiohttp @@ -35,7 +36,7 @@ def http_fetcher(session: aiohttp.ClientSession) -> Fetcher: async def autoindex( url: str, *, - fetcher: Fetcher = None, + fetcher: Optional[Fetcher] = None, index_href_suffix: str = "", ) -> AsyncGenerator[GeneratedIndex, None]: if fetcher is None: diff --git a/repo_autoindex/base.py b/repo_autoindex/base.py index 3fcc330..4362007 100644 --- a/repo_autoindex/base.py +++ b/repo_autoindex/base.py @@ -46,7 +46,7 @@ class Repo(ABC): self.fetcher = fetcher @abstractmethod - async def render_index( + def render_index( self, index_href_suffix: str ) -> AsyncGenerator[GeneratedIndex, None]: pass diff --git a/repo_autoindex/cmd.py b/repo_autoindex/cmd.py index 546a104..b521e75 100644 --- a/repo_autoindex/cmd.py +++ b/repo_autoindex/cmd.py @@ -3,14 +3,13 @@ import asyncio import gzip import logging import os -from collections.abc import AsyncGenerator from repo_autoindex import autoindex LOG = logging.getLogger("autoindex") -async def dump_autoindices(args): +async def dump_autoindices(args: argparse.Namespace) -> None: index_filename = args.index_filename async for index in autoindex(args.url, index_href_suffix=index_filename): os.makedirs(index.relative_dir or ".", exist_ok=True) diff --git a/repo_autoindex/template.py b/repo_autoindex/template.py index 7c24c93..e027b84 100644 --- a/repo_autoindex/template.py +++ b/repo_autoindex/template.py @@ -10,7 +10,7 @@ TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates") class TemplateContext: - def __init__(self): + def __init__(self) -> None: self.env = jinja2.Environment( autoescape=True, loader=jinja2.FileSystemLoader(TEMPLATE_DIR) ) diff --git a/repo_autoindex/tree.py b/repo_autoindex/tree.py index 3af9dfa..168bda4 100644 --- a/repo_autoindex/tree.py +++ b/repo_autoindex/tree.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field, replace +from collections.abc import Iterable from .base import ICON_FOLDER, IndexEntry @@ -11,7 +12,9 @@ class TreeNode: def treeify( - all_entries: list[IndexEntry], relative_dir: str = "", index_href_suffix: str = "" + all_entries: Iterable[IndexEntry], + relative_dir: str = "", + index_href_suffix: str = "", ) -> TreeNode: out = TreeNode(relative_dir=relative_dir) @@ -24,7 +27,7 @@ def treeify( ) ) - entries_by_leading_dir = {} + entries_by_leading_dir: dict[str, list[IndexEntry]] = {} for entry in all_entries: subdir = entry.href.removeprefix(relative_dir + "/") components = subdir.split("/", 1) diff --git a/repo_autoindex/yum.py b/repo_autoindex/yum.py index b3bba9b..3f23d2f 100644 --- a/repo_autoindex/yum.py +++ b/repo_autoindex/yum.py @@ -7,16 +7,28 @@ from typing import Optional, Type from xml.dom.minidom import Element from xml.dom.pulldom import END_ELEMENT, START_ELEMENT, DOMEventStream -from defusedxml import pulldom +from defusedxml import pulldom # type: ignore -from .base import (ICON_FOLDER, ICON_PACKAGE, Fetcher, GeneratedIndex, - IndexEntry, Repo) +from .base import ICON_FOLDER, ICON_PACKAGE, Fetcher, GeneratedIndex, IndexEntry, Repo from .template import TemplateContext from .tree import treeify LOG = logging.getLogger("autoindex") +def get_tag(elem: Element, name: str) -> Element: + elems: list[Element] = elem.getElementsByTagName(name) # type: ignore + return elems[0] + + +def get_text_tag(elem: Element, name: str) -> str: + tagnode = get_tag(elem, name) + child = tagnode.firstChild + # TODO: raise proper error if missing + assert child + return str(child.toxml()) + + @dataclass class Package: href: str @@ -26,10 +38,10 @@ class Package: @classmethod def from_element(cls, elem: Element) -> "Package": return cls( - href=elem.getElementsByTagName("location")[0].attributes["href"].value, + href=get_tag(elem, "location").attributes["href"].value, # TODO: tolerate some of these being absent or wrong. - time=elem.getElementsByTagName("time")[0].attributes["file"].value, - size=elem.getElementsByTagName("size")[0].attributes["package"].value, + time=get_tag(elem, "time").attributes["file"].value, + size=get_tag(elem, "size").attributes["package"].value, ) @property @@ -45,7 +57,7 @@ class Package: def pulldom_elements( xml_str: str, path_matcher, attr_matcher=lambda _: True -) -> Generator[Element]: +) -> Generator[Element, None, None]: stream = pulldom.parseString(xml_str) current_path = [] for event, node in stream: @@ -91,8 +103,11 @@ class YumRepo(Repo): ) ) if len(revision_nodes) == 1: + timestamp_node = revision_nodes[0].firstChild + # TODO: raise proper error + assert timestamp_node time = datetime.datetime.utcfromtimestamp( - float(revision_nodes[0].firstChild.toxml()) + timestamp_node.toxml() ).isoformat() out.append( @@ -100,7 +115,7 @@ class YumRepo(Repo): href="repodata/repomd.xml", text="repomd.xml", time=time, - size=size, + size=str(size), ) ) @@ -111,14 +126,14 @@ class YumRepo(Repo): attr_matcher=lambda attrs: attrs.get("type"), ) ) - data_nodes.sort(key=lambda node: node.attributes["type"].value) + data_nodes.sort(key=lambda node: str(node.attributes["type"].value)) for node in data_nodes: - href = node.getElementsByTagName("location")[0].attributes["href"].value + href = get_tag(node, "location").attributes["href"].value basename = os.path.basename(href) - timestamp = node.getElementsByTagName("timestamp")[0].firstChild.toxml() + timestamp = get_text_tag(node, "timestamp") time = datetime.datetime.utcfromtimestamp(float(timestamp)).isoformat() - size = int(node.getElementsByTagName("size")[0].firstChild.toxml()) + size = int(get_text_tag(node, "size")) out.append( IndexEntry( @@ -144,12 +159,15 @@ class YumRepo(Repo): assert len(primary_nodes) == 1 primary_node = primary_nodes[0] - location = primary_node.getElementsByTagName("location")[0] + location = get_tag(primary_node, "location") href = location.attributes["href"].value primary_url = "/".join([self.base_url, href]) primary_xml = await self.fetcher(primary_url) + # TODO: raise proper error if missing + assert primary_xml + return sorted( [p.index_entry for p in self.__packages_from_primary(primary_xml)], key=lambda e: e.text, @@ -175,7 +193,7 @@ class YumRepo(Repo): self, entries: Iterable[IndexEntry], index_href_suffix: str, - ) -> Generator[GeneratedIndex, None]: + ) -> Generator[GeneratedIndex, None, None]: ctx = TemplateContext() nodes = [treeify(entries, index_href_suffix=index_href_suffix)] while nodes: @@ -197,7 +215,7 @@ class YumRepo(Repo): if repomd_xml is None: # not yum repo - return + return None # it is a yum repo return cls(url, repomd_xml, fetcher) diff --git a/tox.ini b/tox.ini index 8e0288c..be0ab06 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] isolated_build = True -envlist = py39,docs,precommit,mypy +envlist = py39,precommit,mypy envdir = {toxworkdir}/poetry [testenv]