mirror of
https://github.com/release-engineering/repo-autoindex.git
synced 2025-02-23 21:52:52 +00:00
eac74ec1e4
The Fetcher type was designed to return a 'str'. That wasn't a good idea because it implies that every fetched file must be loaded into memory completely. On certain large yum repos, decompressed primary XML can be hundreds of MB, and it's not appropriate to require loading that all into memory at once. Make it support a file-like object (stream of bytes). Since the SAX XML parser supports reading from a stream, this makes it possible to avoid loading everything into memory at once. A test of repo-autoindex CLI against /content/dist/rhel/server/7/7Server/x86_64/os showed major improvement: - before: ~1200MiB - after: ~80MiB Note that achieving the full improvement requires any downstream users of the library (e.g. exodus-gw) to update their Fetcher implementation as well, to stop returning a 'str'.
41 lines
967 B
TOML
41 lines
967 B
TOML
[tool.poetry]
|
|
name = "repo-autoindex"
|
|
version = "1.2.0"
|
|
description = "Generic static HTML indexes of various repository types"
|
|
authors = ["Rohan McGovern <rmcgover@redhat.com>"]
|
|
license = "GPL-3.0-or-later"
|
|
readme = "README.md"
|
|
homepage = "https://github.com/release-engineering/repo-autoindex"
|
|
repository = "https://github.com/release-engineering/repo-autoindex"
|
|
|
|
|
|
[tool.poetry.dependencies]
|
|
python = ">=3.9,<4"
|
|
aiohttp = ">=3.8.1"
|
|
defusedxml = ">=0.7.1"
|
|
Jinja2 = ">=3.1.2"
|
|
|
|
[tool.poetry.dev-dependencies]
|
|
pytest = ">=7.1.2"
|
|
mypy = ">=0.961"
|
|
pytest-asyncio = ">=0.18.3"
|
|
pytest-cov = ">=3.0.0"
|
|
Sphinx = "^5.1.1"
|
|
sphinx-argparse = "^0.3.1"
|
|
|
|
[tool.poetry.scripts]
|
|
"repo-autoindex" = "repo_autoindex._impl.cmd:entrypoint"
|
|
|
|
[tool.poetry.group.dev.dependencies]
|
|
bandit = "^1.7.4"
|
|
safety = "^2.3.1"
|
|
|
|
[tool.isort]
|
|
profile = "black"
|
|
|
|
[build-system]
|
|
requires = ["poetry-core>=1.0.0"]
|
|
build-backend = "poetry.core.masonry.api"
|
|
|
|
[tool.pytest.ini_options]
|
|
asyncio_mode = "auto"
|