From b284ddbd9fd0ee0b2bfd22768d0a86ed090b1167 Mon Sep 17 00:00:00 2001 From: Caleigh Runge-Hottman Date: Thu, 23 Mar 2023 13:58:07 -0400 Subject: [PATCH] Generate kickstart repo index [RHELDST-14528] Due to the presence of a "repodata/repomd.xml" path in a kickstart repo, repo-autoindex previously interpreted kickstart repos as yum repos. As such, a kickstart repo's index would solely consist of two directories: "Packages" and "repodata". While a kickstart repo does contain a yum repo, kickstart repos also contain two additional repo entry points: treeinfo and extra_files.json. Each entry point references additional files that should be included in a kickstart repo's index. These files were previously ignored. Now, when repo-autoindex encounters a kickstart repo, repo-autoindex produces a repo index that reflects the content referenced in all three repo entry points (repomd.xml, treeinfo, extra_files.json). --- docs/index.rst | 5 + repo_autoindex/_impl/api.py | 4 +- repo_autoindex/_impl/kickstart.py | 235 +++++++ repo_autoindex/_impl/yum.py | 8 +- .../Packages/w/walrus-5.21-1.noarch.rpm | Bin 0 -> 2445 bytes tests/sample_kickstart_repo/extra_files.json | 43 ++ tests/sample_kickstart_repo/images/boot.iso | 0 .../sample_kickstart_repo/images/efiboot.img | 0 .../sample_kickstart_repo/images/install.img | 0 .../images/pxeboot/initrd.img | 0 .../images/pxeboot/vmlinuz | 0 ...384ad654e7214266648c37f0b-filelists.xml.gz | Bin 0 -> 243 bytes ...529bbfa22d0c890285ce6aa3129-primary.xml.gz | Bin 0 -> 598 bytes ...e3d28420987132d7d2c176127b9db-other.xml.gz | Bin 0 -> 229 bytes .../sample_kickstart_repo/repodata/repomd.xml | 28 + tests/sample_kickstart_repo/treeinfo | 56 ++ tests/test_kickstart_render_typical.py | 576 ++++++++++++++++++ 17 files changed, 950 insertions(+), 5 deletions(-) create mode 100644 repo_autoindex/_impl/kickstart.py create mode 100644 tests/sample_kickstart_repo/Packages/w/walrus-5.21-1.noarch.rpm create mode 100644 tests/sample_kickstart_repo/extra_files.json create mode 100644 tests/sample_kickstart_repo/images/boot.iso create mode 100644 tests/sample_kickstart_repo/images/efiboot.img create mode 100644 tests/sample_kickstart_repo/images/install.img create mode 100644 tests/sample_kickstart_repo/images/pxeboot/initrd.img create mode 100644 tests/sample_kickstart_repo/images/pxeboot/vmlinuz create mode 100644 tests/sample_kickstart_repo/repodata/33795fed0c0144a7fe732a9ded8d7529940e4a1384ad654e7214266648c37f0b-filelists.xml.gz create mode 100644 tests/sample_kickstart_repo/repodata/3a7a286e13883d497b2e3c7029ceb7c372ff2529bbfa22d0c890285ce6aa3129-primary.xml.gz create mode 100644 tests/sample_kickstart_repo/repodata/834b12e38d809c4b5afd1a7c03ad48c0e15e3d28420987132d7d2c176127b9db-other.xml.gz create mode 100644 tests/sample_kickstart_repo/repodata/repomd.xml create mode 100644 tests/sample_kickstart_repo/treeinfo create mode 100644 tests/test_kickstart_render_typical.py diff --git a/docs/index.rst b/docs/index.rst index 779a841..5be4c64 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ It supports: - yum repositories (``repodata/repomd.xml``) - pulp file repositories (``PULP_MANIFEST``) +- kickstart tree repositories\* (``treeinfo``, ``repodata/repomd.xml``, ``extra_files.json``) ``repo-autoindex`` provides similar functionality to traditional server-generated directory indexes such as httpd's @@ -28,6 +29,10 @@ a few key differences: library to integrate with exotic scenarios such as repositories generated on demand or not stored within a traditional filesystem. +\* ``repo-autoindex`` supports kickstart tree repositories satisfying certain conditions: + +- The kickstart repo contains exactly one yum repo +- The yum repo is located in the root of the kickstart tree repo, at exactly ``.`` Reference: CLI -------------- diff --git a/repo_autoindex/_impl/api.py b/repo_autoindex/_impl/api.py index b094190..4954e54 100644 --- a/repo_autoindex/_impl/api.py +++ b/repo_autoindex/_impl/api.py @@ -8,9 +8,10 @@ import aiohttp from .base import Fetcher, GeneratedIndex, Repo, ContentError, FetcherError from .yum import YumRepo from .pulp import PulpFileRepo +from .kickstart import KickstartRepo LOG = logging.getLogger("repo-autoindex") -REPO_TYPES: list[Type[Repo]] = [YumRepo, PulpFileRepo] +REPO_TYPES: list[Type[Repo]] = [KickstartRepo, YumRepo, PulpFileRepo] def http_fetcher(session: aiohttp.ClientSession) -> Fetcher: @@ -133,6 +134,7 @@ async def autoindex( index_href_suffix=index_href_suffix ): yield page + break except FetcherError as exc: # FetcherErrors are unwrapped to propagate whatever was the original error assert exc.__cause__ diff --git a/repo_autoindex/_impl/kickstart.py b/repo_autoindex/_impl/kickstart.py new file mode 100644 index 0000000..df2572e --- /dev/null +++ b/repo_autoindex/_impl/kickstart.py @@ -0,0 +1,235 @@ +from typing import Optional, Type +from collections.abc import AsyncGenerator +import logging +import configparser +import json +import os + +from .base import Repo, GeneratedIndex, Fetcher, IndexEntry, ICON_OPTICAL, ICON_QCOW +from .template import TemplateContext +from .tree import treeify +from .yum import YumRepo + +LOG = logging.getLogger("repo-autoindex") + + +class KickstartRepo(YumRepo): + def __init__( + self, + base_url: str, + repomd_xml: str, + extra_files: str, + treeinfo: str, + fetcher: Fetcher, + ): + super().__init__(base_url, repomd_xml, fetcher) + self.base_url = base_url + self.fetcher = fetcher + self.entry_point_content = repomd_xml + self.extra_files_content = extra_files + self.treeinfo_content = treeinfo + + async def render_index( + self, index_href_suffix: str + ) -> AsyncGenerator[GeneratedIndex, None]: + all_entries: list[IndexEntry] = [] + + # Parse the treeinfo entry point + LOG.debug("treeinfo: %s", self.treeinfo_content) + all_entries.extend(await self._treeinfo_entries()) + + # Parse the extra_files.json entry point + # + # Legacy kickstart tree repositories do not contain extra_files.json files. + # If the repo does not contain an extra_files.json, do not attempt to process it. + if self.extra_files_content: + LOG.debug("extra_files.json: %s", self.extra_files_content) + all_entries.extend(await self._extra_files_entries()) + + # Parse the yum repo embedded in the kickstart repo + LOG.debug("repomd.xml: %s", self.entry_point_content) + all_entries.extend(await super()._repodata_entries()) + all_entries.extend(await super()._package_entries()) + + ctx = TemplateContext() + nodes = [treeify(all_entries, index_href_suffix=index_href_suffix)] + while nodes: + node = nodes.pop() + yield GeneratedIndex( + content=ctx.render_index(index_entries=node.entries), + relative_dir=node.relative_dir, + ) + nodes.extend(node.children) + + async def _treeinfo_entries(self) -> list[IndexEntry]: + """ + A treeinfo file might look like this: + + [checksums] + images/boot.iso = sha256:f6be6ec48a4a610e25d591dcf98e1777c4274ed58c583fa64d0aea5b3ecffb18 + images/efiboot.img = sha256:94d5500c4ba266ce77b06aa955d9041eea22129737badc6af56c283dcaec1c29 + images/install.img = sha256:46171146377610cfa0deae157bbcc4ea146b3995c9b0c58d9f261ce404468abe + images/pxeboot/initrd.img = sha256:e0cd3966097c175d3aaf406a7f8c094374c69504c7be8f08d8084ab9a8812796 + images/pxeboot/vmlinuz = sha256:370db9a3943d4f46dc079dbaeb7e0cc3910dca069f7eede66d3d7d0d5177f684 + + [general] + ; WARNING.0 = This section provides compatibility with pre-productmd treeinfos. + ; WARNING.1 = Read productmd documentation for details about new format. + arch = x86_64 + family = Red Hat Enterprise Linux + name = Red Hat Enterprise Linux 8.0.0 + packagedir = Packages + platforms = x86_64,xen + repository = . + timestamp = 1554367044 + variant = BaseOS + variants = BaseOS + version = 8.0.0 + + [header] + type = productmd.treeinfo + version = 1.2 + + [images-x86_64] + boot.iso = images/boot.iso + efiboot.img = images/efiboot.img + initrd = images/pxeboot/initrd.img + kernel = images/pxeboot/vmlinuz + + [images-xen] + initrd = images/pxeboot/initrd.img + kernel = images/pxeboot/vmlinuz + + [release] + name = Red Hat Enterprise Linux + short = RHEL + version = 8.0.0 + + [stage2] + mainimage = images/install.img + + [tree] + arch = x86_64 + build_timestamp = 1554367044 + platforms = x86_64,xen + variants = BaseOS + + [variant-BaseOS] + id = BaseOS + name = BaseOS + packages = Packages + repository = . + type = variant + uid = BaseOS + """ + out: list[IndexEntry] = [ + IndexEntry( + href="treeinfo", + text="treeinfo", + size=str(len(self.treeinfo_content)), + ), + ] + + treeinfo = configparser.ConfigParser() + treeinfo.read_string(self.treeinfo_content) + + for image in treeinfo["checksums"]: + entry = IndexEntry( + href=image, + text=os.path.basename(image), + ) + if entry.href.endswith(".iso") or entry.href.endswith(".img"): + entry.icon = ICON_OPTICAL + out.append(entry) + return out + + async def _extra_files_entries(self) -> list[IndexEntry]: + """ + An extra_files.json file might look like this: + + { + "data": [ + { + "checksums": { + "md5": "feb4d252ee63634debea654b446e830b", + "sha1": "a73fad5aeb5642d1b2108885010c4e7a547a1204", + "sha256": "c4117d0e325cde392981626edbd1484c751f0216689a171e4b7547e8800acc21" + }, + "file": "RPM-GPG-KEY-redhat-release", + "size": 5134 + }, + { + "checksums": { + "md5": "3c24137e12ece142a27bbf825c256936", + "sha1": "a72daf8585b41529269cdffcca3a0b3d4e2f21cd", + "sha256": "3f8644b35db4197e7689d0a034bdef2039d92e330e6b22217abfa6b86a1fc0fa" + }, + "file": "RPM-GPG-KEY-redhat-beta", + "size": 1669 + }, + { + "checksums": { + "md5": "b234ee4d69f5fce4486a80fdaf4a4263", + "sha1": "4cc77b90af91e615a64ae04893fdffa7939db84c", + "sha256": "8177f97513213526df2cf6184d8ff986c675afb514d4e68a404010521b880643" + }, + "file": "GPL", + "size": 18092 + }, + { + "checksums": { + "md5": "0c53898068810a989fa59ca0656bdf24", + "sha1": "42d51858642b8a0d10fdf09050266395544ea556", + "sha256": "8f833ce3fbcbcb82e47687a890c043332c88350ddabd606201556e14aaf8fcd9" + }, + "file": "EULA", + "size": 8154 + } + ], + "header": { + "version": "1.0" + } + } + """ + out: list[IndexEntry] = [ + IndexEntry( + href="extra_files.json", + text="extra_files.json", + size=str(len(self.extra_files_content)), + ), + ] + + extra_files = json.loads(self.extra_files_content) + for extra_file in extra_files["data"]: + entry = IndexEntry( + href=extra_file["file"], + text=extra_file["file"], + size=extra_file["size"], + ) + out.append(entry) + return out + + @classmethod + async def probe( + cls: Type["KickstartRepo"], fetcher: Fetcher, url: str + ) -> Optional["KickstartRepo"]: + treeinfo_url = f"{url}/treeinfo" + treeinfo_content = await fetcher(treeinfo_url) + extra_files_url = f"{url}/extra_files.json" + extra_files_content = await fetcher(extra_files_url) or "" + repomd_xml_url = f"{url}/repodata/repomd.xml" + repomd_xml = await fetcher(repomd_xml_url) + + # Modern versions of kickstart repositories (RHEL-8, 9) contain three entry points: + # treeinfo, extra_files.json, and repomd.xml. repo-autoindex requires that a kickstart + # repo contains a treeinfo file and exactly one yum repo located in the root of the + # kickstart tree repo. + # + # Legacy kickstart tree repositories do not contain an extra_files.json file. When + # repo-autoindex encounters a legacy kickstart tree repository, it will attempt to + # produce a repo index. The repo index produced by repo-autoindex will not contain the + # files commonly included in extra_files.json (EULA, GPL, GPG keys). + if treeinfo_content is None or repomd_xml is None: + return None + + return cls(url, repomd_xml, extra_files_content, treeinfo_content, fetcher) diff --git a/repo_autoindex/_impl/yum.py b/repo_autoindex/_impl/yum.py index 5d1a9b7..96dc935 100644 --- a/repo_autoindex/_impl/yum.py +++ b/repo_autoindex/_impl/yum.py @@ -128,13 +128,13 @@ class YumRepo(Repo): LOG.debug("repomd.xml: %s", self.entry_point_content) entries = [] - entries.extend(await self.__repodata_entries()) - entries.extend(await self.__package_entries()) + entries.extend(await self._repodata_entries()) + entries.extend(await self._package_entries()) for page in self.__render_entries(entries, index_href_suffix): yield page - async def __repodata_entries(self) -> list[IndexEntry]: + async def _repodata_entries(self) -> list[IndexEntry]: out = [] # There's always an entry for repomd.xml itself... @@ -189,7 +189,7 @@ class YumRepo(Repo): return out - async def __package_entries(self) -> list[IndexEntry]: + async def _package_entries(self) -> list[IndexEntry]: primary_nodes = list( pulldom_elements( diff --git a/tests/sample_kickstart_repo/Packages/w/walrus-5.21-1.noarch.rpm b/tests/sample_kickstart_repo/Packages/w/walrus-5.21-1.noarch.rpm new file mode 100644 index 0000000000000000000000000000000000000000..1e833d9e2bbe7c02397f08aeba3c709001d71a39 GIT binary patch literal 2445 zcmb7`e^69a6vyu_EMewvO{CNXHSxzT@9i(xQVf?A5ET*?49vH?Z((C$x827Ne~gL) zCE^%pfT7}#s5Z^iB4r9i5{Zx$(1da@C(Ho@$;P1^vF)7QYhz9Q(cQW4dq3}-d*6HK z-gEEv9l3MOi$D=&a~6B1L#CE1SQ(3V{(lh>$v->oINB|u_nsg`oCobNsMr=jN5X@j z@-SS9$Pel;X!v;A1p0b)1A7BUpabx2Q0Om;^!Rx7dRV*$6vp%F2L*qz4OCPh8s{Y; zOZ9=p{rSyZzrA&$L?o_A^CkPsAGBATH-~<(sV1T+cU#Ye^+^@Be(C5=AK8}Tdr91g zc|Ex=T&^4H@;b77{<~i!ANee~{YacwXT^7Zhgm^lel0X@_pI3sFMSg@+&1dhmG=@0 zS0(ip?v5Whx~%2?s!bOq+wT4l_U;#F&eTjAzkBm=U?%sn%x-=#sO)GCS~cbb6T4=Z`qah?HU3YQ zL^hP&cQlQty_j%vqHw)_Wo=P%Y{B?f4w31n4?T>i?b2zwrfi<5T@9?VbP7N z(LarePLjHEs^SvIl>FSrPM&hn+M=&d>o6VAYKNt74RzP=kuot3&jBh)Tt% zRjeVD4K*?$Y7@i|iqEuvi=(N?mz;|IHaT@k)}89CU0btE9hY|mTKpq?nvZ{-*6oZ@ z#GKpsdG_qvzLN*6Ve7}mUyrT%w7YUwneM>sfVeuf-)@~NC;Qy`?^hpR@xFpYN0z?V zGV-hTd2tGFs@)!j+p{P#?mUfumZS31-g=ejZ3`Fio`zDvvYe;=VP zsp2M$6&hw}n=46*KOoiPg`qeoga4_LlmR-kaN|Pr3GsL@D~2 z9G@0{EwS^&^@ZEWt1it=5yS6-d>%&2{$$^})=wJaIv320byOCP-$mRt}7%)Q24 zE>{t>VNDY9SRSZhZOY)*JgArl6G73(2-O!VmPhLz@_>jb_kbb~p^SNeJe)G-0orgZ z?B7RuCgnSzkQd}p9$}tff5;=Z|6NeD$5S2vMSB9}-$8Nw)szQ8(T;hH&wEI@jPf6p z_j%YG$HrPo8T;ba+yD4XZ;`{W1!n}KDd+cK9abFT#>=sWCG>eXZ2S8Pd2EUTQT-sl$I; z;FO4Tq`%I~vK=fWDVCWFV`H)uBq~GT?HRlf9B#8&%mz*{TQeLaRS;~?2L}rd!Oj^{ ztpX=E@kXniv+-7&g_m3HDa2yV$jl)bR?conb&r=@N$@m^RBH?-g@y~!>lI9>R;vvO zF&H?G)2KBXwTjVVtlh2Ef`xQjFp(`)NM$KfS-M6A7a=o)pPRyG z@OHC-7;I)MNm*yM5$&T;c_JpW^Tt$8ka?o$c?zzD@MXHFD5=21;6}5ZHwaey+DBGU zW&h6lh;_$i9nG)LYGfxk(d6E2U9LggJVs6~#5D!QOUZ=5PsK3INrq8L+I*gVqu@E8 zj@*M$YigWdcQ&EDCBD7!Y|GZd+&Xe=g`S+a-4Z@}`>4@3 WyL-C)E1k|pSmQ%PLG3{>bAJMMz);!% literal 0 HcmV?d00001 diff --git a/tests/sample_kickstart_repo/extra_files.json b/tests/sample_kickstart_repo/extra_files.json new file mode 100644 index 0000000..83fcba7 --- /dev/null +++ b/tests/sample_kickstart_repo/extra_files.json @@ -0,0 +1,43 @@ +{ + "data": [ + { + "checksums": { + "md5": "feb4d252ee63634debea654b446e830b", + "sha1": "a73fad5aeb5642d1b2108885010c4e7a547a1204", + "sha256": "c4117d0e325cde392981626edbd1484c751f0216689a171e4b7547e8800acc21" + }, + "file": "RPM-GPG-KEY-redhat-release", + "size": 5134 + }, + { + "checksums": { + "md5": "3c24137e12ece142a27bbf825c256936", + "sha1": "a72daf8585b41529269cdffcca3a0b3d4e2f21cd", + "sha256": "3f8644b35db4197e7689d0a034bdef2039d92e330e6b22217abfa6b86a1fc0fa" + }, + "file": "RPM-GPG-KEY-redhat-beta", + "size": 1669 + }, + { + "checksums": { + "md5": "b234ee4d69f5fce4486a80fdaf4a4263", + "sha1": "4cc77b90af91e615a64ae04893fdffa7939db84c", + "sha256": "8177f97513213526df2cf6184d8ff986c675afb514d4e68a404010521b880643" + }, + "file": "GPL", + "size": 18092 + }, + { + "checksums": { + "md5": "0c53898068810a989fa59ca0656bdf24", + "sha1": "42d51858642b8a0d10fdf09050266395544ea556", + "sha256": "8f833ce3fbcbcb82e47687a890c043332c88350ddabd606201556e14aaf8fcd9" + }, + "file": "EULA", + "size": 8154 + } + ], + "header": { + "version": "1.0" + } +} diff --git a/tests/sample_kickstart_repo/images/boot.iso b/tests/sample_kickstart_repo/images/boot.iso new file mode 100644 index 0000000..e69de29 diff --git a/tests/sample_kickstart_repo/images/efiboot.img b/tests/sample_kickstart_repo/images/efiboot.img new file mode 100644 index 0000000..e69de29 diff --git a/tests/sample_kickstart_repo/images/install.img b/tests/sample_kickstart_repo/images/install.img new file mode 100644 index 0000000..e69de29 diff --git a/tests/sample_kickstart_repo/images/pxeboot/initrd.img b/tests/sample_kickstart_repo/images/pxeboot/initrd.img new file mode 100644 index 0000000..e69de29 diff --git a/tests/sample_kickstart_repo/images/pxeboot/vmlinuz b/tests/sample_kickstart_repo/images/pxeboot/vmlinuz new file mode 100644 index 0000000..e69de29 diff --git a/tests/sample_kickstart_repo/repodata/33795fed0c0144a7fe732a9ded8d7529940e4a1384ad654e7214266648c37f0b-filelists.xml.gz b/tests/sample_kickstart_repo/repodata/33795fed0c0144a7fe732a9ded8d7529940e4a1384ad654e7214266648c37f0b-filelists.xml.gz new file mode 100644 index 0000000000000000000000000000000000000000..77d3cec661439d97b1a512d3470bf484e0e53d4a GIT binary patch literal 243 zcmVhiwFP!0000013gbma)cld-RBfk?<0buk`iZ^6C?*fY-T7CEycv+2Pf97 z`uF;M+S@UC@`LN<=AeLOIS>qH9$dI9`2PBQZ{XIwwbps`Zi^d9_^>H>#JGG=>Rs56 zY}iN4aMEjs(+`8hbd6*g?`}|lO;PKr5XMMpWmW4sw?*0Ws_C^B z9CKSTZkWhZ+o0s7wW80ZWmby1upCIx6Ds)Ce%+IrUd=;6n9rKz2x+fy1ef_drGROp tKxCW&S<#=q=o5LG?r5AA`p=fdBeu^d$v3@>pY1DM_Xh+{9&JDY002!$cZL7} literal 0 HcmV?d00001 diff --git a/tests/sample_kickstart_repo/repodata/3a7a286e13883d497b2e3c7029ceb7c372ff2529bbfa22d0c890285ce6aa3129-primary.xml.gz b/tests/sample_kickstart_repo/repodata/3a7a286e13883d497b2e3c7029ceb7c372ff2529bbfa22d0c890285ce6aa3129-primary.xml.gz new file mode 100644 index 0000000000000000000000000000000000000000..9455510ef5907ba659f3ff09bee7ebf4630400b2 GIT binary patch literal 598 zcmV-c0;&BUiwFP!000001BH`KbKD>dhVS_m4EHtb&$Tl)nH*Y)RWuCtlz@`D?$oKGKfZt1<@AByl><0Y z3nvKT%2u>>ZoFr#@6_}X)YBOPs3*2kOzL6Kiq7=C8HaZ>kfCH0@+?k(=Z6%FE-rD` z5i05len&{D!~pzR^k%XpyE#58jBL5meks|_@zJF-0;85~MX@`KR5T5ukP-vEuf%So zY(bvwG%VF@E21>RE1geWUD2OU-zbMXK8P$%r7Vi3$PVJLZ{p+_W%*HvG=hDTgpmx> zJ(@WbQPMQ&abGlHQ>2HuiAuH>xmCn<5a!AsNsZQZ?Vjj{tlh1M$7`_Cbfd#|{+IF= z^JxoWc*$N(AD>Lmmzg;0jF7E%B50u2MvS1x9s+GnCG(u;T{i$}x*oL&v$ROUB1=%i z}sIXp|%=C7rTP^gSS-M3nd;br-)|WJ^}BrSLZ=t%0VZ zD86OSY;JAmhqeoYdB%Z%4s%kY%^;jN(ZTS2-$_sw_{T4wFDUxk&<0&VVXJ)JUYu?B zWb`!hPs)K&;Mn6h;t98_**RkdwuGAwt+tNa;TPF`0!p(O~)nJe-CrlNCsN}67r k@Yg+krSJa~Jcj*yc*)iwyuLDTuD_fA0m#(_c4q_t03ppUSpWb4 literal 0 HcmV?d00001 diff --git a/tests/sample_kickstart_repo/repodata/834b12e38d809c4b5afd1a7c03ad48c0e15e3d28420987132d7d2c176127b9db-other.xml.gz b/tests/sample_kickstart_repo/repodata/834b12e38d809c4b5afd1a7c03ad48c0e15e3d28420987132d7d2c176127b9db-other.xml.gz new file mode 100644 index 0000000000000000000000000000000000000000..1c896caf79e35554ec052f941394158ac2e19ed7 GIT binary patch literal 229 zcmVqlo7b(MgGHI72prtp_Tg#q>+|Ei!pHrsp7)N|mUe>3hs|QQ@5@by zu@A>9X^(*uZVxf>PA4V)c!Y)AKu?am#CYXuVHuu%Ymuw0q$1PK8RJb+QknWJZ&Xn= zL|Sp`bE%xvX_OTkm3yz7)OhKQE;FwX0!?i3hsO2T5NLIs#W0^Wz5}SQV8F|Ku41@2 fv8WRz5mp{gUm_Y4m$Os;N4@_7j*tA;9RUCU^389# literal 0 HcmV?d00001 diff --git a/tests/sample_kickstart_repo/repodata/repomd.xml b/tests/sample_kickstart_repo/repodata/repomd.xml new file mode 100644 index 0000000..3dd3bd0 --- /dev/null +++ b/tests/sample_kickstart_repo/repodata/repomd.xml @@ -0,0 +1,28 @@ + + + 1659419679 + + 3a7a286e13883d497b2e3c7029ceb7c372ff2529bbfa22d0c890285ce6aa3129 + ad4149ec99b72282ab4891ea5d224db02cc3d7e0ad5c1bdaba56c21cbd4ab132 + + 1659419679 + 598 + 1127 + + + 33795fed0c0144a7fe732a9ded8d7529940e4a1384ad654e7214266648c37f0b + ae8aa2cca2e1eba056ed56a66da2b1f6cdb142e465a13bb55f603c7481239e39 + + 1659419679 + 243 + 320 + + + 834b12e38d809c4b5afd1a7c03ad48c0e15e3d28420987132d7d2c176127b9db + ee1c6e87c3b7ebfa2e85d9b56e245ef097a0d928794da75ab63d43ac5593d9d0 + + 1659419679 + 229 + 285 + + diff --git a/tests/sample_kickstart_repo/treeinfo b/tests/sample_kickstart_repo/treeinfo new file mode 100644 index 0000000..708c381 --- /dev/null +++ b/tests/sample_kickstart_repo/treeinfo @@ -0,0 +1,56 @@ +[checksums] +images/boot.iso = sha256:f6be6ec48a4a610e25d591dcf98e1777c4274ed58c583fa64d0aea5b3ecffb18 +images/efiboot.img = sha256:94d5500c4ba266ce77b06aa955d9041eea22129737badc6af56c283dcaec1c29 +images/install.img = sha256:46171146377610cfa0deae157bbcc4ea146b3995c9b0c58d9f261ce404468abe +images/pxeboot/initrd.img = sha256:e0cd3966097c175d3aaf406a7f8c094374c69504c7be8f08d8084ab9a8812796 +images/pxeboot/vmlinuz = sha256:370db9a3943d4f46dc079dbaeb7e0cc3910dca069f7eede66d3d7d0d5177f684 + +[general] +; WARNING.0 = This section provides compatibility with pre-productmd treeinfos. +; WARNING.1 = Read productmd documentation for details about new format. +arch = x86_64 +family = Red Hat Enterprise Linux +name = Red Hat Enterprise Linux 8.0.0 +packagedir = Packages +platforms = x86_64,xen +repository = . +timestamp = 1554367044 +variant = BaseOS +variants = BaseOS +version = 8.0.0 + +[header] +type = productmd.treeinfo +version = 1.2 + +[images-x86_64] +boot.iso = images/boot.iso +efiboot.img = images/efiboot.img +initrd = images/pxeboot/initrd.img +kernel = images/pxeboot/vmlinuz + +[images-xen] +initrd = images/pxeboot/initrd.img +kernel = images/pxeboot/vmlinuz + +[release] +name = Red Hat Enterprise Linux +short = RHEL +version = 8.0.0 + +[stage2] +mainimage = images/install.img + +[tree] +arch = x86_64 +build_timestamp = 1554367044 +platforms = x86_64,xen +variants = BaseOS + +[variant-BaseOS] +id = BaseOS +name = BaseOS +packages = Packages +repository = . +type = variant +uid = BaseOS diff --git a/tests/test_kickstart_render_typical.py b/tests/test_kickstart_render_typical.py new file mode 100644 index 0000000..e771960 --- /dev/null +++ b/tests/test_kickstart_render_typical.py @@ -0,0 +1,576 @@ +from typing import Optional +import textwrap + +from repo_autoindex import autoindex +from repo_autoindex._impl.base import GeneratedIndex + +REPOMD_XML = textwrap.dedent( + """ + + + 1657165688 + + d4888f04f95ac067af4d997d35c6d345cbe398563d777d017a3634c9ed6148cf + 6fc4eddd4e9de89246efba3815b8a9dec9dfe168e4fd3104cc792dff908a0f62 + + 1657165688 + 2932 + 16585 + + + 284769ec79daa9e0a3b0129bb6260cc6271c90c4fe02b43dfa7cdf7635fb803f + 72f89223c8b0f6c7a2ee6ed7fbd16ee0bb395ca68260038bb3895265af84c29f + + 1657165688 + 4621 + 36911 + + + 36c2195bbee0c39ee080969abc6fd59d943c3471114cfd43c6e776ac20d7ed21 + 39f52cf295db14e863abcd7b2eede8e6c5e39ac9b2f194349459d29cd492c90f + + 1657165688 + 1408 + 8432 + + + 55e6bfd00e889c5c1f9a3c9fb35a660158bc5d975ae082d434f3cf81cc2c0c21 + b2692c49d1d98d68e764e29108d8a81a3dfd9e04fa7665115853a029396d118d + + 1657165688 + 7609 + 114688 + 10 + + + de63a509812c37f7736fcef0b79e9c55dfe67a2d77006f74fdc442935103e9e6 + 40eb5d53fe547c98d470813256c9bfc8a239b13697d8eb824a1485c9e186a0e3 + + 1657165688 + 10323 + 65536 + 10 + + + 9aa39b62df200cb3784dea24092d0c1c686afff0cd0990c2ec7a61afe8896e1c + 3e5cefb10ce805b827e12ca3b4839bba873dc9403fd92b60a364bf6f312bd972 + + 1657165688 + 2758 + 32768 + 10 + + +""" +).strip() + +PRIMARY_XML = textwrap.dedent( + """ + + + + wireplumber + x86_64 + + 539a773f3f39a7b2b5f971bdd0063f7d4201aab00920f380962e935356dc4d3a + A modular session/policy manager for PipeWire + WirePlumber is a modular session/policy manager for PipeWire and a +GObject-based high-level library that wraps PipeWire's API, providing +convenience for writing the daemon's modules as well as external tools for +managing PipeWire. + Fedora Project + https://pipewire.pages.freedesktop.org/wireplumber/ + + + wireplumber-libs + x86_64 + + 1f0d373bd1b8af6b4b7baab1c89e4820aa8cd8691f51fca4fccac9785fe715ea + Libraries for WirePlumber clients + This package contains the runtime libraries for any application that wishes +to interface with WirePlumber. + Fedora Project + https://pipewire.pages.freedesktop.org/wireplumber/ + + + xfce4-panel + x86_64 + + 1eecad127499d557f9d97562a1c65d9c881f3f63431546007a9ed714997b909c + Next generation panel for Xfce + This package includes the panel for the Xfce desktop environment. + Fedora Project + http://www.xfce.org/ + + + xfce4-power-manager + x86_64 + + 48697b6e83646e702d83523acd4a25df546129a1a11f3fbb81724c30d58e9c21 + Power management for the Xfce desktop environment + Xfce Power Manager uses the information and facilities provided by HAL to +display icons and handle user callbacks in an interactive Xfce session. +Xfce Power Preferences allows authorised users to set policy and change +preferences. + Fedora Project + http://goodies.xfce.org/projects/applications/xfce4-power-manager + + + xfce4-terminal + x86_64 + + 6b6d0d941c16988b4c68ae473f1af141dedafe691922c0c88f6f3ef82baeef79 + Terminal Emulator for the Xfce Desktop environment + Xfce4-terminal is a lightweight and easy to use terminal emulator application +with many advanced features including drop down, tabs, unlimited scrolling, +full colors, fonts, transparent backgrounds, and more. + Fedora Project + http://docs.xfce.org/apps/terminal/start + + +""" +).strip() + +TREEINFO = """[checksums] +images/boot.iso = sha256:f6be6ec48a4a610e25d591dcf98e1777c4274ed58c583fa64d0aea5b3ecffb18 +images/efiboot.img = sha256:94d5500c4ba266ce77b06aa955d9041eea22129737badc6af56c283dcaec1c29 +images/install.img = sha256:46171146377610cfa0deae157bbcc4ea146b3995c9b0c58d9f261ce404468abe +images/pxeboot/initrd.img = sha256:e0cd3966097c175d3aaf406a7f8c094374c69504c7be8f08d8084ab9a8812796 +images/pxeboot/vmlinuz = sha256:370db9a3943d4f46dc079dbaeb7e0cc3910dca069f7eede66d3d7d0d5177f684 + +[general] +; WARNING.0 = This section provides compatibility with pre-productmd treeinfos. +; WARNING.1 = Read productmd documentation for details about new format. +arch = x86_64 +family = Red Hat Enterprise Linux +name = Red Hat Enterprise Linux 8.0.0 +packagedir = Packages +platforms = x86_64,xen +repository = . +timestamp = 1554367044 +variant = BaseOS +variants = BaseOS +version = 8.0.0 + +[header] +type = productmd.treeinfo +version = 1.2 + +[images-x86_64] +boot.iso = images/boot.iso +efiboot.img = images/efiboot.img +initrd = images/pxeboot/initrd.img +kernel = images/pxeboot/vmlinuz + +[images-xen] +initrd = images/pxeboot/initrd.img +kernel = images/pxeboot/vmlinuz + +[release] +name = Red Hat Enterprise Linux +short = RHEL +version = 8.0.0 + +[stage2] +mainimage = images/install.img + +[tree] +arch = x86_64 +build_timestamp = 1554367044 +platforms = x86_64,xen +variants = BaseOS + +[variant-BaseOS] +id = BaseOS +name = BaseOS +packages = Packages +repository = . +type = variant +uid = BaseOS""" + +EXTRA_FILES_JSON = """{ + "data": [ + { + "checksums": { + "md5": "feb4d252ee63634debea654b446e830b", + "sha1": "a73fad5aeb5642d1b2108885010c4e7a547a1204", + "sha256": "c4117d0e325cde392981626edbd1484c751f0216689a171e4b7547e8800acc21" + }, + "file": "RPM-GPG-KEY-redhat-release", + "size": 5134 + }, + { + "checksums": { + "md5": "3c24137e12ece142a27bbf825c256936", + "sha1": "a72daf8585b41529269cdffcca3a0b3d4e2f21cd", + "sha256": "3f8644b35db4197e7689d0a034bdef2039d92e330e6b22217abfa6b86a1fc0fa" + }, + "file": "RPM-GPG-KEY-redhat-beta", + "size": 1669 + }, + { + "checksums": { + "md5": "b234ee4d69f5fce4486a80fdaf4a4263", + "sha1": "4cc77b90af91e615a64ae04893fdffa7939db84c", + "sha256": "8177f97513213526df2cf6184d8ff986c675afb514d4e68a404010521b880643" + }, + "file": "GPL", + "size": 18092 + }, + { + "checksums": { + "md5": "0c53898068810a989fa59ca0656bdf24", + "sha1": "42d51858642b8a0d10fdf09050266395544ea556", + "sha256": "8f833ce3fbcbcb82e47687a890c043332c88350ddabd606201556e14aaf8fcd9" + }, + "file": "EULA", + "size": 8154 + } + ], + "header": { + "version": "1.0" + } +}""" + + +class StaticFetcher: + def __init__(self): + self.content: dict[str, str] = {} + + async def __call__(self, url: str) -> Optional[str]: + return self.content.get(url) + + +async def test_typical_index(): + fetcher = StaticFetcher() + + fetcher.content["https://example.com/repodata/repomd.xml"] = REPOMD_XML + fetcher.content[ + "https://example.com/repodata/d4888f04f95ac067af4d997d35c6d345cbe398563d777d017a3634c9ed6148cf-primary.xml.gz" + ] = PRIMARY_XML + fetcher.content["https://example.com/treeinfo"] = TREEINFO + fetcher.content["https://example.com/extra_files.json"] = EXTRA_FILES_JSON + + entries: list[GeneratedIndex] = [] + async for entry in autoindex("https://example.com", fetcher=fetcher): + print(f"Found one entry: {entry.relative_dir}") + entries.append(entry) + + # It should generate some entries + assert entries + + entries.sort(key=lambda e: e.relative_dir) + + # First check that the directory structure was reproduced. + assert [e.relative_dir for e in entries] == [ + "", + "images", + "images/pxeboot", + "packages", + "packages/w", + "packages/x", + "repodata", + ] + + by_relative_dir: dict[str, GeneratedIndex] = {} + for entry in entries: + by_relative_dir[entry.relative_dir] = entry + + # Sanity check a few links expected to appear in each. + assert '' in by_relative_dir[""].content + assert '' in by_relative_dir[""].content + + assert '' in by_relative_dir["packages"].content + assert '' in by_relative_dir["packages"].content + + assert '' in by_relative_dir[""].content + assert '' in by_relative_dir["images"].content + + assert ( + '' + in by_relative_dir["repodata"].content + ) + + assert ( + '' + in by_relative_dir["packages/w"].content + ) + assert ( + '' + in by_relative_dir["packages/x"].content + ) + + assert '' in by_relative_dir[""].content + + assert '' in by_relative_dir[""].content + + assert '' in by_relative_dir[""].content + + assert '' in by_relative_dir[""].content + + assert '' in by_relative_dir[""].content + + assert '' in by_relative_dir[""].content + + assert '' in by_relative_dir["images"].content + + assert '' in by_relative_dir["images"].content + + assert '' in by_relative_dir["images/pxeboot"].content