diff --git a/.packit.yaml b/.packit.yaml index 496c263..682f5c9 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -22,12 +22,19 @@ packages: upstream_tag_template: 'dist-git-{version}' paths: - dist-git - + dist-git-client: + specfile_path: dist-git-client.spec + upstream_package_name: dist-git-client + downstream_package_name: dist-git-client + upstream_tag_template: 'dist-git-client-{version}' + paths: + - dist-git-client jobs: - &copr job: copr_build packages: - dist-git + - dist-git-client trigger: pull_request metadata: targets: diff --git a/.tito/packages/dist-git-client b/.tito/packages/dist-git-client new file mode 100644 index 0000000..f04c59a --- /dev/null +++ b/.tito/packages/dist-git-client @@ -0,0 +1 @@ +1.0-1 dist-git-client/ diff --git a/dist-git-client/LICENSE b/dist-git-client/LICENSE new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/dist-git-client/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/dist-git-client/bin/dist-git-client b/dist-git-client/bin/dist-git-client new file mode 100755 index 0000000..2b9f0ab --- /dev/null +++ b/dist-git-client/bin/dist-git-client @@ -0,0 +1,10 @@ +#! /usr/bin/python3 + +""" +From within a git checkout, try to download files from dist-git lookaside cache. +""" + +from dist_git_client import main + +if __name__ == "__main__": + main() diff --git a/dist-git-client/dist-git-client b/dist-git-client/dist-git-client new file mode 100755 index 0000000..f507e42 --- /dev/null +++ b/dist-git-client/dist-git-client @@ -0,0 +1,22 @@ +#! /bin/sh + +# Run copr-disgit-client script directly from git, TESTING ONLY SCRIPT! +# Copyright (C) 2020 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +absdir="$(dirname "$(readlink -f "$0")")" +export PYTHONPATH="$absdir${PYTHONPATH+:$PYTHONPATH}" +python3 "$absdir/bin/dist-git-client" "$@" diff --git a/dist-git-client/dist-git-client.spec b/dist-git-client/dist-git-client.spec new file mode 100644 index 0000000..ab7d49a --- /dev/null +++ b/dist-git-client/dist-git-client.spec @@ -0,0 +1,80 @@ +# SPEC file overview: +# https://docs.fedoraproject.org/en-US/quick-docs/creating-rpm-packages/#con_rpm-spec-file-overview +# Fedora packaging guidelines: +# https://docs.fedoraproject.org/en-US/packaging-guidelines/ + + +Name: dist-git-client +Version: 1.0 +Release: 1%{?dist} +Summary: Get sources for RPM builds from DistGit repositories +BuildArch: noarch + +License: GPL-2.0-or-later +URL: https://github.com/release-engineering/dist-git.git +Source0: %name-%version.tar.gz + +Requires: curl +Requires: /usr/bin/git + +BuildRequires: python3-pytest +BuildRequires: python3-rpm-macros +BuildRequires: /usr/bin/argparse-manpage +BuildRequires: /usr/bin/git + +%if 0%{?fedora} || 0%{?rhel} > 9 +Requires: python3-rpmautospec +BuildRequires: python3-rpmautospec +%endif + +%description +A simple, configurable python utility that is able to clone package sources from +a DistGit repository, download sources from the corresponding lookaside cache +locations, and generate source RPMs. + +The utility is able to automatically map the .git/config clone URL into the +corresponding DistGit instance configuration. + + +%prep +%setup -q + + +%build + + + +%install +install -d %{buildroot}%{_bindir} +install -d %{buildroot}%{_mandir}/man1 +install -d %{buildroot}%{_sysconfdir}/dist-git-client +install -d %{buildroot}%{python3_sitelib} +install -p -m 755 bin/dist-git-client %buildroot%_bindir +argparse-manpage --pyfile dist_git_client.py \ + --function _get_argparser \ + --author "Copr Team" \ + --author-email "copr-team@redhat.com" \ + --url %url --project-name Copr \ +> %{buildroot}%{_mandir}/man1/dist-git-client.1 +install -p -m 644 etc/default.ini \ + %{buildroot}%{_sysconfdir}/dist-git-client +install -p -m 644 dist_git_client.py %{buildroot}%{python3_sitelib} + + +%check +PYTHON=python3 ./run_tests.sh -vv --no-coverage + + +%files +%license LICENSE +%_bindir/dist-git-client +%_mandir/man1/dist-git-client.1* +%dir %_sysconfdir/dist-git-client +%config %_sysconfdir/dist-git-client/default.ini +%python3_sitelib/dist_git_client.* +%python3_sitelib/__pycache__/dist_git_client* + + +%changelog +* Thu Jun 06 2024 Pavel Raiskup 1.1-0 +- new package built with tito diff --git a/dist-git-client/dist_git_client.py b/dist-git-client/dist_git_client.py new file mode 100644 index 0000000..bcfb7ca --- /dev/null +++ b/dist-git-client/dist_git_client.py @@ -0,0 +1,501 @@ +""" +dist-git-client code, moved to a python module to simplify unit-testing +""" + +import argparse +import configparser +import errno +import glob +import logging +import shlex +import os +import shutil +import subprocess +import sys +from urllib.parse import urlparse + +try: + from rpmautospec import ( + specfile_uses_rpmautospec as rpmautospec_used, + process_distgit as rpmautospec_expand, + ) +except ImportError: + rpmautospec_used = lambda _: False + + +def log_cmd(command, comment="Running command"): + """ Dump the command to stderr so it can be c&p to shell """ + command = ' '.join([shlex.quote(x) for x in command]) + logging.info("%s: %s", comment, command) + + +def check_output(cmd, comment="Reading stdout from command"): + """ el6 compatible subprocess.check_output() """ + log_cmd(cmd, comment) + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, _) = process.communicate() + if process.returncode: + raise RuntimeError("Exit non-zero: {0}".format(process.returncode)) + return stdout + +def call(cmd, comment="Calling"): + """ wrap sp.call() with logging info """ + log_cmd(cmd, comment) + return subprocess.call(cmd) + +def check_call(cmd, comment="Checked call"): + """ wrap sp.check_call() with logging info """ + log_cmd(cmd, comment) + subprocess.check_call(cmd) + +def _load_config(directory): + config = configparser.ConfigParser() + files = glob.glob(os.path.join(directory, "*.ini")) + logging.debug("Files %s in config directory %s", files, directory) + config.read(files) + + config_dict = { + "instances": {}, + "clone_host_map": {}, + } + + instances = config_dict["instances"] + for section_name in config.sections(): + section = config[section_name] + instance = instances[section_name] = {} + for key in section.keys(): + # array-like config options + if key in ["clone_hostnames", "path_prefixes"]: + hostnames = section[key].split() + instance[key] = [h.strip() for h in hostnames] + else: + instance[key] = section[key] + + for key in ["sources", "specs"]: + if key in instance: + continue + instance[key] = "." + + if "sources_file" not in instance: + instance["sources_file"] = "sources" + + if "default_sum" not in instance: + instance["default_sum"] = "md5" + + for host in instance["clone_hostnames"]: + if host not in config_dict["clone_host_map"]: + config_dict["clone_host_map"][host] = {} + host_dict = config_dict["clone_host_map"][host] + for prefix in instance.get("path_prefixes", ["DEFAULT"]): + if prefix in host_dict: + msg = "Duplicate prefix {0} for {1} hostname".format( + prefix, host, + ) + raise RuntimeError(msg) + host_dict[prefix] = instance + + return config_dict + + +def download(url, filename): + """ Download URL as FILENAME using curl command """ + + if not hasattr(download, "curl_has_retry_all_errors"): + # Drop this once EL8 is not a thing to support + output = check_output(["curl", "--help", "all"]) + # method's static variable to avoid using 'global' + download.curl_has_retry_all_errors = b"--retry-all-errors" in output + + command = [ + "curl", + "-H", "Pragma:", + "-o", filename, + "--location", + "--connect-timeout", "60", + "--retry", "3", "--retry-delay", "10", + "--remote-time", + "--show-error", + "--fail", + ] + + if download.curl_has_retry_all_errors: + command += ["--retry-all-errors"] + + if call(command + [url]): + raise RuntimeError("Can't download file {0}".format(filename)) + + +def mkdir_p(path): + """ mimic 'mkdir -p ' command """ + try: + os.makedirs(path) + except OSError as err: + if err.errno != errno.EEXIST: + raise + + +def download_file_and_check(url, params, distgit_config): + """ Download given URL (if not yet downloaded), and try the checksum """ + filename = params["filename"] + sum_binary = params["hashtype"] + "sum" + + mkdir_p(distgit_config["sources"]) + + if not os.path.exists(filename): + logging.info("Downloading %s", filename) + download(url, filename) + else: + logging.info("File %s already exists", filename) + + sum_command = [sum_binary, filename] + output = check_output(sum_command).decode("utf-8") + checksum, _ = output.strip().split() + if checksum != params["hash"]: + raise RuntimeError("Check-sum {0} is wrong, expected: {1}".format( + checksum, + params["hash"], + )) + + +def _detect_clone_url(): + git_config = ".git/config" + if not os.path.exists(git_config): + msg = "{0} not found, $PWD is not a git repository".format(git_config) + raise RuntimeError(msg) + + git_conf_reader = configparser.ConfigParser() + git_conf_reader.read(git_config) + return git_conf_reader['remote "origin"']["url"] + + +def get_distgit_config(config, forked_from=None): + """ + Given the '.git/config' file from current directory, return the + appropriate part of dist-git configuration. + Returns tuple: (urlparse(clone_url), distgit_config) + """ + url = forked_from + if not url: + url = _detect_clone_url() + parsed_url = urlparse(url) + if parsed_url.hostname is None: + hostname = "localhost" + else: + hostname = parsed_url.hostname + + prefixes = config["clone_host_map"][hostname] + prefix_found = None + for prefix in prefixes.keys(): + if not parsed_url.path.startswith(prefix): + continue + prefix_found = prefix + + if not prefix_found: + if "DEFAULT" not in prefixes: + raise RuntimeError("Path {0} does not match any of 'path_prefixes' " + "for '{1}' hostname".format(parsed_url.path, + hostname)) + prefix_found = "DEFAULT" + + return parsed_url, prefixes[prefix_found] + + +def get_spec(distgit_config): + """ + Find the specfile name inside distgit_config["specs"] directory + """ + spec_dir = distgit_config["specs"] + specfiles = glob.glob(os.path.join(spec_dir, '*.spec')) + if len(specfiles) != 1: + abs_spec_dir = os.path.join(os.getcwd(), spec_dir) + message = "Exactly one spec file expected in {0} directory, {1} found".format( + abs_spec_dir, len(specfiles), + ) + raise RuntimeError(message) + specfile = os.path.basename(specfiles[0]) + return specfile + + +def sources(args, config): + """ + Locate the sources, and download them from the appropriate dist-git + lookaside cache. + """ + parsed_url, distgit_config = get_distgit_config(config, args.forked_from) + namespace = parsed_url.path.strip('/').split('/') + # drop the last {name}.git part + repo_name = namespace.pop() + if repo_name.endswith(".git"): + repo_name = repo_name[:-4] + namespace = list(reversed(namespace)) + + output = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + output = output.decode("utf-8").strip() + if output == "HEAD": + output = check_output(["git", "rev-parse", "HEAD"]) + output = output.decode("utf-8").strip() + refspec = output + specfile = get_spec(distgit_config) + name = specfile[:-5] + sources_file = distgit_config["sources_file"].format(name=name) + if not os.path.exists(sources_file): + logging.info("'%s' file not found, download skipped", sources_file) + return + + logging.info("Reading sources specification file: %s", sources_file) + with open(sources_file, 'r') as sfd: + while True: + line = sfd.readline() + if not line: + break + + kwargs = { + "name": repo_name, + "refspec": refspec, + "namespace": namespace, + } + + source_spec = line.split() + if not source_spec: + # line full of white-spaces, skip + continue + + if len(source_spec) == 2: + # old md5/sha1 format: 0ced6f20b9fa1bea588005b5ad4b52c1 tar-1.26.tar.xz + kwargs["hashtype"] = distgit_config["default_sum"].lower() + kwargs["hash"] = source_spec[0] + kwargs["filename"] = source_spec[1] + elif len(source_spec) == 4: + # SHA512 (tar-1.30.tar.xz.sig) = + kwargs["hashtype"] = source_spec[0].lower() + kwargs["hash"] = source_spec[3] + filename = os.path.basename(source_spec[1]) + kwargs["filename"] = filename.strip('()') + else: + msg = "Weird sources line: {0}".format(line) + raise RuntimeError(msg) + + url_file = '/'.join([ + distgit_config["lookaside_location"], + distgit_config["lookaside_uri_pattern"].format(**kwargs) + ]) + + download_file_and_check(url_file, kwargs, distgit_config) + + +def handle_autospec(spec_abspath, spec_basename, args): + """ + When %auto* macros are used in SPEC_ABSPATH, expand them into a separate + spec file within ARGS.OUTPUTDIR, and return the absolute filename of the + specfile. When %auto* macros are not used, return SPEC_ABSPATH unchanged. + """ + result = spec_abspath + if rpmautospec_used(spec_abspath): + git_dir = check_output(["git", "rev-parse", "--git-dir"]) + git_dir = git_dir.decode("utf-8").strip() + if os.path.exists(os.path.join(git_dir, "shallow")): + # Hack. The rpmautospec doesn't support shallow clones: + # https://pagure.io/fedora-infra/rpmautospec/issue/227 + logging.info("rpmautospec requires full clone => --unshallow") + check_call(["git", "fetch", "--unshallow"]) + + # Expand the %auto* macros, and create the separate spec file in the + # output directory. + output_spec = os.path.join(args.outputdir, spec_basename) + rpmautospec_expand(spec_abspath, output_spec) + result = output_spec + return result + + +def srpm(args, config): + """ + Using the appropriate dist-git configuration, generate source RPM + file. This requires running 'def sources()' first. + """ + _, distgit_config = get_distgit_config(config, args.forked_from) + + cwd = os.getcwd() + sources_dir = os.path.join(cwd, distgit_config["sources"]) + specs = os.path.join(cwd, distgit_config["specs"]) + spec = get_spec(distgit_config) + + mkdir_p(args.outputdir) + + spec_abspath = os.path.join(specs, spec) + spec_abspath = handle_autospec(spec_abspath, spec, args) + + if args.mock_chroot: + command = [ + "mock", "--buildsrpm", + "-r", args.mock_chroot, + "--spec", spec_abspath, + "--sources", sources_dir, + "--resultdir", args.outputdir, + ] + else: + command = [ + "rpmbuild", "-bs", spec_abspath, + "--define", "dist %nil", + "--define", "_sourcedir {0}".format(sources_dir), + "--define", "_srcrpmdir {0}".format(args.outputdir), + "--define", "_disable_source_fetch 1", + ] + + if args.dry_run or 'COPR_DISTGIT_CLIENT_DRY_RUN' in os.environ: + log_cmd(command, comment="Dry run") + else: + check_call(command) + + +def clone(args, config): + """ + Automatically clone a package from a given DistGit instance + """ + distgit = config["instances"][args.dist_git] + parts = distgit.get("cloning_pattern_package_parts") + if parts: + expected = parts.split() + have = args.package.split("/") + if len(expected) != len(have): + raise RuntimeError( + "Package '{0}' has a wrong format, {1} " + "slash-separated parts are expected: {2}".format( + args.package, len(expected), + "/".join(expected), + )) + + clone_url = distgit["cloning_pattern"].format(package=args.package) + check_call([ + "git", "clone", clone_url, + ]) + +def _get_argparser(): + parser = argparse.ArgumentParser(prog="dist-git-client", + description="""\ +A simple, configurable python utility that is able to download sources from +various dist-git instances, and generate source RPMs. +The utility is able to automatically map the "origin" .git/config clone URL +(or --forked-from URL, if specified) to a corresponding dist-git instance +configured in /etc/dist-git-client directory. +""") + + # main parser + default_confdir = os.environ.get("COPR_DISTGIT_CLIENT_CONFDIR", + "/etc/dist-git-client") + parser.add_argument( + "--configdir", default=default_confdir, + help="Where to load configuration files from") + parser.add_argument( + "--loglevel", default="info", + help="Python logging level, e.g. debug, info, error") + parser.add_argument( + "--forked-from", + metavar="CLONE_URL", + help=("Specify that this git clone directory is a dist-git repository " + "fork. If used, the default clone url detection from the " + ".git/config file is disabled and CLONE_URL is used instead. " + "This specified CLONE_URL is used to detect the appropriate " + "lookaside cache pattern to download the sources.")) + + subparsers = parser.add_subparsers( + title="actions", dest="action") + + # sources parser + subparsers.add_parser( + "sources", + description=( + "Using the 'url' .git/config, detect where the right DistGit " + "lookaside cache exists, and download the corresponding source " + "files."), + help="Download sources from the lookaside cache") + + # srpm parser + srpm_parser = subparsers.add_parser( + "srpm", + help="Generate a source RPM", + description=( + "Generate a source RPM from the downloaded source files " + "by 'sources' command, please run 'sources' first."), + ) + srpm_parser.add_argument( + "--outputdir", + default="/tmp", + help="Where to store the resulting source RPM") + srpm_parser.add_argument( + "--mock-chroot", + help=("Generate the SRPM in mock buildroot instead of on host. The " + "argument is passed down to mock as the 'mock -r|--root' " + "argument."), + ) + srpm_parser.add_argument( + "--dry-run", action="store_true", + help=("Don't produce the SRPM, just print the command which would be " + "otherwise called"), + ) + + clone_parser = subparsers.add_parser( + "clone", + help="Clone package from a DistGit source", + ) + + clone_parser.add_argument( + "--dist-git", + default="fedora", + help=("The DistGit ID as configured in /etc/dist-git-client/"), + ) + + clone_parser.add_argument( + "package", + default="fedora", + help=("Package name specification. For some DistGit " + "instances this consists of multiple parts separated " + "by slash, e.g. for '--dist-git=fedora-copr' use " + "'@copr/copr-dev/copr-cli'."), + ) + + return parser + + +def unittests_init_git(files=None): + """ + Initialize .git/ directory. This method is only used for unit-testing. + """ + check_output(["git", "init", ".", "-b", "main"]) + shutil.rmtree(".git/hooks") + check_output(["git", "config", "user.email", "you@example.com"]) + check_output(["git", "config", "user.name", "Your Name"]) + check_output(["git", "config", "advice.detachedHead", "false"]) + + for filename, content in files: + dirname = os.path.dirname(filename) + try: + os.makedirs(dirname) + except OSError: + pass + with open(filename, "w", encoding="utf-8") as filed: + filed.write(content) + check_output(["git", "add", filename]) + + check_output(["git", "commit", "-m", "initial"]) + + +def main(): + """ The entrypoint for the whole logic """ + args = _get_argparser().parse_args() + logging.basicConfig( + level=getattr(logging, args.loglevel.upper()), + format="%(levelname)s: %(message)s", + ) + config = _load_config(args.configdir) + + try: + if args.action == "srpm": + srpm(args, config) + elif args.action == "clone": + clone(args, config) + else: + sources(args, config) + except RuntimeError as err: + logging.error("%s", err) + sys.exit(1) diff --git a/dist-git-client/etc/default.ini b/dist-git-client/etc/default.ini new file mode 100644 index 0000000..63a4429 --- /dev/null +++ b/dist-git-client/etc/default.ini @@ -0,0 +1,86 @@ +# *Match* the given repository clone_url's with an appropriate DistGit +# configuration (== identify the corresponding lookaside cache location). +# +# Any *.ini file in /etc/copr-distgit-client directory is parsed. The format is +# just INI (python ConfigParser format). The section names denote the DistGit +# instances' IDs. Each section can have the following options: +# +# clone_hostnames: List[string] +# List of possible hostnames to consider when matching this configuration. +# E.g. "example.com" matches both "git://example.com/foo" and +# "https://example.com/foo.git" URLs. +# +# path_prefixes: List[string] (optional) +# List of possible path prefixes to consider. Complements the +# "clone_hostnames" option above. When specified, **both** +# "clone_hostnames" and 'path_prefixes' must match to use the +# corresponding configuration section. E.g. "/foo/bar" prefix matches +# the "https://example.com/foo/bar/rpms/component.git" clone_url. +# When not specified, **any** prefix is accepted on matched hostname. +# +# sources_file: String (optional, pathname) +# Expected 'sources' file location for this DistGit instance (relative to +# the git root directory). Defaault = './sources'. +# +# specs: String (optional, pathname) +# Expected spec file directory, relative to the git root directory. +# Default = '.' (spec files stored directly in the git root). +# +# sources: String (optional, pathname) +# Where the source files (referenced by 'sources_file') should be +# downloaded. Default = '.' (git root directory). +# +# default_sum: String (capital, optional) +# Up2date 'sources' files explicitly denote the checksum type used for +# given files using lines like "SHA512 () = ". But still, +# even the "old" sources file format is supported with lines like +# " ". This option is to define what sum type is expected +# in this DistGit instance (Default = MD5). +# +# lookaside_location: String (hostname) +# Url of the storage where to download the sources from. +# +# lookaside_uri_pattern: String +# Relative path on the 'lookaside_location' where the files should be +# found, given the info parsed from 'sources' file. Possible fields are +# "name" (component), "filename", "hashtype" (e.g. 'md5'), "hash" +# (checksum), "namespace" (array, prefix before the component name). + +[fedora] +clone_hostnames = + pkgs.fedoraproject.org + src.fedoraproject.org +lookaside_location = https://src.fedoraproject.org +lookaside_uri_pattern = repo/pkgs/rpms/{name}/{filename}/{hashtype}/{hash}/{filename} +cloning_pattern = https://src.fedoraproject.org/rpms/{package}.git + +[centos] +clone_hostnames = git.centos.org +lookaside_location = https://git.centos.org +sources_file = .{name}.metadata +specs = SPECS +sources = SOURCES +default_sum = SHA1 +lookaside_uri_pattern = sources/{name}/{refspec}/{hash} +cloning_pattern = https://git.centos.org/rpms/{package}.git + +[fedora-copr] +clone_hostnames = copr-dist-git.fedorainfracloud.org +lookaside_location = https://copr-dist-git.fedorainfracloud.org +lookaside_uri_pattern = repo/pkgs/{namespace[1]}/{namespace[0]}/{name}/{filename}/{hashtype}/{hash}/{filename} +cloning_pattern_package_parts = owner_name project_name package_name +cloning_pattern=https://copr-dist-git.fedorainfracloud.org/git/{package} + +[fedora-copr-dev] +clone_hostnames = copr-dist-git-dev.fedorainfracloud.org +lookaside_location = https://copr-dist-git-dev.fedorainfracloud.org +lookaside_uri_pattern = repo/pkgs/{namespace[1]}/{namespace[0]}/{name}/{filename}/{hashtype}/{hash}/{filename} +cloning_pattern_package_parts = owner_name project_name package_name +cloning_pattern=https://copr-dist-git-dev.fedorainfracloud.org/git/{package} + +[centos-stream] +clone_hostnames = gitlab.com +path_prefixes = /redhat/centos-stream/rpms +lookaside_location = https://sources.stream.centos.org +lookaside_uri_pattern = sources/rpms/{name}/{filename}/{hashtype}/{hash}/{filename} +cloning_pattern = https://gitlab.com/redhat/centos-stream/rpms/{package}.git diff --git a/dist-git-client/run_tests.sh b/dist-git-client/run_tests.sh new file mode 100755 index 0000000..1d934d6 --- /dev/null +++ b/dist-git-client/run_tests.sh @@ -0,0 +1,18 @@ +#! /bin/bash + +set -e + +args=() + +coverage=( --cov-report term-missing --cov bin --cov dist_git_client ) +for arg; do + case $arg in + --no-coverage) coverage=() ;; + *) args+=( "$arg" ) ;; + esac +done + +abspath=$(readlink -f .) +export PYTHONPATH="${PYTHONPATH+$PYTHONPATH:}$abspath" +export PATH=$(readlink -f bin):$PATH +"${PYTHON:-python3}" -m pytest -s tests "${coverage[@]}" "${args[@]}" diff --git a/dist-git-client/tags b/dist-git-client/tags new file mode 100644 index 0000000..c996022 --- /dev/null +++ b/dist-git-client/tags @@ -0,0 +1,176 @@ +!_TAG_EXTRA_DESCRIPTION anonymous /Include tags for non-named objects like lambda/ +!_TAG_EXTRA_DESCRIPTION fileScope /Include tags of file scope/ +!_TAG_EXTRA_DESCRIPTION pseudo /Include pseudo tags/ +!_TAG_EXTRA_DESCRIPTION qualified /Include an extra class-qualified tag entry for each tag/ +!_TAG_EXTRA_DESCRIPTION subparser /Include tags generated by subparsers/ +!_TAG_FIELD_DESCRIPTION access /Access (or export) of class members/ +!_TAG_FIELD_DESCRIPTION epoch /the last modified time of the input file (only for F\/file kind tag)/ +!_TAG_FIELD_DESCRIPTION file /File-restricted scoping/ +!_TAG_FIELD_DESCRIPTION inherits /Inheritance information/ +!_TAG_FIELD_DESCRIPTION input /input file/ +!_TAG_FIELD_DESCRIPTION name /tag name/ +!_TAG_FIELD_DESCRIPTION pattern /pattern/ +!_TAG_FIELD_DESCRIPTION signature /Signature of routine (e.g. prototype or parameter list)/ +!_TAG_FIELD_DESCRIPTION typeref /Type and name of a variable or typedef/ +!_TAG_FIELD_DESCRIPTION!Python nameref /the original name for the tag/ +!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ +!_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ +!_TAG_KIND_DESCRIPTION!Iniconf k,key /keys/ +!_TAG_KIND_DESCRIPTION!Iniconf s,section /sections/ +!_TAG_KIND_DESCRIPTION!Markdown S,subsection /level 2 sections/ +!_TAG_KIND_DESCRIPTION!Markdown T,l4subsection /level 4 sections/ +!_TAG_KIND_DESCRIPTION!Markdown c,chapter /chapters/ +!_TAG_KIND_DESCRIPTION!Markdown n,footnote /footnotes/ +!_TAG_KIND_DESCRIPTION!Markdown s,section /sections/ +!_TAG_KIND_DESCRIPTION!Markdown t,subsubsection /level 3 sections/ +!_TAG_KIND_DESCRIPTION!Markdown u,l5subsection /level 5 sections/ +!_TAG_KIND_DESCRIPTION!Python I,namespace /name referring a module defined in other file/ +!_TAG_KIND_DESCRIPTION!Python Y,unknown /name referring a class\/variable\/function\/module defined in other module/ +!_TAG_KIND_DESCRIPTION!Python c,class /classes/ +!_TAG_KIND_DESCRIPTION!Python f,function /functions/ +!_TAG_KIND_DESCRIPTION!Python i,module /modules/ +!_TAG_KIND_DESCRIPTION!Python m,member /class members/ +!_TAG_KIND_DESCRIPTION!Python v,variable /variables/ +!_TAG_KIND_DESCRIPTION!RpmSpec g,global /global macros/ +!_TAG_KIND_DESCRIPTION!RpmSpec m,macro /macros/ +!_TAG_KIND_DESCRIPTION!RpmSpec p,package /packages/ +!_TAG_KIND_DESCRIPTION!RpmSpec p,patch /patch files/ +!_TAG_KIND_DESCRIPTION!RpmSpec t,tag /tags/ +!_TAG_KIND_DESCRIPTION!Sh a,alias /aliases/ +!_TAG_KIND_DESCRIPTION!Sh f,function /functions/ +!_TAG_KIND_DESCRIPTION!Sh h,heredoc /label for here document/ +!_TAG_KIND_DESCRIPTION!Sh s,script /script files/ +!_TAG_OUTPUT_EXCMD mixed /number, pattern, mixed, or combineV2/ +!_TAG_OUTPUT_FILESEP slash /slash or backslash/ +!_TAG_OUTPUT_MODE u-ctags /u-ctags or e-ctags/ +!_TAG_OUTPUT_VERSION 0.0 /current.age/ +!_TAG_PARSER_VERSION!Iniconf 0.0 /current.age/ +!_TAG_PARSER_VERSION!Markdown 0.0 /current.age/ +!_TAG_PARSER_VERSION!Python 0.0 /current.age/ +!_TAG_PARSER_VERSION!RpmSpec 0.0 /current.age/ +!_TAG_PARSER_VERSION!Sh 0.0 /current.age/ +!_TAG_PATTERN_LENGTH_LIMIT 96 /0 for no limit/ +!_TAG_PROC_CWD /home/praiskup/rh/projects/copr/copr/worktree/praiskup/dist-git-client/dist-git-client/ // +!_TAG_PROGRAM_AUTHOR Universal Ctags Team // +!_TAG_PROGRAM_NAME Universal Ctags /Derived from Exuberant Ctags/ +!_TAG_PROGRAM_URL https://ctags.io/ /official site/ +!_TAG_PROGRAM_VERSION 6.0.0 // +!_TAG_ROLE_DESCRIPTION!Python!module imported /imported modules/ +!_TAG_ROLE_DESCRIPTION!Python!module indirectlyImported /module imported in alternative name/ +!_TAG_ROLE_DESCRIPTION!Python!module namespace /namespace from where classes\/variables\/functions are imported/ +!_TAG_ROLE_DESCRIPTION!Python!unknown imported /imported from the other module/ +!_TAG_ROLE_DESCRIPTION!Python!unknown indirectlyImported /classes\/variables\/functions\/modules imported in alternative name/ +!_TAG_ROLE_DESCRIPTION!RpmSpec!macro undef /undefined/ +!_TAG_ROLE_DESCRIPTION!RpmSpec!patch decl /declared for applying later/ +!_TAG_ROLE_DESCRIPTION!Sh!heredoc endmarker /end marker/ +!_TAG_ROLE_DESCRIPTION!Sh!script loaded /loaded/ +BuildArch dist-git-client.spec /^BuildArch: noarch$/;" t +BuildRequires dist-git-client.spec /^BuildRequires: \/usr\/bin\/argparse-manpage$/;" t +BuildRequires dist-git-client.spec /^BuildRequires: python-rpm-macros$/;" t +BuildRequires dist-git-client.spec /^BuildRequires: python3-rpmautospec$/;" t +License dist-git-client.spec /^License: GPL-2.0-or-later$/;" t +Name dist-git-client.spec /^Name: dist-git-client$/;" t +Release dist-git-client.spec /^Release: 1%{?dist}$/;" t +Requires dist-git-client.spec /^Requires: %{_bindir}\/git$/;" t +Requires dist-git-client.spec /^Requires: curl$/;" t +Requires dist-git-client.spec /^Requires: python3-rpmautospec$/;" t +Requires dist-git-client.spec /^Requires: python3-six$/;" t +Source0 dist-git-client.spec /^Source0: %name-%version.tar.gz$/;" t +Summary dist-git-client.spec /^Summary: Get sources for RPM builds from DistGit repositories$/;" t +TestDistGitDownload tests/test_dist_git_client.py /^class TestDistGitDownload(object):$/;" c inherits:object access:public +TestDistGitDownload.args tests/test_dist_git_client.py /^ args = None$/;" v class:TestDistGitDownload access:public +TestDistGitDownload.config tests/test_dist_git_client.py /^ config = None$/;" v class:TestDistGitDownload access:public +TestDistGitDownload.config_dir tests/test_dist_git_client.py /^ config_dir = None$/;" v class:TestDistGitDownload access:public +TestDistGitDownload.setup_method tests/test_dist_git_client.py /^ def setup_method(self, method):$/;" m class:TestDistGitDownload access:public signature:(self, method) +TestDistGitDownload.setup_method._Args tests/test_dist_git_client.py /^ class _Args:$/;" c member:TestDistGitDownload.setup_method file: inherits: access:private +TestDistGitDownload.setup_method._Args.dry_run tests/test_dist_git_client.py /^ dry_run = False$/;" v class:TestDistGitDownload.setup_method._Args access:public +TestDistGitDownload.setup_method._Args.forked_from tests/test_dist_git_client.py /^ forked_from = None$/;" v class:TestDistGitDownload.setup_method._Args access:public +TestDistGitDownload.teardown_method tests/test_dist_git_client.py /^ def teardown_method(self, method):$/;" m class:TestDistGitDownload access:public signature:(self, method) +TestDistGitDownload.test_centos tests/test_dist_git_client.py /^ def test_centos(self, download):$/;" m class:TestDistGitDownload access:public signature:(self, download) +TestDistGitDownload.test_centos_download tests/test_dist_git_client.py /^ def test_centos_download(self, patched_check_call):$/;" m class:TestDistGitDownload access:public signature:(self, patched_check_call) +TestDistGitDownload.test_copr_distgit tests/test_dist_git_client.py /^ def test_copr_distgit(self, download):$/;" m class:TestDistGitDownload access:public signature:(self, download) +TestDistGitDownload.test_duplicate_prefix tests/test_dist_git_client.py /^ def test_duplicate_prefix(self):$/;" m class:TestDistGitDownload access:public signature:(self) +TestDistGitDownload.test_fedora_new tests/test_dist_git_client.py /^ def test_fedora_new(self, download, base):$/;" m class:TestDistGitDownload access:public signature:(self, download, base) +TestDistGitDownload.test_fedora_old tests/test_dist_git_client.py /^ def test_fedora_old(self, download, base):$/;" m class:TestDistGitDownload access:public signature:(self, download, base) +TestDistGitDownload.test_load_prefix tests/test_dist_git_client.py /^ def test_load_prefix(self):$/;" m class:TestDistGitDownload access:public signature:(self) +TestDistGitDownload.test_load_prefix_fail tests/test_dist_git_client.py /^ def test_load_prefix_fail(self):$/;" m class:TestDistGitDownload access:public signature:(self) +TestDistGitDownload.test_no_git_config tests/test_dist_git_client.py /^ def test_no_git_config(self):$/;" m class:TestDistGitDownload access:public signature:(self) +TestDistGitDownload.test_no_spec tests/test_dist_git_client.py /^ def test_no_spec(self):$/;" m class:TestDistGitDownload access:public signature:(self) +TestDistGitDownload.workdir tests/test_dist_git_client.py /^ workdir = None$/;" v class:TestDistGitDownload access:public +URL dist-git-client.spec /^URL: https:\/\/github.com\/release-engineering\/dist-git.git$/;" t +Version dist-git-client.spec /^Version: 1.1$/;" t +_Args tests/test_dist_git_client.py /^ class _Args:$/;" c member:TestDistGitDownload.setup_method file: inherits: access:private +_detect_clone_url dist_git_client.py /^def _detect_clone_url():$/;" f access:protected signature:() +_get_argparser dist_git_client.py /^def _get_argparser():$/;" f access:protected signature:() +_load_config dist_git_client.py /^def _load_config(directory):$/;" f access:protected signature:(directory) +args tests/test_dist_git_client.py /^ args = None$/;" v class:TestDistGitDownload access:public +call dist_git_client.py /^def call(cmd, comment="Calling"):$/;" f access:public signature:(cmd, comment="Calling") +centos etc/default.ini /^[centos]$/;" s +centos-stream etc/default.ini /^[centos-stream]$/;" s +check_call dist_git_client.py /^def check_call(cmd, comment="Checked call"):$/;" f access:public signature:(cmd, comment="Checked call") +check_output dist_git_client.py /^def check_output(cmd, comment="Reading stdout from command"):$/;" f access:public signature:(cmd, comment="Reading stdout from command") +clone dist_git_client.py /^def clone(args, config):$/;" f access:public signature:(args, config) +clone_hostnames etc/default.ini /^clone_hostnames = copr-dist-git-dev.fedorainfracloud.org$/;" k section:fedora-copr-dev +clone_hostnames etc/default.ini /^clone_hostnames = copr-dist-git.fedorainfracloud.org$/;" k section:fedora-copr +clone_hostnames etc/default.ini /^clone_hostnames = git.centos.org$/;" k section:centos +clone_hostnames etc/default.ini /^clone_hostnames = gitlab.com$/;" k section:centos-stream +clone_hostnames etc/default.ini /^clone_hostnames =$/;" k section:fedora +cloning_pattern etc/default.ini /^cloning_pattern = https:\/\/git.centos.org\/rpms\/{package}.git$/;" k section:centos +cloning_pattern etc/default.ini /^cloning_pattern = https:\/\/gitlab.com\/redhat\/centos-stream\/rpms\/{package}.git$/;" k section:centos-stream +cloning_pattern etc/default.ini /^cloning_pattern = https:\/\/src.fedoraproject.org\/rpms\/{package}.git$/;" k section:fedora +cloning_pattern etc/default.ini /^cloning_pattern=https:\/\/copr-dist-git-dev.fedorainfracloud.org\/git\/{package}$/;" k section:fedora-copr-dev +cloning_pattern etc/default.ini /^cloning_pattern=https:\/\/copr-dist-git.fedorainfracloud.org\/git\/{package}$/;" k section:fedora-copr +cloning_pattern_package_parts etc/default.ini /^cloning_pattern_package_parts = owner_name project_name package_name$/;" k section:fedora-copr +cloning_pattern_package_parts etc/default.ini /^cloning_pattern_package_parts = owner_name project_name package_name$/;" k section:fedora-copr-dev +config tests/test_dist_git_client.py /^ config = None$/;" v class:TestDistGitDownload access:public +config_dir tests/test_dist_git_client.py /^ config_dir = None$/;" v class:TestDistGitDownload access:public +default_sum etc/default.ini /^default_sum = SHA1$/;" k section:centos +dist-git-client dist-git-client.spec /^Name: dist-git-client$/;" p +download dist_git_client.py /^def download(url, filename):$/;" f access:public signature:(url, filename) +download_file_and_check dist_git_client.py /^def download_file_and_check(url, params, distgit_config):$/;" f access:public signature:(url, params, distgit_config) +dry_run tests/test_dist_git_client.py /^ dry_run = False$/;" v class:TestDistGitDownload.setup_method._Args access:public +fedora etc/default.ini /^[fedora]$/;" s +fedora-copr etc/default.ini /^[fedora-copr]$/;" s +fedora-copr-dev etc/default.ini /^[fedora-copr-dev]$/;" s +forked_from tests/test_dist_git_client.py /^ forked_from = None$/;" v class:TestDistGitDownload.setup_method._Args access:public +get_distgit_config dist_git_client.py /^def get_distgit_config(config, forked_from=None):$/;" f access:public signature:(config, forked_from=None) +get_spec dist_git_client.py /^def get_spec(distgit_config):$/;" f access:public signature:(distgit_config) +git_origin_url tests/test_dist_git_client.py /^def git_origin_url(url):$/;" f access:public signature:(url) +handle_autospec dist_git_client.py /^def handle_autospec(spec_abspath, spec_basename, args):$/;" f access:public signature:(spec_abspath, spec_basename, args) +log_cmd dist_git_client.py /^def log_cmd(command, comment="Running command"):$/;" f access:public signature:(command, comment="Running command") +lookaside_location etc/default.ini /^lookaside_location = https:\/\/copr-dist-git-dev.fedorainfracloud.org$/;" k section:fedora-copr-dev +lookaside_location etc/default.ini /^lookaside_location = https:\/\/copr-dist-git.fedorainfracloud.org$/;" k section:fedora-copr +lookaside_location etc/default.ini /^lookaside_location = https:\/\/git.centos.org$/;" k section:centos +lookaside_location etc/default.ini /^lookaside_location = https:\/\/sources.stream.centos.org$/;" k section:centos-stream +lookaside_location etc/default.ini /^lookaside_location = https:\/\/src.fedoraproject.org$/;" k section:fedora +lookaside_uri_pattern etc/default.ini /^lookaside_uri_pattern = repo\/pkgs\/rpms\/{name}\/{filename}\/{hashtype}\/{hash}\/{filename}$/;" k section:fedora +lookaside_uri_pattern etc/default.ini /^lookaside_uri_pattern = repo\/pkgs\/{namespace[1]}\/{namespace[0]}\/{name}\/{filename}\/{hashtyp/;" k section:fedora-copr +lookaside_uri_pattern etc/default.ini /^lookaside_uri_pattern = repo\/pkgs\/{namespace[1]}\/{namespace[0]}\/{name}\/{filename}\/{hashtyp/;" k section:fedora-copr-dev +lookaside_uri_pattern etc/default.ini /^lookaside_uri_pattern = sources\/rpms\/{name}\/{filename}\/{hashtype}\/{hash}\/{filename}$/;" k section:centos-stream +lookaside_uri_pattern etc/default.ini /^lookaside_uri_pattern = sources\/{name}\/{refspec}\/{hash}$/;" k section:centos +main dist_git_client.py /^def main():$/;" f access:public signature:() +mkdir_p dist_git_client.py /^def mkdir_p(path):$/;" f access:public signature:(path) +path_prefixes etc/default.ini /^path_prefixes = \/redhat\/centos-stream\/rpms$/;" k section:centos-stream +pytest cache directory .pytest_cache/README.md /^# pytest cache directory #$/;" c +rpmautospec_expand dist_git_client.py /^ process_distgit as rpmautospec_expand,$/;" Y access:public nameref:unknown:process_distgit +rpmautospec_used dist_git_client.py /^ specfile_uses_rpmautospec as rpmautospec_used,$/;" Y access:public nameref:unknown:specfile_uses_rpmautospec +rpmautospec_used dist_git_client.py /^ rpmautospec_used = lambda _: False$/;" f access:public signature:(_) +setup_method tests/test_dist_git_client.py /^ def setup_method(self, method):$/;" m class:TestDistGitDownload access:public signature:(self, method) +sources dist_git_client.py /^def sources(args, config):$/;" f access:public signature:(args, config) +sources etc/default.ini /^sources = SOURCES$/;" k section:centos +sources_file etc/default.ini /^sources_file = .{name}.metadata$/;" k section:centos +specs etc/default.ini /^specs = SPECS$/;" k section:centos +srpm dist_git_client.py /^def srpm(args, config):$/;" f access:public signature:(args, config) +teardown_method tests/test_dist_git_client.py /^ def teardown_method(self, method):$/;" m class:TestDistGitDownload access:public signature:(self, method) +test_centos tests/test_dist_git_client.py /^ def test_centos(self, download):$/;" m class:TestDistGitDownload access:public signature:(self, download) +test_centos_download tests/test_dist_git_client.py /^ def test_centos_download(self, patched_check_call):$/;" m class:TestDistGitDownload access:public signature:(self, patched_check_call) +test_copr_distgit tests/test_dist_git_client.py /^ def test_copr_distgit(self, download):$/;" m class:TestDistGitDownload access:public signature:(self, download) +test_duplicate_prefix tests/test_dist_git_client.py /^ def test_duplicate_prefix(self):$/;" m class:TestDistGitDownload access:public signature:(self) +test_fedora_new tests/test_dist_git_client.py /^ def test_fedora_new(self, download, base):$/;" m class:TestDistGitDownload access:public signature:(self, download, base) +test_fedora_old tests/test_dist_git_client.py /^ def test_fedora_old(self, download, base):$/;" m class:TestDistGitDownload access:public signature:(self, download, base) +test_load_prefix tests/test_dist_git_client.py /^ def test_load_prefix(self):$/;" m class:TestDistGitDownload access:public signature:(self) +test_load_prefix_fail tests/test_dist_git_client.py /^ def test_load_prefix_fail(self):$/;" m class:TestDistGitDownload access:public signature:(self) +test_no_git_config tests/test_dist_git_client.py /^ def test_no_git_config(self):$/;" m class:TestDistGitDownload access:public signature:(self) +test_no_spec tests/test_dist_git_client.py /^ def test_no_spec(self):$/;" m class:TestDistGitDownload access:public signature:(self) +tests_init_git dist_git_client.py /^def tests_init_git(files=None):$/;" f access:public signature:(files=None) +workdir tests/test_dist_git_client.py /^ workdir = None$/;" v class:TestDistGitDownload access:public diff --git a/dist-git-client/tests/__pycache__/test_dist_git_client.cpython-312-pytest-7.4.3.pyc b/dist-git-client/tests/__pycache__/test_dist_git_client.cpython-312-pytest-7.4.3.pyc new file mode 100644 index 0000000..aafd604 Binary files /dev/null and b/dist-git-client/tests/__pycache__/test_dist_git_client.cpython-312-pytest-7.4.3.pyc differ diff --git a/dist-git-client/tests/test_dist_git_client.py b/dist-git-client/tests/test_dist_git_client.py new file mode 100644 index 0000000..9b4df4f --- /dev/null +++ b/dist-git-client/tests/test_dist_git_client.py @@ -0,0 +1,208 @@ +""" +dist-git-client testsuite +""" + +import os +import shutil +import tempfile + +import pytest + +try: + from unittest import mock +except ImportError: + import mock + +from dist_git_client import (sources, srpm, _load_config, check_output, + _detect_clone_url, get_distgit_config, unittests_init_git, +) + +# pylint: disable=useless-object-inheritance + +def git_origin_url(url): + """ setup .git/config with core.origin.url == URL """ + with open(".git/config", "a+") as gcf: + gcf.write('[remote "origin"]\n') + gcf.write('url = {0}\n'.format(url)) + + +class TestDistGitDownload(object): + """ Test the 'sources()' method """ + config = None + args = None + workdir = None + config_dir = None + + def setup_method(self, method): + _unused_but_needed_for_el6 = (method) + testdir = os.path.dirname(__file__) + projdir = os.path.dirname(testdir) + self.config_dir = os.path.join(projdir, 'etc') + self.config = _load_config(self.config_dir) + class _Args: + # pylint: disable=too-few-public-methods + dry_run = False + forked_from = None + self.args = _Args() + self.workdir = tempfile.mkdtemp(prefix="dist-git-client-test-") + os.chdir(self.workdir) + + def teardown_method(self, method): + _unused_but_needed_for_el6 = (method) + shutil.rmtree(self.workdir) + + + @mock.patch('dist_git_client.download_file_and_check') + def test_copr_distgit(self, download): + unittests_init_git([ + # .spec in .git, in Copr it is possible + ("test.spec", ""), + ("sources", "2102fd0602de72e58765adcbf92349d8 retrace-server-git-955.3e4742a.tar.gz\n"), + ]) + git_origin_url("https://copr-dist-git.fedorainfracloud.org/git/@abrt/retrace-server-devel/retrace-server.git") + sources(self.args, self.config) + assert len(download.call_args_list) == 1 + assert download.call_args_list[0][0][0] == ( + "https://copr-dist-git.fedorainfracloud.org/repo/pkgs/" + "@abrt/retrace-server-devel/retrace-server/retrace-server-git-955.3e4742a.tar.gz/" + "md5/2102fd0602de72e58765adcbf92349d8/retrace-server-git-955.3e4742a.tar.gz" + ) + + @pytest.mark.parametrize('base', ["tar", "tar/"]) + @mock.patch('dist_git_client.download_file_and_check') + def test_fedora_old(self, download, base): + """ + Old sources format + ssh clone + """ + unittests_init_git([ + ("tar.spec", ""), + ("sources", "0ced6f20b9fa1bea588005b5ad4b52c1 tar-1.26.tar.xz\n"), + ]) + git_origin_url("ssh://praiskup@pkgs.fedoraproject.org/rpms/" + base) + sources(self.args, self.config) + assert len(download.call_args_list) == 1 + assert download.call_args_list[0][0][0] == ( + "https://src.fedoraproject.org/repo/pkgs/rpms/" + "tar/tar-1.26.tar.xz/md5/0ced6f20b9fa1bea588005b5ad4b52c1/tar-1.26.tar.xz" + ) + + @pytest.mark.parametrize('base', ["tar.git", "tar.git/"]) + @mock.patch('dist_git_client.download_file_and_check') + def test_fedora_new(self, download, base): + """ + New sources format + anonymous clone + """ + sha512 = ( + "1bd13854009b6ee08958481738e6bf661e40216a2befe461d06b4b350eb882e43" + "1b3a4eeea7ca1d35d37102df76194c9d933df2b18b3c5401350e9fc17017750" + ) + unittests_init_git([ + ("tar.spec", ""), + ("sources", "SHA512 (tar-1.32.tar.xz) = {0}\n".format(sha512)), + ]) + git_origin_url("https://src.fedoraproject.org/rpms/" + base) + sources(self.args, self.config) + assert len(download.call_args_list) == 1 + url = ( + "https://src.fedoraproject.org/repo/pkgs/rpms/" + "tar/tar-1.32.tar.xz/sha512/{sha512}/tar-1.32.tar.xz" + ).format(sha512=sha512) + assert download.call_args_list[0][0][0] == url + + @mock.patch('dist_git_client.download_file_and_check') + def test_centos(self, download): + """ + Anonymous centos clone + """ + unittests_init_git([ + ("SPECS/centpkg-minimal.spec", ""), + (".centpkg-minimal.metadata", "cf9ce8d900768ed352a6f19a2857e64403643545 SOURCES/centpkg-minimal.tar.gz\n"), + ]) + git_origin_url("https://git.centos.org/rpms/centpkg-minimal.git") + sources(self.args, self.config) + assert len(download.call_args_list) == 1 + assert download.call_args_list[0][0][0] == ( + "https://git.centos.org/sources/centpkg-minimal/main/" + "cf9ce8d900768ed352a6f19a2857e64403643545" + ) + assert download.call_args_list[0][0][2]["sources"] == "SOURCES" + assert download.call_args_list[0][0][1]["hashtype"] == "sha1" + + oldref = check_output(["git", "rev-parse", "HEAD"]).decode("utf-8") + oldref = oldref.strip() + + # create new commit, and checkout back (so --show-current is not set) + check_output(["git", "commit", "--allow-empty", "-m", "empty"]) + check_output(["git", "checkout", "-q", oldref]) + + sources(self.args, self.config) + assert download.call_args_list[1][0][0] == ( + "https://git.centos.org/sources/centpkg-minimal/{0}/" + "cf9ce8d900768ed352a6f19a2857e64403643545" + ).format(oldref) + + + @mock.patch("dist_git_client.subprocess.check_call") + def test_centos_download(self, patched_check_call): + unittests_init_git([ + ("SPECS/centpkg-minimal.spec", ""), + (".centpkg-minimal.metadata", "cf9ce8d900768ed352a6f19a2857e64403643545 SOURCES/centpkg-minimal.tar.gz\n"), + ]) + git_origin_url("https://git.centos.org/rpms/centpkg-minimal.git") + setattr(self.args, "outputdir", os.path.join(self.workdir, "result")) + setattr(self.args, "mock_chroot", None) + srpm(self.args, self.config) + assert patched_check_call.call_args_list[0][0][0] == [ + 'rpmbuild', '-bs', + os.path.join(self.workdir, "SPECS", "centpkg-minimal.spec"), + '--define', 'dist %nil', + '--define', '_sourcedir ' + self.workdir + '/SOURCES', + '--define', '_srcrpmdir ' + self.workdir + '/result', + '--define', '_disable_source_fetch 1', + ] + + def test_duplicate_prefix(self): + modified_dir = os.path.join(self.workdir, "config") + shutil.copytree(self.config_dir, modified_dir) + modified_file = os.path.join(modified_dir, "default.ini") + with open(modified_file, "a+") as fmodify: + fmodify.write( + "\n\n[hack]\n" + "clone_hostnames = gitlab.com\n" + "path_prefixes = /redhat/centos-stream/rpms\n" + ) + with pytest.raises(RuntimeError) as err: + _load_config(modified_dir) + assert "Duplicate prefix /redhat" in str(err) + + def test_no_git_config(self): + with pytest.raises(RuntimeError) as err: + _detect_clone_url() + assert "is not a git" in str(err) + + def test_load_prefix(self): + prefixed_url = "git://gitlab.com/redhat/centos-stream/rpms/test.git" + _, config = get_distgit_config(self.config, forked_from=prefixed_url) + assert config["lookaside_location"] == "https://sources.stream.centos.org" + + def test_load_prefix_fail(self): + prefixed_url = "git://gitlab.com/non-existent/centos-stream/rpms/test.git" + with pytest.raises(RuntimeError) as err: + get_distgit_config(self.config, forked_from=prefixed_url) + msg = "Path /non-existent/centos-stream/rpms/test.git does not " + \ + "match any of 'path_prefixes' for 'gitlab.com' hostname" + assert msg in str(err) + + def test_no_spec(self): + unittests_init_git([ + ("sources", "0ced6f20b9fa1bea588005b5ad4b52c1 tar-1.26.tar.xz\n"), + ]) + git_origin_url("ssh://praiskup@pkgs.fedoraproject.org/rpms/tar") + with pytest.raises(RuntimeError) as err: + sources(self.args, self.config) + strings = [ + "directory, 0 found", + "Exactly one spec file expected in", + ] + for string in strings: + assert string in str(err)