centos/centpkg
1
0
Fork 0
mirror of https://git.centos.org/centos/centpkg.git synced 2025-02-23 08:12:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Reformat centpkg src with black

Browse source 
Signed-off-by: Stephen Gallagher <sgallagh@redhat.com>
This commit is contained in:
Stephen Gallagher 2024-09-13 11:32:14 -04:00
parent ff58b37f1f
commit 85cce45045
6 changed files with 467 additions and 297 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

164
src/centpkg/__init__.py
View file

@ -1,8 +1,8 @@
# pylint: disable=line-too-long,abstract-class-not-used
'''
"""
Top level function library for centpkg
'''
"""
# Author(s):
# Jesse Keating <jkeating@redhat.com>
@ -28,7 +28,7 @@ from . import cli # noqa
from .lookaside import StreamLookasideCache, SIGLookasideCache, CLLookasideCache
_DEFAULT_VERSION = '9'
_DEFAULT_VERSION = "9"
class DistGitDirectory(object):
@ -40,17 +40,17 @@ class DistGitDirectory(object):
distrobranch = False
sigbranch = False
repo = None
git_origin_substr = 'git@gitlab.com/redhat/centos-stream'
git_origin_substr = "git@gitlab.com/redhat/centos-stream"
def __init__(self, branchtext, repo_path=None):
if repo_path:
# self.repo = git.cmd.Git(repo_path)
self.repo = git.repo.Repo(repo_path)
rhelbranchre = r'rhel-(?P<major>\d+)\.(?P<minor>\d+)(?:\.(?P<appstream>\d+))?'
sigtobranchre = r'c(?P<centosversion>\d+[s]?)-sig-(?P<signame>\w+)-?(?P<projectname>\w+)?-?(?P<releasename>\w+)?'
distrobranchre = r'c(?P<centosversion>\d+)-?(?P<projectname>\w+)?'
javabranchre = r'openjdk-portable-centos-(?P<centosversion>\d+)'
oldbranchre = r'(?P<signame>\w+)(?P<centosversion>\d)'
rhelbranchre = r"rhel-(?P<major>\d+)\.(?P<minor>\d+)(?:\.(?P<appstream>\d+))?"
sigtobranchre = r"c(?P<centosversion>\d+[s]?)-sig-(?P<signame>\w+)-?(?P<projectname>\w+)?-?(?P<releasename>\w+)?"
distrobranchre = r"c(?P<centosversion>\d+)-?(?P<projectname>\w+)?"
javabranchre = r"openjdk-portable-centos-(?P<centosversion>\d+)"
oldbranchre = r"(?P<signame>\w+)(?P<centosversion>\d)"
rhelmatch = re.search(rhelbranchre, branchtext)
sigmatch = re.match(sigtobranchre, branchtext)
distromatch = re.match(distrobranchre, branchtext)
@ -59,55 +59,60 @@ class DistGitDirectory(object):
if rhelmatch:
gd = rhelmatch.groupdict()
self.distrobranch = True
self.signame = 'centos'
self.centosversion = gd['major']
self.signame = "centos"
self.centosversion = gd["major"]
elif sigmatch:
gd = sigmatch.groupdict()
self.sigbranch = True
self.signame = gd['signame']
self.centosversion = gd['centosversion']
self.signame = gd["signame"]
self.centosversion = gd["centosversion"]
# Users have the option to specify (or not specify) common in their
# git repos. Ww need to handle these cases because common is not a
# project nor is it a release.
if gd['projectname'] != 'common':
self.projectname = gd['projectname']
if gd['releasename'] != 'common':
self.releasename = gd['releasename']
if gd["projectname"] != "common":
self.projectname = gd["projectname"]
if gd["releasename"] != "common":
self.releasename = gd["releasename"]
elif distromatch:
gd = distromatch.groupdict()
self.distrobranch = True
self.signame = 'centos'
self.centosversion = gd['centosversion']
self.signame = "centos"
self.centosversion = gd["centosversion"]
if gd['projectname'] != 'common':
self.projectname = gd['projectname']
if gd["projectname"] != "common":
self.projectname = gd["projectname"]
elif javamatch:
gd = javamatch.groupdict()
self.distrobranch = True
self.signame = 'centos'
self.centosversion = gd['centosversion']
self.signame = "centos"
self.centosversion = gd["centosversion"]
elif oldbranchmatch:
warnings.warn("This branch is deprecated and will be removed soon",
DeprecationWarning)
warnings.warn(
"This branch is deprecated and will be removed soon", DeprecationWarning
)
else:
if not self.is_fork():
warnings.warn('Unable to determine if this is a fork or not. Proceeding, but you should double check.')
warnings.warn(
"Unable to determine if this is a fork or not. Proceeding, but you should double check."
)
else:
self.distrobranch = True
self.signame = 'centos'
self.projectname = self.get_origin().split('_')[-1].replace('.git', '')
self.signame = "centos"
self.projectname = self.get_origin().split("_")[-1].replace(".git", "")
warnings.warn('Remote "origin" was detected as a fork, ignoring branch name checking')
warnings.warn(
'Remote "origin" was detected as a fork, ignoring branch name checking'
)
def get_origin(self):
if self.repo is None:
return ''
if 'origin' not in self.repo.remotes:
return ''
urls = [u for u in self.repo.remotes['origin'].urls]
return ""
if "origin" not in self.repo.remotes:
return ""
urls = [u for u in self.repo.remotes["origin"].urls]
if len(urls) == 0:
return ''
return ""
return urls[0]
def is_fork(self):
@ -124,42 +129,47 @@ class DistGitDirectory(object):
return False
return self.git_origin_substr not in self.get_origin()
@property
def target(self):
projectorcommon = self.projectname
releaseorcommon = self.releasename
if self.distrobranch:
if self.centosversion not in ('6', '7'):
return 'c{}s-candidate'.format(self.centosversion)
if self.centosversion not in ("6", "7"):
return "c{}s-candidate".format(self.centosversion)
else:
return '-'.join(filter(None, ['c'+self.centosversion,
projectorcommon]))
return "-".join(
filter(None, ["c" + self.centosversion, projectorcommon])
)
if not releaseorcommon:
if not projectorcommon or projectorcommon == 'common':
projectorcommon = 'common'
if not projectorcommon or projectorcommon == "common":
projectorcommon = "common"
else:
releaseorcommon = 'common'
releaseorcommon = "common"
return '-'.join(filter(None, [self.signame+self.centosversion,
projectorcommon, releaseorcommon])) + '-el{0}'.format(self.centosversion)
return "-".join(
filter(
None,
[self.signame + self.centosversion, projectorcommon, releaseorcommon],
)
) + "-el{0}".format(self.centosversion)
class Commands(Commands):
'''
"""
For the pyrpkg commands with centpkg behavior
'''
"""
def __init__(self, *args, **kwargs):
'''
"""
Init the object and some configuration details.
'''
"""
super(Commands, self).__init__(*args, **kwargs)
# For MD5 we want to use the old format of source files, the BSD format
# should only be used when configured for SHA512
self.source_entry_type = 'bsd' if self.lookasidehash != 'md5' else 'old'
self.branchre = 'c\d{1,}(s)?(tream)?|master'
self.source_entry_type = "bsd" if self.lookasidehash != "md5" else "old"
self.branchre = "c\d{1,}(s)?(tream)?|master"
@property
def distgitdir(self):
@ -168,19 +178,23 @@ class Commands(Commands):
@cached_property
def lookasidecache(self):
if self.layout.sources_file_template == "sources":
return StreamLookasideCache(self.lookasidehash,
return StreamLookasideCache(
self.lookasidehash,
self.lookaside,
self.lookaside_cgi,
)
else:
if self.distgitdir.sigbranch:
return SIGLookasideCache(self.lookasidehash,
return SIGLookasideCache(
self.lookasidehash,
self.lookaside,
self.lookaside_cgi,
self.repo_name,
self.branch_merge)
self.branch_merge,
)
else:
return CLLookasideCache(self.lookasidehash,
return CLLookasideCache(
self.lookasidehash,
self.lookaside,
self.lookaside_cgi,
self.repo_name,
@ -189,27 +203,41 @@ class Commands(Commands):
# redefined loaders
def load_rpmdefines(self):
'''
"""
Populate rpmdefines based on branch data
'''
"""
if not self.distgitdir.centosversion:
raise rpkgError('Could not get the OS version from the branch:{0}'.format(self.branch_merge))
raise rpkgError(
"Could not get the OS version from the branch:{0}".format(
self.branch_merge
)
)
self._distvar = self.distgitdir.centosversion
self._distval = self._distvar.replace('.', '_')
self._distval = self._distvar.replace(".", "_")
self._disttag = 'el%s' % self._distval
self._rpmdefines = ['--define', '_sourcedir %s' % self.layout.sourcedir,
'--define', '_specdir %s' % self.layout.specdir,
'--define', '_builddir %s' % self.layout.builddir,
'--define', '_srcrpmdir %s' % self.layout.srcrpmdir,
'--define', '_rpmdir %s' % self.layout.rpmdir,
'--define', 'dist .%s' % self._disttag,
self._disttag = "el%s" % self._distval
self._rpmdefines = [
"--define",
"_sourcedir %s" % self.layout.sourcedir,
"--define",
"_specdir %s" % self.layout.specdir,
"--define",
"_builddir %s" % self.layout.builddir,
"--define",
"_srcrpmdir %s" % self.layout.srcrpmdir,
"--define",
"_rpmdir %s" % self.layout.rpmdir,
"--define",
"dist .%s" % self._disttag,
# int and float this to remove the decimal
'--define', '%s 1' % self._disttag,
"--define",
"%s 1" % self._disttag,
# This is so the rhel macro is set for spec files
'--define', 'rhel %s' % self._distval.split('_')[0]]
"--define",
"rhel %s" % self._distval.split("_")[0],
]
self.log.debug("RPMDefines: %s" % self._rpmdefines)
def construct_build_url(self, *args, **kwargs):
@ -218,7 +246,7 @@ class Commands(Commands):
In CentOS/Fedora Koji, anonymous URL should have prefix "git+https://"
"""
url = super(Commands, self).construct_build_url(*args, **kwargs)
return 'git+{0}'.format(url)
return "git+{0}".format(url)
def load_target(self):
"""This sets the target attribute (used for mock and koji)"""

37
src/centpkg/__main__.py
View file

@ -1,6 +1,7 @@
'''
"""
The main behavior of centpkg
'''
"""
#
# Author(s):
# Jesse Keating <jkeating@redhat.com>
@ -32,7 +33,8 @@ def main():
program_name = os.path.basename(sys.argv[0])
default_user_config_path = os.path.join(
os.path.expanduser('~'), '.config', 'rpkg', '%s.conf' % program_name)
os.path.expanduser("~"), ".config", "rpkg", "%s.conf" % program_name
)
# Setup an argparser and parse the known commands to get the config file
@ -41,16 +43,23 @@ def main():
# --user-config
parser = pyrpkg.cli.ArgumentParser(add_help=False, allow_abbrev=False)
parser.add_argument('-C', '--config', help='Specify a config file to use',
default='/etc/rpkg/%s.conf' % program_name)
parser.add_argument('--user-config', help='Specify a user config file to use',
default=default_user_config_path)
parser.add_argument(
"-C",
"--config",
help="Specify a config file to use",
default="/etc/rpkg/%s.conf" % program_name,
)
parser.add_argument(
"--user-config",
help="Specify a user config file to use",
default=default_user_config_path,
)
(args, other) = parser.parse_known_args()
# Make sure we have a sane config file
if not os.path.exists(args.config) and not other[-1] in ['--help', '-h', 'help']:
sys.stderr.write('Invalid config file %s\n' % args.config)
if not os.path.exists(args.config) and not other[-1] in ["--help", "-h", "help"]:
sys.stderr.write("Invalid config file %s\n" % args.config)
sys.exit(1)
# Setup a configuration object and read config file data
@ -59,14 +68,14 @@ def main():
config.read(args.user_config)
client = centpkg.cli.centpkgClient(config, name=program_name)
client.do_imports(site='centpkg')
client.do_imports(site="centpkg")
client.parse_cmdline()
# This is due to a difference argparse behavior to Python 2 version.
# In Python 3, argparse will proceed to here without reporting
# "too few arguments". Instead, client.args does not have attribute
# command.
if not hasattr(client.args, 'command'):
if not hasattr(client.args, "command"):
client.parser.print_help()
sys.exit(1)
@ -74,7 +83,7 @@ def main():
try:
client.args.path = pyrpkg.utils.getcwd()
except Exception:
print('Could not get current path, have you deleted it?')
print("Could not get current path, have you deleted it?")
sys.exit(1)
# setup the logger -- This logger will take things of INFO or DEBUG and
@ -98,9 +107,9 @@ def main():
except KeyboardInterrupt:
pass
except Exception as e:
if getattr(client.args, 'debug', False):
if getattr(client.args, "debug", False):
raise
log.error('Could not execute %s: %s' % (client.args.command.__name__, e))
log.error("Could not execute %s: %s" % (client.args.command.__name__, e))
sys.exit(1)

17
src/centpkg/centos_cert.py
View file

@ -12,24 +12,29 @@ import datetime
class centos_cert_error(Exception):
pass
def _open_cert():
"""
Read in the certificate so we dont duplicate the code
"""
# Make sure we can even read the thing.
cert_file = os.path.join(os.path.expanduser('~'), ".koji", "client.crt")
cert_file = os.path.join(os.path.expanduser("~"), ".koji", "client.crt")
if not os.access(cert_file, os.R_OK):
raise centos_cert_error("""!!! cannot read your centos cert file !!!
!!! Ensure the file is readable and try again !!!""")
raw_cert = open(cert_file, 'rb').read()
raise centos_cert_error(
"""!!! cannot read your centos cert file !!!
!!! Ensure the file is readable and try again !!!"""
)
raw_cert = open(cert_file, "rb").read()
try:
my_cert = x509.load_pem_x509_certificate(raw_cert)
except TypeError:
# it was required to specify a backend prior to cryptography 3.1
from cryptography.hazmat.backends import default_backend
my_cert = x509.load_pem_x509_certificate(raw_cert, default_backend())
return my_cert
def verify_cert():
"""
Check that the user cert is valid.
@ -40,10 +45,10 @@ def verify_cert():
my_cert = _open_cert()
warn = datetime.datetime.now() + datetime.timedelta(days=21)
print(my_cert.not_valid_after.strftime('cert expires: %Y-%m-%d'))
print(my_cert.not_valid_after.strftime("cert expires: %Y-%m-%d"))
if my_cert.not_valid_after < warn:
print('WARNING: Your cert expires soon.')
print("WARNING: Your cert expires soon.")
def certificate_expired():

232
src/centpkg/cli.py
View file

@ -1,6 +1,7 @@
'''
"""
Command line behavior for centpkg
'''
"""
#
# Author(s):
# Jesse Keating <jkeating@redhat.com>
@ -30,21 +31,21 @@ import json
import os
import sys
_DEFAULT_API_BASE_URL = 'https://gitlab.com'
_DEFAULT_API_BASE_URL = "https://gitlab.com"
_DEFAULT_PARENT_NAMESPACE = {
'centpkg': 'redhat/centos-stream',
'centpkg-sig': 'CentOS',
"centpkg": "redhat/centos-stream",
"centpkg-sig": "CentOS",
}
class centpkgClient(cliClient):
def __init__(self, config, name='centpkg'):
def __init__(self, config, name="centpkg"):
self.DEFAULT_CLI_NAME = name
# Save the config for utilities such as get_repo_name()
centpkg.utils.dist_git_config = config
centpkg.utils.dist_git_config.add_section('__default')
centpkg.utils.dist_git_config.set('__default', 'cli_name', name)
centpkg.utils.dist_git_config.add_section("__default")
centpkg.utils.dist_git_config.set("__default", "cli_name", name)
super(centpkgClient, self).__init__(config, name)
@ -56,14 +57,17 @@ class centpkgClient(cliClient):
self.register_current_state()
def register_do_fork(self):
help_msg = 'Create a new fork of the current repository'
distgit_section = '{0}.distgit'.format(self.name)
help_msg = "Create a new fork of the current repository"
distgit_section = "{0}.distgit".format(self.name)
# uses default dist-git url in case section is not present
try:
distgit_api_base_url = config_get_safely(self.config, distgit_section, "apibaseurl")
distgit_api_base_url = config_get_safely(
self.config, distgit_section, "apibaseurl"
)
except rpkgError:
distgit_api_base_url = _DEFAULT_API_BASE_URL
description = textwrap.dedent('''
description = textwrap.dedent(
"""
Create a new fork of the current repository
Before the operation, you need to generate an API token at
@ -77,13 +81,17 @@ class centpkgClient(cliClient):
Below is a basic example of the command to fork a current repository:
{0} fork
'''.format(self.name, urlparse(distgit_api_base_url).netloc))
""".format(
self.name, urlparse(distgit_api_base_url).netloc
)
)
fork_parser = self.subparsers.add_parser(
'fork',
"fork",
formatter_class=argparse.RawDescriptionHelpFormatter,
help=help_msg,
description=description)
description=description,
)
fork_parser.set_defaults(command=self.do_distgit_fork)
def do_distgit_fork(self):
@ -91,9 +99,11 @@ class centpkgClient(cliClient):
That includes creating fork itself using API call and then adding
remote tracked repository
"""
distgit_section = '{0}.distgit'.format(self.name)
distgit_api_base_url = config_get_safely(self.config, distgit_section, "apibaseurl")
distgit_token = config_get_safely(self.config, distgit_section, 'token')
distgit_section = "{0}.distgit".format(self.name)
distgit_api_base_url = config_get_safely(
self.config, distgit_section, "apibaseurl"
)
distgit_token = config_get_safely(self.config, distgit_section, "token")
ret, repo_path = do_fork(
logger=self.log,
@ -105,8 +115,8 @@ class centpkgClient(cliClient):
)
# assemble url of the repo in web browser
fork_url = '{0}/{1}'.format(
distgit_api_base_url.rstrip('/'),
fork_url = "{0}/{1}".format(
distgit_api_base_url.rstrip("/"),
repo_path,
)
@ -117,11 +127,11 @@ class centpkgClient(cliClient):
self.log.info(msg.format(fork_url))
distgit_remote_base_url = self.config.get(
'{0}'.format(self.name),
"{0}".format(self.name),
"gitbaseurl",
vars={'repo': '{0}/{1}'.format(self.cmd.ns, self.cmd.repo_name)},
vars={"repo": "{0}/{1}".format(self.cmd.ns, self.cmd.repo_name)},
)
remote_name = repo_path.split('/')[0]
remote_name = repo_path.split("/")[0]
ret = do_add_remote(
base_url=distgit_api_base_url,
@ -153,19 +163,27 @@ class centpkgClient(cliClient):
# Get our internal only variables
cfg = ConfigParser()
cfg.read(internal_config_file)
pp_api_url = config_get_safely(cfg, "centpkg.internal", 'pp_api_url')
gitbz_query_url = config_get_safely(cfg, "centpkg.internal", 'gitbz_query_url')
rhel_dist_git = config_get_safely(cfg, "centpkg.internal", 'rhel_dist_git')
pp_api_url = config_get_safely(cfg, "centpkg.internal", "pp_api_url")
gitbz_query_url = config_get_safely(
cfg, "centpkg.internal", "gitbz_query_url"
)
rhel_dist_git = config_get_safely(cfg, "centpkg.internal", "rhel_dist_git")
# Find out divergent branch and stabalization
self.log.info("Checking current state:")
stream_version = self.cmd.target.split('-')[0]
stream_version = self.cmd.target.split("-")[0]
rhel_version = centpkg.utils.stream_mapping(stream_version)
try:
x_version, active_y, is_beta, in_stabilization = centpkg.utils.determine_active_y_version(rhel_version, pp_api_url)
x_version, active_y, is_beta, in_stabilization = (
centpkg.utils.determine_active_y_version(rhel_version, pp_api_url)
)
except AssertionError as e:
self.log.error(" Error: centpkg cannot determine the development phase.")
self.log.error(" Please file an issue at https://git.centos.org/centos/centpkg")
self.log.error(
" Error: centpkg cannot determine the development phase."
)
self.log.error(
" Please file an issue at https://git.centos.org/centos/centpkg"
)
self.log.error(" Workaround: Use the --rhel-target option")
self.log.error("Exiting")
raise SystemExit(1)
@ -175,11 +193,8 @@ class centpkgClient(cliClient):
divergent_branch = True
else:
divergent_branch = centpkg.utils.does_divergent_branch_exist(
self.cmd.repo_name,
x_version,
active_y,
rhel_dist_git,
"rpms")
self.cmd.repo_name, x_version, active_y, rhel_dist_git, "rpms"
)
# Good to know
if in_stabilization:
self.log.info(" we are in stabilization mode.")
@ -193,7 +208,9 @@ class centpkgClient(cliClient):
self.log.info(" a divergent branch was not found.")
else:
self.log.error("NO INTERNAL CONFIGURATION")
self.log.error("Hint: If you are internal, install the rhel-packager package from https://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-9/compose/BaseOS/x86_64/os/Packages/")
self.log.error(
"Hint: If you are internal, install the rhel-packager package from https://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-9/compose/BaseOS/x86_64/os/Packages/"
)
self.log.error("Exiting")
raise SystemExit(1)
@ -201,9 +218,10 @@ class centpkgClient(cliClient):
def register_build(self):
# Do all the work from the super class
super(centpkgClient, self).register_build()
build_parser = self.subparsers.choices['build']
build_parser = self.subparsers.choices["build"]
build_parser.formatter_class = argparse.RawDescriptionHelpFormatter
build_parser.description = textwrap.dedent('''
build_parser.description = textwrap.dedent(
"""
{0}
centpkg now sets the rhel metadata with --rhel-target.
@ -214,60 +232,93 @@ class centpkgClient(cliClient):
* latest - This will always build for the next Y-stream release
* none - This will not build in brew. No rhel metadata is set.
'''.format('\n'.join(textwrap.wrap(build_parser.description))))
""".format(
"\n".join(textwrap.wrap(build_parser.description))
)
)
# Now add our additional option
build_parser.add_argument(
'--rhel-target',
choices=['exception', 'zstream', 'latest', 'none'],
help='Set the rhel-target metadata')
"--rhel-target",
choices=["exception", "zstream", "latest", "none"],
help="Set the rhel-target metadata",
)
# Overloaded _build
def _build(self, sets=None):
# Only do rhel-target if we are centpkg, not centpkg-sig, or if we are doing scratch
if not os.path.basename(sys.argv[0]).endswith('-sig') and not self.args.scratch:
if not os.path.basename(sys.argv[0]).endswith("-sig") and not self.args.scratch:
# Only run if we have internal configuraions
internal_config_file = "/etc/rpkg/centpkg_internal.conf"
if os.path.exists(internal_config_file):
# If rhel-target is set, no need to check, just use it
if hasattr(self.args, 'rhel_target') and self.args.rhel_target:
if hasattr(self.args, "rhel_target") and self.args.rhel_target:
# If rhel-target set to none, do nothing with metadata
if self.args.rhel_target.lower() != "none":
# If custom-user-metadata set, add onto it
if hasattr(self.args, 'custom_user_metadata') and self.args.custom_user_metadata:
if (
hasattr(self.args, "custom_user_metadata")
and self.args.custom_user_metadata
):
try:
temp_custom_user_metadata = json.loads(self.args.custom_user_metadata)
temp_custom_user_metadata = json.loads(
self.args.custom_user_metadata
)
# Use ValueError instead of json.JSONDecodeError for Python 2 and 3 compatibility
except ValueError as e:
self.parser.error("--custom-user-metadata is not valid JSON: %s" % e)
self.parser.error(
"--custom-user-metadata is not valid JSON: %s" % e
)
if not isinstance(temp_custom_user_metadata, dict):
self.parser.error("--custom-user-metadata must be a JSON object")
temp_custom_user_metadata["rhel-target"] = self.args.rhel_target
self.parser.error(
"--custom-user-metadata must be a JSON object"
)
temp_custom_user_metadata["rhel-target"] = (
self.args.rhel_target
)
else:
temp_custom_user_metadata = {"rhel-target": self.args.rhel_target}
self.args.custom_user_metadata = json.dumps(temp_custom_user_metadata)
temp_custom_user_metadata = {
"rhel-target": self.args.rhel_target
}
self.args.custom_user_metadata = json.dumps(
temp_custom_user_metadata
)
else:
# Get our internal only variables
cfg = ConfigParser()
cfg.read(internal_config_file)
pp_api_url = config_get_safely(cfg, "centpkg.internal", 'pp_api_url')
gitbz_query_url = config_get_safely(cfg, "centpkg.internal", 'gitbz_query_url')
rhel_dist_git = config_get_safely(cfg, "centpkg.internal", 'rhel_dist_git')
pp_api_url = config_get_safely(
cfg, "centpkg.internal", "pp_api_url"
)
gitbz_query_url = config_get_safely(
cfg, "centpkg.internal", "gitbz_query_url"
)
rhel_dist_git = config_get_safely(
cfg, "centpkg.internal", "rhel_dist_git"
)
# Find out divergent branch and stabalization
self.log.info("Checking rhel-target information:")
stream_version = self.cmd.target.split('-')[0]
stream_version = self.cmd.target.split("-")[0]
rhel_version = centpkg.utils.stream_mapping(stream_version)
try:
x_version, active_y, is_beta, in_stabilization = centpkg.utils.determine_active_y_version(rhel_version, pp_api_url)
x_version, active_y, is_beta, in_stabilization = (
centpkg.utils.determine_active_y_version(
rhel_version, pp_api_url
)
)
except AssertionError as e:
self.log.error(" Error: centpkg cannot determine the development phase.")
self.log.error(" Please file an issue at https://git.centos.org/centos/centpkg")
self.log.error(
" Error: centpkg cannot determine the development phase."
)
self.log.error(
" Please file an issue at https://git.centos.org/centos/centpkg"
)
self.log.error(" Workaround: Use the --rhel-target option")
self.log.error("Exiting")
raise SystemExit(1)
@ -282,7 +333,8 @@ class centpkgClient(cliClient):
x_version,
active_y,
rhel_dist_git,
"rpms")
"rpms",
)
# Good to know
if divergent_branch and not is_beta:
@ -297,51 +349,74 @@ class centpkgClient(cliClient):
self.log.info(" we are not in stabilization mode.")
# Update args.custom_user_metadata
if hasattr(self.args, 'custom_user_metadata') and self.args.custom_user_metadata:
if (
hasattr(self.args, "custom_user_metadata")
and self.args.custom_user_metadata
):
try:
temp_custom_user_metadata = json.loads(self.args.custom_user_metadata)
temp_custom_user_metadata = json.loads(
self.args.custom_user_metadata
)
# Use ValueError instead of json.JSONDecodeError for Python 2 and 3 compatibility
except ValueError as e:
self.parser.error("--custom-user-metadata is not valid JSON: %s" % e)
self.parser.error(
"--custom-user-metadata is not valid JSON: %s" % e
)
if not isinstance(temp_custom_user_metadata, dict):
self.parser.error("--custom-user-metadata must be a JSON object")
self.parser.error(
"--custom-user-metadata must be a JSON object"
)
if divergent_branch:
temp_custom_user_metadata["rhel-target"] = "latest"
elif not divergent_branch and not in_stabilization:
temp_custom_user_metadata["rhel-target"] = "zstream"
else:
self.log.error("We are currently in Stabalization mode")
self.log.error("You must either set the rhel-target (--rhel-target)")
self.log.error(
"You must either set the rhel-target (--rhel-target)"
)
self.log.error("or branch for the previous version.")
self.log.error("Exiting")
raise SystemExit(1)
self.args.custom_user_metadata = json.dumps(temp_custom_user_metadata)
self.args.custom_user_metadata = json.dumps(
temp_custom_user_metadata
)
else:
if divergent_branch:
self.args.custom_user_metadata = '{"rhel-target": "latest"}'
elif not divergent_branch and not in_stabilization:
self.args.custom_user_metadata = '{"rhel-target": "zstream"}'
self.args.custom_user_metadata = (
'{"rhel-target": "zstream"}'
)
else:
self.log.error("We are currently in Stabalization mode")
self.log.error("You must either set the rhel-target (--rhel-target)")
self.log.error(
"You must either set the rhel-target (--rhel-target)"
)
self.log.error("or branch for the previous version.")
self.log.error("Exiting")
raise SystemExit(1)
# Good to know
self.log.info(" rhel-target: " + json.loads(self.args.custom_user_metadata)['rhel-target'])
self.log.info(
" rhel-target: "
+ json.loads(self.args.custom_user_metadata)["rhel-target"]
)
else:
self.log.error("NO INTERNAL CONFIGURATION")
self.log.error("Only scratch builds are allowed without internal configurations")
self.log.error("Hint: Install the rhel-packager package from https://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-9/compose/BaseOS/x86_64/os/Packages/ if you want to build the package via internal (Red Hat) configuration.")
self.log.error(
"Only scratch builds are allowed without internal configurations"
)
self.log.error(
"Hint: Install the rhel-packager package from https://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-9/compose/BaseOS/x86_64/os/Packages/ if you want to build the package via internal (Red Hat) configuration."
)
self.log.error("Exiting")
raise SystemExit(1)
# Proceed with build
return super(centpkgClient, self)._build(sets)
def register_request_gated_side_tag(self):
"""Register command line parser for subcommand request-gated-side-tag"""
parser = self.subparsers.add_parser(
@ -365,17 +440,26 @@ class centpkgClient(cliClient):
project_name_with_namespace = parent_namespace + "/" + self.args.repo[0]
short_name = self.args.repo[0].split("/")[-1]
else:
project_name_with_namespace = parent_namespace+"/rpms/"+self.args.repo[0]
project_name_with_namespace = (
parent_namespace + "/rpms/" + self.args.repo[0]
)
short_name = self.args.repo[0]
try:
project = gl.projects.get(project_name_with_namespace)
except gitlab.exceptions.GitlabGetError as e:
self.log.error("There is no repository with the given name. Did you spell it correctly?")
self.log.error(
"There is no repository with the given name. Did you spell it correctly?"
)
self.log.error("Fatal: ")
self.log.error(e)
raise SystemExit(1)
if not project.name == short_name:
self.log.error("Fatal: ")
self.log.error("Project name is wrong: " + short_name + " it should be: " + project.name)
self.log.error(
"Project name is wrong: "
+ short_name
+ " it should be: "
+ project.name
)
raise SystemExit(1)
super(centpkgClient, self).clone()

97
src/centpkg/lookaside.py
View file

@ -58,16 +58,19 @@ class StreamLookasideCache(CGILookasideCache):
It inherits most of its behavior from `pyrpkg.lookasideCGILookasideCache`.
"""
def __init__(self, hashtype, download_url, upload_url, client_cert=None, ca_cert=None):
def __init__(
self, hashtype, download_url, upload_url, client_cert=None, ca_cert=None
):
super(StreamLookasideCache, self).__init__(
hashtype, download_url, upload_url,
client_cert=client_cert, ca_cert=ca_cert)
hashtype, download_url, upload_url, client_cert=client_cert, ca_cert=ca_cert
)
def get_download_url(self, name, filename, hash, hashtype=None, **kwargs):
_name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
return super(StreamLookasideCache, self).get_download_url(
_name, filename, hash, hashtype=hashtype, **kwargs)
_name, filename, hash, hashtype=hashtype, **kwargs
)
def remote_file_exists(self, name, filename, hashstr):
"""
@ -98,10 +101,10 @@ class StreamLookasideCache(CGILookasideCache):
try:
status = super(StreamLookasideCache, self).remote_file_exists(
_name, filename, hashstr)
_name, filename, hashstr
)
except UploadError as e:
self.log.error('Error checking for %s at %s'
% (filename, self.upload_url))
self.log.error("Error checking for %s at %s" % (filename, self.upload_url))
self.log.error(e)
raise SystemExit(1)
@ -139,8 +142,7 @@ class StreamLookasideCache(CGILookasideCache):
"""
_name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
return super(StreamLookasideCache, self).upload(
_name, filename, hashstr)
return super(StreamLookasideCache, self).upload(_name, filename, hashstr)
def download(self, name, filename, hashstr, outfile, hashtype=None, **kwargs):
"""
@ -178,7 +180,8 @@ class StreamLookasideCache(CGILookasideCache):
_name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
return super(StreamLookasideCache, self).download(
_name, filename, hashstr, outfile, hashtype=hashtype, **kwargs)
_name, filename, hashstr, outfile, hashtype=hashtype, **kwargs
)
class CLLookasideCache(CGILookasideCache):
@ -190,22 +193,23 @@ class CLLookasideCache(CGILookasideCache):
def __init__(self, hashtype, download_url, upload_url, name, branch):
super(CLLookasideCache, self).__init__(
hashtype, download_url, upload_url, name, branch)
hashtype, download_url, upload_url, name, branch
)
self.name = name
self.branch = branch
def get_download_url(self, name, filename, hash, hashtype=None, **kwargs):
self.download_path='%(name)s/%(branch)s/%(hash)s'
self.download_path = "%(name)s/%(branch)s/%(hash)s"
if "/" in name:
real_name = name.split("/")[-1]
else:
real_name = name
path_dict = {
'name': real_name,
'filename': filename,
'branch': self.branch,
'hash': hash,
'hashtype': hashtype
"name": real_name,
"filename": filename,
"branch": self.branch,
"hash": hash,
"hashtype": hashtype,
}
path = self.download_path % path_dict
return os.path.join(self.download_url, path)
@ -217,25 +221,36 @@ class SIGLookasideCache(CGILookasideCache):
It inherits most of its behavior from `pyrpkg.lookasideCGILookasideCache`.
"""
def __init__(self, hashtype, download_url, upload_url, name, branch, client_cert=None, ca_cert=None):
def __init__(
self,
hashtype,
download_url,
upload_url,
name,
branch,
client_cert=None,
ca_cert=None,
):
super(SIGLookasideCache, self).__init__(
hashtype, download_url, upload_url, client_cert=client_cert, ca_cert=ca_cert)
hashtype, download_url, upload_url, client_cert=client_cert, ca_cert=ca_cert
)
self.name = name
self.branch = branch
def get_download_url(self, name, filename, hash, hashtype=None, **kwargs):
download_path = '%(name)s/%(branch)s/%(hash)s'
download_path = "%(name)s/%(branch)s/%(hash)s"
if "/" in name:
real_name = name.split("/")[-1]
else:
real_name = name
path_dict = {
'name': real_name,
'filename': filename,
'branch': self.branch,
'hash': hash,
'hashtype': hashtype
"name": real_name,
"filename": filename,
"branch": self.branch,
"hash": hash,
"hashtype": hashtype,
}
path = download_path % path_dict
return os.path.join(self.download_url, path)
@ -256,9 +271,11 @@ class SIGLookasideCache(CGILookasideCache):
# https://bugzilla.redhat.com/show_bug.cgi?id=1241059
_name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
post_data = [('name', _name),
('%ssum' % self.hashtype, hash),
('filename', filename)]
post_data = [
("name", _name),
("%ssum" % self.hashtype, hash),
("filename", filename),
]
with io.BytesIO() as buf:
c = pycurl.Curl()
@ -270,8 +287,7 @@ class SIGLookasideCache(CGILookasideCache):
if os.path.exists(self.client_cert):
c.setopt(pycurl.SSLCERT, self.client_cert)
else:
self.log.warning("Missing certificate: %s"
% self.client_cert)
self.log.warning("Missing certificate: %s" % self.client_cert)
if self.ca_cert is not None:
if os.path.exists(self.ca_cert):
@ -280,7 +296,7 @@ class SIGLookasideCache(CGILookasideCache):
self.log.warning("Missing certificate: %s", self.ca_cert)
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
c.setopt(pycurl.USERPWD, ':')
c.setopt(pycurl.USERPWD, ":")
try:
c.perform()
@ -299,16 +315,15 @@ class SIGLookasideCache(CGILookasideCache):
# Lookaside CGI script returns these strings depending on whether
# or not the file exists:
if output == b'Available':
if output == b"Available":
return True
if output == b'Missing':
if output == b"Missing":
return False
# Something unexpected happened
self.log.debug(output)
raise UploadError('Error checking for %s at %s'
% (filename, self.upload_url))
raise UploadError("Error checking for %s at %s" % (filename, self.upload_url))
def upload(self, name, filepath, hash):
"""Upload a source file
@ -326,9 +341,11 @@ class SIGLookasideCache(CGILookasideCache):
return
self.log.info("Uploading: %s", filepath)
post_data = [('name', name),
('%ssum' % self.hashtype, hash),
('file', (pycurl.FORM_FILE, filepath))]
post_data = [
("name", name),
("%ssum" % self.hashtype, hash),
("file", (pycurl.FORM_FILE, filepath)),
]
with io.BytesIO() as buf:
c = pycurl.Curl()
@ -351,7 +368,7 @@ class SIGLookasideCache(CGILookasideCache):
self.log.warning("Missing certificate: %s", self.ca_cert)
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
c.setopt(pycurl.USERPWD, ':')
c.setopt(pycurl.USERPWD, ":")
try:
c.perform()
@ -366,7 +383,7 @@ class SIGLookasideCache(CGILookasideCache):
output = buf.getvalue().strip()
# Get back a new line, after displaying the download progress
sys.stdout.write('\n')
sys.stdout.write("\n")
sys.stdout.flush()
if status != 200:

131
src/centpkg/utils.py
View file

@ -28,6 +28,7 @@ import git as gitpython
dist_git_config = None
def do_fork(logger, base_url, token, repo_name, namespace, cli_name):
"""
Creates a fork of the project.
@ -40,27 +41,30 @@ def do_fork(logger, base_url, token, repo_name, namespace, cli_name):
:return: a tuple consisting of whether the fork needed to be created (bool)
and the fork path (string)
"""
api_url = '{0}/api/v4'.format(base_url.rstrip('/'))
api_url = "{0}/api/v4".format(base_url.rstrip("/"))
project_id = quote_plus("redhat/centos-stream/{0}/{1}".format(namespace, repo_name))
fork_url = '{0}/projects/{1}/fork'.format(api_url, project_id)
fork_url = "{0}/projects/{1}/fork".format(api_url, project_id)
headers = {
'PRIVATE-TOKEN': token,
'Accept': 'application/json',
'Content-Type': 'application/json'
"PRIVATE-TOKEN": token,
"Accept": "application/json",
"Content-Type": "application/json",
}
# define a new repository name/path to avoid collision with other projects
safe_name = "centos_{0}_{1}".format(namespace, repo_name)
payload = json.dumps({
'name': safe_name, # name of the project after forking
'path': safe_name,
})
payload = json.dumps(
{
"name": safe_name, # name of the project after forking
"path": safe_name,
}
)
try:
rv = requests.post(
fork_url, headers=headers, data=payload, timeout=60)
rv = requests.post(fork_url, headers=headers, data=payload, timeout=60)
except ConnectionError as error:
error_msg = ('The connection to API failed while trying to '
'create a new fork. The error was: {0}'.format(str(error)))
error_msg = (
"The connection to API failed while trying to "
"create a new fork. The error was: {0}".format(str(error))
)
raise rpkgError(error_msg)
try:
@ -71,49 +75,64 @@ def do_fork(logger, base_url, token, repo_name, namespace, cli_name):
pass
if rv.ok:
fork_id = rv.json()['id']
fork_id = rv.json()["id"]
try:
# Unprotect c9s in fork
rv = requests.delete('{0}/projects/{1}/protected_branches/{2}'.format(api_url, fork_id, 'c9s'), headers=headers)
rv = requests.delete(
"{0}/projects/{1}/protected_branches/{2}".format(
api_url, fork_id, "c9s"
),
headers=headers,
)
except ConnectionError as error:
error_msg = ('The connection to API failed while trying to unprotect c9s branch'
'in the fork. The error was: {0}'.format(str(error)))
error_msg = (
"The connection to API failed while trying to unprotect c9s branch"
"in the fork. The error was: {0}".format(str(error))
)
raise rpkgError(error_msg)
try:
# Reprotect c9s to disable pushes
# Only maintainers in gitlab are allowed to push with the following config
# In CS, every pkg maintainer is considered as a developer in gitlab
data = {'id': fork_id,
'name': 'c9s',
'allowed_to_push': [{'access_level': 40}],
'allowed_to_merge': [{'access_level': 40}],
data = {
"id": fork_id,
"name": "c9s",
"allowed_to_push": [{"access_level": 40}],
"allowed_to_merge": [{"access_level": 40}],
}
rv = requests.post('{0}/projects/{1}/protected_branches'.format(api_url, fork_id), json=data, headers=headers)
rv = requests.post(
"{0}/projects/{1}/protected_branches".format(api_url, fork_id),
json=data,
headers=headers,
)
except ConnectionError as error:
error_msg = ('The connection to API failed while trying to reprotect c9s branch'
'in the fork fork. The error was: {0}'.format(str(error)))
error_msg = (
"The connection to API failed while trying to reprotect c9s branch"
"in the fork fork. The error was: {0}".format(str(error))
)
raise rpkgError(error_msg)
base_error_msg = ('The following error occurred while creating a new fork: {0}')
base_error_msg = "The following error occurred while creating a new fork: {0}"
if not rv.ok:
# fork was already created
if rv.status_code == 409 or rv.reason == "Conflict":
# When the repo already exists, the return doesn't contain the repo
# path or username. Make one more API call to get the username of
# the token to construct the repo path.
rv = requests.get('{0}/user'.format(api_url), headers=headers)
username = rv.json()['username']
return False, '{0}/{1}'.format(username, safe_name)
rv = requests.get("{0}/user".format(api_url), headers=headers)
username = rv.json()["username"]
return False, "{0}/{1}".format(username, safe_name)
# show hint for invalid, expired or revoked token
elif rv.status_code == 401 or rv.reason == "Unauthorized":
base_error_msg += '\nFor invalid or expired token refer to ' \
'"{0} fork -h" to set a token in your user ' \
'configuration.'.format(cli_name)
base_error_msg += (
"\nFor invalid or expired token refer to "
'"{0} fork -h" to set a token in your user '
"configuration.".format(cli_name)
)
raise rpkgError(base_error_msg.format(rv.text))
return True, rv_json['path_with_namespace']
return True, rv_json["path_with_namespace"]
def do_add_remote(base_url, remote_base_url, repo, repo_path, remote_name):
@ -127,7 +146,7 @@ def do_add_remote(base_url, remote_base_url, repo, repo_path, remote_name):
:return: a bool; True if remote was created, False when already exists
"""
parsed_url = urlparse(remote_base_url)
remote_url = '{0}://{1}/{2}.git'.format(
remote_url = "{0}://{1}/{2}.git".format(
parsed_url.scheme,
parsed_url.netloc,
repo_path,
@ -142,7 +161,8 @@ def do_add_remote(base_url, remote_base_url, repo, repo_path, remote_name):
repo.create_remote(remote_name, url=remote_url)
except git.exc.GitCommandError as e:
error_msg = "During create remote:\n {0}\n {1}".format(
" ".join(e.command), e.stderr)
" ".join(e.command), e.stderr
)
raise rpkgError(error_msg)
return True
@ -194,9 +214,11 @@ def get_canonical_repo_name(config, repo_url):
"""
# Look up the repo and query for forked_from_project
cli_name = config_get_safely(dist_git_config, '__default', 'cli_name')
distgit_section = '{0}.distgit'.format(cli_name)
distgit_api_base_url = config_get_safely(dist_git_config, distgit_section, "apibaseurl")
cli_name = config_get_safely(dist_git_config, "__default", "cli_name")
distgit_section = "{0}.distgit".format(cli_name)
distgit_api_base_url = config_get_safely(
dist_git_config, distgit_section, "apibaseurl"
)
parsed_repo_url = urlparse(repo_url)
if not parsed_repo_url.scheme and repo_url.startswith("git@"):
@ -210,17 +232,18 @@ def get_canonical_repo_name(config, repo_url):
faked_url = "git+ssh://{0}".format(repo_url.replace(":", "/", 1))
parsed_repo_url = urlparse(faked_url)
try:
distgit_token = config_get_safely(dist_git_config, distgit_section, 'token')
distgit_token = config_get_safely(dist_git_config, distgit_section, "token")
api_url = '{0}/api/v4'.format(distgit_api_base_url.rstrip('/'))
project_url = '{0}/projects/{1}'.format(api_url, quote_plus(parsed_repo_url.path.lstrip('/')))
api_url = "{0}/api/v4".format(distgit_api_base_url.rstrip("/"))
project_url = "{0}/projects/{1}".format(
api_url, quote_plus(parsed_repo_url.path.lstrip("/"))
)
headers = {
'PRIVATE-TOKEN': distgit_token,
'Accept': 'application/json',
'Content-Type': 'application/json'
"PRIVATE-TOKEN": distgit_token,
"Accept": "application/json",
"Content-Type": "application/json",
}
rv = requests.get(project_url, headers=headers)
@ -229,7 +252,7 @@ def get_canonical_repo_name(config, repo_url):
# Extract response json for debugging
rv_json = rv.json()
canonical_repo_name = rv_json['forked_from_project']['name']
canonical_repo_name = rv_json["forked_from_project"]["name"]
except HTTPError as e:
# We got a 4xx or 5xx error code from the URL lookup
@ -246,9 +269,10 @@ def get_canonical_repo_name(config, repo_url):
raise rpkgError("Insufficient Gitlab API permissions. Missing token?")
# Chop off a trailing .git if any
return canonical_repo_name.rsplit('.git', 1)[0]
return canonical_repo_name.rsplit(".git", 1)[0]
def get_repo_name(name, org='rpms'):
def get_repo_name(name, org="rpms"):
"""
Try to parse the repository name in case it is a git url.
@ -273,7 +297,8 @@ def get_repo_name(name, org='rpms'):
# This is probably a renamed fork, so try to find the fork's parent
repo_name = get_canonical_repo_name(dist_git_config, name)
return '%s/%s' % (org, repo_name)
return "%s/%s" % (org, repo_name)
def stream_mapping(csname):
"""
@ -299,7 +324,10 @@ def stream_mapping(csname):
return "rhel-11"
return None
def does_divergent_branch_exist(repo_name, x_version, active_y, rhel_dist_git, namespace):
def does_divergent_branch_exist(
repo_name, x_version, active_y, rhel_dist_git, namespace
):
logger = logging.getLogger(__name__)
# Determine if the Y-1 branch exists for this repo
@ -334,7 +362,7 @@ def does_divergent_branch_exist(repo_name, x_version, active_y, rhel_dist_git, n
def _datesplit(isodate):
date_string_tuple = isodate.split('-')
date_string_tuple = isodate.split("-")
return [int(x) for x in date_string_tuple]
@ -419,4 +447,3 @@ def determine_active_y_version(rhel_version, api_url):
)
return x_version, active_y_version, beta, in_exception_phase