centos/centpkg
1
0
Fork 0
mirror of https://git.centos.org/centos/centpkg.git synced 2025-02-23 16:22:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Reformat centpkg src with black

Browse source 
Signed-off-by: Stephen Gallagher <sgallagh@redhat.com>
This commit is contained in:
Stephen Gallagher 2024-09-13 11:32:14 -04:00
parent ff58b37f1f
commit 85cce45045
6 changed files with 467 additions and 297 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

198
src/centpkg/__init__.py
View file

@ -1,8 +1,8 @@
# pylint: disable=line-too-long,abstract-class-not-used # pylint: disable=line-too-long,abstract-class-not-used
''' """
Top level function library for centpkg Top level function library for centpkg
''' """
# Author(s): # Author(s):
# Jesse Keating <jkeating@redhat.com> # Jesse Keating <jkeating@redhat.com>
@ -28,7 +28,7 @@ from . import cli # noqa
from .lookaside import StreamLookasideCache, SIGLookasideCache, CLLookasideCache from .lookaside import StreamLookasideCache, SIGLookasideCache, CLLookasideCache
_DEFAULT_VERSION = '9' _DEFAULT_VERSION = "9"
class DistGitDirectory(object): class DistGitDirectory(object):
@ -40,17 +40,17 @@ class DistGitDirectory(object):
distrobranch = False distrobranch = False
sigbranch = False sigbranch = False
repo = None repo = None
git_origin_substr = 'git@gitlab.com/redhat/centos-stream' git_origin_substr = "git@gitlab.com/redhat/centos-stream"
def __init__(self, branchtext, repo_path=None): def __init__(self, branchtext, repo_path=None):
if repo_path: if repo_path:
# self.repo = git.cmd.Git(repo_path) # self.repo = git.cmd.Git(repo_path)
self.repo = git.repo.Repo(repo_path) self.repo = git.repo.Repo(repo_path)
rhelbranchre = r'rhel-(?P<major>\d+)\.(?P<minor>\d+)(?:\.(?P<appstream>\d+))?' rhelbranchre = r"rhel-(?P<major>\d+)\.(?P<minor>\d+)(?:\.(?P<appstream>\d+))?"
sigtobranchre = r'c(?P<centosversion>\d+[s]?)-sig-(?P<signame>\w+)-?(?P<projectname>\w+)?-?(?P<releasename>\w+)?' sigtobranchre = r"c(?P<centosversion>\d+[s]?)-sig-(?P<signame>\w+)-?(?P<projectname>\w+)?-?(?P<releasename>\w+)?"
distrobranchre = r'c(?P<centosversion>\d+)-?(?P<projectname>\w+)?' distrobranchre = r"c(?P<centosversion>\d+)-?(?P<projectname>\w+)?"
javabranchre = r'openjdk-portable-centos-(?P<centosversion>\d+)' javabranchre = r"openjdk-portable-centos-(?P<centosversion>\d+)"
oldbranchre = r'(?P<signame>\w+)(?P<centosversion>\d)' oldbranchre = r"(?P<signame>\w+)(?P<centosversion>\d)"
rhelmatch = re.search(rhelbranchre, branchtext) rhelmatch = re.search(rhelbranchre, branchtext)
sigmatch = re.match(sigtobranchre, branchtext) sigmatch = re.match(sigtobranchre, branchtext)
distromatch = re.match(distrobranchre, branchtext) distromatch = re.match(distrobranchre, branchtext)
@ -59,55 +59,60 @@ class DistGitDirectory(object):
if rhelmatch: if rhelmatch:
gd = rhelmatch.groupdict() gd = rhelmatch.groupdict()
self.distrobranch = True self.distrobranch = True
self.signame = 'centos' self.signame = "centos"
self.centosversion = gd['major'] self.centosversion = gd["major"]
elif sigmatch: elif sigmatch:
gd = sigmatch.groupdict() gd = sigmatch.groupdict()
self.sigbranch = True self.sigbranch = True
self.signame = gd['signame'] self.signame = gd["signame"]
self.centosversion = gd['centosversion'] self.centosversion = gd["centosversion"]
# Users have the option to specify (or not specify) common in their # Users have the option to specify (or not specify) common in their
# git repos. Ww need to handle these cases because common is not a # git repos. Ww need to handle these cases because common is not a
# project nor is it a release. # project nor is it a release.
if gd['projectname'] != 'common': if gd["projectname"] != "common":
self.projectname = gd['projectname'] self.projectname = gd["projectname"]
if gd['releasename'] != 'common': if gd["releasename"] != "common":
self.releasename = gd['releasename'] self.releasename = gd["releasename"]
elif distromatch: elif distromatch:
gd = distromatch.groupdict() gd = distromatch.groupdict()
self.distrobranch = True self.distrobranch = True
self.signame = 'centos' self.signame = "centos"
self.centosversion = gd['centosversion'] self.centosversion = gd["centosversion"]
if gd['projectname'] != 'common': if gd["projectname"] != "common":
self.projectname = gd['projectname'] self.projectname = gd["projectname"]
elif javamatch: elif javamatch:
gd = javamatch.groupdict() gd = javamatch.groupdict()
self.distrobranch = True self.distrobranch = True
self.signame = 'centos' self.signame = "centos"
self.centosversion = gd['centosversion'] self.centosversion = gd["centosversion"]
elif oldbranchmatch: elif oldbranchmatch:
warnings.warn("This branch is deprecated and will be removed soon", warnings.warn(
DeprecationWarning) "This branch is deprecated and will be removed soon", DeprecationWarning
)
else: else:
if not self.is_fork(): if not self.is_fork():
warnings.warn('Unable to determine if this is a fork or not. Proceeding, but you should double check.') warnings.warn(
"Unable to determine if this is a fork or not. Proceeding, but you should double check."
)
else: else:
self.distrobranch = True self.distrobranch = True
self.signame = 'centos' self.signame = "centos"
self.projectname = self.get_origin().split('_')[-1].replace('.git', '') self.projectname = self.get_origin().split("_")[-1].replace(".git", "")
warnings.warn('Remote "origin" was detected as a fork, ignoring branch name checking') warnings.warn(
'Remote "origin" was detected as a fork, ignoring branch name checking'
)
def get_origin(self): def get_origin(self):
if self.repo is None: if self.repo is None:
return '' return ""
if 'origin' not in self.repo.remotes: if "origin" not in self.repo.remotes:
return '' return ""
urls = [u for u in self.repo.remotes['origin'].urls] urls = [u for u in self.repo.remotes["origin"].urls]
if len(urls) == 0: if len(urls) == 0:
return '' return ""
return urls[0] return urls[0]
def is_fork(self): def is_fork(self):
@ -124,42 +129,47 @@ class DistGitDirectory(object):
return False return False
return self.git_origin_substr not in self.get_origin() return self.git_origin_substr not in self.get_origin()
@property @property
def target(self): def target(self):
projectorcommon = self.projectname projectorcommon = self.projectname
releaseorcommon = self.releasename releaseorcommon = self.releasename
if self.distrobranch: if self.distrobranch:
if self.centosversion not in ('6', '7'): if self.centosversion not in ("6", "7"):
return 'c{}s-candidate'.format(self.centosversion) return "c{}s-candidate".format(self.centosversion)
else: else:
return '-'.join(filter(None, ['c'+self.centosversion, return "-".join(
projectorcommon])) filter(None, ["c" + self.centosversion, projectorcommon])
)
if not releaseorcommon: if not releaseorcommon:
if not projectorcommon or projectorcommon == 'common': if not projectorcommon or projectorcommon == "common":
projectorcommon = 'common' projectorcommon = "common"
else: else:
releaseorcommon = 'common' releaseorcommon = "common"
return '-'.join(filter(None, [self.signame+self.centosversion, return "-".join(
projectorcommon, releaseorcommon])) + '-el{0}'.format(self.centosversion) filter(
None,
[self.signame + self.centosversion, projectorcommon, releaseorcommon],
)
) + "-el{0}".format(self.centosversion)
class Commands(Commands): class Commands(Commands):
''' """
For the pyrpkg commands with centpkg behavior For the pyrpkg commands with centpkg behavior
''' """
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
''' """
Init the object and some configuration details. Init the object and some configuration details.
''' """
super(Commands, self).__init__(*args, **kwargs) super(Commands, self).__init__(*args, **kwargs)
# For MD5 we want to use the old format of source files, the BSD format # For MD5 we want to use the old format of source files, the BSD format
# should only be used when configured for SHA512 # should only be used when configured for SHA512
self.source_entry_type = 'bsd' if self.lookasidehash != 'md5' else 'old' self.source_entry_type = "bsd" if self.lookasidehash != "md5" else "old"
self.branchre = 'c\d{1,}(s)?(tream)?|master' self.branchre = "c\d{1,}(s)?(tream)?|master"
@property @property
def distgitdir(self): def distgitdir(self):
@ -168,48 +178,66 @@ class Commands(Commands):
@cached_property @cached_property
def lookasidecache(self): def lookasidecache(self):
if self.layout.sources_file_template == "sources": if self.layout.sources_file_template == "sources":
return StreamLookasideCache(self.lookasidehash, return StreamLookasideCache(
self.lookaside, self.lookasidehash,
self.lookaside_cgi, self.lookaside,
) self.lookaside_cgi,
)
else: else:
if self.distgitdir.sigbranch: if self.distgitdir.sigbranch:
return SIGLookasideCache(self.lookasidehash, return SIGLookasideCache(
self.lookaside, self.lookasidehash,
self.lookaside_cgi, self.lookaside,
self.repo_name, self.lookaside_cgi,
self.branch_merge) self.repo_name,
self.branch_merge,
)
else: else:
return CLLookasideCache(self.lookasidehash, return CLLookasideCache(
self.lookaside, self.lookasidehash,
self.lookaside_cgi, self.lookaside,
self.repo_name, self.lookaside_cgi,
self.branch_merge, self.repo_name,
) self.branch_merge,
)
# redefined loaders # redefined loaders
def load_rpmdefines(self): def load_rpmdefines(self):
''' """
Populate rpmdefines based on branch data Populate rpmdefines based on branch data
''' """
if not self.distgitdir.centosversion: if not self.distgitdir.centosversion:
raise rpkgError('Could not get the OS version from the branch:{0}'.format(self.branch_merge)) raise rpkgError(
"Could not get the OS version from the branch:{0}".format(
self.branch_merge
)
)
self._distvar = self.distgitdir.centosversion self._distvar = self.distgitdir.centosversion
self._distval = self._distvar.replace('.', '_') self._distval = self._distvar.replace(".", "_")
self._disttag = 'el%s' % self._distval self._disttag = "el%s" % self._distval
self._rpmdefines = ['--define', '_sourcedir %s' % self.layout.sourcedir, self._rpmdefines = [
'--define', '_specdir %s' % self.layout.specdir, "--define",
'--define', '_builddir %s' % self.layout.builddir, "_sourcedir %s" % self.layout.sourcedir,
'--define', '_srcrpmdir %s' % self.layout.srcrpmdir, "--define",
'--define', '_rpmdir %s' % self.layout.rpmdir, "_specdir %s" % self.layout.specdir,
'--define', 'dist .%s' % self._disttag, "--define",
# int and float this to remove the decimal "_builddir %s" % self.layout.builddir,
'--define', '%s 1' % self._disttag, "--define",
# This is so the rhel macro is set for spec files "_srcrpmdir %s" % self.layout.srcrpmdir,
'--define', 'rhel %s' % self._distval.split('_')[0]] "--define",
"_rpmdir %s" % self.layout.rpmdir,
"--define",
"dist .%s" % self._disttag,
# int and float this to remove the decimal
"--define",
"%s 1" % self._disttag,
# This is so the rhel macro is set for spec files
"--define",
"rhel %s" % self._distval.split("_")[0],
]
self.log.debug("RPMDefines: %s" % self._rpmdefines) self.log.debug("RPMDefines: %s" % self._rpmdefines)
def construct_build_url(self, *args, **kwargs): def construct_build_url(self, *args, **kwargs):
@ -218,9 +246,9 @@ class Commands(Commands):
In CentOS/Fedora Koji, anonymous URL should have prefix "git+https://" In CentOS/Fedora Koji, anonymous URL should have prefix "git+https://"
""" """
url = super(Commands, self).construct_build_url(*args, **kwargs) url = super(Commands, self).construct_build_url(*args, **kwargs)
return 'git+{0}'.format(url) return "git+{0}".format(url)
def load_target(self): def load_target(self):
""" This sets the target attribute (used for mock and koji) """ """This sets the target attribute (used for mock and koji)"""
self._target = self.distgitdir.target self._target = self.distgitdir.target

37
src/centpkg/__main__.py
View file

@ -1,6 +1,7 @@
''' """
The main behavior of centpkg The main behavior of centpkg
''' """
# #
# Author(s): # Author(s):
# Jesse Keating <jkeating@redhat.com> # Jesse Keating <jkeating@redhat.com>
@ -32,7 +33,8 @@ def main():
program_name = os.path.basename(sys.argv[0]) program_name = os.path.basename(sys.argv[0])
default_user_config_path = os.path.join( default_user_config_path = os.path.join(
os.path.expanduser('~'), '.config', 'rpkg', '%s.conf' % program_name) os.path.expanduser("~"), ".config", "rpkg", "%s.conf" % program_name
)
# Setup an argparser and parse the known commands to get the config file # Setup an argparser and parse the known commands to get the config file
@ -41,16 +43,23 @@ def main():
# --user-config # --user-config
parser = pyrpkg.cli.ArgumentParser(add_help=False, allow_abbrev=False) parser = pyrpkg.cli.ArgumentParser(add_help=False, allow_abbrev=False)
parser.add_argument('-C', '--config', help='Specify a config file to use', parser.add_argument(
default='/etc/rpkg/%s.conf' % program_name) "-C",
parser.add_argument('--user-config', help='Specify a user config file to use', "--config",
default=default_user_config_path) help="Specify a config file to use",
default="/etc/rpkg/%s.conf" % program_name,
)
parser.add_argument(
"--user-config",
help="Specify a user config file to use",
default=default_user_config_path,
)
(args, other) = parser.parse_known_args() (args, other) = parser.parse_known_args()
# Make sure we have a sane config file # Make sure we have a sane config file
if not os.path.exists(args.config) and not other[-1] in ['--help', '-h', 'help']: if not os.path.exists(args.config) and not other[-1] in ["--help", "-h", "help"]:
sys.stderr.write('Invalid config file %s\n' % args.config) sys.stderr.write("Invalid config file %s\n" % args.config)
sys.exit(1) sys.exit(1)
# Setup a configuration object and read config file data # Setup a configuration object and read config file data
@ -59,14 +68,14 @@ def main():
config.read(args.user_config) config.read(args.user_config)
client = centpkg.cli.centpkgClient(config, name=program_name) client = centpkg.cli.centpkgClient(config, name=program_name)
client.do_imports(site='centpkg') client.do_imports(site="centpkg")
client.parse_cmdline() client.parse_cmdline()
# This is due to a difference argparse behavior to Python 2 version. # This is due to a difference argparse behavior to Python 2 version.
# In Python 3, argparse will proceed to here without reporting # In Python 3, argparse will proceed to here without reporting
# "too few arguments". Instead, client.args does not have attribute # "too few arguments". Instead, client.args does not have attribute
# command. # command.
if not hasattr(client.args, 'command'): if not hasattr(client.args, "command"):
client.parser.print_help() client.parser.print_help()
sys.exit(1) sys.exit(1)
@ -74,7 +83,7 @@ def main():
try: try:
client.args.path = pyrpkg.utils.getcwd() client.args.path = pyrpkg.utils.getcwd()
except Exception: except Exception:
print('Could not get current path, have you deleted it?') print("Could not get current path, have you deleted it?")
sys.exit(1) sys.exit(1)
# setup the logger -- This logger will take things of INFO or DEBUG and # setup the logger -- This logger will take things of INFO or DEBUG and
@ -98,9 +107,9 @@ def main():
except KeyboardInterrupt: except KeyboardInterrupt:
pass pass
except Exception as e: except Exception as e:
if getattr(client.args, 'debug', False): if getattr(client.args, "debug", False):
raise raise
log.error('Could not execute %s: %s' % (client.args.command.__name__, e)) log.error("Could not execute %s: %s" % (client.args.command.__name__, e))
sys.exit(1) sys.exit(1)

21
src/centpkg/centos_cert.py
View file

@ -12,24 +12,29 @@ import datetime
class centos_cert_error(Exception): class centos_cert_error(Exception):
pass pass
def _open_cert(): def _open_cert():
""" """
Read in the certificate so we dont duplicate the code Read in the certificate so we dont duplicate the code
""" """
# Make sure we can even read the thing. # Make sure we can even read the thing.
cert_file = os.path.join(os.path.expanduser('~'), ".koji", "client.crt") cert_file = os.path.join(os.path.expanduser("~"), ".koji", "client.crt")
if not os.access(cert_file, os.R_OK): if not os.access(cert_file, os.R_OK):
raise centos_cert_error("""!!! cannot read your centos cert file !!! raise centos_cert_error(
!!! Ensure the file is readable and try again !!!""") """!!! cannot read your centos cert file !!!
raw_cert = open(cert_file, 'rb').read() !!! Ensure the file is readable and try again !!!"""
)
raw_cert = open(cert_file, "rb").read()
try: try:
my_cert = x509.load_pem_x509_certificate(raw_cert) my_cert = x509.load_pem_x509_certificate(raw_cert)
except TypeError: except TypeError:
# it was required to specify a backend prior to cryptography 3.1 # it was required to specify a backend prior to cryptography 3.1
from cryptography.hazmat.backends import default_backend from cryptography.hazmat.backends import default_backend
my_cert = x509.load_pem_x509_certificate(raw_cert, default_backend()) my_cert = x509.load_pem_x509_certificate(raw_cert, default_backend())
return my_cert return my_cert
def verify_cert(): def verify_cert():
""" """
Check that the user cert is valid. Check that the user cert is valid.
@ -40,10 +45,10 @@ def verify_cert():
my_cert = _open_cert() my_cert = _open_cert()
warn = datetime.datetime.now() + datetime.timedelta(days=21) warn = datetime.datetime.now() + datetime.timedelta(days=21)
print(my_cert.not_valid_after.strftime('cert expires: %Y-%m-%d')) print(my_cert.not_valid_after.strftime("cert expires: %Y-%m-%d"))
if my_cert.not_valid_after < warn: if my_cert.not_valid_after < warn:
print('WARNING: Your cert expires soon.') print("WARNING: Your cert expires soon.")
def certificate_expired(): def certificate_expired():

254
src/centpkg/cli.py
View file

@ -1,6 +1,7 @@
''' """
Command line behavior for centpkg Command line behavior for centpkg
''' """
# #
# Author(s): # Author(s):
# Jesse Keating <jkeating@redhat.com> # Jesse Keating <jkeating@redhat.com>
@ -30,21 +31,21 @@ import json
import os import os
import sys import sys
_DEFAULT_API_BASE_URL = 'https://gitlab.com' _DEFAULT_API_BASE_URL = "https://gitlab.com"
_DEFAULT_PARENT_NAMESPACE = { _DEFAULT_PARENT_NAMESPACE = {
'centpkg': 'redhat/centos-stream', "centpkg": "redhat/centos-stream",
'centpkg-sig': 'CentOS', "centpkg-sig": "CentOS",
} }
class centpkgClient(cliClient): class centpkgClient(cliClient):
def __init__(self, config, name='centpkg'): def __init__(self, config, name="centpkg"):
self.DEFAULT_CLI_NAME = name self.DEFAULT_CLI_NAME = name
# Save the config for utilities such as get_repo_name() # Save the config for utilities such as get_repo_name()
centpkg.utils.dist_git_config = config centpkg.utils.dist_git_config = config
centpkg.utils.dist_git_config.add_section('__default') centpkg.utils.dist_git_config.add_section("__default")
centpkg.utils.dist_git_config.set('__default', 'cli_name', name) centpkg.utils.dist_git_config.set("__default", "cli_name", name)
super(centpkgClient, self).__init__(config, name) super(centpkgClient, self).__init__(config, name)
@ -56,14 +57,17 @@ class centpkgClient(cliClient):
self.register_current_state() self.register_current_state()
def register_do_fork(self): def register_do_fork(self):
help_msg = 'Create a new fork of the current repository' help_msg = "Create a new fork of the current repository"
distgit_section = '{0}.distgit'.format(self.name) distgit_section = "{0}.distgit".format(self.name)
# uses default dist-git url in case section is not present # uses default dist-git url in case section is not present
try: try:
distgit_api_base_url = config_get_safely(self.config, distgit_section, "apibaseurl") distgit_api_base_url = config_get_safely(
self.config, distgit_section, "apibaseurl"
)
except rpkgError: except rpkgError:
distgit_api_base_url = _DEFAULT_API_BASE_URL distgit_api_base_url = _DEFAULT_API_BASE_URL
description = textwrap.dedent(''' description = textwrap.dedent(
"""
Create a new fork of the current repository Create a new fork of the current repository
Before the operation, you need to generate an API token at Before the operation, you need to generate an API token at
@ -77,13 +81,17 @@ class centpkgClient(cliClient):
Below is a basic example of the command to fork a current repository: Below is a basic example of the command to fork a current repository:
{0} fork {0} fork
'''.format(self.name, urlparse(distgit_api_base_url).netloc)) """.format(
self.name, urlparse(distgit_api_base_url).netloc
)
)
fork_parser = self.subparsers.add_parser( fork_parser = self.subparsers.add_parser(
'fork', "fork",
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
help=help_msg, help=help_msg,
description=description) description=description,
)
fork_parser.set_defaults(command=self.do_distgit_fork) fork_parser.set_defaults(command=self.do_distgit_fork)
def do_distgit_fork(self): def do_distgit_fork(self):
@ -91,9 +99,11 @@ class centpkgClient(cliClient):
That includes creating fork itself using API call and then adding That includes creating fork itself using API call and then adding
remote tracked repository remote tracked repository
""" """
distgit_section = '{0}.distgit'.format(self.name) distgit_section = "{0}.distgit".format(self.name)
distgit_api_base_url = config_get_safely(self.config, distgit_section, "apibaseurl") distgit_api_base_url = config_get_safely(
distgit_token = config_get_safely(self.config, distgit_section, 'token') self.config, distgit_section, "apibaseurl"
)
distgit_token = config_get_safely(self.config, distgit_section, "token")
ret, repo_path = do_fork( ret, repo_path = do_fork(
logger=self.log, logger=self.log,
@ -105,8 +115,8 @@ class centpkgClient(cliClient):
) )
# assemble url of the repo in web browser # assemble url of the repo in web browser
fork_url = '{0}/{1}'.format( fork_url = "{0}/{1}".format(
distgit_api_base_url.rstrip('/'), distgit_api_base_url.rstrip("/"),
repo_path, repo_path,
) )
@ -117,11 +127,11 @@ class centpkgClient(cliClient):
self.log.info(msg.format(fork_url)) self.log.info(msg.format(fork_url))
distgit_remote_base_url = self.config.get( distgit_remote_base_url = self.config.get(
'{0}'.format(self.name), "{0}".format(self.name),
"gitbaseurl", "gitbaseurl",
vars={'repo': '{0}/{1}'.format(self.cmd.ns, self.cmd.repo_name)}, vars={"repo": "{0}/{1}".format(self.cmd.ns, self.cmd.repo_name)},
) )
remote_name = repo_path.split('/')[0] remote_name = repo_path.split("/")[0]
ret = do_add_remote( ret = do_add_remote(
base_url=distgit_api_base_url, base_url=distgit_api_base_url,
@ -153,19 +163,27 @@ class centpkgClient(cliClient):
# Get our internal only variables # Get our internal only variables
cfg = ConfigParser() cfg = ConfigParser()
cfg.read(internal_config_file) cfg.read(internal_config_file)
pp_api_url = config_get_safely(cfg, "centpkg.internal", 'pp_api_url') pp_api_url = config_get_safely(cfg, "centpkg.internal", "pp_api_url")
gitbz_query_url = config_get_safely(cfg, "centpkg.internal", 'gitbz_query_url') gitbz_query_url = config_get_safely(
rhel_dist_git = config_get_safely(cfg, "centpkg.internal", 'rhel_dist_git') cfg, "centpkg.internal", "gitbz_query_url"
)
rhel_dist_git = config_get_safely(cfg, "centpkg.internal", "rhel_dist_git")
# Find out divergent branch and stabalization # Find out divergent branch and stabalization
self.log.info("Checking current state:") self.log.info("Checking current state:")
stream_version = self.cmd.target.split('-')[0] stream_version = self.cmd.target.split("-")[0]
rhel_version = centpkg.utils.stream_mapping(stream_version) rhel_version = centpkg.utils.stream_mapping(stream_version)
try: try:
x_version, active_y, is_beta, in_stabilization = centpkg.utils.determine_active_y_version(rhel_version, pp_api_url) x_version, active_y, is_beta, in_stabilization = (
centpkg.utils.determine_active_y_version(rhel_version, pp_api_url)
)
except AssertionError as e: except AssertionError as e:
self.log.error(" Error: centpkg cannot determine the development phase.") self.log.error(
self.log.error(" Please file an issue at https://git.centos.org/centos/centpkg") " Error: centpkg cannot determine the development phase."
)
self.log.error(
" Please file an issue at https://git.centos.org/centos/centpkg"
)
self.log.error(" Workaround: Use the --rhel-target option") self.log.error(" Workaround: Use the --rhel-target option")
self.log.error("Exiting") self.log.error("Exiting")
raise SystemExit(1) raise SystemExit(1)
@ -175,13 +193,10 @@ class centpkgClient(cliClient):
divergent_branch = True divergent_branch = True
else: else:
divergent_branch = centpkg.utils.does_divergent_branch_exist( divergent_branch = centpkg.utils.does_divergent_branch_exist(
self.cmd.repo_name, self.cmd.repo_name, x_version, active_y, rhel_dist_git, "rpms"
x_version, )
active_y,
rhel_dist_git,
"rpms")
# Good to know # Good to know
if in_stabilization : if in_stabilization:
self.log.info(" we are in stabilization mode.") self.log.info(" we are in stabilization mode.")
else: else:
self.log.info(" we are not in stabilization mode.") self.log.info(" we are not in stabilization mode.")
@ -193,7 +208,9 @@ class centpkgClient(cliClient):
self.log.info(" a divergent branch was not found.") self.log.info(" a divergent branch was not found.")
else: else:
self.log.error("NO INTERNAL CONFIGURATION") self.log.error("NO INTERNAL CONFIGURATION")
self.log.error("Hint: If you are internal, install the rhel-packager package from https://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-9/compose/BaseOS/x86_64/os/Packages/") self.log.error(
"Hint: If you are internal, install the rhel-packager package from https://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-9/compose/BaseOS/x86_64/os/Packages/"
)
self.log.error("Exiting") self.log.error("Exiting")
raise SystemExit(1) raise SystemExit(1)
@ -201,9 +218,10 @@ class centpkgClient(cliClient):
def register_build(self): def register_build(self):
# Do all the work from the super class # Do all the work from the super class
super(centpkgClient, self).register_build() super(centpkgClient, self).register_build()
build_parser = self.subparsers.choices['build'] build_parser = self.subparsers.choices["build"]
build_parser.formatter_class = argparse.RawDescriptionHelpFormatter build_parser.formatter_class = argparse.RawDescriptionHelpFormatter
build_parser.description = textwrap.dedent(''' build_parser.description = textwrap.dedent(
"""
{0} {0}
centpkg now sets the rhel metadata with --rhel-target. centpkg now sets the rhel metadata with --rhel-target.
@ -214,60 +232,93 @@ class centpkgClient(cliClient):
* latest - This will always build for the next Y-stream release * latest - This will always build for the next Y-stream release
* none - This will not build in brew. No rhel metadata is set. * none - This will not build in brew. No rhel metadata is set.
'''.format('\n'.join(textwrap.wrap(build_parser.description)))) """.format(
"\n".join(textwrap.wrap(build_parser.description))
)
)
# Now add our additional option # Now add our additional option
build_parser.add_argument( build_parser.add_argument(
'--rhel-target', "--rhel-target",
choices=['exception', 'zstream', 'latest', 'none'], choices=["exception", "zstream", "latest", "none"],
help='Set the rhel-target metadata') help="Set the rhel-target metadata",
)
# Overloaded _build # Overloaded _build
def _build(self, sets=None): def _build(self, sets=None):
# Only do rhel-target if we are centpkg, not centpkg-sig, or if we are doing scratch # Only do rhel-target if we are centpkg, not centpkg-sig, or if we are doing scratch
if not os.path.basename(sys.argv[0]).endswith('-sig') and not self.args.scratch: if not os.path.basename(sys.argv[0]).endswith("-sig") and not self.args.scratch:
# Only run if we have internal configuraions # Only run if we have internal configuraions
internal_config_file = "/etc/rpkg/centpkg_internal.conf" internal_config_file = "/etc/rpkg/centpkg_internal.conf"
if os.path.exists(internal_config_file): if os.path.exists(internal_config_file):
# If rhel-target is set, no need to check, just use it # If rhel-target is set, no need to check, just use it
if hasattr(self.args, 'rhel_target') and self.args.rhel_target: if hasattr(self.args, "rhel_target") and self.args.rhel_target:
# If rhel-target set to none, do nothing with metadata # If rhel-target set to none, do nothing with metadata
if self.args.rhel_target.lower() != "none": if self.args.rhel_target.lower() != "none":
# If custom-user-metadata set, add onto it # If custom-user-metadata set, add onto it
if hasattr(self.args, 'custom_user_metadata') and self.args.custom_user_metadata: if (
hasattr(self.args, "custom_user_metadata")
and self.args.custom_user_metadata
):
try: try:
temp_custom_user_metadata = json.loads(self.args.custom_user_metadata) temp_custom_user_metadata = json.loads(
self.args.custom_user_metadata
)
# Use ValueError instead of json.JSONDecodeError for Python 2 and 3 compatibility # Use ValueError instead of json.JSONDecodeError for Python 2 and 3 compatibility
except ValueError as e: except ValueError as e:
self.parser.error("--custom-user-metadata is not valid JSON: %s" % e) self.parser.error(
"--custom-user-metadata is not valid JSON: %s" % e
)
if not isinstance(temp_custom_user_metadata, dict): if not isinstance(temp_custom_user_metadata, dict):
self.parser.error("--custom-user-metadata must be a JSON object") self.parser.error(
temp_custom_user_metadata["rhel-target"] = self.args.rhel_target "--custom-user-metadata must be a JSON object"
)
temp_custom_user_metadata["rhel-target"] = (
self.args.rhel_target
)
else: else:
temp_custom_user_metadata = {"rhel-target": self.args.rhel_target} temp_custom_user_metadata = {
self.args.custom_user_metadata = json.dumps(temp_custom_user_metadata) "rhel-target": self.args.rhel_target
}
self.args.custom_user_metadata = json.dumps(
temp_custom_user_metadata
)
else: else:
# Get our internal only variables # Get our internal only variables
cfg = ConfigParser() cfg = ConfigParser()
cfg.read(internal_config_file) cfg.read(internal_config_file)
pp_api_url = config_get_safely(cfg, "centpkg.internal", 'pp_api_url') pp_api_url = config_get_safely(
gitbz_query_url = config_get_safely(cfg, "centpkg.internal", 'gitbz_query_url') cfg, "centpkg.internal", "pp_api_url"
rhel_dist_git = config_get_safely(cfg, "centpkg.internal", 'rhel_dist_git') )
gitbz_query_url = config_get_safely(
cfg, "centpkg.internal", "gitbz_query_url"
)
rhel_dist_git = config_get_safely(
cfg, "centpkg.internal", "rhel_dist_git"
)
# Find out divergent branch and stabalization # Find out divergent branch and stabalization
self.log.info("Checking rhel-target information:") self.log.info("Checking rhel-target information:")
stream_version = self.cmd.target.split('-')[0] stream_version = self.cmd.target.split("-")[0]
rhel_version = centpkg.utils.stream_mapping(stream_version) rhel_version = centpkg.utils.stream_mapping(stream_version)
try: try:
x_version, active_y, is_beta, in_stabilization = centpkg.utils.determine_active_y_version(rhel_version, pp_api_url) x_version, active_y, is_beta, in_stabilization = (
centpkg.utils.determine_active_y_version(
rhel_version, pp_api_url
)
)
except AssertionError as e: except AssertionError as e:
self.log.error(" Error: centpkg cannot determine the development phase.") self.log.error(
self.log.error(" Please file an issue at https://git.centos.org/centos/centpkg") " Error: centpkg cannot determine the development phase."
)
self.log.error(
" Please file an issue at https://git.centos.org/centos/centpkg"
)
self.log.error(" Workaround: Use the --rhel-target option") self.log.error(" Workaround: Use the --rhel-target option")
self.log.error("Exiting") self.log.error("Exiting")
raise SystemExit(1) raise SystemExit(1)
@ -278,11 +329,12 @@ class centpkgClient(cliClient):
divergent_branch = True divergent_branch = True
else: else:
divergent_branch = centpkg.utils.does_divergent_branch_exist( divergent_branch = centpkg.utils.does_divergent_branch_exist(
self.cmd.repo_name, self.cmd.repo_name,
x_version, x_version,
active_y, active_y,
rhel_dist_git, rhel_dist_git,
"rpms") "rpms",
)
# Good to know # Good to know
if divergent_branch and not is_beta: if divergent_branch and not is_beta:
@ -291,57 +343,80 @@ class centpkgClient(cliClient):
self.log.info(" we are working on a beta release.") self.log.info(" we are working on a beta release.")
else: else:
self.log.info(" a divergent branch was not found.") self.log.info(" a divergent branch was not found.")
if in_stabilization : if in_stabilization:
self.log.info(" we are in stabilization mode.") self.log.info(" we are in stabilization mode.")
else: else:
self.log.info(" we are not in stabilization mode.") self.log.info(" we are not in stabilization mode.")
# Update args.custom_user_metadata # Update args.custom_user_metadata
if hasattr(self.args, 'custom_user_metadata') and self.args.custom_user_metadata: if (
hasattr(self.args, "custom_user_metadata")
and self.args.custom_user_metadata
):
try: try:
temp_custom_user_metadata = json.loads(self.args.custom_user_metadata) temp_custom_user_metadata = json.loads(
self.args.custom_user_metadata
)
# Use ValueError instead of json.JSONDecodeError for Python 2 and 3 compatibility # Use ValueError instead of json.JSONDecodeError for Python 2 and 3 compatibility
except ValueError as e: except ValueError as e:
self.parser.error("--custom-user-metadata is not valid JSON: %s" % e) self.parser.error(
"--custom-user-metadata is not valid JSON: %s" % e
)
if not isinstance(temp_custom_user_metadata, dict): if not isinstance(temp_custom_user_metadata, dict):
self.parser.error("--custom-user-metadata must be a JSON object") self.parser.error(
"--custom-user-metadata must be a JSON object"
)
if divergent_branch: if divergent_branch:
temp_custom_user_metadata["rhel-target"] = "latest" temp_custom_user_metadata["rhel-target"] = "latest"
elif not divergent_branch and not in_stabilization : elif not divergent_branch and not in_stabilization:
temp_custom_user_metadata["rhel-target"] = "zstream" temp_custom_user_metadata["rhel-target"] = "zstream"
else: else:
self.log.error("We are currently in Stabalization mode") self.log.error("We are currently in Stabalization mode")
self.log.error("You must either set the rhel-target (--rhel-target)") self.log.error(
"You must either set the rhel-target (--rhel-target)"
)
self.log.error("or branch for the previous version.") self.log.error("or branch for the previous version.")
self.log.error("Exiting") self.log.error("Exiting")
raise SystemExit(1) raise SystemExit(1)
self.args.custom_user_metadata = json.dumps(temp_custom_user_metadata) self.args.custom_user_metadata = json.dumps(
temp_custom_user_metadata
)
else: else:
if divergent_branch: if divergent_branch:
self.args.custom_user_metadata = '{"rhel-target": "latest"}' self.args.custom_user_metadata = '{"rhel-target": "latest"}'
elif not divergent_branch and not in_stabilization : elif not divergent_branch and not in_stabilization:
self.args.custom_user_metadata = '{"rhel-target": "zstream"}' self.args.custom_user_metadata = (
'{"rhel-target": "zstream"}'
)
else: else:
self.log.error("We are currently in Stabalization mode") self.log.error("We are currently in Stabalization mode")
self.log.error("You must either set the rhel-target (--rhel-target)") self.log.error(
"You must either set the rhel-target (--rhel-target)"
)
self.log.error("or branch for the previous version.") self.log.error("or branch for the previous version.")
self.log.error("Exiting") self.log.error("Exiting")
raise SystemExit(1) raise SystemExit(1)
# Good to know # Good to know
self.log.info(" rhel-target: " + json.loads(self.args.custom_user_metadata)['rhel-target']) self.log.info(
" rhel-target: "
+ json.loads(self.args.custom_user_metadata)["rhel-target"]
)
else: else:
self.log.error("NO INTERNAL CONFIGURATION") self.log.error("NO INTERNAL CONFIGURATION")
self.log.error("Only scratch builds are allowed without internal configurations") self.log.error(
self.log.error("Hint: Install the rhel-packager package from https://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-9/compose/BaseOS/x86_64/os/Packages/ if you want to build the package via internal (Red Hat) configuration.") "Only scratch builds are allowed without internal configurations"
)
self.log.error(
"Hint: Install the rhel-packager package from https://download.devel.redhat.com/rel-eng/RCMTOOLS/latest-RCMTOOLS-2-RHEL-9/compose/BaseOS/x86_64/os/Packages/ if you want to build the package via internal (Red Hat) configuration."
)
self.log.error("Exiting") self.log.error("Exiting")
raise SystemExit(1) raise SystemExit(1)
# Proceed with build # Proceed with build
return super(centpkgClient, self)._build(sets) return super(centpkgClient, self)._build(sets)
def register_request_gated_side_tag(self): def register_request_gated_side_tag(self):
"""Register command line parser for subcommand request-gated-side-tag""" """Register command line parser for subcommand request-gated-side-tag"""
parser = self.subparsers.add_parser( parser = self.subparsers.add_parser(
@ -362,20 +437,29 @@ class centpkgClient(cliClient):
parent_namespace = _DEFAULT_PARENT_NAMESPACE[self.DEFAULT_CLI_NAME] parent_namespace = _DEFAULT_PARENT_NAMESPACE[self.DEFAULT_CLI_NAME]
if "/" in self.args.repo[0]: if "/" in self.args.repo[0]:
project_name_with_namespace = parent_namespace+"/"+self.args.repo[0] project_name_with_namespace = parent_namespace + "/" + self.args.repo[0]
short_name=self.args.repo[0].split("/")[-1] short_name = self.args.repo[0].split("/")[-1]
else: else:
project_name_with_namespace = parent_namespace+"/rpms/"+self.args.repo[0] project_name_with_namespace = (
short_name=self.args.repo[0] parent_namespace + "/rpms/" + self.args.repo[0]
)
short_name = self.args.repo[0]
try: try:
project = gl.projects.get(project_name_with_namespace) project = gl.projects.get(project_name_with_namespace)
except gitlab.exceptions.GitlabGetError as e: except gitlab.exceptions.GitlabGetError as e:
self.log.error("There is no repository with the given name. Did you spell it correctly?") self.log.error(
"There is no repository with the given name. Did you spell it correctly?"
)
self.log.error("Fatal: ") self.log.error("Fatal: ")
self.log.error(e) self.log.error(e)
raise SystemExit(1) raise SystemExit(1)
if not project.name == short_name: if not project.name == short_name:
self.log.error("Fatal: ") self.log.error("Fatal: ")
self.log.error("Project name is wrong: " + short_name + " it should be: " + project.name) self.log.error(
"Project name is wrong: "
+ short_name
+ " it should be: "
+ project.name
)
raise SystemExit(1) raise SystemExit(1)
super(centpkgClient, self).clone() super(centpkgClient, self).clone()

111
src/centpkg/lookaside.py
View file

@ -58,16 +58,19 @@ class StreamLookasideCache(CGILookasideCache):
It inherits most of its behavior from `pyrpkg.lookasideCGILookasideCache`. It inherits most of its behavior from `pyrpkg.lookasideCGILookasideCache`.
""" """
def __init__(self, hashtype, download_url, upload_url, client_cert=None, ca_cert=None): def __init__(
self, hashtype, download_url, upload_url, client_cert=None, ca_cert=None
):
super(StreamLookasideCache, self).__init__( super(StreamLookasideCache, self).__init__(
hashtype, download_url, upload_url, hashtype, download_url, upload_url, client_cert=client_cert, ca_cert=ca_cert
client_cert=client_cert, ca_cert=ca_cert) )
def get_download_url(self, name, filename, hash, hashtype=None, **kwargs): def get_download_url(self, name, filename, hash, hashtype=None, **kwargs):
_name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name _name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
return super(StreamLookasideCache, self).get_download_url( return super(StreamLookasideCache, self).get_download_url(
_name, filename, hash, hashtype=hashtype, **kwargs) _name, filename, hash, hashtype=hashtype, **kwargs
)
def remote_file_exists(self, name, filename, hashstr): def remote_file_exists(self, name, filename, hashstr):
""" """
@ -76,7 +79,7 @@ class StreamLookasideCache(CGILookasideCache):
This method inherits the behavior of its parent class from pyrpkg. This method inherits the behavior of its parent class from pyrpkg.
It uses the internal `utils.get_repo_name` method to parse the name in case It uses the internal `utils.get_repo_name` method to parse the name in case
it is a scm url. it is a scm url.
Parameters Parameters
---------- ----------
@ -98,10 +101,10 @@ class StreamLookasideCache(CGILookasideCache):
try: try:
status = super(StreamLookasideCache, self).remote_file_exists( status = super(StreamLookasideCache, self).remote_file_exists(
_name, filename, hashstr) _name, filename, hashstr
)
except UploadError as e: except UploadError as e:
self.log.error('Error checking for %s at %s' self.log.error("Error checking for %s at %s" % (filename, self.upload_url))
% (filename, self.upload_url))
self.log.error(e) self.log.error(e)
raise SystemExit(1) raise SystemExit(1)
@ -114,7 +117,7 @@ class StreamLookasideCache(CGILookasideCache):
This method inherits the behavior of its parent class from pyrpkg. This method inherits the behavior of its parent class from pyrpkg.
It uses the internal `utils.get_repo_name` method to parse the name in case It uses the internal `utils.get_repo_name` method to parse the name in case
it is a scm url. it is a scm url.
Parameters Parameters
---------- ----------
@ -139,8 +142,7 @@ class StreamLookasideCache(CGILookasideCache):
""" """
_name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name _name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
return super(StreamLookasideCache, self).upload( return super(StreamLookasideCache, self).upload(_name, filename, hashstr)
_name, filename, hashstr)
def download(self, name, filename, hashstr, outfile, hashtype=None, **kwargs): def download(self, name, filename, hashstr, outfile, hashtype=None, **kwargs):
""" """
@ -149,7 +151,7 @@ class StreamLookasideCache(CGILookasideCache):
This method inherits the behavior of its parent class from pyrpkg. This method inherits the behavior of its parent class from pyrpkg.
It uses the internal `utils.get_repo_name` method to parse the name in case It uses the internal `utils.get_repo_name` method to parse the name in case
it is a scm url. it is a scm url.
Parameters Parameters
---------- ----------
@ -178,7 +180,8 @@ class StreamLookasideCache(CGILookasideCache):
_name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name _name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
return super(StreamLookasideCache, self).download( return super(StreamLookasideCache, self).download(
_name, filename, hashstr, outfile, hashtype=hashtype, **kwargs) _name, filename, hashstr, outfile, hashtype=hashtype, **kwargs
)
class CLLookasideCache(CGILookasideCache): class CLLookasideCache(CGILookasideCache):
@ -190,22 +193,23 @@ class CLLookasideCache(CGILookasideCache):
def __init__(self, hashtype, download_url, upload_url, name, branch): def __init__(self, hashtype, download_url, upload_url, name, branch):
super(CLLookasideCache, self).__init__( super(CLLookasideCache, self).__init__(
hashtype, download_url, upload_url, name, branch) hashtype, download_url, upload_url, name, branch
)
self.name = name self.name = name
self.branch = branch self.branch = branch
def get_download_url(self, name, filename, hash, hashtype=None, **kwargs): def get_download_url(self, name, filename, hash, hashtype=None, **kwargs):
self.download_path='%(name)s/%(branch)s/%(hash)s' self.download_path = "%(name)s/%(branch)s/%(hash)s"
if "/" in name: if "/" in name:
real_name = name.split("/")[-1] real_name = name.split("/")[-1]
else: else:
real_name = name real_name = name
path_dict = { path_dict = {
'name': real_name, "name": real_name,
'filename': filename, "filename": filename,
'branch': self.branch, "branch": self.branch,
'hash': hash, "hash": hash,
'hashtype': hashtype "hashtype": hashtype,
} }
path = self.download_path % path_dict path = self.download_path % path_dict
return os.path.join(self.download_url, path) return os.path.join(self.download_url, path)
@ -217,25 +221,36 @@ class SIGLookasideCache(CGILookasideCache):
It inherits most of its behavior from `pyrpkg.lookasideCGILookasideCache`. It inherits most of its behavior from `pyrpkg.lookasideCGILookasideCache`.
""" """
def __init__(self, hashtype, download_url, upload_url, name, branch, client_cert=None, ca_cert=None):
def __init__(
self,
hashtype,
download_url,
upload_url,
name,
branch,
client_cert=None,
ca_cert=None,
):
super(SIGLookasideCache, self).__init__( super(SIGLookasideCache, self).__init__(
hashtype, download_url, upload_url, client_cert=client_cert, ca_cert=ca_cert) hashtype, download_url, upload_url, client_cert=client_cert, ca_cert=ca_cert
)
self.name = name self.name = name
self.branch = branch self.branch = branch
def get_download_url(self, name, filename, hash, hashtype=None, **kwargs): def get_download_url(self, name, filename, hash, hashtype=None, **kwargs):
download_path = '%(name)s/%(branch)s/%(hash)s' download_path = "%(name)s/%(branch)s/%(hash)s"
if "/" in name: if "/" in name:
real_name = name.split("/")[-1] real_name = name.split("/")[-1]
else: else:
real_name = name real_name = name
path_dict = { path_dict = {
'name': real_name, "name": real_name,
'filename': filename, "filename": filename,
'branch': self.branch, "branch": self.branch,
'hash': hash, "hash": hash,
'hashtype': hashtype "hashtype": hashtype,
} }
path = download_path % path_dict path = download_path % path_dict
return os.path.join(self.download_url, path) return os.path.join(self.download_url, path)
@ -256,9 +271,11 @@ class SIGLookasideCache(CGILookasideCache):
# https://bugzilla.redhat.com/show_bug.cgi?id=1241059 # https://bugzilla.redhat.com/show_bug.cgi?id=1241059
_name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name _name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
post_data = [('name', _name), post_data = [
('%ssum' % self.hashtype, hash), ("name", _name),
('filename', filename)] ("%ssum" % self.hashtype, hash),
("filename", filename),
]
with io.BytesIO() as buf: with io.BytesIO() as buf:
c = pycurl.Curl() c = pycurl.Curl()
@ -270,8 +287,7 @@ class SIGLookasideCache(CGILookasideCache):
if os.path.exists(self.client_cert): if os.path.exists(self.client_cert):
c.setopt(pycurl.SSLCERT, self.client_cert) c.setopt(pycurl.SSLCERT, self.client_cert)
else: else:
self.log.warning("Missing certificate: %s" self.log.warning("Missing certificate: %s" % self.client_cert)
% self.client_cert)
if self.ca_cert is not None: if self.ca_cert is not None:
if os.path.exists(self.ca_cert): if os.path.exists(self.ca_cert):
@ -280,7 +296,7 @@ class SIGLookasideCache(CGILookasideCache):
self.log.warning("Missing certificate: %s", self.ca_cert) self.log.warning("Missing certificate: %s", self.ca_cert)
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE) c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
c.setopt(pycurl.USERPWD, ':') c.setopt(pycurl.USERPWD, ":")
try: try:
c.perform() c.perform()
@ -299,16 +315,15 @@ class SIGLookasideCache(CGILookasideCache):
# Lookaside CGI script returns these strings depending on whether # Lookaside CGI script returns these strings depending on whether
# or not the file exists: # or not the file exists:
if output == b'Available': if output == b"Available":
return True return True
if output == b'Missing': if output == b"Missing":
return False return False
# Something unexpected happened # Something unexpected happened
self.log.debug(output) self.log.debug(output)
raise UploadError('Error checking for %s at %s' raise UploadError("Error checking for %s at %s" % (filename, self.upload_url))
% (filename, self.upload_url))
def upload(self, name, filepath, hash): def upload(self, name, filepath, hash):
"""Upload a source file """Upload a source file
@ -326,9 +341,11 @@ class SIGLookasideCache(CGILookasideCache):
return return
self.log.info("Uploading: %s", filepath) self.log.info("Uploading: %s", filepath)
post_data = [('name', name), post_data = [
('%ssum' % self.hashtype, hash), ("name", name),
('file', (pycurl.FORM_FILE, filepath))] ("%ssum" % self.hashtype, hash),
("file", (pycurl.FORM_FILE, filepath)),
]
with io.BytesIO() as buf: with io.BytesIO() as buf:
c = pycurl.Curl() c = pycurl.Curl()
@ -351,7 +368,7 @@ class SIGLookasideCache(CGILookasideCache):
self.log.warning("Missing certificate: %s", self.ca_cert) self.log.warning("Missing certificate: %s", self.ca_cert)
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE) c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
c.setopt(pycurl.USERPWD, ':') c.setopt(pycurl.USERPWD, ":")
try: try:
c.perform() c.perform()
@ -366,7 +383,7 @@ class SIGLookasideCache(CGILookasideCache):
output = buf.getvalue().strip() output = buf.getvalue().strip()
# Get back a new line, after displaying the download progress # Get back a new line, after displaying the download progress
sys.stdout.write('\n') sys.stdout.write("\n")
sys.stdout.flush() sys.stdout.flush()
if status != 200: if status != 200:

143
src/centpkg/utils.py
View file

@ -28,6 +28,7 @@ import git as gitpython
dist_git_config = None dist_git_config = None
def do_fork(logger, base_url, token, repo_name, namespace, cli_name): def do_fork(logger, base_url, token, repo_name, namespace, cli_name):
""" """
Creates a fork of the project. Creates a fork of the project.
@ -40,27 +41,30 @@ def do_fork(logger, base_url, token, repo_name, namespace, cli_name):
:return: a tuple consisting of whether the fork needed to be created (bool) :return: a tuple consisting of whether the fork needed to be created (bool)
and the fork path (string) and the fork path (string)
""" """
api_url = '{0}/api/v4'.format(base_url.rstrip('/')) api_url = "{0}/api/v4".format(base_url.rstrip("/"))
project_id = quote_plus("redhat/centos-stream/{0}/{1}".format(namespace, repo_name)) project_id = quote_plus("redhat/centos-stream/{0}/{1}".format(namespace, repo_name))
fork_url = '{0}/projects/{1}/fork'.format(api_url, project_id) fork_url = "{0}/projects/{1}/fork".format(api_url, project_id)
headers = { headers = {
'PRIVATE-TOKEN': token, "PRIVATE-TOKEN": token,
'Accept': 'application/json', "Accept": "application/json",
'Content-Type': 'application/json' "Content-Type": "application/json",
} }
# define a new repository name/path to avoid collision with other projects # define a new repository name/path to avoid collision with other projects
safe_name = "centos_{0}_{1}".format(namespace, repo_name) safe_name = "centos_{0}_{1}".format(namespace, repo_name)
payload = json.dumps({ payload = json.dumps(
'name': safe_name, # name of the project after forking {
'path': safe_name, "name": safe_name, # name of the project after forking
}) "path": safe_name,
}
)
try: try:
rv = requests.post( rv = requests.post(fork_url, headers=headers, data=payload, timeout=60)
fork_url, headers=headers, data=payload, timeout=60)
except ConnectionError as error: except ConnectionError as error:
error_msg = ('The connection to API failed while trying to ' error_msg = (
'create a new fork. The error was: {0}'.format(str(error))) "The connection to API failed while trying to "
"create a new fork. The error was: {0}".format(str(error))
)
raise rpkgError(error_msg) raise rpkgError(error_msg)
try: try:
@ -71,49 +75,64 @@ def do_fork(logger, base_url, token, repo_name, namespace, cli_name):
pass pass
if rv.ok: if rv.ok:
fork_id = rv.json()['id'] fork_id = rv.json()["id"]
try: try:
# Unprotect c9s in fork # Unprotect c9s in fork
rv = requests.delete('{0}/projects/{1}/protected_branches/{2}'.format(api_url, fork_id, 'c9s'), headers=headers) rv = requests.delete(
"{0}/projects/{1}/protected_branches/{2}".format(
api_url, fork_id, "c9s"
),
headers=headers,
)
except ConnectionError as error: except ConnectionError as error:
error_msg = ('The connection to API failed while trying to unprotect c9s branch' error_msg = (
'in the fork. The error was: {0}'.format(str(error))) "The connection to API failed while trying to unprotect c9s branch"
"in the fork. The error was: {0}".format(str(error))
)
raise rpkgError(error_msg) raise rpkgError(error_msg)
try: try:
# Reprotect c9s to disable pushes # Reprotect c9s to disable pushes
# Only maintainers in gitlab are allowed to push with the following config # Only maintainers in gitlab are allowed to push with the following config
# In CS, every pkg maintainer is considered as a developer in gitlab # In CS, every pkg maintainer is considered as a developer in gitlab
data = {'id': fork_id, data = {
'name': 'c9s', "id": fork_id,
'allowed_to_push': [{'access_level': 40}], "name": "c9s",
'allowed_to_merge': [{'access_level': 40}], "allowed_to_push": [{"access_level": 40}],
} "allowed_to_merge": [{"access_level": 40}],
rv = requests.post('{0}/projects/{1}/protected_branches'.format(api_url, fork_id), json=data, headers=headers) }
rv = requests.post(
"{0}/projects/{1}/protected_branches".format(api_url, fork_id),
json=data,
headers=headers,
)
except ConnectionError as error: except ConnectionError as error:
error_msg = ('The connection to API failed while trying to reprotect c9s branch' error_msg = (
'in the fork fork. The error was: {0}'.format(str(error))) "The connection to API failed while trying to reprotect c9s branch"
"in the fork fork. The error was: {0}".format(str(error))
)
raise rpkgError(error_msg) raise rpkgError(error_msg)
base_error_msg = ('The following error occurred while creating a new fork: {0}') base_error_msg = "The following error occurred while creating a new fork: {0}"
if not rv.ok: if not rv.ok:
# fork was already created # fork was already created
if rv.status_code == 409 or rv.reason == "Conflict": if rv.status_code == 409 or rv.reason == "Conflict":
# When the repo already exists, the return doesn't contain the repo # When the repo already exists, the return doesn't contain the repo
# path or username. Make one more API call to get the username of # path or username. Make one more API call to get the username of
# the token to construct the repo path. # the token to construct the repo path.
rv = requests.get('{0}/user'.format(api_url), headers=headers) rv = requests.get("{0}/user".format(api_url), headers=headers)
username = rv.json()['username'] username = rv.json()["username"]
return False, '{0}/{1}'.format(username, safe_name) return False, "{0}/{1}".format(username, safe_name)
# show hint for invalid, expired or revoked token # show hint for invalid, expired or revoked token
elif rv.status_code == 401 or rv.reason == "Unauthorized": elif rv.status_code == 401 or rv.reason == "Unauthorized":
base_error_msg += '\nFor invalid or expired token refer to ' \ base_error_msg += (
'"{0} fork -h" to set a token in your user ' \ "\nFor invalid or expired token refer to "
'configuration.'.format(cli_name) '"{0} fork -h" to set a token in your user '
"configuration.".format(cli_name)
)
raise rpkgError(base_error_msg.format(rv.text)) raise rpkgError(base_error_msg.format(rv.text))
return True, rv_json['path_with_namespace'] return True, rv_json["path_with_namespace"]
def do_add_remote(base_url, remote_base_url, repo, repo_path, remote_name): def do_add_remote(base_url, remote_base_url, repo, repo_path, remote_name):
@ -127,7 +146,7 @@ def do_add_remote(base_url, remote_base_url, repo, repo_path, remote_name):
:return: a bool; True if remote was created, False when already exists :return: a bool; True if remote was created, False when already exists
""" """
parsed_url = urlparse(remote_base_url) parsed_url = urlparse(remote_base_url)
remote_url = '{0}://{1}/{2}.git'.format( remote_url = "{0}://{1}/{2}.git".format(
parsed_url.scheme, parsed_url.scheme,
parsed_url.netloc, parsed_url.netloc,
repo_path, repo_path,
@ -142,7 +161,8 @@ def do_add_remote(base_url, remote_base_url, repo, repo_path, remote_name):
repo.create_remote(remote_name, url=remote_url) repo.create_remote(remote_name, url=remote_url)
except git.exc.GitCommandError as e: except git.exc.GitCommandError as e:
error_msg = "During create remote:\n {0}\n {1}".format( error_msg = "During create remote:\n {0}\n {1}".format(
" ".join(e.command), e.stderr) " ".join(e.command), e.stderr
)
raise rpkgError(error_msg) raise rpkgError(error_msg)
return True return True
@ -194,9 +214,11 @@ def get_canonical_repo_name(config, repo_url):
""" """
# Look up the repo and query for forked_from_project # Look up the repo and query for forked_from_project
cli_name = config_get_safely(dist_git_config, '__default', 'cli_name') cli_name = config_get_safely(dist_git_config, "__default", "cli_name")
distgit_section = '{0}.distgit'.format(cli_name) distgit_section = "{0}.distgit".format(cli_name)
distgit_api_base_url = config_get_safely(dist_git_config, distgit_section, "apibaseurl") distgit_api_base_url = config_get_safely(
dist_git_config, distgit_section, "apibaseurl"
)
parsed_repo_url = urlparse(repo_url) parsed_repo_url = urlparse(repo_url)
if not parsed_repo_url.scheme and repo_url.startswith("git@"): if not parsed_repo_url.scheme and repo_url.startswith("git@"):
@ -210,17 +232,18 @@ def get_canonical_repo_name(config, repo_url):
faked_url = "git+ssh://{0}".format(repo_url.replace(":", "/", 1)) faked_url = "git+ssh://{0}".format(repo_url.replace(":", "/", 1))
parsed_repo_url = urlparse(faked_url) parsed_repo_url = urlparse(faked_url)
try: try:
distgit_token = config_get_safely(dist_git_config, distgit_section, 'token') distgit_token = config_get_safely(dist_git_config, distgit_section, "token")
api_url = '{0}/api/v4'.format(distgit_api_base_url.rstrip('/')) api_url = "{0}/api/v4".format(distgit_api_base_url.rstrip("/"))
project_url = '{0}/projects/{1}'.format(api_url, quote_plus(parsed_repo_url.path.lstrip('/'))) project_url = "{0}/projects/{1}".format(
api_url, quote_plus(parsed_repo_url.path.lstrip("/"))
)
headers = { headers = {
'PRIVATE-TOKEN': distgit_token, "PRIVATE-TOKEN": distgit_token,
'Accept': 'application/json', "Accept": "application/json",
'Content-Type': 'application/json' "Content-Type": "application/json",
} }
rv = requests.get(project_url, headers=headers) rv = requests.get(project_url, headers=headers)
@ -229,7 +252,7 @@ def get_canonical_repo_name(config, repo_url):
# Extract response json for debugging # Extract response json for debugging
rv_json = rv.json() rv_json = rv.json()
canonical_repo_name = rv_json['forked_from_project']['name'] canonical_repo_name = rv_json["forked_from_project"]["name"]
except HTTPError as e: except HTTPError as e:
# We got a 4xx or 5xx error code from the URL lookup # We got a 4xx or 5xx error code from the URL lookup
@ -246,9 +269,10 @@ def get_canonical_repo_name(config, repo_url):
raise rpkgError("Insufficient Gitlab API permissions. Missing token?") raise rpkgError("Insufficient Gitlab API permissions. Missing token?")
# Chop off a trailing .git if any # Chop off a trailing .git if any
return canonical_repo_name.rsplit('.git', 1)[0] return canonical_repo_name.rsplit(".git", 1)[0]
def get_repo_name(name, org='rpms'):
def get_repo_name(name, org="rpms"):
""" """
Try to parse the repository name in case it is a git url. Try to parse the repository name in case it is a git url.
@ -273,7 +297,8 @@ def get_repo_name(name, org='rpms'):
# This is probably a renamed fork, so try to find the fork's parent # This is probably a renamed fork, so try to find the fork's parent
repo_name = get_canonical_repo_name(dist_git_config, name) repo_name = get_canonical_repo_name(dist_git_config, name)
return '%s/%s' % (org, repo_name) return "%s/%s" % (org, repo_name)
def stream_mapping(csname): def stream_mapping(csname):
""" """
@ -289,17 +314,20 @@ def stream_mapping(csname):
str str
Correspoinding RHEL name. Correspoinding RHEL name.
""" """
if csname == "c8s" or csname == "cs8" : if csname == "c8s" or csname == "cs8":
return "rhel-8" return "rhel-8"
if csname == "c9s" or csname == "cs9" : if csname == "c9s" or csname == "cs9":
return "rhel-9" return "rhel-9"
if csname == "c10s" or csname == "cs10" : if csname == "c10s" or csname == "cs10":
return "rhel-10" return "rhel-10"
if csname == "c11s" or csname == "cs11" : if csname == "c11s" or csname == "cs11":
return "rhel-11" return "rhel-11"
return None return None
def does_divergent_branch_exist(repo_name, x_version, active_y, rhel_dist_git, namespace):
def does_divergent_branch_exist(
repo_name, x_version, active_y, rhel_dist_git, namespace
):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Determine if the Y-1 branch exists for this repo # Determine if the Y-1 branch exists for this repo
@ -334,8 +362,8 @@ def does_divergent_branch_exist(repo_name, x_version, active_y, rhel_dist_git, n
def _datesplit(isodate): def _datesplit(isodate):
date_string_tuple = isodate.split('-') date_string_tuple = isodate.split("-")
return [ int(x) for x in date_string_tuple ] return [int(x) for x in date_string_tuple]
def parse_rhel_shortname(shortname): def parse_rhel_shortname(shortname):
@ -419,4 +447,3 @@ def determine_active_y_version(rhel_version, api_url):
) )
return x_version, active_y_version, beta, in_exception_phase return x_version, active_y_version, beta, in_exception_phase