centos/centpkg
1
0
Fork 0
mirror of https://git.centos.org/centos/centpkg.git synced 2025-02-23 16:22:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

big update for a new rpkg

Browse source
This commit is contained in:
Brian Stinson 2018-11-14 22:28:34 -06:00
parent 894f00cb6f
commit a394b15127
7 changed files with 248 additions and 219 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
centpkg.spec
View file

@ -31,7 +31,7 @@ BuildRequires: pyrpkg
Provides the centpkg command for working with dist-git
%prep
%setup -q
%setup -q -c
%build

2
setup.py
View file

@ -7,7 +7,7 @@ setup(
name="centpkg",
version=__version__,
author="Brian Stinson",
author_email="bstinson@ksu.edu",
author_email="brian@bstinson.com",
description="CentOS Plugin to rpkg for managing RPM package sources",
url="http://bitbucket.org/bstinsonmhk/centpkg.git",
license="GPLv2+",

14
src/centpkg.conf
View file

@ -1,9 +1,13 @@
[centpkg]
lookaside = https://git.centos.org/sources/
lookaside = https://git.stg.centos.org/sources
lookasidehash = sha1
lookaside_cgi = https://localhost/repo/pkgs/upload.cgi #Not Implemented
gitbaseurl = https://%(user)s@git.centos.org/git/rpms/%(module)s
anongiturl = git://git.centos.org/rpms/%(module)s
lookaside_cgi = https://git.stg.centos.org/sources/upload.cgi
lookaside_request_params = branch
distgit_namespaced = True
distgit_namespaces = rpms
gitbaseurl = https://%(user)s@git.stg.centos.org/%(repo)s.git
anongiturl = https://git.stg.centos.org/%(repo)s
branchre = .+\d$|.+\d-.+|master$
kojiconfig = /etc/koji.conf.d/cbs-koji.conf
kojiprofile = cbs
build_client = cbs
clone_config =

207
src/centpkg/__init__.py
View file

@ -23,7 +23,10 @@ import urlparse
import warnings
from pyrpkg import Commands, rpkgError
from pyrpkg.layout import ExplodedSRPMLayout, DistGitLayout
from centos import centos_cert
from .lookaside import CentOSLookasideCache
from pyrpkg.utils import cached_property
from . import cli
@ -36,7 +39,7 @@ class DistGitDirectory(object):
distrobranch = False
def __init__(self, branchtext):
sigtobranchre = r'sig-(?P<signame>\w+)(?P<centosversion>\d)-?(?P<projectname>\w+)?-?(?P<releasename>\w+)?'
sigtobranchre = r'c(?P<centosversion>\d+)-sig-(?P<signame>\w+)-?(?P<projectname>\w+)?-?(?P<releasename>\w+)?'
distrobranchre = r'c(?P<centosversion>\d+)-?(?P<projectname>\w+)?'
oldbranchre = r'(?P<signame>\w+)(?P<centosversion>\d)'
sigmatch = re.match(sigtobranchre, branchtext)
@ -86,23 +89,30 @@ class DistGitDirectory(object):
return '-'.join(filter(None, [self.signame+self.centosversion,
projectorcommon, releaseorcommon])) + '-el{0}'.format(self.centosversion)
class Commands(Commands):
'''
For the pyrpkg commands with centpkg behavior
'''
def __init__(self, path, lookaside, lookasidehash, lookaside_cgi,
gitbaseurl, anongiturl, branchre, kojiconfig,
build_client, user=None, dist=None, target=None,
quiet=False, distgit_namespaced=False):
def __init__(self, *args, **kwargs):
'''
Init the object and some configuration details.
'''
super(Commands, self).__init__(path, lookaside, lookasidehash,
lookaside_cgi, gitbaseurl, anongiturl,
branchre, kojiconfig, build_client,
user, dist, target, quiet, distgit_namespaced)
super(Commands, self).__init__(*args, **kwargs)
self.distgitdir = DistGitDirectory(self.branch_merge)
self.source_entry_type = 'old'
@property
def distgitdir(self):
return DistGitDirectory(self.branch_merge)
@cached_property
def lookasidecache(self):
return CentOSLookasideCache(self.lookasidehash,
self.lookaside,
self.lookaside_cgi,
self.repo_name,
self.branch_merge)
# redefined loaders
def load_rpmdefines(self):
@ -113,39 +123,25 @@ class Commands(Commands):
if not self.distgitdir.centosversion:
raise rpkgError('Could not get the OS version from the branch:{0}'.format(self.branch_merge))
self._distval = self.distgitdir.centosversion
self._distval = self._distval.replace('.', '_')
self._distvar = self.distgitdir.centosversion
self._distval = self._distvar.replace('.', '_')
self._disttag = 'el%s' % self._distval
self._rpmdefines = ["--define '_topdir {0}'".format(self.path),
"--define '_srcrpmdir {0}'".format(self.path),
"--define '_rpmdir {0}'".format(self.path),
"--define 'dist .{0}'".format(self._disttag),
self._rpmdefines = ["--define '_sourcedir %s'" % self.layout.sourcedir,
"--define '_specdir %s'" % self.layout.specdir,
"--define '_builddir %s'" % self.layout.builddir,
"--define '_srcrpmdir %s'" % self.layout.srcrpmdir,
"--define '_rpmdir %s'" % self.layout.rpmdir,
"--define 'dist .%s'" % self._disttag,
# int and float this to remove the decimal
"--define '{0} 1'".format(self._disttag)]
def load_spec(self):
"""This sets the spec attribute"""
# We are not using the upstream load_spec because the file structure is
# hard-coded
# Get a list of files in the path we're looking at
files = os.listdir(os.path.join(self.path,'SPECS'))
# Search the files for the first one that ends with ".spec"
for __f in files:
if __f.endswith('.spec') and not __f.startswith('.'):
self._spec = os.path.join('SPECS', __f)
return
raise rpkgError('No spec file found.')
"--define '%s 1'" % self._disttag]
self.log.debug("RPMDefines: %s" % self._rpmdefines)
def load_target(self):
""" This sets the target attribute (used for mock and koji) """
# Distribution branches start with c and may or may not end in -plus
# otherwise, it's a sig branch
# otherwise, it's a sig branch
if not self.distgitdir.distrobranch:
# send distribution packages to build on the the bananas tags for now
self._target = 'bananas{0}-{1}'.format(self.distval, self.disttag)
@ -156,157 +152,26 @@ class Commands(Commands):
def load_user(self):
try:
self._user = centos_cert.CentOSUserCert().CN
except:
except Exception:
print >>sys.stderr, "Could not load user from cert file"
super(Commands, self).load_user()
# These are the commands defined in the base pyrpkg.Commands class
# and have been implemented here
def sources(self, outdir=None):
"""Download source files"""
# See also:
# https://lists.fedoraproject.org/pipermail/buildsys/2014-July/004313.html
#
# in 'super' the sources function expects a file named 'sources' to be in the base directory.
# A patch has been sent to upstream to allow a more flexible location.
#
# This code doesn't work due to:
# archive.strip().split(' ', 1) # patch provided to upstream to fix
#
# url = '%s/%s/%s/%s/%s' % (self.lookaside, self.module_name,
# file.replace(' ', '%20'),
# csum, file.replace(' ', '%20'))
#
#os.symlink(os.path.join(self.path, '.{0}.metadata'.format(self.module_name)), os.path.join(self.path, 'sources'))
#super(Commands, self).sources(outdir=None)
#os.unlink(os.path.join(self.path, 'sources'))
# The following is copied from rpkg/__init__.py:sources with minor changes
try:
archives = open(os.path.join(self.path,
'.{0}.metadata'.format(self.module_name)),
'r').readlines()
except IOError, e:
raise rpkgError('%s is not a valid repo: %s' % (self.path, e))
# Default to putting the files where the module is
if not outdir:
outdir = self.path
for archive in archives:
try:
# This strip / split is kind a ugly, but checksums shouldn't have
# two spaces in them. sources file might need more structure in the
# future
csum, file = archive.strip().split(None, 1)
except ValueError:
raise rpkgError('Malformed sources file.')
# The default lookaside hash is stored in centpkg.conf, but there is
# a mix of md5 and sha sums in the CentOS lookaside, here we divine
# which one we are using
sum_lengths = { 128: 'sha512',
64: 'sha256',
40: 'sha1',
32: 'md5',
}
self.lookasidehash = sum_lengths[len(csum)]
# If a directory is specified in the metadata file, append it to
# outdir
if os.path.dirname(file):
outdir = os.path.join(self.path, os.path.dirname(file))
file = os.path.basename(file)
# Create the output directory if it's not checked into git
if not os.path.exists(outdir):
self.log.info("Creating OUTDIR: {0}".format(outdir))
os.makedirs(outdir)
outfile = os.path.join(outdir, file)
# See if we already have a valid copy downloaded
if os.path.exists(outfile):
if self.lookasidecache.file_is_valid(outfile, csum, self.lookasidehash):
continue
self.log.info("Downloading %s" % (file))
filepath = '%s/%s/%s' % (self.module_name,
self.branch_merge,
csum,
)
url = urlparse.urljoin(self.lookaside, filepath)
command = ['curl', '-H', 'Pragma:', '-o', outfile, '-R', '-S', '--fail']
if self.quiet:
command.append('-s')
command.append(url)
self._run_command(command)
if not self.lookasidecache.file_is_valid(outfile, csum, self.lookasidehash):
raise rpkgError('%s failed checksum' % file)
return
def get_latest_commit(self, *args, **kwargs):
raise NotImplementedError("get_latest_commit is not yet implemented in centpkg")
def gitbuildhash(self, *args, **kwargs):
raise NotImplementedError("gitbuildhash is not yet implemented in centpkg")
def upload(self, *args, **kwargs):
if not self.distgitdir.distrobranch:
self.source_entry_type = 'bsd'
return super(Commands, self).upload(*args, **kwargs)
def import_srpm(self, *args, **kwargs):
raise NotImplementedError("import_srpm is not yet implemented in centpkg")
def new(self, *args, **kwargs):
raise NotImplementedError("new is not yet implemented in centpkg")
def patch(self, *args, **kwargs):
raise NotImplementedError("patch is not yet implemented in centpkg")
def push(self, *args, **kwargs):
raise NotImplementedError("push is not yet implemented in centpkg")
def file_exists(self, *args, **kwargs):
raise NotImplementedError("file_exists is not yet implemented in centpkg")
def upload_file(self, *args, **kwargs):
raise NotImplementedError("upload_file is not yet implemented in centpkg")
def install(self, *args, **kwargs):
raise NotImplementedError("install is not yet implemented in centpkg")
def lint(self, *args, **kwargs):
raise NotImplementedError("lint is not yet implemented in centpkg")
def upload(self, *args, **kwargs):
raise NotImplementedError("upload is not yet implemented in centpkg")
def prep(self, *args, **kwargs):
raise NotImplementedError("prep is not yet implemented in centpkg")
def unused_patches(self):
"""Discover patches checked into source control that are not used
Returns a list of unused patches, which may be empty.
"""
# Create a list for unused patches
unused = []
# Get the content of spec into memory for fast searching
spec = open(self.spec, 'r').read()
# Replace %{name} with the package name
spec = spec.replace("%{name}", self.module_name)
# Replace %{version} with the package version
spec = spec.replace("%{version}", self.ver)
# Get a list of files tracked in source control
files = self.repo.git.ls_files('--exclude-standard').split()
for file in map(os.path.basename, files):
# throw out non patches
if not file.endswith(('.patch', '.diff')):
continue
if file not in spec:
unused.append(file)
return unused

14
src/centpkg/__main__.py
View file

@ -19,8 +19,10 @@ import logging
import ConfigParser
import argparse
import fedpkg
import pyrpkg
import centpkg
from pyrpkg.layout import Layout, ExplodedSRPMLayout, DistGitLayout
def main():
'''
@ -40,8 +42,16 @@ def main():
config = ConfigParser.SafeConfigParser()
config.read(args.config)
client = centpkg.cli.centpkgClient(config)
client.do_imports(site='centpkg')
layout = Layout.load()
if isinstance(layout, ExplodedSRPMLayout):
client = centpkg.cli.centpkgClient(config)
elif isinstance(layout, DistGitLayout):
config.read('/etc/rpkg/fedpkg.conf')
client = fedpkg.cli.fedpkgClient(config, name='fedpkg')
else:
raise ValueError("Not a site we know about")
client.do_imports(site=client.DEFAULT_CLI_NAME)
client.parse_cmdline()
if not client.args.path:

43
src/centpkg/cli.py
View file

@ -13,46 +13,11 @@
# option) any later version. See http://www.gnu.org/copyleft/gpl.html for
# the full text of the license.
import sys
import os
import logging
from __future__ import print_function
from pyrpkg.cli import cliClient
class centpkgClient(cliClient):
'''
Where we import our custom stuff
'''
def __init__(self, config, name='centpkg'):
'''init'''
def __init__(self, config, name=None):
self.DEFAULT_CLI_NAME = 'centpkg'
super(centpkgClient, self).__init__(config, name)
if __name__ == '__main__':
client = centpkgClient()
client.do_imports()
client.parse_cmdline()
if not client.args.path:
try:
client.args.path = os.getcwd()
except OSError as err_msg:
print('Could not get current path')
print(err_msg)
sys.exit(1)
log = client.site.log
client.setupLogging(log)
if client.args.v:
log.setLevel(logging.DEBUG)
elif client.args.q:
log.setLevel(logging.WARNING)
else:
log.setLevel(logging.INFO)
# Run the necessary command
try:
client.args.command()
except KeyboardInterrupt:
pass

185
src/centpkg/lookaside.py Normal file
View file

@ -0,0 +1,185 @@
# Copyright (c) 2018 - Red Hat Inc.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version. See http://www.gnu.org/copyleft/gpl.html for
# the full text of the license.
"""Interact with the CentOS lookaside cache
We need to override the pyrpkg.lookasidecache module to handle our custom
download path.
"""
import hashlib
import io
import logging
import os
import sys
import pycurl
import six
from pyrpkg.errors import DownloadError, InvalidHashType, UploadError
from pyrpkg.lookaside import CGILookasideCache
from six.moves import http_client
class CentOSLookasideCache(CGILookasideCache):
def __init__(self, hashtype, download_url, upload_url, name, branch):
super(CentOSLookasideCache, self).__init__(
hashtype, download_url, upload_url, client_cert="/home/bstinson/.centos.cert")
self.branch = branch
self.download_path = (
'%(name)s/%(branch)s/%(hash)s')
def remote_file_exists(self, name, filename, hash):
"""Verify whether a file exists on the lookaside cache
:param str name: The name of the module. (usually the name of the
SRPM). This can include the namespace as well (depending on what
the server side expects).
:param str filename: The name of the file to check for.
:param str hash: The known good hash of the file.
"""
# RHEL 7 ships pycurl that does not accept unicode. When given unicode
# type it would explode with "unsupported second type in tuple". Let's
# convert to str just to be sure.
# https://bugzilla.redhat.com/show_bug.cgi?id=1241059
if six.PY2 and isinstance(filename, six.text_type):
filename = filename.encode('utf-8')
if six.PY2 and isinstance(self.branch, six.text_type):
self.branch = self.branch.encode('utf-8')
post_data = [('name', name),
('%ssum' % self.hashtype, hash),
('branch', self.branch),
('filename', filename)]
with io.BytesIO() as buf:
c = pycurl.Curl()
c.setopt(pycurl.URL, self.upload_url)
c.setopt(pycurl.WRITEFUNCTION, buf.write)
c.setopt(pycurl.HTTPPOST, post_data)
if self.client_cert is not None:
if os.path.exists(self.client_cert):
c.setopt(pycurl.SSLCERT, self.client_cert)
else:
self.log.warning("Missing certificate: %s"
% self.client_cert)
if self.ca_cert is not None:
if os.path.exists(self.ca_cert):
c.setopt(pycurl.CAINFO, self.ca_cert)
else:
self.log.warning("Missing certificate: %s", self.ca_cert)
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
c.setopt(pycurl.USERPWD, ':')
try:
c.perform()
status = c.getinfo(pycurl.RESPONSE_CODE)
except Exception as e:
raise UploadError(e)
finally:
c.close()
output = buf.getvalue().strip()
if status != 200:
self.raise_upload_error(status)
# Lookaside CGI script returns these strings depending on whether
# or not the file exists:
if output == b'Available':
return True
if output == b'Missing':
return False
# Something unexpected happened
self.log.debug(output)
raise UploadError('Error checking for %s at %s'
% (filename, self.upload_url))
def upload(self, name, filepath, hash):
"""Upload a source file
:param str name: The name of the module. (usually the name of the SRPM)
This can include the namespace as well (depending on what the
server side expects).
:param str filepath: The full path to the file to upload.
:param str hash: The known good hash of the file.
"""
filename = os.path.basename(filepath)
# As in remote_file_exists, we need to convert unicode strings to str
if six.PY2:
if isinstance(name, six.text_type):
name = name.encode('utf-8')
if isinstance(filepath, six.text_type):
filepath = filepath.encode('utf-8')
if self.remote_file_exists(name, filename, hash):
self.log.info("File already uploaded: %s", filepath)
return
self.log.info("Uploading: %s", filepath)
post_data = [('name', name),
('%ssum' % self.hashtype, hash),
('branch', self.branch),
('file', (pycurl.FORM_FILE, filepath))]
with io.BytesIO() as buf:
c = pycurl.Curl()
c.setopt(pycurl.URL, self.upload_url)
c.setopt(pycurl.NOPROGRESS, False)
c.setopt(pycurl.PROGRESSFUNCTION, self.print_progress)
c.setopt(pycurl.WRITEFUNCTION, buf.write)
c.setopt(pycurl.HTTPPOST, post_data)
if self.client_cert is not None:
if os.path.exists(self.client_cert):
c.setopt(pycurl.SSLCERT, self.client_cert)
else:
self.log.warning("Missing certificate: %s", self.client_cert)
if self.ca_cert is not None:
if os.path.exists(self.ca_cert):
c.setopt(pycurl.CAINFO, self.ca_cert)
else:
self.log.warning("Missing certificate: %s", self.ca_cert)
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
c.setopt(pycurl.USERPWD, ':')
try:
c.perform()
status = c.getinfo(pycurl.RESPONSE_CODE)
except Exception as e:
raise UploadError(e)
finally:
c.close()
output = buf.getvalue().strip()
# Get back a new line, after displaying the download progress
sys.stdout.write('\n')
sys.stdout.flush()
if status != 200:
self.raise_upload_error(status)
if output:
self.log.debug(output)