From 88e0d00e70714b1a42f4198007e4c75a956e1e8a Mon Sep 17 00:00:00 2001 From: Adam Samalik Date: Wed, 25 Mar 2015 12:44:27 +0100 Subject: [PATCH] config: genacls and pkgdb_sync_git_branches --- files/dist_git_main.conf | 27 +++++++++++ files/genacls.pkgdb | 78 ++++++++++++-------------------- files/pkgdb_sync_git_branches.py | 41 ++++------------- 3 files changed, 66 insertions(+), 80 deletions(-) create mode 100644 files/dist_git_main.conf diff --git a/files/dist_git_main.conf b/files/dist_git_main.conf new file mode 100644 index 0000000..577831c --- /dev/null +++ b/files/dist_git_main.conf @@ -0,0 +1,27 @@ +[acls] + +# user groups on the server associated with dist-git +# default: cvsadmin,fedora-arm,fedora-sparc,fedora-ia64,fedora-s390,fedora-ppc,provenpackager +user_groups=cvsadmin,fedora-arm,fedora-sparc,fedora-ia64,fedora-s390,fedora-ppc,provenpackager + +# user groups with write access to all repositories +# default: cvsadmin,fedora-arm,fedora-s390,fedora-ppc +admin_groups=cvsadmin,fedora-arm,fedora-s390,fedora-ppc + +# comma separated list of active branches to create ACLs for +# default: olpc2,olpc3,el4,el5,el6,el7,epel7,f11,f12,f13,f14,f15,f16,f17,f18,f19,f20,f21,f22,master +active_branches=olpc2,olpc3,el4,el5,el6,el7,epel7,f11,f12,f13,f14,f15,f16,f17,f18,f19,f20,f21,f22,master + +# a "regex"ish list of the reserved branches +# default: f[0-9][0-9],epel[0-9],epel[0-9][0-9],el[0-9],olpc[0-9] +reserved_branches=f[0-9][0-9],epel[0-9],epel[0-9][0-9],el[0-9],olpc[0-9] + +# url of package database - acls (data in json format) +# default: https://admin.fedoraproject.org/pkgdb/api/vcs?format=json +pkgdb_acls_url=https://admin.fedoraproject.org/pkgdb/api/vcs?format=json + +# url of package database - groups (data in json format) +# default: https://admin.fedoraproject.org/pkgdb/api/groups?format=json +pkgdb_groups_url=https://admin.fedoraproject.org/pkgdb/api/groups?format=json + + diff --git a/files/genacls.pkgdb b/files/genacls.pkgdb index b46276e..255ad05 100644 --- a/files/genacls.pkgdb +++ b/files/genacls.pkgdb @@ -9,37 +9,28 @@ import grp import sys import requests +from ConfigParser import ConfigParser + +def _get_conf(cp, section, option, default): + if cp.has_section(section) and cp.has_option(section, option): + return cp.get(section, option) + return default + if __name__ == '__main__': - # Get the users in various groups - TRUSTED = grp.getgrnam('cvsadmin')[3] - ARM = grp.getgrnam('fedora-arm')[3] - SPARC = grp.getgrnam('fedora-sparc')[3] - IA64 = grp.getgrnam('fedora-ia64')[3] - S390 = grp.getgrnam('fedora-s390')[3] - PPC = grp.getgrnam('fedora-ppc')[3] - PROVEN = grp.getgrnam('provenpackager')[3] + config = ConfigParser() + config.read("/etc/dist-git/dist-git.conf") - # Set the active branches to create ACLs for - # Give them the git branch eqiv until pkgdb follows suite - ACTIVE = {'OLPC-2': 'olpc2', 'OLPC-3': 'olpc3', 'EL-4': 'el4', - 'EL-5': 'el5', 'el5': 'el5', 'el6': 'el6', 'EL-6': 'el6', - 'epel7': 'epel7', - 'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15': - 'f15', 'f16': 'f16', 'f17': 'f17', 'f18': 'f18', 'f19': 'f19', - 'f20': 'f20', 'f21': 'f21', 'f22': 'f22', - 'devel': 'master', 'master': 'master'} + user_groups = _get_conf(config, "acls", "user_groups", "").split(",") + admin_groups = _get_conf(config, "acls", "admin_groups", "").split(",") + ACTIVE = _get_conf(config, "acls", "active_branches", "").split(",") + RESERVED = _get_conf(config, "acls", "reserved_branches", "").split(",") + pkgdb_acls_url = _get_conf(config, "acls", "pkgdb_acls_url", "") + pkgdb_groups_url = _get_conf(config, "acls", "pkgdb_groups_url", "") - # Create a "regex"ish list 0f the reserved branches - RESERVED = ['f[0-9][0-9]', 'epel[0-9]', 'epel[0-9][0-9]', 'el[0-9]', 'olpc[0-9]'] # Read the ACL information from the packageDB -{% if env == 'staging' %} - url = 'https://admin.stg.fedoraproject.org/pkgdb/api/vcs?format=json' -{% else %} - url = 'https://admin.fedoraproject.org/pkgdb/api/vcs?format=json' -{% endif %} - data = requests.get(url).json() + data = requests.get(pkgdb_acls_url).json() # Get a list of all the packages acls = data['packageAcls'] @@ -47,31 +38,24 @@ if __name__ == '__main__': pkglist.sort() # sanity check - if len(pkglist) < 2500: - sys.exit(1) + #if len(pkglist) < 2500: + # sys.exit(1) + + # get the list of all groups + pkgdb_groups = requests.get(pkgdb_groups_url).json() # print out our user groups - print '@admins = %s' % ' '.join(TRUSTED) - print '@provenpackager = %s' % ' '.join(PROVEN) - print '@fedora-arm = %s' % ' '.join(ARM) - print '@fedora-s390 = %s' % ' '.join(S390) - print '@fedora-ppc = %s' % ' '.join(PPC) + for group in user_groups + pkgdb_groups["groups"]: + print "@{0} = {1}".format(group, " ".join(grp.getgrnam(group)[3])) - # Get a list of all the groups -{% if env == 'staging' %} - groups = requests.get('https://admin.stg.fedoraproject.org/pkgdb/api/groups?format=json').json() -{% else %} - groups = requests.get('https://admin.fedoraproject.org/pkgdb/api/groups?format=json').json() -{% endif %} - for group in groups['groups']: - print '@%s = %s' % (group, ' '.join(grp.getgrnam(group)[3])) # Give a little space before moving onto the permissions print '' # print our default permissions print 'repo @all' print ' - VREF/update-block-push-origin = @all' - print ' RWC = @admins @fedora-arm @fedora-s390 @fedora-ppc' + if admin_groups: + print ' RWC = @{}'.format(" @".join(admin_groups)) print ' R = @all' #print ' RW private- = @all' # dont' enable the above until we prevent building for real from private- @@ -86,13 +70,12 @@ if __name__ == '__main__': branches = acls[pkg].keys() branches.sort() for branch in branches: - if not branch in ACTIVE.keys(): + if not branch in ACTIVE: continue if 'packager' in acls[pkg][branch]['commit']['groups']: # If the packager group is defined, everyone has access - buffer.append(' RWC %s = @all' % (ACTIVE[branch])) - branchAcls.setdefault('@all', []).append((pkg, - ACTIVE[branch])) + buffer.append(' RWC %s = @all' % (branch)) + branchAcls.setdefault('@all', []).append((pkg, branch)) if branch == 'master': masters.append('@all') if '@all' not in writers: @@ -117,9 +100,8 @@ if __name__ == '__main__': # Print the committers to the acl for this package-branch committers = ' '.join(committers) buffer.append(' RWC %s = %s' % - (ACTIVE[branch], committers)) - branchAcls.setdefault(committers, []).append((pkg, - ACTIVE[branch])) + (branch, committers)) + branchAcls.setdefault(committers, []).append((pkg, branch)) print print 'repo %s' % pkg diff --git a/files/pkgdb_sync_git_branches.py b/files/pkgdb_sync_git_branches.py index 9055e48..dcfee5e 100644 --- a/files/pkgdb_sync_git_branches.py +++ b/files/pkgdb_sync_git_branches.py @@ -32,8 +32,7 @@ import subprocess import time import requests - -import fedmsg +from ConfigParser import ConfigParser # Do some off-the-bat configuration of fedmsg. # 1) since this is a one-off script and not a daemon, it needs to connect @@ -43,16 +42,14 @@ import fedmsg # read access to. Contrast that with the 'scm' certificate which # everyone in the 'packager' group has access to. -config = fedmsg.config.load_config([], None) -config['active'] = True -config['endpoints']['relay_inbound'] = config['relay_inbound'] -fedmsg.init(name='relay_inbound', cert_prefix='shell', **config) +def _get_conf(cp, section, option, default): + if cp.has_section(section) and cp.has_option(section, option): + return cp.get(section, option) + return default -{% if env == 'staging' %} -PKGDB_URL = 'https://admin.stg.fedoraproject.org/pkgdb' -{% else %} -PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb' -{% endif %} +config = ConfigParser() +config.read("/etc/dist-git/dist-git.conf") +PKGDB_URL = _get_conf(config, "acls", "pkgdb_acls_url", "") GIT_FOLDER = '/srv/git/rpms/' MKBRANCH = '/usr/local/bin/mkbranch' @@ -122,15 +119,6 @@ def _create_branch(pkgname, branch, existing_branches): try: _invoke(MKBRANCH, [branch, pkgname]) - fedmsg.publish( - topic='branch', - modname='git', - msg=dict( - agent='pkgdb', - name=pkgname, - branch=branch, - ), - ) except ProcessError, e: if e.returnCode == 255: # This is a warning, not an error @@ -145,9 +133,7 @@ def pkgdb_pkg_branch(): :return: a dict[pkg_name] = [pkg_branches] :rtype: dict """ - url = '%s/api/vcs' % PKGDB_URL - req = requests.get(url, params={'format': 'json'}) - data = req.json() + data = requests.get(PKGDB_URL).json() output = {} for pkg in data['packageAcls']: @@ -196,15 +182,6 @@ def branch_package(pkgname, requested_branches, existing_branches): _invoke(SETUP_PACKAGE, [pkgname]) if 'master' in requested_branches: requested_branches.remove('master') # SETUP_PACKAGE creates master - fedmsg.publish( - topic='branch', - modname='git', - msg=dict( - agent='pkgdb', - name=pkgname, - branch='master', - ), - ) # Create all the required branches for the package # Use the translated branch name until pkgdb falls inline