diff --git a/beaker-tests/files/etc/rpkg/rhpkg.conf b/beaker-tests/files/etc/rpkg/rhpkg.conf new file mode 100644 index 0000000..fa0bbde --- /dev/null +++ b/beaker-tests/files/etc/rpkg/rhpkg.conf @@ -0,0 +1,37 @@ +[rhpkg] +lookaside = http://pkgs.example.org/repo/pkgs +lookasidehash = md5 +lookaside_cgi = http://pkgs.example.org/repo/pkgs/upload.cgi +gitbaseurl = ssh://clime@pkgs.example.org/var/lib/dist-git/git/%(module)s +anongiturl = git://pkgs.example.org/%(module)s + +branchre = rhel +kojiprofile = brew +build_client = brew +kerberos_realms = REDHAT.COM +distgit_namespaced = True +lookaside_namespaced = True + +[rhpkg.dist-git] +commit_url = http://pkgs.devel.redhat.com/cgit/%(module)s/commit/?id=%(commit_hash)s + +[rhpkg.covscan] +executable = covscan + +[rhpkg.errata] +url = https://errata.devel.redhat.com/ + +[rhpkg.UMB] +brokers = amqps://some-server.redhat.com:5671 amqps://some-server.ext.phx2.redhat.com:5671 + +[rhpkg.test-rpmdiff] +hub = https://rpmdiff-hub.host.prod.eng.bos.redhat.com/api/v1/ + +[rhpkg.bugzilla] +url = https://bugzilla.redhat.com/xmlrpc.cgi + +[rhpkg.remotes] +fedora_cli = fedpkg +fedora_dist_git = pkgs.fedoraproject.org +centos_cli = centpkg +centos_dist_git = git.centos.org diff --git a/beaker-tests/tests/rhpkg-test/dist-git.conf b/beaker-tests/tests/rhpkg-test/dist-git.conf new file mode 100644 index 0000000..b1302ee --- /dev/null +++ b/beaker-tests/tests/rhpkg-test/dist-git.conf @@ -0,0 +1,16 @@ +[dist-git] +git_author_name = Release Engineering +git_author_email = rel-eng@redhat.com + +cache_dir = /var/lib/dist-git/cache +gitroot_dir = /var/lib/dist-git/git + +gitolite = False +grok = False + +default_namespace = rpms + +[upload] +fedmsgs = False +old_paths = True +nomd5 = False diff --git a/beaker-tests/tests/rhpkg-test/run.sh b/beaker-tests/tests/rhpkg-test/run.sh new file mode 100755 index 0000000..45353be --- /dev/null +++ b/beaker-tests/tests/rhpkg-test/run.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +. /usr/bin/rhts-environment.sh || exit 1 +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +export SCRIPTDIR="$( builtin cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export CWD=`pwd` +export TEST_CWD=`mktemp -d` + +function pkgs_cmd { + ssh -o 'StrictHostKeyChecking no' clime@pkgs.example.org $1 +} + +rlJournalStart + rlPhaseStartSetup rhpkgTest + pkgs_cmd 'git config --global user.email "clime@redhat.com"' + pkgs_cmd 'git config --global user.name "clime"' + pkgs_cmd '/usr/share/dist-git/setup_git_package rpms/prunerepo' + scp -o 'StrictHostKeyChecking no' $SCRIPTDIR/dist-git.conf root@pkgs.example.org:/etc/dist-git/dist-git.conf + rlPhaseEnd + + rlPhaseStartTest rhpkgTest + cd $TEST_CWD + echo "Running tests in $TEST_CWD" + + # clone repo using rhpkg + rlRun "rhpkg -v clone rpms/prunerepo" + + cd prunerepo + git config user.email "somebody@example.com" + git config user.name "Some name" + + # upload into lookaside and working tree update + rlRun "rhpkg -v import --skip-diffs $SCRIPTDIR/../../data/prunerepo-1.1-1.fc23.src.rpm" + + # test of presence of the uploaded file + rlRun 'wget http://pkgs.example.org/repo/pkgs/rpms/prunerepo/prunerepo-1.1.tar.gz/c5af09c7fb2c05e556898c93c62b1e35/prunerepo-1.1.tar.gz' + + # commit of spec and updated sources and push into the git repo + rlRun "git add -A && git commit -m 'test commit'" + rlRun "rhpkg push" + + # delete imported tarball + rm prunerepo-1.1.tar.gz + + # get srpm file using rhpkg + rlRun "rhpkg -v --release rhel-8 srpm" + + cd $CWD + rlPhaseEnd + + rlPhaseStartCleanup rhpkgTest + pkgs_cmd 'rm -rf /var/lib/dist-git/git/rpms/prunerepo.git' + pkgs_cmd 'sudo rm -rf /var/lib/dist-git/cache/lookaside/pkgs/rpms/prunerepo' + rm -rf $TEST_CWD + rlPhaseEnd +rlJournalEnd &> /dev/null diff --git a/docs/scripts/httpd/upload-rh.cgi b/docs/scripts/httpd/upload-rh.cgi new file mode 100644 index 0000000..da9c771 --- /dev/null +++ b/docs/scripts/httpd/upload-rh.cgi @@ -0,0 +1,208 @@ +#!/usr/bin/python +# +# CGI script to handle file updates for the rpms git repository. There +# is nothing really complex here other than tedious checking of our +# every step along the way... +# + +import os +import sys +import errno +import cgi +import stat +import hashlib +import tempfile + +# reading buffer size +BUFFER_SIZE = 4096 +GITREPO = "/srv/git" +DEFAULT_NAMESPACE = 'rpms' + +# Lookaside cache directory +CACHE_DIR = '/srv/cache/lookaside' + +form = cgi.FieldStorage() +os.umask(002) + +def log_msg(*msgs): + sys.stderr.write(' '.join(map(str,msgs)) + '\n') + +# abort running the script +def send_error(text): + print "Content-type: text/plain\n" + print text + sys.exit(1) + +# prepare to exit graciously +def send_ok(text): + print "Content-Type: text/plain" + print + if text: + print text + +# check and validate that all the fields are present +def check_form(var, strip=True): + if not form.has_key(var): + send_error("required field '%s' is not present" % (var,)) + ret = form.getvalue(var) + if type(ret) == type([]): + send_error("Multiple values given for '%s'. Aborting" % (var,)) + if strip: + ret = os.path.basename(ret) # this is a path component + return ret + + +def hardlink(src, dst): + """Create a hardlink, making sure the target directory exists.""" + makedirs(os.path.dirname(dst)) + if os.path.exists(dst): + # The file already exists, let's hardlink over it. + os.unlink(dst) + os.link(src, dst) + log_msg('ln %s %s' % (src, dst)) + + +def makedirs(path, mode=0o2775): + """Create a directory with all parents. If it already exists, then do + nothing.""" + try: + os.makedirs(path, mode=mode) + log_msg('mkdir -p %s' % path) + except OSError as exc: + if exc.errno != errno.EEXIST: + send_error('Failed to create directory: %s' % exc) + + +def check_file_exists(path, filename, upload): + """Send message if exists and exit the script.""" + if not os.access(path, os.F_OK | os.R_OK): + # File does not exist, nothing to do here. + return + + if upload is None: + # File exists and we're just checking. + message = "Available" + else: + # File exists and we wanted to upload it. Just say it's there already. + upload.file.close() + s = os.stat(path) + message = "File %s already exists\nFile: %s Size: %d" % ( + filename, path, s[stat.ST_SIZE]) + send_ok(message) + sys.exit(0) + + +# Get correct hashing algorithm +if 'sha512sum' in form: + checksum = check_form('sha512sum') + hash_type = 'sha512' +elif 'md5sum' in form: + checksum = check_form('md5sum') + hash_type = 'md5' +else: + send_error('Required checksum is not present') + + +NAME = check_form("name", strip=False) + +if '/' not in NAME: + NAME = '%s/%s' % (DEFAULT_NAMESPACE, NAME) + +# Is this a submission or a test? +FILE = None +FILENAME = None +if form.has_key("filename"): + # check the presence of the file + FILENAME = check_form("filename") +else: + if form.has_key("file"): + FILE = form["file"] + if not FILE.file: + send_error("No file given for upload. Aborting") + try: + FILENAME = os.path.basename(FILE.filename) + except: + send_error("Could not extract the filename for upload. Aborting") + else: + send_error("required field '%s' is not present" % ("file", )) + +# Now that all the fields are valid,, figure out our operating environment +if not os.environ.has_key("SCRIPT_FILENAME"): + send_error("My running environment is funky. Aborting") + +# try to see if we already have this file... +file_dest = "%s/%s/%s/%s/%s/%s" % (CACHE_DIR, NAME, FILENAME, hash_type, checksum, FILENAME) +old_path = "%s/%s/%s/%s/%s" % (CACHE_DIR, NAME, FILENAME, checksum, FILENAME) + +check_file_exists(file_dest, FILENAME, FILE) +if hash_type == 'md5': + # If we're using MD5, handle the case of the file only existing in the old + # path. Once the new checksum is fully deployed, this condition will become + # obsolete. + check_file_exists(old_path, FILENAME, FILE) + +# just checking? +if FILE is None: + send_ok("Missing") + sys.exit(-9) + +# if a directory exists, check that it has the proper permissions +def check_dir(tmpdir, wok=os.W_OK): + if not os.access(tmpdir, os.F_OK): + return 0 + if not os.access(tmpdir, os.R_OK|wok|os.X_OK): + send_error("Unable to write to %s repository." % ( + tmpdir,)) + if not os.path.isdir(tmpdir): + send_error("Path %s is not a directory." % (tmpdir,)) + return 1 + +if not os.environ.has_key("SCRIPT_FILENAME"): + send_error("My running environment is funky. Aborting") +# the module's top level directory +my_moddir = "%s/%s" % (CACHE_DIR, NAME) +my_filedir = "%s/%s" % (my_moddir, FILENAME) +hash_dir = "%s/%s/%s" % (my_filedir, hash_type, checksum) + +# first test if the module really exists +if not check_dir("%s/%s.git" % (GITREPO, NAME), 0): + log_msg("Unknown module", NAME) + send_ok("Module '%s' does not exist!" % (NAME,)) + sys.exit(-9) + +makedirs(my_moddir) + +# grab a temporary filename and dump our file in there +tempfile.tempdir = my_moddir +tmpfile = tempfile.mktemp(checksum) +tmpfd = open(tmpfile, "wb+") +# now read the whole file in +m = hashlib.new(hash_type) +FILELENGTH=0 +while 1: + s = FILE.file.read(BUFFER_SIZE) + if not s: + break + tmpfd.write(s) + m.update(s) + FILELENGTH = FILELENGTH + len(s) +# now we're done reading, check the MD5 sum of what we got +tmpfd.close() +check_checksum = m.hexdigest() +if checksum != check_checksum: + send_error("%s check failed. Received %s instead of %s" % ( + hash_type.upper(), check_checksum, checksum)) +# wow, even the checksum matches. make sure full path is valid now +makedirs(hash_dir) +# and move our file to the final location +os.rename(tmpfile, file_dest) +log_msg("Stored %s (%s bytes)" % (file_dest, FILELENGTH)) +send_ok("File %s Size %d STORED OK" % (FILENAME, FILELENGTH)) + +# The file was uploaded with MD5, so we hardlink it to the old location +# (without hash type). This is a temporary workaround to make sure old clients +# can access files they uploaded. +if hash_type == 'md5': + hardlink(file_dest, old_path) + +sys.exit(0)