release-engineering/tito
1
0
Fork 0
mirror of https://github.com/rpm-software-management/tito.git synced 2025-02-23 12:12:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'bzflagcheck'

Browse source 
Conflicts:
	hacking/titotest-centos-5.9/Dockerfile
This commit is contained in:
Devan Goodwin 2014-06-23 14:50:27 -03:00
commit cdf54b6ec5
13 changed files with 707 additions and 395 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
hacking/titotest-centos-5.9/Dockerfile
View file

@ -10,7 +10,7 @@ RUN yum -y install git rpm-build \
python-devel python-nose python-setuptools python-pep8 \
python-pip \
docbook-style-xsl \
libxslt asciidoc tar createrepo which
libxslt asciidoc tar createrepo which python-bugzilla
RUN pip-python install mock --upgrade

2
hacking/titotest-centos-6.4/Dockerfile
View file

@ -8,7 +8,7 @@ MAINTAINER Paul Morgan <jumanjiman@gmail.com>
RUN yum -y install git rpm-build python-devel python-nose \
libxslt asciidoc python-setuptools tar createrepo which
RUN rpm -Uvh http://ftp.linux.ncsu.edu/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
RUN yum -y install python-pep8 git-annex python-mock
RUN yum -y install python-pep8 git-annex python-mock python-bugzilla
# Remove yum metadata.
RUN yum clean all

2
hacking/titotest-fedora-20/Dockerfile
View file

@ -13,7 +13,7 @@ RUN yum -y install git rpm-build libxslt tar \
python-devel python-nose python-setuptools python-pep8 \
python-mock \
python3-devel python3-nose python3-setuptools python3-pep8 rpm-python3 python3-mock \
createrepo git-annex which
createrepo git-annex which python-bugzilla python3-bugzilla
# Remove yum metadata.
RUN yum clean all

8
releasers.conf.5.asciidoc
View file

@ -66,13 +66,15 @@ are then synced to the first branch your releaser lists. After this tito will
git merge the first branch into all other listed branches, triggering builds
in each.
+
WARNING: Highly experimental, very prone to failure if merging master into
your branches is likely to cause a conflict. You will need to cleanup manually
if this occurs.
The 'required_bz_flags' property can be specified to have tito check Red Hat Bugzilla to see if each bug number extracted from the changelog has appropriate flags. If it does not, it will be skipped in the commit message. If no bugs are found with the required tags, a 'placeholder_bz' can be specified (see below), otherwise the release will abort.
+
The 'placeholder_bz' property can be specified to use if no bugs were found in the changelog with the required flags.
+
[fedora-git]
releaser = tito.release.FedoraGitReleaser
branches = master el5 el6 f14 f15 f16
required_bz_flags = myos-1.1.0+ pm_ack+
placeholder_bz = 100000
+
If you would like to build (ie - koji) against a different target than what is
default for the FedoraGit/DistGit branch currently being worked on it can be

139
src/tito/common.py
View file

@ -16,11 +16,13 @@ Common operations.
import os
import re
import sys
import traceback
import subprocess
import shlex
from bugzilla.rhbugzilla import RHBugzilla
from tito.compat import *
from tito.exception import TitoException
from tito.exception import RunCommandException
DEFAULT_BUILD_DIR = "/tmp/tito"
@ -54,32 +56,123 @@ def extract_sources(spec_file_lines):
return filenames
def extract_bzs(output):
class MissingBugzillaCredsException(TitoException):
pass
class BugzillaExtractor(object):
"""
Parses the output of CVS diff or a series of git commit log entries,
looking for new lines which look like a commit of the format:
Parses output of a dist-git commit diff looking for changelog
entries that look like they reference bugzilla commits.
######: Commit message
Returns a list of lines of text similar to:
Resolves: #XXXXXX - Commit message
Optionally can check bugzilla for required flags on each bug.
"""
regex = re.compile(r"^- (\d*)\s?[:-]+\s?(.*)")
diff_regex = re.compile(r"^(\+- )+(\d*)\s?[:-]+\s?(.*)")
bzs = []
for line in output.split("\n"):
match = re.match(regex, line)
match2 = re.match(diff_regex, line)
if match:
bzs.append((match.group(1), match.group(2)))
elif match2:
bzs.append((match2.group(2), match2.group(3)))
def __init__(self, diff_output, required_flags=None,
placeholder_bz=None):
output = []
for bz in bzs:
output.append("Resolves: #%s - %s" % (bz[0], bz[1]))
return output
self.diff_output = diff_output
self.required_flags = required_flags
self.placeholder_bz = placeholder_bz
# Tuples of bugzilla ID + commit message we extracted:
self.bzs = []
def extract(self):
self.bzs = self._extract_bzs()
if self.required_flags:
self._check_for_bugzilla_creds()
self.bzs = self._filter_bzs_with_flags()
return self._format_lines()
def _check_for_bugzilla_creds(self):
if not os.path.exists(os.path.expanduser("~/.bugzillarc")):
raise MissingBugzillaCredsException("Missing ~/.bugzillarc")
else:
debug("Found bugzilla credentials in ~/.bugzillarc")
def _extract_bzs(self):
"""
Parses the output of CVS diff or a series of git commit log entries,
looking for new lines which look like a commit of the format:
######: Commit message
Returns a list of lines of text similar to:
Resolves: #XXXXXX - Commit message
If the releaser specifies any required bugzilla flags we will
check each bug found and see if it has all required flags. If not
we skip it. If we end up with *no* bugs with the required flags
our build is likely to fail, so we look for a placeholder bugzilla
defined in relaser config and use that instead if possible, otherwise
error out.
Returns a list of lines to write to the commit message as is.
"""
regex = re.compile(r"^- (\d*)\s?[:-]+\s?(.*)")
diff_regex = re.compile(r"^(\+- )+(\d*)\s?[:-]+\s?(.*)")
bzs = []
for line in self.diff_output.split("\n"):
match = re.match(regex, line)
match2 = re.match(diff_regex, line)
if match:
bzs.append((match.group(1), match.group(2)))
elif match2:
bzs.append((match2.group(2), match2.group(3)))
return bzs
def _format_lines(self):
output = []
for bz in self.bzs:
output.append("Resolves: #%s - %s" % (bz[0], bz[1]))
if len(output) == 0 and self.required_flags:
# No bugzillas had required flags, use a placeholder if
# we have one, otherwise we have to error out.
if self.placeholder_bz:
print("No bugs with required flags were found, using placeholder: %s" % self.placeholder_bz)
output.append("Related: #%s" % self.placeholder_bz)
else:
error_out("No bugzillas found with required flags: %s" %
self.required_flags)
return output
def _filter_bzs_with_flags(self):
print("Checking flags on bugs: %s" % self.bzs)
print(" required flags: %s" % self.required_flags)
# TODO: Would be nice to load bugs in bulk here but for now we'll
# keep it simple.
filtered_bzs = []
for bz_tuple in self.bzs:
bug_id = bz_tuple[0]
try:
bug = self._load_bug(bug_id)
except xmlrpclib.Fault:
print("WARNING: Bug %s does not seem to exist." % bug_id)
continue
debug("Bug %s has flags: %s" % (bug_id, bug.flags))
flags_missing = False
for flag in self.required_flags:
if bug.get_flag_status(flag[0:-1]) != flag[-1]:
print("WARNING: Bug %s missing required flag: %s" %
(bug_id, flag))
flags_missing = True
break
else:
debug("Bug %s has required flag: %s" %
(bug_id, flag))
if not flags_missing:
filtered_bzs.append(bz_tuple)
return filtered_bzs
def _load_bug(self, bug_id):
bugzilla = RHBugzilla(url='https://bugzilla.redhat.com/xmlrpc.cgi')
return bugzilla.getbug(bug_id, include_fields=['id', 'flags'])
def error_out(error_msgs):

2
src/tito/compat.py
View file

@ -22,11 +22,13 @@ if PY2:
from ConfigParser import NoOptionError
from ConfigParser import RawConfigParser
from StringIO import StringIO
import xmlrpclib
else:
import subprocess
from configparser import NoOptionError
from configparser import RawConfigParser
from io import StringIO
import xmlrpc.client as xmlrpclib
def getstatusoutput(cmd):

3
src/tito/release/__init__.py
View file

@ -2,10 +2,9 @@ from tito.release.main import \
Releaser, \
RsyncReleaser, \
YumRepoReleaser, \
FedoraGitReleaser, \
DistGitReleaser, \
KojiReleaser, \
KojiGitReleaser
from tito.release.distgit import FedoraGitReleaser, DistGitReleaser
from tito.release.obs import ObsReleaser
from tito.release.copr import CoprReleaser

399
src/tito/release/distgit.py Normal file
View file

@ -0,0 +1,399 @@
# Copyright (c) 2013 Red Hat, Inc.
#
# This software is licensed to you under the GNU General Public License,
# version 2 (GPLv2). There is NO WARRANTY for this software, express or
# implied, including the implied warranties of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
# along with this software; if not, see
# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
#
# Red Hat trademarks are not licensed under GPLv2. No permission is
# granted to use or replicate Red Hat trademarks that are incorporated
# in this software or its documentation.
import os
import os.path
import subprocess
import sys
import tempfile
from tito.common import run_command, BugzillaExtractor, debug, extract_sources, \
MissingBugzillaCredsException, error_out
from tito.compat import getoutput, getstatusoutput, write
from tito.release import Releaser
from tito.release.main import PROTECTED_BUILD_SYS_FILES
from tito.buildparser import BuildTargetParser
class FedoraGitReleaser(Releaser):
REQUIRED_CONFIG = ['branches']
cli_tool = "fedpkg"
def __init__(self, name=None, tag=None, build_dir=None,
config=None, user_config=None,
target=None, releaser_config=None, no_cleanup=False,
test=False, auto_accept=False, **kwargs):
Releaser.__init__(self, name, tag, build_dir, config,
user_config, target, releaser_config, no_cleanup, test,
auto_accept, **kwargs)
self.git_branches = \
self.releaser_config.get(self.target, "branches").split(" ")
if self.config.has_option(self.target, "remote_git_name"):
overwrite_checkout = self.config.get(self.target, "remote_git_name")
else:
overwrite_checkout = None
if overwrite_checkout:
self.project_name = overwrite_checkout
self.package_workdir = os.path.join(self.working_dir,
self.project_name)
build_target_parser = BuildTargetParser(self.releaser_config, self.target,
self.git_branches)
self.build_targets = build_target_parser.get_build_targets()
# Files we should copy to git during a release:
self.copy_extensions = (".spec", ".patch")
def release(self, dry_run=False, no_build=False, scratch=False):
self.dry_run = dry_run
self.no_build = no_build
self._git_release()
def _get_build_target_for_branch(self, branch):
if branch in self.build_targets:
return self.build_targets[branch]
return None
def _git_release(self):
getoutput("mkdir -p %s" % self.working_dir)
os.chdir(self.working_dir)
run_command("%s clone %s" % (self.cli_tool, self.project_name))
project_checkout = os.path.join(self.working_dir, self.project_name)
os.chdir(project_checkout)
run_command("%s switch-branch %s" % (self.cli_tool, self.git_branches[0]))
self.builder.tgz()
if self.test:
self.builder._setup_test_specfile()
self._git_sync_files(project_checkout)
self._git_upload_sources(project_checkout)
self._git_user_confirm_commit(project_checkout)
def _get_bz_flags(self):
required_bz_flags = None
if self.releaser_config.has_option(self.target,
'required_bz_flags'):
required_bz_flags = self.releaser_config.get(self.target,
'required_bz_flags').split(" ")
debug("Found required flags: %s" % required_bz_flags)
placeholder_bz = None
if self.releaser_config.has_option(self.target,
'placeholder_bz'):
placeholder_bz = self.releaser_config.get(self.target,
'placeholder_bz')
debug("Found placeholder bugzilla: %s" % placeholder_bz)
return (required_bz_flags, placeholder_bz)
def _confirm_commit_msg(self, diff_output):
"""
Generates a commit message in a temporary file, gives the user a
chance to edit it, and returns the filename to the caller.
"""
fd, name = tempfile.mkstemp()
debug("Storing commit message in temp file: %s" % name)
write(fd, "Update %s to %s\n" % (self.project_name,
self.builder.build_version))
# Write out Resolves line for all bugzillas we see in commit diff:
# TODO: move to DistGitBuilder only?
try:
(required_bz_flags, placeholder_bz) = self._get_bz_flags()
extractor = BugzillaExtractor(diff_output,
required_flags=required_bz_flags,
placeholder_bz=placeholder_bz)
for line in extractor.extract():
write(fd, line + "\n")
except MissingBugzillaCredsException:
error_out([
"Releaser specifies required flags but you have not configured",
"a ~/.bugzillarc with your bugzilla credentials.",
"Example:",
"",
"[bugzilla.redhat.com]",
"user = dgoodwin@redhat.com",
"password = mypassword"])
print("")
print("##### Commit message: #####")
print("")
os.lseek(fd, 0, 0)
file = os.fdopen(fd)
for line in file.readlines():
print(line)
file.close()
print("")
print("###############################")
print("")
if self._ask_yes_no("Would you like to edit this commit message? [y/n] ", False):
debug("Opening editor for user to edit commit message in: %s" % name)
editor = 'vi'
if "EDITOR" in os.environ:
editor = os.environ["EDITOR"]
subprocess.call(editor.split() + [name])
return name
def _git_user_confirm_commit(self, project_checkout):
""" Prompt user if they wish to proceed with commit. """
print("")
text = "Running 'git diff' in: %s" % project_checkout
print("#" * len(text))
print(text)
print("#" * len(text))
print("")
main_branch = self.git_branches[0]
os.chdir(project_checkout)
# Newer versions of git don't seem to want --cached here? Try both:
(status, diff_output) = getstatusoutput("git diff --cached")
if diff_output.strip() == "":
debug("git diff --cached returned nothing, falling back to git diff.")
(status, diff_output) = getstatusoutput("git diff")
if diff_output.strip() == "":
print("No changes in main branch, skipping commit for: %s" % main_branch)
else:
print(diff_output)
print("")
print("##### Please review the above diff #####")
if not self._ask_yes_no("Do you wish to proceed with commit? [y/n] "):
print("Fine, you're on your own!")
self.cleanup()
sys.exit(1)
print("Proceeding with commit.")
commit_msg_file = self._confirm_commit_msg(diff_output)
cmd = '%s commit -F %s' % (self.cli_tool,
commit_msg_file)
debug("git commit command: %s" % cmd)
print
if self.dry_run:
self.print_dry_run_warning(cmd)
else:
print("Proceeding with commit.")
os.chdir(self.package_workdir)
output = run_command(cmd)
os.unlink(commit_msg_file)
cmd = "%s push" % self.cli_tool
if self.dry_run:
self.print_dry_run_warning(cmd)
else:
# Push
print(cmd)
run_command(cmd)
if not self.no_build:
self._build(main_branch)
for branch in self.git_branches[1:]:
print("Merging branch: '%s' -> '%s'" % (main_branch, branch))
run_command("%s switch-branch %s" % (self.cli_tool, branch))
self._merge(main_branch)
cmd = "git push origin %s:%s" % (branch, branch)
if self.dry_run:
self.print_dry_run_warning(cmd)
else:
print(cmd)
run_command(cmd)
if not self.no_build:
self._build(branch)
print
def _merge(self, main_branch):
try:
run_command("git merge %s" % main_branch)
except:
print
print("WARNING!!! Conflicts occurred during merge.")
print
print("You are being dropped to a shell in the working directory.")
print
print("Please resolve this by doing the following:")
print
print(" 1. List the conflicting files: git ls-files --unmerged")
print(" 2. Edit each resolving the conflict and then: git add FILENAME")
print(" 4. Commit the result when you are done: git commit")
print(" 4. Return to the tito release: exit")
print
# TODO: maybe prompt y/n here
os.system(os.environ['SHELL'])
def _build(self, branch):
""" Submit a Fedora build from current directory. """
target_param = ""
build_target = self._get_build_target_for_branch(branch)
if build_target:
target_param = "--target %s" % build_target
build_cmd = "%s build --nowait %s" % (self.cli_tool, target_param)
if self.dry_run:
self.print_dry_run_warning(build_cmd)
return
print("Submitting build: %s" % build_cmd)
(status, output) = getstatusoutput(build_cmd)
if status > 0:
if "already been built" in output:
print("Build has been submitted previously, continuing...")
else:
sys.stderr.write("ERROR: Unable to submit build.\n")
sys.stderr.write(" Status code: %s\n" % status)
sys.stderr.write(" Output: %s\n" % output)
sys.exit(1)
# Print the task ID and URL:
for line in extract_task_info(output):
print(line)
def _git_upload_sources(self, project_checkout):
"""
Upload any tarballs to the lookaside directory. (if necessary)
Uses the "fedpkg new-sources" command.
"""
if not self.builder.sources:
debug("No sources need to be uploaded.")
return
print("Uploading sources to lookaside:")
os.chdir(project_checkout)
cmd = '%s new-sources %s' % (self.cli_tool, " ".join(self.builder.sources))
debug(cmd)
if self.dry_run:
self.print_dry_run_warning(cmd)
return
output = run_command(cmd)
debug(output)
debug("Removing write-only permission on:")
for filename in self.builder.sources:
run_command("chmod u+w %s" % filename)
def _list_files_to_copy(self):
"""
Returns a list of the full file paths for each file that should be
copied from our git project into the build system checkout. This
is used to sync files to git during a release.
i.e. spec file, .patches.
It is assumed that any file found in the build system checkout
but not in this list, and not in the protected files list, should
probably be cleaned up.
"""
# Include the spec file explicitly, in the case of SatelliteBuilder
# we modify and then use a spec file copy from a different location.
files_to_copy = [self.builder.spec_file] # full paths
f = open(self.builder.spec_file, 'r')
lines = f.readlines()
f.close()
source_filenames = extract_sources(lines)
debug("Watching for source filenames: %s" % source_filenames)
for filename in os.listdir(self.builder.rpmbuild_gitcopy):
full_filepath = os.path.join(self.builder.rpmbuild_gitcopy, filename)
if os.path.isdir(full_filepath):
# skip it
continue
if filename in PROTECTED_BUILD_SYS_FILES:
debug(" skipping: %s (protected file)" % filename)
continue
elif filename.endswith(".spec"):
# Skip the spec file, we already copy this explicitly as it
# can come from a couple different locations depending on which
# builder is in use.
continue
# Check if file looks like it matches a Source line in the spec file:
if filename in source_filenames:
debug(" copying: %s" % filename)
files_to_copy.append(full_filepath)
continue
# Check if file ends with something this builder subclass wants
# to copy:
copy_it = False
for extension in self.copy_extensions:
if filename.endswith(extension):
copy_it = True
continue
if copy_it:
debug(" copying: %s" % filename)
files_to_copy.append(full_filepath)
return files_to_copy
def _git_sync_files(self, project_checkout):
"""
Copy files from our git into each git build branch and add them.
A list of safe files is used to protect critical files both from
being overwritten by a git file of the same name, as well as being
deleted after.
"""
# Build the list of all files we will copy:
debug("Searching for files to copy to build system git:")
files_to_copy = self._list_files_to_copy()
os.chdir(project_checkout)
new, copied, old = \
self._sync_files(files_to_copy, project_checkout)
os.chdir(project_checkout)
# Git add everything:
for add_file in (new + copied):
run_command("git add %s" % add_file)
# Cleanup obsolete files:
for cleanup_file in old:
# Can't delete via full path, must not chdir:
run_command("git rm %s" % cleanup_file)
class DistGitReleaser(FedoraGitReleaser):
cli_tool = "rhpkg"
def extract_task_info(output):
""" Extracts task ID and URL from koji/brew build output. """
task_lines = []
for line in output.splitlines():
if "Created task" in line:
task_lines.append(line)
elif "Task info" in line:
task_lines.append(line)
return task_lines

344
src/tito/release/main.py
View file

@ -17,8 +17,6 @@ Code for submitting builds for release.
import copy
import os
import sys
import tempfile
import subprocess
import rpm
from tempfile import mkdtemp
@ -26,7 +24,6 @@ import shutil
from tito.common import *
from tito.compat import *
from tito.buildparser import BuildTargetParser
from tito.exception import TitoException
from tito.config_object import ConfigObject
@ -38,17 +35,6 @@ PROTECTED_BUILD_SYS_FILES = ('branch', 'Makefile', 'sources', ".git", ".gitignor
RSYNC_USERNAME = 'RSYNC_USERNAME' # environment variable name
def extract_task_info(output):
""" Extracts task ID and URL from koji/brew build output. """
task_lines = []
for line in output.splitlines():
if "Created task" in line:
task_lines.append(line)
elif "Task info" in line:
task_lines.append(line)
return task_lines
class Releaser(ConfigObject):
"""
Parent class of all releasers.
@ -426,336 +412,6 @@ class YumRepoReleaser(RsyncReleaser):
filename))
class FedoraGitReleaser(Releaser):
REQUIRED_CONFIG = ['branches']
cli_tool = "fedpkg"
def __init__(self, name=None, tag=None, build_dir=None,
config=None, user_config=None,
target=None, releaser_config=None, no_cleanup=False,
test=False, auto_accept=False, **kwargs):
Releaser.__init__(self, name, tag, build_dir, config,
user_config, target, releaser_config, no_cleanup, test,
auto_accept, **kwargs)
self.git_branches = \
self.releaser_config.get(self.target, "branches").split(" ")
if self.config.has_option(self.target, "remote_git_name"):
overwrite_checkout = self.config.get(self.target, "remote_git_name")
else:
overwrite_checkout = None
if overwrite_checkout:
self.project_name = overwrite_checkout
self.package_workdir = os.path.join(self.working_dir,
self.project_name)
build_target_parser = BuildTargetParser(self.releaser_config, self.target,
self.git_branches)
self.build_targets = build_target_parser.get_build_targets()
# Files we should copy to git during a release:
self.copy_extensions = (".spec", ".patch")
def release(self, dry_run=False, no_build=False, scratch=False):
self.dry_run = dry_run
self.no_build = no_build
self._git_release()
def _get_build_target_for_branch(self, branch):
if branch in self.build_targets:
return self.build_targets[branch]
return None
def _git_release(self):
getoutput("mkdir -p %s" % self.working_dir)
os.chdir(self.working_dir)
run_command("%s clone %s" % (self.cli_tool, self.project_name))
project_checkout = os.path.join(self.working_dir, self.project_name)
os.chdir(project_checkout)
run_command("%s switch-branch %s" % (self.cli_tool, self.git_branches[0]))
self.builder.tgz()
if self.test:
self.builder._setup_test_specfile()
self._git_sync_files(project_checkout)
self._git_upload_sources(project_checkout)
self._git_user_confirm_commit(project_checkout)
def _confirm_commit_msg(self, diff_output):
"""
Generates a commit message in a temporary file, gives the user a
chance to edit it, and returns the filename to the caller.
"""
fd, name = tempfile.mkstemp()
debug("Storing commit message in temp file: %s" % name)
write(fd, "Update %s to %s\n" % (self.project_name,
self.builder.build_version))
# Write out Resolves line for all bugzillas we see in commit diff:
for line in extract_bzs(diff_output):
write(fd, line + "\n")
print("")
print("##### Commit message: #####")
print("")
os.lseek(fd, 0, 0)
file = os.fdopen(fd)
for line in file.readlines():
print(line)
file.close()
print("")
print("###############################")
print("")
if self._ask_yes_no("Would you like to edit this commit message? [y/n] ", False):
debug("Opening editor for user to edit commit message in: %s" % name)
editor = 'vi'
if "EDITOR" in os.environ:
editor = os.environ["EDITOR"]
subprocess.call(editor.split() + [name])
return name
def _git_user_confirm_commit(self, project_checkout):
""" Prompt user if they wish to proceed with commit. """
print("")
text = "Running 'git diff' in: %s" % project_checkout
print("#" * len(text))
print(text)
print("#" * len(text))
print("")
main_branch = self.git_branches[0]
os.chdir(project_checkout)
# Newer versions of git don't seem to want --cached here? Try both:
(status, diff_output) = getstatusoutput("git diff --cached")
if diff_output.strip() == "":
debug("git diff --cached returned nothing, falling back to git diff.")
(status, diff_output) = getstatusoutput("git diff")
if diff_output.strip() == "":
print("No changes in main branch, skipping commit for: %s" % main_branch)
else:
print(diff_output)
print("")
print("##### Please review the above diff #####")
if not self._ask_yes_no("Do you wish to proceed with commit? [y/n] "):
print("Fine, you're on your own!")
self.cleanup()
sys.exit(1)
print("Proceeding with commit.")
commit_msg_file = self._confirm_commit_msg(diff_output)
cmd = '%s commit -F %s' % (self.cli_tool,
commit_msg_file)
debug("git commit command: %s" % cmd)
print
if self.dry_run:
self.print_dry_run_warning(cmd)
else:
print("Proceeding with commit.")
os.chdir(self.package_workdir)
output = run_command(cmd)
os.unlink(commit_msg_file)
cmd = "%s push" % self.cli_tool
if self.dry_run:
self.print_dry_run_warning(cmd)
else:
# Push
print(cmd)
run_command(cmd)
if not self.no_build:
self._build(main_branch)
for branch in self.git_branches[1:]:
print("Merging branch: '%s' -> '%s'" % (main_branch, branch))
run_command("%s switch-branch %s" % (self.cli_tool, branch))
self._merge(main_branch)
cmd = "git push origin %s:%s" % (branch, branch)
if self.dry_run:
self.print_dry_run_warning(cmd)
else:
print(cmd)
run_command(cmd)
if not self.no_build:
self._build(branch)
print
def _merge(self, main_branch):
try:
run_command("git merge %s" % main_branch)
except:
print
print("WARNING!!! Conflicts occurred during merge.")
print
print("You are being dropped to a shell in the working directory.")
print
print("Please resolve this by doing the following:")
print
print(" 1. List the conflicting files: git ls-files --unmerged")
print(" 2. Edit each resolving the conflict and then: git add FILENAME")
print(" 4. Commit the result when you are done: git commit")
print(" 4. Return to the tito release: exit")
print
# TODO: maybe prompt y/n here
os.system(os.environ['SHELL'])
def _build(self, branch):
""" Submit a Fedora build from current directory. """
target_param = ""
build_target = self._get_build_target_for_branch(branch)
if build_target:
target_param = "--target %s" % build_target
build_cmd = "%s build --nowait %s" % (self.cli_tool, target_param)
if self.dry_run:
self.print_dry_run_warning(build_cmd)
return
print("Submitting build: %s" % build_cmd)
(status, output) = getstatusoutput(build_cmd)
if status > 0:
if "already been built" in output:
print("Build has been submitted previously, continuing...")
else:
sys.stderr.write("ERROR: Unable to submit build.\n")
sys.stderr.write(" Status code: %s\n" % status)
sys.stderr.write(" Output: %s\n" % output)
sys.exit(1)
# Print the task ID and URL:
for line in extract_task_info(output):
print(line)
def _git_upload_sources(self, project_checkout):
"""
Upload any tarballs to the lookaside directory. (if necessary)
Uses the "fedpkg new-sources" command.
"""
if not self.builder.sources:
debug("No sources need to be uploaded.")
return
print("Uploading sources to lookaside:")
os.chdir(project_checkout)
cmd = '%s new-sources %s' % (self.cli_tool, " ".join(self.builder.sources))
debug(cmd)
if self.dry_run:
self.print_dry_run_warning(cmd)
return
output = run_command(cmd)
debug(output)
debug("Removing write-only permission on:")
for filename in self.builder.sources:
run_command("chmod u+w %s" % filename)
def _list_files_to_copy(self):
"""
Returns a list of the full file paths for each file that should be
copied from our git project into the build system checkout. This
is used to sync files to git during a release.
i.e. spec file, .patches.
It is assumed that any file found in the build system checkout
but not in this list, and not in the protected files list, should
probably be cleaned up.
"""
# Include the spec file explicitly, in the case of SatelliteBuilder
# we modify and then use a spec file copy from a different location.
files_to_copy = [self.builder.spec_file] # full paths
f = open(self.builder.spec_file, 'r')
lines = f.readlines()
f.close()
source_filenames = extract_sources(lines)
debug("Watching for source filenames: %s" % source_filenames)
for filename in os.listdir(self.builder.rpmbuild_gitcopy):
full_filepath = os.path.join(self.builder.rpmbuild_gitcopy, filename)
if os.path.isdir(full_filepath):
# skip it
continue
if filename in PROTECTED_BUILD_SYS_FILES:
debug(" skipping: %s (protected file)" % filename)
continue
elif filename.endswith(".spec"):
# Skip the spec file, we already copy this explicitly as it
# can come from a couple different locations depending on which
# builder is in use.
continue
# Check if file looks like it matches a Source line in the spec file:
if filename in source_filenames:
debug(" copying: %s" % filename)
files_to_copy.append(full_filepath)
continue
# Check if file ends with something this builder subclass wants
# to copy:
copy_it = False
for extension in self.copy_extensions:
if filename.endswith(extension):
copy_it = True
continue
if copy_it:
debug(" copying: %s" % filename)
files_to_copy.append(full_filepath)
return files_to_copy
def _git_sync_files(self, project_checkout):
"""
Copy files from our git into each git build branch and add them.
A list of safe files is used to protect critical files both from
being overwritten by a git file of the same name, as well as being
deleted after.
"""
# Build the list of all files we will copy:
debug("Searching for files to copy to build system git:")
files_to_copy = self._list_files_to_copy()
os.chdir(project_checkout)
new, copied, old = \
self._sync_files(files_to_copy, project_checkout)
os.chdir(project_checkout)
# Git add everything:
for add_file in (new + copied):
run_command("git add %s" % add_file)
# Cleanup obsolete files:
for cleanup_file in old:
# Can't delete via full path, must not chdir:
run_command("git rm %s" % cleanup_file)
class DistGitReleaser(FedoraGitReleaser):
cli_tool = "rhpkg"
class KojiReleaser(Releaser):
"""
Releaser for the Koji build system.

5
src/tito/release/obs.py
View file

@ -16,7 +16,7 @@ import tempfile
import subprocess
import sys
from tito.common import run_command, debug, extract_bzs
from tito.common import run_command, debug, BugzillaExtractor
from tito.compat import *
from tito.release import Releaser
@ -73,7 +73,8 @@ class ObsReleaser(Releaser):
write(fd, "Update %s to %s\n" % (self.obs_package_name,
self.builder.build_version))
# Write out Resolves line for all bugzillas we see in commit diff:
for line in extract_bzs(diff_output):
extractor = BugzillaExtractor(diff_output)
for line in extractor.extract():
write(fd, line + "\n")
print("")

2
test/functional/release_yum_tests.py
View file

@ -1,4 +1,4 @@
#
# Copyright (c) 2008-2014 Red Hat, Inc.
#
# This software is licensed to you under the GNU General Public License,

193
test/unit/common-tests.py
View file

@ -105,6 +105,24 @@ class CommonTests(unittest.TestCase):
def test_run_command_print(self):
self.assertEquals('', run_command_print("sleep 0.1"))
def test_rpmbuild_claims_to_be_successful(self):
succeeded_result = "success"
output = "Wrote: %s" % succeeded_result
success_line = find_wrote_in_rpmbuild_output(output)
self.assertEquals(succeeded_result, success_line[0])
def test_rpmbuild_which_ended_with_error_is_described_with_the_analyzed_line(self):
output = "some error output from rpmbuild\n" \
"next error line"
common.error_out = Mock()
find_wrote_in_rpmbuild_output(output)
common.error_out.assert_called_once_with("Unable to locate 'Wrote: ' lines in rpmbuild output: '%s'" % output)
class VersionMathTest(unittest.TestCase):
def test_increase_version_minor(self):
@ -182,61 +200,202 @@ class ExtractBugzillasTest(unittest.TestCase):
def test_single_line(self):
commit_log = "- 123456: Did something interesting."
results = extract_bzs(commit_log)
extractor = BugzillaExtractor(commit_log)
results = extractor.extract()
self.assertEquals(1, len(results))
self.assertEquals("Resolves: #123456 - Did something interesting.",
results[0])
def test_single_with_dash(self):
commit_log = "- 123456 - Did something interesting."
results = extract_bzs(commit_log)
extractor = BugzillaExtractor(commit_log)
results = extractor.extract()
self.assertEquals(1, len(results))
self.assertEquals("Resolves: #123456 - Did something interesting.",
results[0])
def test_single_with_no_spaces(self):
commit_log = "- 123456-Did something interesting."
results = extract_bzs(commit_log)
extractor = BugzillaExtractor(commit_log)
results = extractor.extract()
self.assertEquals(1, len(results))
self.assertEquals("Resolves: #123456 - Did something interesting.",
results[0])
def test_diff_format(self):
commit_log = "+- 123456: Did something interesting."
results = extract_bzs(commit_log)
extractor = BugzillaExtractor(commit_log)
results = extractor.extract()
self.assertEquals(1, len(results))
self.assertEquals("Resolves: #123456 - Did something interesting.",
results[0])
def test_single_line_no_bz(self):
commit_log = "- Did something interesting."
results = extract_bzs(commit_log)
extractor = BugzillaExtractor(commit_log)
results = extractor.extract()
self.assertEquals(0, len(results))
def test_multi_line(self):
commit_log = "- 123456: Did something interesting.\n- Another commit.\n" \
"- 456789: A third commit."
results = extract_bzs(commit_log)
extractor = BugzillaExtractor(commit_log)
results = extractor.extract()
self.assertEquals(2, len(results))
self.assertEquals("Resolves: #123456 - Did something interesting.",
results[0])
self.assertEquals("Resolves: #456789 - A third commit.",
results[1])
def test_rpmbuild_cailms_to_be_successul(self):
succeeded_result = "success"
output = "Wrote: %s" % succeeded_result
def test_single_required_flag_found(self):
success_line = find_wrote_in_rpmbuild_output(output)
extractor = BugzillaExtractor("", required_flags=[
'myos-1.0+', 'pm_ack+'])
bug1 = ('123456', 'Did something interesting.')
extractor._extract_bzs = Mock(return_value=[
bug1])
self.assertEquals(succeeded_result, success_line[0])
extractor._load_bug = Mock(
return_value=MockBug(bug1[0], ['myos-1.0+', 'pm_ack+']))
def test_rpmbuild_which_ended_with_error_is_described_with_the_analyzed_line(self):
output = "some error output from rpmbuild\n" \
"next error line"
results = extractor.extract()
common.error_out = Mock()
self.assertEquals(1, len(extractor.bzs))
self.assertEquals(bug1[0], extractor.bzs[0][0])
self.assertEquals(bug1[1], extractor.bzs[0][1])
find_wrote_in_rpmbuild_output(output)
self.assertEquals(1, len(results))
self.assertEquals("Resolves: #123456 - Did something interesting.",
results[0])
common.error_out.assert_called_once_with("Unable to locate 'Wrote: ' lines in rpmbuild output: '%s'" % output)
def test_required_flags_found(self):
extractor = BugzillaExtractor("", required_flags=[
'myos-1.0+', 'pm_ack+'])
bug1 = ('123456', 'Did something interesting.')
bug2 = ('444555', 'Something else.')
bug3 = ('987654', 'Such amaze!')
extractor._extract_bzs = Mock(return_value=[
bug1, bug2, bug3])
bug_mocks = [
MockBug(bug1[0], ['myos-1.0+', 'pm_ack+']),
MockBug(bug2[0], ['myos-2.0?', 'pm_ack?']),
MockBug(bug3[0], ['myos-1.0+', 'pm_ack+'])]
def next_bug(*args):
return bug_mocks.pop(0)
extractor._load_bug = Mock(side_effect=next_bug)
results = extractor.extract()
self.assertEquals(2, len(extractor.bzs))
self.assertEquals(bug1[0], extractor.bzs[0][0])
self.assertEquals(bug1[1], extractor.bzs[0][1])
self.assertEquals(bug3[0], extractor.bzs[1][0])
self.assertEquals(bug3[1], extractor.bzs[1][1])
self.assertEquals(2, len(results))
self.assertEquals("Resolves: #123456 - Did something interesting.",
results[0])
self.assertEquals("Resolves: #987654 - Such amaze!",
results[1])
def test_required_flags_missing(self):
extractor = BugzillaExtractor("", required_flags=[
'myos-2.0+'])
bug1 = ('123456', 'Did something interesting.')
bug2 = ('444555', 'Something else.')
bug3 = ('987654', 'Such amaze!')
extractor._extract_bzs = Mock(return_value=[
bug1, bug2, bug3])
bug_mocks = [
MockBug(bug1[0], ['myos-1.0+', 'pm_ack+']),
MockBug(bug2[0], ['myos-2.0?', 'pm_ack?']),
MockBug(bug3[0], ['myos-1.0+', 'pm_ack+'])]
def next_bug(*args):
return bug_mocks.pop(0)
extractor._load_bug = Mock(side_effect=next_bug)
results = extractor.extract()
self.assertEquals(0, len(extractor.bzs))
self.assertEquals(0, len(results))
def test_required_flags_missing_with_placeholder(self):
extractor = BugzillaExtractor("", required_flags=[
'myos-2.0+'], placeholder_bz="54321")
bug1 = ('123456', 'Did something interesting.')
extractor._extract_bzs = Mock(return_value=[
bug1])
extractor._load_bug = Mock(
return_value=MockBug(bug1[0], ['myos-1.0+', 'pm_ack+']))
results = extractor.extract()
self.assertEquals(0, len(extractor.bzs))
self.assertEquals(1, len(results))
self.assertEquals("Related: #54321", results[0])
def test_same_id_multiple_times(self):
extractor = BugzillaExtractor("", required_flags=[
'myos-1.0+', 'pm_ack+'])
bug1 = ('123456', 'Did something interesting.')
bug3 = ('123456', 'Oops, lets try again.')
extractor._extract_bzs = Mock(return_value=[
bug1, bug3])
extractor._load_bug = Mock(
return_value=MockBug(bug1[0], ['myos-1.0+', 'pm_ack+']))
results = extractor.extract()
self.assertEquals(2, len(extractor.bzs))
self.assertEquals(bug1[0], extractor.bzs[0][0])
self.assertEquals(bug1[1], extractor.bzs[0][1])
self.assertEquals(bug3[0], extractor.bzs[1][0])
self.assertEquals(bug3[1], extractor.bzs[1][1])
self.assertEquals(2, len(results))
self.assertEquals("Resolves: #123456 - Did something interesting.",
results[0])
self.assertEquals("Resolves: #123456 - Oops, lets try again.",
results[1])
def test_bug_doesnt_exist(self):
extractor = BugzillaExtractor("", required_flags=[
'myos-1.0+', 'pm_ack+'])
bug1 = ('123456', 'Did something interesting.')
extractor._extract_bzs = Mock(return_value=[
bug1])
from tito.compat import xmlrpclib
extractor._load_bug = Mock(side_effect=xmlrpclib.Fault("", ""))
results = extractor.extract()
self.assertEquals(0, len(extractor.bzs))
self.assertEquals(0, len(results))
class MockBug(object):
def __init__(self, bug_id, flags):
self.flags = {}
for flag in flags:
self.flags[flag[0:-1]] = flag[-1]
def get_flag_status(self, flag):
if flag in self.flags:
return self.flags[flag]
else:
return None

1
tito.spec
View file

@ -19,6 +19,7 @@ BuildRequires: docbook-style-xsl
BuildRequires: libxslt
Requires: python-setuptools
Requires: python-bugzilla
Requires: rpm-build
Requires: rpmlint
Requires: fedpkg