Merge branch 'master' into sourcefetch

Conflicts:
	test/functional/singleproject_tests.py

Technically not a merge, did a bunch of work only to discover I was mid-merge when I did it.

src/tito/builder/externalsrc.py

@@ -44,17 +44,129 @@ class ExternalSourceBuilder(ConfigObject, BuilderBase):
             error_out("ExternalSourceBuilder does not support building "
                     "specific tags.")
 
-        # Project directory where we started this build:
-        self.start_dir = os.getcwd()
-
         self.build_tag = '%s-%s' % (self.project_name,
                 get_spec_version_and_release(self.start_dir,
                     '%s.spec' % self.project_name))
 
+    def tgz(self):
+        self.ran_tgz = True
+        self._create_build_dirs()
+
+        print("Fetching sources...")
+        source_strat = KeywordArgSourceStrategy(self)
+        source_strat.fetch()
+        self.sources = source_strat.sources
+        self.spec_file = source_strat.spec_file
+
+        # Copy every normal file in the directory we ran tito from. This
+        # will pick up any sources that were sitting around locally.
+        # TODO: how to copy only sources?
+        #files_in_src_dir = [f for f in os.listdir(self.start_dir) \
+                #        if os.path.isfile(os.path.join(self.start_dir, f)) ]
+        #print files_in_src_dir
+        #for f in files_in_src_dir:
+        #    shutil.copyfile(os.path.join(self.start_dir, f),
+        #            os.path.join(self.rpmbuild_sourcedir, f))
+        # TODO: extract version/release from filename?
+        # TODO: what filename?
+        #cmd = "/usr/bin/spectool --list-files '%s' | awk '{print $2}' |xargs -l1 --no-run-if-empty basename " % self.spec_file
+        #result = run_command(cmd)
+        #self.sources = map(lambda x: os.path.join(self.rpmbuild_sourcedir, x), result.split("\n"))
+
+    def _get_rpmbuild_dir_options(self):
+        return ('--define "_sourcedir %s" --define "_builddir %s" '
+            '--define "_srcrpmdir %s" --define "_rpmdir %s" ' % (
+            self.rpmbuild_sourcedir, self.rpmbuild_builddir,
+            self.rpmbuild_basedir, self.rpmbuild_basedir))
+
+
+class SourceStrategy(object):
+    """
+    Base class for source strategies. These are responsible for fetching the
+    sources the builder will use, and determining what version/release we're
+    building.
+
+    This is created and run in the tgz step of the builder. It will be passed
+    a reference to the builder calling it, which will be important for accessing
+    a lot of required information.
+
+    Ideally sources and the spec file to be used should be copied into
+    builder.rpmbuild_sourcedir, which will be cleaned up automatically after
+    the builder runs.
+    """
+    def __init__(self, builder):
+        """
+        Defines fields that should be set when a sub-class runs fetch.
+        """
+        self.builder = builder
+
+        # Full path to the spec file we'll actually use to build, should be a
+        # copy, never a live spec file in a git repo as sometimes it will be
+        # modified:
+        self.spec_file = None
+
+        # Will contain the full path to each source we gather:
+        self.sources = []
+
+        # The version we're building:
+        self.version = None
+
+        # The release we're building:
+        self.release = None
+
+    def fetch(self):
+        raise NotImplementedError()
+
+
+class KeywordArgSourceStrategy(SourceStrategy):
+    """
+    Assumes the builder was passed an explicit argument specifying which source
+    file(s) to use.
+    """
+    def fetch(self):
+
         # Assuming we're still in the start directory, get the absolute path
         # to all sources specified:
-        self.manual_sources = [os.path.abspath(s) for s in kwargs['sources']]
+        manual_sources = [os.path.abspath(s) for s in \
-        debug("Got sources: %s" % self.manual_sources)
+                self.builder.kwargs['sources']]
+        debug("Got sources: %s" % manual_sources)
+
+        # Copy the live spec from our starting location. Unlike most builders,
+        # we are not using a copy from a past git commit.
+        self.spec_file = os.path.join(self.builder.rpmbuild_sourcedir,
+                    '%s.spec' % self.builder.project_name)
+        shutil.copyfile(
+                os.path.join(self.builder.start_dir, '%s.spec' %
+                    self.builder.project_name),
+                self.spec_file)
+        print("  %s.spec" % self.builder.project_name)
+
+        # TODO: Make this a configurable strategy:
+        i = 0
+        replacements = []
+        for s in manual_sources:
+            base_name = os.path.basename(s)
+            dest_filepath = os.path.join(self.builder.rpmbuild_sourcedir,
+                    base_name)
+            shutil.copyfile(s, dest_filepath)
+            self.sources.append(dest_filepath)
+
+            # Add a line to replace in the spec for each source:
+            source_regex = re.compile("^(source%s:\s*)(.+)$" % i, re.IGNORECASE)
+            new_line = "Source%s: %s" % (i, base_name)
+            replacements.append((source_regex, new_line))
+
+        # Replace version and release in spec:
+        version_regex = re.compile("^(version:\s*)(.+)$", re.IGNORECASE)
+        release_regex = re.compile("^(release:\s*)(.+)$", re.IGNORECASE)
+
+        (self.version, self.release) = self._get_version_and_release()
+        print("Building version: %s" % self.version)
+        print("Building release: %s" % self.release)
+        replacements.append((version_regex, "Version: %s\n" % self.version))
+        replacements.append((release_regex, "Release: %s\n" % self.release))
+
+        self.replace_in_spec(replacements)
 
     def _get_version_and_release(self):
         """
@@ -80,62 +192,6 @@ class ExternalSourceBuilder(ConfigObject, BuilderBase):
 
         return (version, release)
 
-    def tgz(self):
-        self.ran_tgz = True
-        self._create_build_dirs()
-
-        print("Fetching sources...")
-
-        # Copy the live spec from our starting location. Unlike most builders,
-        # we are not using a copy from a past git commit.
-        self.spec_file = os.path.join(self.rpmbuild_sourcedir,
-                    '%s.spec' % self.project_name)
-        shutil.copyfile(
-                os.path.join(self.start_dir, '%s.spec' % self.project_name),
-                self.spec_file)
-        print("  %s.spec" % self.project_name)
-
-        # TODO: Make this a configurable strategy:
-        i = 0
-        replacements = []
-        for s in self.manual_sources:
-            base_name = os.path.basename(s)
-            dest_filepath = os.path.join(self.rpmbuild_sourcedir, base_name)
-            shutil.copyfile(s, dest_filepath)
-            self.sources.append(dest_filepath)
-
-            # Add a line to replace in the spec for each source:
-            source_regex = re.compile("^(source%s:\s*)(.+)$" % i, re.IGNORECASE)
-            new_line = "Source%s: %s" % (i, base_name)
-            replacements.append((source_regex, new_line))
-
-        # Replace version and release in spec:
-        version_regex = re.compile("^(version:\s*)(.+)$", re.IGNORECASE)
-        release_regex = re.compile("^(release:\s*)(.+)$", re.IGNORECASE)
-
-        (version, release) = self._get_version_and_release()
-        print("Building version: %s" % version)
-        print("Building release: %s" % release)
-        replacements.append((version_regex, "Version: %s\n" % version))
-        replacements.append((release_regex, "Release: %s\n" % release))
-
-        self.replace_in_spec(replacements)
-
-        # Copy every normal file in the directory we ran tito from. This
-        # will pick up any sources that were sitting around locally.
-        # TODO: how to copy only sources?
-        #files_in_src_dir = [f for f in os.listdir(self.start_dir) \
-                #        if os.path.isfile(os.path.join(self.start_dir, f)) ]
-        #print files_in_src_dir
-        #for f in files_in_src_dir:
-        #    shutil.copyfile(os.path.join(self.start_dir, f),
-        #            os.path.join(self.rpmbuild_sourcedir, f))
-        # TODO: extract version/release from filename?
-        # TODO: what filename?
-        #cmd = "/usr/bin/spectool --list-files '%s' | awk '{print $2}' |xargs -l1 --no-run-if-empty basename " % self.spec_file
-        #result = run_command(cmd)
-        #self.sources = map(lambda x: os.path.join(self.rpmbuild_sourcedir, x), result.split("\n"))
-
     def replace_in_spec(self, replacements):
         """
         Replace lines in the spec file using the given replacements.
@@ -158,9 +214,5 @@ class ExternalSourceBuilder(ConfigObject, BuilderBase):
         out_f.close()
         shutil.move(self.spec_file + ".new", self.spec_file)
 
-    def _get_rpmbuild_dir_options(self):
+
-        return ('--define "_sourcedir %s" --define "_builddir %s" '
-            '--define "_srcrpmdir %s" --define "_rpmdir %s" ' % (
-            self.rpmbuild_sourcedir, self.rpmbuild_builddir,
-            self.rpmbuild_basedir, self.rpmbuild_basedir))
 

src/tito/builder/main.py

@@ -47,9 +47,13 @@ class BuilderBase(object):
             pkg_config=None, global_config=None, user_config=None,
             args=None, **kwargs):
 
+        # Project directory where we started this build:
+        self.start_dir = os.getcwd()
+
         self.project_name = name
         self.user_config = user_config
         self.args = args
+        self.kwargs = kwargs
 
         # Optional keyword arguments:
         self.dist = self._get_optional_arg(kwargs, 'dist', None)
@@ -181,6 +185,7 @@ class BuilderBase(object):
             '--define "_binary_filedigest_algorithm md5" %s %s %s --clean '
             '-ba %s' % (rpmbuild_options,
                 self._get_rpmbuild_dir_options(), define_dist, self.spec_file))
+        debug(cmd)
         try:
             output = run_command(cmd)
         except (KeyboardInterrupt, SystemExit):
@@ -469,8 +474,9 @@ class Builder(ConfigObject, BuilderBase):
             self.ran_setup_test_specfile = True
 
     def _get_rpmbuild_dir_options(self):
-        return ('--define "_sourcedir %s" --define "_builddir %s" --define '
+        return ('--define "_topdir %s" --define "_sourcedir %s" --define "_builddir %s" --define '
             '"_srcrpmdir %s" --define "_rpmdir %s" ' % (
+            self.rpmbuild_dir,
             self.rpmbuild_sourcedir, self.rpmbuild_builddir,
             self.rpmbuild_basedir, self.rpmbuild_basedir))
 
@@ -539,8 +545,9 @@ class NoTgzBuilder(Builder):
         dir, use the git copy we create as the sources directory when
         building package so everything can be found:
         """
-        return ('--define "_sourcedir %s" --define "_builddir %s" '
+        return ('--define "_topdir %s" --define "_sourcedir %s" --define "_builddir %s" '
             '--define "_srcrpmdir %s" --define "_rpmdir %s" ' % (
+            self.rpmbuild_dir,
             self.rpmbuild_gitcopy, self.rpmbuild_builddir,
             self.rpmbuild_basedir, self.rpmbuild_basedir))
 
@@ -927,8 +934,9 @@ class UpstreamBuilder(NoTgzBuilder):
         dir, use the git copy we create as the sources directory when
         building package so everything can be found:
         """
-        return ('--define "_sourcedir %s" --define "_builddir %s" '
+        return ('--define "_topdir %s" --define "_sourcedir %s" --define "_builddir %s" '
             '--define "_srcrpmdir %s" --define "_rpmdir %s" ' % (
+            self.rpmbuild_dir,
             self.rpmbuild_sourcedir, self.rpmbuild_builddir,
             self.rpmbuild_basedir, self.rpmbuild_basedir))
 

test/functional/externalsrc_tests.py

@@ -0,0 +1,64 @@
+#
+# Copyright (c) 2008-2014 Red Hat, Inc.
+#
+# This software is licensed to you under the GNU General Public License,
+# version 2 (GPLv2). There is NO WARRANTY for this software, express or
+# implied, including the implied warranties of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
+# along with this software; if not, see
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
+#
+# Red Hat trademarks are not licensed under GPLv2. No permission is
+# granted to use or replicate Red Hat trademarks that are incorporated
+# in this software or its documentation.
+"""
+Functional Tests for the ExternalSource builder.
+"""
+
+import os
+import tempfile
+
+from tito.common import run_command
+from fixture import TitoGitTestFixture, tito
+
+EXT_SRC_PKG = "extsrc"
+
+class ExternalSourceBuilderTests(TitoGitTestFixture):
+
+    def setUp(self):
+        TitoGitTestFixture.setUp(self)
+        self.pkg_dir = os.path.join(self.repo_dir, EXT_SRC_PKG)
+        spec = os.path.join(os.path.dirname(__file__), "specs/extsrc.spec")
+        self.create_project_from_spec(EXT_SRC_PKG, pkg_dir=self.pkg_dir,
+                spec=spec,
+                builder='tito.builder.ExternalSourceBuilder')
+        self.source_filename = 'extsrc-0.0.2.tar.gz'
+        os.chdir(self.pkg_dir)
+
+        # Make a fake source file, do we need something more real?
+        run_command('touch %s' % self.source_filename)
+
+        self.output_dir = tempfile.mkdtemp("-titotestoutput")
+
+    def tearDown(self):
+        TitoGitTestFixture.tearDown(self)
+        #shutil.rmtree(self.output_dir)
+
+    def test_simple_build_no_tag(self):
+        # We have not tagged here. Build --rpm should just work:
+        self.assertFalse(os.path.exists(
+            os.path.join(self.pkg_dir, 'rel-eng/packages/extsrc')))
+        tito('build --rpm --output=%s --no-cleanup --source=%s --debug' %
+                (self.output_dir, self.source_filename))
+        self.assertTrue(os.path.exists(
+            os.path.join(self.output_dir, 'extsrc-0.0.2-1.fc20.src.rpm')))
+        self.assertTrue(os.path.exists(
+            os.path.join(self.output_dir, 'noarch/extsrc-0.0.2-1.fc20.noarch.rpm')))
+
+    def test_tag_rejected(self):
+        self.assertRaises(SystemExit, tito,
+                'build --tag=extsrc-0.0.1-1 --rpm --output=%s --source=%s' %
+                (self.output_dir, self.source_filename))
+
+
+

test/functional/fixture.py

@@ -129,7 +129,7 @@ class TitoGitTestFixture(unittest.TestCase):
         index.commit('Setting offline.')
 
     def tearDown(self):
-        #shutil.rmtree(self.repo_dir)
+        shutil.rmtree(self.repo_dir)
         pass
 
     def write_file(self, path, contents):

test/functional/multiproject_tests.py

@@ -59,45 +59,6 @@ module Iteng
   end
 """
 
-EXT_SRC_PKG = "extsrc"
-
-class ExternalSourceBuilderTests(TitoGitTestFixture):
-
-    def setUp(self):
-        TitoGitTestFixture.setUp(self)
-        self.pkg_dir = os.path.join(self.repo_dir, EXT_SRC_PKG)
-        spec = os.path.join(os.path.dirname(__file__), "specs/extsrc.spec")
-        self.create_project_from_spec(EXT_SRC_PKG, pkg_dir=self.pkg_dir,
-                spec=spec,
-                builder='tito.builder.ExternalSourceBuilder')
-        self.source_filename = 'extsrc-0.0.2.tar.gz'
-        os.chdir(self.pkg_dir)
-
-        # Make a fake source file, do we need something more real?
-        run_command('touch %s' % self.source_filename)
-
-        self.output_dir = tempfile.mkdtemp("-titotestoutput")
-
-    def tearDown(self):
-        TitoGitTestFixture.tearDown(self)
-        #shutil.rmtree(self.output_dir)
-
-    def test_simple_build_no_tag(self):
-        # We have not tagged here. Build --rpm should just work:
-        self.assertFalse(os.path.exists(
-            os.path.join(self.pkg_dir, 'rel-eng/packages/extsrc')))
-        tito('build --rpm --output=%s --no-cleanup --source=%s --debug' %
-                (self.output_dir, self.source_filename))
-        self.assertTrue(os.path.exists(
-            os.path.join(self.output_dir, 'extsrc-0.0.2-1.fc20.src.rpm')))
-        self.assertTrue(os.path.exists(
-            os.path.join(self.output_dir, 'noarch/extsrc-0.0.2-1.fc20.noarch.rpm')))
-
-    def test_tag_rejected(self):
-        self.assertRaises(SystemExit, tito,
-                'build --tag=extsrc-0.0.1-1 --rpm --output=%s --source=%s' %
-                (self.output_dir, self.source_filename))
-
 
 class MultiProjectTests(TitoGitTestFixture):
 
@@ -191,4 +152,3 @@ class MultiProjectTests(TitoGitTestFixture):
         os.chdir(os.path.join(self.repo_dir, 'pkg1'))
         artifacts = tito('build --rpm')
         self.assertEquals(3, len(artifacts))
-        print artifacts

test/functional/singleproject_tests.py

@@ -61,21 +61,22 @@ class SingleProjectTests(TitoGitTestFixture):
     def test_latest_tgz(self):
         tito("build --tgz -o %s" % self.repo_dir)
 
-    def test_tag_tgz(self):
+    def test_build_tgz_tag(self):
         tito("build --tgz --tag=%s-0.0.1-1 -o %s" % (PKG_NAME,
             self.repo_dir))
         self.assertTrue(os.path.exists(os.path.join(self.repo_dir,
             "%s-0.0.1.tar.gz" % PKG_NAME)))
 
-    def test_latest_srpm(self):
+    def test_build_latest_srpm(self):
         tito("build --srpm")
 
-    def test_tag_srpm(self):
+    def test_build_srpm_tag(self):
         tito("build --srpm --tag=%s-0.0.1-1 -o %s" % (PKG_NAME, self.repo_dir))
 
-    def test_latest_rpm(self):
+    def test_build_latest_rpm(self):
         tito("build --rpm -o %s" % self.repo_dir)
 
-    def test_tag_rpm(self):
+    def test_build_rpm_tag(self):
         tito("build --rpm --tag=%s-0.0.1-1 -o %s" % (PKG_NAME,
             self.repo_dir))
+