DistGitReleasers: don't nuke external sources with fetch_sources

Previously only 'tito build --srpm|--rpm' worked well with fetch_sources
where we rely on RPM's (rpmbuild) built-in download mechanism.  It did
not work well with 'tito release' and DistGit releasers, though.

The thing is that we don't use rpmbuild at all while doing DistGit
releases BUT we still want those files to be (a) downloaded to
%_sourcedir and then (b) uploaded to DistGit lookaside cache.  Because
we did not download those files before, every extra source file was
entirely ignored, or even worse, if the file was already uploaded in
the target lookaside cache, the file was removed from sources.

This issue has bitten us during the 'copr-backend' releases several
times before, and now it starts also in 'resalloc-aws' releases.

Unfortunately for this fix, we don't read/use the "buildconfig" options
while instantiating Builder objects from Releaser classes, therefore
I had to add an explicit read for the 'fetch_sources' config.

Closes: PR#420
Relates: #407

src/tito/builder/main.py

@@ -34,7 +34,7 @@ from tito.common import scl_to_rpm_option, get_latest_tagged_version, \
     find_cheetah_template_file, render_cheetah, replace_spec_release, \
     find_spec_like_file, warn_out, get_commit_timestamp, chdir, mkdir_p, \
     find_git_root, info_out, munge_specfile, BUILDCONFIG_SECTION
-from tito.compat import getstatusoutput, getoutput, urlparse
+from tito.compat import getstatusoutput, getoutput, urlparse, urlretrieve
 from tito.exception import RunCommandException
 from tito.exception import TitoException
 from tito.config_object import ConfigObject
@@ -79,7 +79,12 @@ class BuilderBase(object):
 
         self.quiet = self._get_optional_arg(kwargs, 'quiet', False)
         self.verbose = self._get_optional_arg(kwargs, 'verbose', False)
-        self.fetch_sources = self._get_optional_arg(kwargs, 'fetch_sources', False)
+
+        # The default for the builder instantiated from Releasers
+        config_fetch_default = False
+        if self.config.has_option(BUILDCONFIG_SECTION, "fetch_sources"):
+            config_fetch_default = self.config.get(BUILDCONFIG_SECTION, "fetch_sources")
+        self.fetch_sources = self._get_optional_arg(kwargs, 'fetch_sources', config_fetch_default)
 
         rpmbuildopts = self._get_optional_arg(args, 'rpmbuild_options', None)
         if rpmbuildopts:
@@ -499,10 +504,29 @@ class Builder(ConfigObject, BuilderBase):
         # Strip extra dashes if one of the params is empty
         return tag_format.format(**kwargs).strip('-')
 
+    def copy_and_download_extra_sources(self):
+        """
+        Copy extra %{SOURCEX} files to the SOURCE folder, and when
+        self.fetch_sources is configured download external sources as well.
+        """
+        return self._copy_extra_sources(download_sources=True)
+
     def copy_extra_sources(self):
         """
         Copy extra %{SOURCEX} files to the SOURCE folder.
         """
+        return self._copy_extra_sources()
+
+    def _download_one_source(self, source):
+        target = os.path.join(
+            self.rpmbuild_sourcedir,
+            os.path.basename(source),
+        )
+        info_out("Downloading %s into %s" % (source, target))
+        urlretrieve(source, target)
+        self.sources.append(target)
+
+    def _copy_extra_sources(self, download_sources=False):
         cmd = "spectool -S '%s' --define '_sourcedir %s' 2> /dev/null | awk '{print $2}'"\
             % (self.spec_file, self.start_dir)
         sources = getoutput(cmd).split("\n")
@@ -515,9 +539,17 @@ class Builder(ConfigObject, BuilderBase):
 
             parse = urlparse(source)
             if parse.scheme and parse.netloc:
-                # If macro `%_disable_source_fetch 0` is defined, the file will
-                # be automatically downloaded by rpmbuild to the SOURCES
-                # directory. We can safely skip it here.
+                # For self.srpm() and self.rpm() we call
+                # self.download_sources() and later rpmbuild.  We know we can
+                # rely on defined macro `%_disable_source_fetch 0`, and the
+                # file will be automatically downloaded by rpmbuild to the
+                # SOURCES directory. We can safely skip the download here.
+                # For other use-cases (e.g. releasing to Fedora DistGit) we
+                # don't run rpmbuild at all, and we need to download the file
+                # right now (and add it to self.sources).
+                if self.fetch_sources and download_sources:
+                    self._download_one_source(source)
+                    continue
                 debug('Source "%s" is not a local file. Skiping.' % source)
                 continue
 

src/tito/compat.py

@@ -27,6 +27,7 @@ if PY2:
     from ConfigParser import RawConfigParser
     from StringIO import StringIO
     from urlparse import urlparse
+    from urlparse import urlretrieve
     import xmlrpclib
     text_type = unicode
     binary_type = str
@@ -36,6 +37,7 @@ else:
     from configparser import RawConfigParser
     from io import StringIO
     from urllib.parse import urlparse
+    from urllib.request import urlretrieve
     from contextlib import redirect_stdout
     import xmlrpc.client as xmlrpclib
     text_type = str

src/tito/release/distgit.py

@@ -105,7 +105,7 @@ class FedoraGitReleaser(Releaser):
         # Mead builds need to be in the git_root.  Other builders are agnostic.
         with chdir(self.git_root):
             self.builder.tgz()
-            self.builder.copy_extra_sources()
+            self.builder.copy_and_download_extra_sources()
 
         if self.test:
             self.builder._setup_test_specfile()
@@ -492,7 +492,7 @@ class CentosGitReleaser(FedoraGitReleaser):
         # Mead builds need to be in the git_root.  Other builders are agnostic.
         with chdir(self.git_root):
             self.builder.tgz()
-            self.builder.copy_extra_sources()
+            self.builder.copy_and_download_extra_sources()
 
         if self.test:
             self.builder._setup_test_specfile()