Add support for a different lookaside structure

There is a will to offer SIGs the possibility to use a similar lookaside
structure as the one used by CentOS Stream and Fedora which is not tied
to the name of the archive but to its hash.
The idea though is to offer the new structure as opt-in and thus keep
the old structure working.
We thus need to adjust the get_sources.sh script to support both
structure.
This commit makes it so, it allows both exploded-srpm and flat dist-git
structures to use either the old or new lookaside cache structure. The
way this is achieved is simply to first call the URL corresponding to
the old lookaside structure. If that call returns a http code in the 200
range, then the script stops, otherwise, the script will call the URL
corresponding to the new lookaside structure.

This commit also makes consistent the different curl calls and add
--retry 5 on all of them (which does not work for 404 replies, so
using the new structure will not results in 6 requests to the old one
before moving on, but just 1).

Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

get_sources.sh

@@ -77,7 +77,7 @@ while [[ 0 -eq 0 ]]; do
             shift
            ;;
          -b )
-            # Check this particular branch 
+            # Check this particular branch
             BRANCH=$2
             shift
             shift
@@ -173,9 +173,19 @@ if [[ -s sources ]]; then
           for br in "${branches[@]}"
           do
             br=$(echo ${br}| sed -e s'|remotes/origin/||')
+            # Try the branch-specific lookaside structure
             url="${SURL}/$pkgname/${br}/$hash"
             echo "Retrieving ${url}"
-            curl -L ${QUIET} -f "${url}" -o "$tarball" && break
+            HTTP_CODE=$(curl -L ${QUIET} -H Pragma: -o "./$tarball" -R -S --fail --retry 5 "${url}" --write-out "%{http_code}" || true)
+            echo "Returned ${HTTP_CODE}"
+            if [[ ${HTTP_CODE} -gt 199 && ${HTTP_CODE} -lt 300 ]] ; then
+               echo "bailing"
+               break
+            fi
+            # Try the hash-specific lookaside structure
+            url="${SURL}/$pkgname/$tarball/$hashtype/$hash/$tarball"
+            echo "Retrieving ${url}"
+            curl -L ${QUIET} -H Pragma: -o "./$tarball" -R -S --fail --retry 5 "${url}" && break
           done
         else
           echo "$filename exists. skipping"
@@ -231,21 +241,22 @@ else
         # zero byte file
         touch ${fname}
       else
+        hashType=$(weakHashDetection ${fsha})
+        if [ "${hashType}" == "unknown" ]; then
+          echo 'Failure: Hash type unknown.' >&2
+          exit 1;
+        fi
+        hashName=$(echo ${hashType}| sed -e s'|sum||')
+
         if [ ${CHECK} -eq 1 ]; then
-          hashType=$(weakHashDetection ${fsha})
-          if [ "${hashType}" == "unknown" ]; then
-            echo 'Failure: Hash type unknown.' >&2
+          which ${hashType} >/dev/null 2>&1
+          if [[ $? -ne 0 ]]; then
+            echo "Failure: You need ${hashType} in PATH." >&2
             exit 1;
-          else
-            which ${hashType} >/dev/null 2>&1
-            if [[ $? -ne 0 ]]; then
-              echo "Failure: You need ${hashType} in PATH." >&2
-              exit 1;
-            fi
           fi
         fi
         if [ -e ${fname} -a ${CHECK} -eq 1 ]; then
-	    # check hash sum and force download if wrong
+            # check hash sum and force download if wrong
             downsum=$(${hashType} ${fname} | awk '{print $1}')
             if [ "${fsha}" != "${downsum}" ]; then
                 rm -f ${fname}
@@ -255,9 +266,19 @@ else
           for br in "${branches[@]}"
           do
             br=$(echo ${br}| sed -e s'|remotes/origin/||')
+            # Try the branch-specific lookaside structure
             url="${SURL}/${pn}/${br}/${fsha}"
             echo "Retrieving ${url}"
-            curl -L ${QUIET} -f "${url}" -o "${fname}" && break
+            HTTP_CODE=$(curl -L ${QUIET} -H Pragma: -o "${fname}" -R -S --fail --retry 5 "${url}" --write-out "%{http_code}" || true)
+            echo "Returned ${HTTP_CODE}"
+            if [[ ${HTTP_CODE} -gt 199 && ${HTTP_CODE} -lt 300 ]] ; then
+               echo "bailing"
+               break
+            fi
+            # Try the hash-specific lookaside structure
+            url="${SURL}/$pn/$fname/${hashName}/$fsha/$fname"
+            echo "Retrieving ${url}"
+            curl -L ${QUIET} -H Pragma: -o "${fname}" -R -S --fail --retry 5 "${url}" && break
           done
         else
           echo "${fname} exists. skipping"