init
This commit is contained in:
commit
3fc28826f0
3 changed files with 311 additions and 0 deletions
4
Makefile
Normal file
4
Makefile
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
.PHONY: sources
|
||||
|
||||
sources:
|
||||
../common/get_sources.sh
|
||||
1
README.md
Normal file
1
README.md
Normal file
|
|
@ -0,0 +1 @@
|
|||
# common
|
||||
306
get_sources.sh
Executable file
306
get_sources.sh
Executable file
|
|
@ -0,0 +1,306 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Might want to drop this in ~/bin/ and chmod u+x it
|
||||
#
|
||||
|
||||
# Initial Author: Karanbir Singh <kbsingh@centos.org>
|
||||
# Updates:
|
||||
# Viacheslav Anzhiganov <vanzhiganov@outlook.com>
|
||||
# Mike McLean <mikem@redhat.com>
|
||||
# Pat Riehecky <riehecky@fnal.gov>
|
||||
# Tyler Parsons <tparsons@fnal.gov>
|
||||
# Tuomo Soini <tis@foobar.fi>
|
||||
set -eux
|
||||
|
||||
|
||||
#####################################################################
|
||||
usage() {
|
||||
echo '' >&2
|
||||
echo "$0 [-hcq] [-b branch] [--surl url]" >&2
|
||||
echo '' >&2
|
||||
echo 'Script to parse the non-text sources metadata file' >&2
|
||||
echo ' and download the required files from the lookaside' >&2
|
||||
echo ' cache.' >&2
|
||||
echo '' >&2
|
||||
echo 'PLEASE NOTE: this script is non-destructive, it wont' >&2
|
||||
echo ' replace files that already exist, regardless of' >&2
|
||||
echo ' their state, allowing you to have work-in-progress' >&2
|
||||
echo ' content that wont get overwritten.' >&2
|
||||
echo '' >&2
|
||||
echo 'You need to run this from inside a sources git repo' >&2
|
||||
echo '' >&2
|
||||
echo ' -h: This help message' >&2
|
||||
echo '' >&2
|
||||
echo " $0 -b c7" >&2
|
||||
echo " $0 -q -b c7" >&2
|
||||
echo " $0 -c -b remotes/origin/c7" >&2
|
||||
echo " $0 -c -b c7 --surl '$SURL'" >&2
|
||||
echo " $0" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
#####################################################################
|
||||
|
||||
SURL="http://koji.5.128.248.163.sslip.io:5001/repo/pkgs/rpms"
|
||||
|
||||
QUIET=0
|
||||
BRANCH=''
|
||||
CHECK=0
|
||||
|
||||
# for setting any overrides, such as SURL, default BRANCH, or force CHECK
|
||||
if [ -f /etc/centos-git-common ]; then
|
||||
. /etc/centos-git-common
|
||||
fi
|
||||
|
||||
#####################################################################
|
||||
# setup args in the right order for making getopt evaluation
|
||||
# nice and easy. You'll need to read the manpages for more info
|
||||
# utilizing 'while' construct rather than 'for arg' to avoid unnecessary
|
||||
# shifting of program args
|
||||
args=$(getopt -o hcqb: -l surl: -- "$@")
|
||||
eval set -- "$args"
|
||||
|
||||
while [[ 0 -eq 0 ]]; do
|
||||
case $1 in
|
||||
-- )
|
||||
# end of getopt args, shift off the -- and get out of the loop
|
||||
shift
|
||||
break
|
||||
;;
|
||||
-c )
|
||||
# verify the sha1sum of the downloaded file
|
||||
CHECK=1
|
||||
shift
|
||||
;;
|
||||
-q )
|
||||
# suppress warnings
|
||||
QUIET=1
|
||||
shift
|
||||
;;
|
||||
-b )
|
||||
# Check this particular branch
|
||||
BRANCH=$2
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
--surl )
|
||||
# override sources url
|
||||
SURL=$2
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
-h )
|
||||
# get help
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# set curl options this way so defaults can be set in /etc/centos-git-common
|
||||
# across multiple scripts
|
||||
if [[ ${QUIET} -eq 1 ]]; then
|
||||
QUIET='--silent'
|
||||
else
|
||||
QUIET=''
|
||||
fi
|
||||
|
||||
command -v git >/dev/null 2>&1
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo 'You need git in PATH' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
command -v curl >/dev/null 2>&1
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo 'You need curl in PATH' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d .git ] && ([ ! -d SPECS ] || [[ ! -s sources ]] ); then
|
||||
echo 'You need to run this from inside a sources git repo' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# sort out our branch
|
||||
if [ -n "$BRANCH" ]
|
||||
then
|
||||
branches=("$BRANCH")
|
||||
else
|
||||
# generate a list of all branches containing current HEAD
|
||||
branches=()
|
||||
while IFS='' read -r line
|
||||
do
|
||||
# input from: git branch --contains HEAD
|
||||
branch="${line:2}"
|
||||
if [[ "$branch" =~ "detached from" ]]
|
||||
then
|
||||
# ignore detached heads
|
||||
continue
|
||||
fi
|
||||
if [ ".${line:0:1}" = ".*" ]
|
||||
then
|
||||
# current branch, put it first
|
||||
branches=("${branch}" ${branches[@]+"${branches[@]}"})
|
||||
else
|
||||
branches=(${branches[@]+"${branches[@]}"} "${branch}")
|
||||
fi
|
||||
done <<< "$(git branch -r --contains HEAD | sed 's#origin/##g')"
|
||||
fi
|
||||
|
||||
if [[ -f sources ]]; then
|
||||
echo "Flat layout style"
|
||||
if [[ ! -s sources ]]; then
|
||||
echo "Empty sources file -- nothing to check"
|
||||
else
|
||||
# This section is for the "flat" dist-git layout, where the spec file and
|
||||
# patches are all present at the top level directory and the sha of the tarball
|
||||
# present in a 'sources' file.
|
||||
# This code was re-used from the fedpkg-pkg minimal project which is licensed
|
||||
# under GPLv3 or any later version.
|
||||
|
||||
pkgname=$(basename "$PWD")
|
||||
# Read first word of first line. For old MD5 format it's the 32 character
|
||||
# hash. Otherwise let's assume the sources have the BSD format where lines
|
||||
# start with hash type.
|
||||
hashtype="$(head -n1 sources | cut -d' ' -f1 | tr '[:upper:]' '[:lower:]')"
|
||||
# The format is
|
||||
# SHA512 (filename) = ABCDEF
|
||||
# We don't care about the equals sign. We also assume all hashes are
|
||||
# the same type, so we don't need to read it again for each line.
|
||||
while read -r _ filename _ hash || [[ -n "$filename" && -n "$hash" ]]; do
|
||||
if [ -z "$filename" ] || [ -z "$hash" ]; then
|
||||
continue
|
||||
fi
|
||||
# Remove parenthesis around tarball name
|
||||
filename=${filename#(}
|
||||
tarball=${filename%)}
|
||||
if [ ! -e "$tarball" ]; then
|
||||
for br in "${branches[@]}"
|
||||
do
|
||||
br=$(echo ${br}| sed -e s'|remotes/origin/||')
|
||||
# Try the branch-specific lookaside structure
|
||||
url="${SURL}/$pkgname/${br}/$hash"
|
||||
echo "Retrieving ${url}"
|
||||
HTTP_CODE=$(curl -L ${QUIET} -H Pragma: -o "./$tarball" -R -S --fail --retry 5 "${url}" --write-out "%{http_code}" || true)
|
||||
echo "Returned ${HTTP_CODE}"
|
||||
if [[ ${HTTP_CODE} -gt 199 && ${HTTP_CODE} -lt 300 ]] ; then
|
||||
echo "bailing"
|
||||
break
|
||||
fi
|
||||
# Try the hash-specific lookaside structure
|
||||
tarball_bs=$(basename $tarball)
|
||||
url="${SURL}/$pkgname/$tarball_bs/$hashtype/$hash/$tarball_bs"
|
||||
echo "Retrieving ${url}"
|
||||
curl -L ${QUIET} -H Pragma: -o "./$tarball" -R -S --fail --retry 5 "${url}" && break
|
||||
done
|
||||
else
|
||||
echo "$filename exists. skipping"
|
||||
fi
|
||||
done < sources
|
||||
"${hashtype}sum" -c sources
|
||||
fi
|
||||
else
|
||||
echo "Exploded SRPM layout style"
|
||||
# This section is for the "non-flat" dist-git layout, where the spec file
|
||||
# is stored in a SPECS folder, the patches in a SOURCES folder and the sha
|
||||
# of the tarball of the project is present in a '.<pkg_name>.metadata' file.
|
||||
|
||||
mkdir -p SOURCES
|
||||
# should go into a function section at some point
|
||||
weakHashDetection () {
|
||||
strHash=${1};
|
||||
case $((`echo ${strHash}|wc -m` - 1 )) in
|
||||
128)
|
||||
hashBin='sha512sum'
|
||||
;;
|
||||
64)
|
||||
hashBin='sha256sum'
|
||||
;;
|
||||
40)
|
||||
hashBin='sha1sum'
|
||||
;;
|
||||
32)
|
||||
hashBin='md5sum'
|
||||
;;
|
||||
*)
|
||||
hashBin='unknown'
|
||||
;;
|
||||
esac
|
||||
echo ${hashBin};
|
||||
}
|
||||
|
||||
# check metadata file and extract package name
|
||||
shopt -s nullglob
|
||||
set -- .*.metadata
|
||||
if (( $# == 0 ))
|
||||
then
|
||||
echo 'Missing metadata. Please run from inside a sources git repo' >&2
|
||||
exit 1
|
||||
elif (( $# > 1 ))
|
||||
then
|
||||
echo "Warning: multiple metadata files found. Using $1"
|
||||
fi
|
||||
meta=$1
|
||||
pn=${meta%.metadata}
|
||||
pn=${pn#.}
|
||||
|
||||
while read -r fsha fname ; do
|
||||
if [ ".${fsha}" = ".da39a3ee5e6b4b0d3255bfef95601890afd80709" ]; then
|
||||
# zero byte file
|
||||
touch ${fname}
|
||||
else
|
||||
hashType=$(weakHashDetection ${fsha})
|
||||
if [ "${hashType}" == "unknown" ]; then
|
||||
echo 'Failure: Hash type unknown.' >&2
|
||||
exit 1;
|
||||
fi
|
||||
hashName=$(echo ${hashType}| sed -e s'|sum||')
|
||||
|
||||
if [ ${CHECK} -eq 1 ]; then
|
||||
which ${hashType} >/dev/null 2>&1
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Failure: You need ${hashType} in PATH." >&2
|
||||
exit 1;
|
||||
fi
|
||||
fi
|
||||
if [ -e ${fname} -a ${CHECK} -eq 1 ]; then
|
||||
# check hash sum and force download if wrong
|
||||
downsum=$(${hashType} ${fname} | awk '{print $1}')
|
||||
if [ "${fsha}" != "${downsum}" ]; then
|
||||
rm -f ${fname}
|
||||
fi
|
||||
fi
|
||||
if [ ! -e "${fname}" ]; then
|
||||
for br in "${branches[@]}"
|
||||
do
|
||||
br=$(echo ${br}| sed -e s'|remotes/origin/||')
|
||||
# Try the branch-specific lookaside structure
|
||||
url="${SURL}/${pn}/${br}/${fsha}"
|
||||
echo "Retrieving ${url}"
|
||||
HTTP_CODE=$(curl -L ${QUIET} -H Pragma: -o "${fname}" -R -S --fail --retry 5 "${url}" --write-out "%{http_code}" || true)
|
||||
echo "Returned ${HTTP_CODE}"
|
||||
if [[ ${HTTP_CODE} -gt 199 && ${HTTP_CODE} -lt 300 ]] ; then
|
||||
echo "bailing"
|
||||
break
|
||||
fi
|
||||
# Try the hash-specific lookaside structure
|
||||
fname_bs=$(basename $fname)
|
||||
url="${SURL}/$pn/$fname_bs/${hashName}/$fsha/$fname_bs"
|
||||
echo "Retrieving ${url}"
|
||||
curl -L ${QUIET} -H Pragma: -o "${fname}" -R -S --fail --retry 5 "${url}" && break
|
||||
done
|
||||
else
|
||||
echo "${fname} exists. skipping"
|
||||
fi
|
||||
if [ ${CHECK} -eq 1 ]; then
|
||||
downsum=$(${hashType} ${fname} | awk '{print $1}')
|
||||
if [ "${fsha}" != "${downsum}" ]; then
|
||||
rm -f ${fname}
|
||||
echo "Failure: ${fname} hash does not match hash from the .metadata file" >&2
|
||||
exit 1;
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done < "${meta}"
|
||||
|
||||
fi
|
||||
Loading…
Add table
Reference in a new issue