From 4fefd7f1d5140ece54d2d2f9a96ea080eaa7fb3e Mon Sep 17 00:00:00 2001 From: clime Date: Wed, 5 Oct 2016 12:43:12 +0200 Subject: [PATCH] reconfiguring package to fit into FedoraInfra --- configs/cron/cgit_pkg_list.cron | 3 - configs/cron/dist_git_sync.cron | 3 - configs/gitolite/gitolite.rc | 9 +- configs/httpd/dist-git/git-smart-http.conf | 9 +- .../dist-git/lookaside-upload.conf.example | 97 ++----- configs/httpd/dist-git/lookaside.conf | 4 +- configs/systemd/dist-git@.service | 2 +- dist-git.spec | 143 ++++------ scripts/dist-git/cgit_pkg_list.sh | 23 -- scripts/dist-git/dist_git_sync.sh | 18 -- scripts/dist-git/{git_branch.sh => mkbranch} | 15 +- scripts/dist-git/mkbranch_branching | 158 ++++++++++ scripts/dist-git/pkgdb_gen_gitolite_conf.py | 122 -------- scripts/dist-git/pkgdb_sync_git_branches.py | 269 ------------------ .../{git_package.sh => setup_git_package} | 63 ++-- scripts/git/hooks/update-block-push-origin | 15 - scripts/httpd/upload.cgi | 196 +++++++------ selinux/dist_git.fc | 6 +- 18 files changed, 408 insertions(+), 747 deletions(-) delete mode 100644 configs/cron/cgit_pkg_list.cron delete mode 100644 configs/cron/dist_git_sync.cron delete mode 100644 scripts/dist-git/cgit_pkg_list.sh delete mode 100644 scripts/dist-git/dist_git_sync.sh rename scripts/dist-git/{git_branch.sh => mkbranch} (91%) create mode 100644 scripts/dist-git/mkbranch_branching delete mode 100644 scripts/dist-git/pkgdb_gen_gitolite_conf.py delete mode 100644 scripts/dist-git/pkgdb_sync_git_branches.py rename scripts/dist-git/{git_package.sh => setup_git_package} (67%) delete mode 100644 scripts/git/hooks/update-block-push-origin diff --git a/configs/cron/cgit_pkg_list.cron b/configs/cron/cgit_pkg_list.cron deleted file mode 100644 index 1c46895..0000000 --- a/configs/cron/cgit_pkg_list.cron +++ /dev/null @@ -1,3 +0,0 @@ -# This would allow to refresh the cgit repos every 10 minutes -# -# */10 * * * * root /usr/share/dist-git/cgit_pkg_list.sh diff --git a/configs/cron/dist_git_sync.cron b/configs/cron/dist_git_sync.cron deleted file mode 100644 index 8889fbe..0000000 --- a/configs/cron/dist_git_sync.cron +++ /dev/null @@ -1,3 +0,0 @@ -# Sync the repositories with a Package Database -# -# 02 10 * * * root /usr/share/dist-git/dist_git_sync.sh diff --git a/configs/gitolite/gitolite.rc b/configs/gitolite/gitolite.rc index 0dbc99d..35426ff 100644 --- a/configs/gitolite/gitolite.rc +++ b/configs/gitolite/gitolite.rc @@ -12,6 +12,11 @@ # "list of non-core programs shipped with gitolite" in the master index) or # directly in the corresponding source file. + +# Hack! +# Pull in our repo aliases generated by genacls.sh +use lib ('/etc/gitolite/'); + %RC = ( # ------------------------------------------------------------------ @@ -67,7 +72,7 @@ # suggested locations for site-local gitolite code (see cust.html) # this one is managed directly on the server - LOCAL_CODE => "/var/lib/dist-git/gitolite/local", + LOCAL_CODE => "/etc/gitolite/local", # or you can use this, which lets you put everything in a subdirectory # called "local" in your gitolite-admin repo. For a SECURITY WARNING @@ -122,7 +127,7 @@ 'no-auto-create', # access a repo by another (possibly legacy) name - # 'Alias', + 'Alias', # give some users direct shell access. See documentation in # sts.html for details on the following two choices. diff --git a/configs/httpd/dist-git/git-smart-http.conf b/configs/httpd/dist-git/git-smart-http.conf index 1af1ee4..96ddf10 100644 --- a/configs/httpd/dist-git/git-smart-http.conf +++ b/configs/httpd/dist-git/git-smart-http.conf @@ -1,3 +1,10 @@ -SetEnv GIT_PROJECT_ROOT /var/lib/dist-git/git/rpms +SetEnv GIT_PROJECT_ROOT /srv/git/repositories SetEnv GIT_HTTP_EXPORT_ALL ScriptAlias /git/ /usr/libexec/git-core/git-http-backend/ + + + Options ExecCGI Indexes + Order allow,deny + Allow from all + Require all granted + diff --git a/configs/httpd/dist-git/lookaside-upload.conf.example b/configs/httpd/dist-git/lookaside-upload.conf.example index 89a033d..8c812b4 100644 --- a/configs/httpd/dist-git/lookaside-upload.conf.example +++ b/configs/httpd/dist-git/lookaside-upload.conf.example @@ -1,4 +1,4 @@ -Alias /repo/ /var/lib/dist-git/cache/lookaside/ +Alias /repo/ /srv/cache/lookaside/ # default SSL configuration... Listen 443 @@ -15,93 +15,54 @@ SSLCryptoDevice builtin # SSL host # This alias must come before the /repo/ one to avoid being overridden. - ScriptAlias /repo/pkgs/upload.cgi /var/lib/dist-git/web/upload.cgi + ScriptAlias /repo/pkgs/upload.cgi /srv/web/upload.cgi - Alias /repo/ /var/lib/dist-git/cache/lookaside/ - - #ServerName pkgs.fedoraproject.org - #ServerAdmin webmaster@fedoraproject.org + Alias /repo/ /srv/cache/lookaside/ + ServerName pkgs.fedoraproject.org + ServerAdmin webmaster@fedoraproject.org SSLEngine on - # Server Certificate: - # Point SSLCertificateFile at a PEM encoded certificate. If - # the certificate is encrypted, then you will be prompted for a - # pass phrase. Note that a kill -HUP will prompt again. A new - # certificate can be generated using the genkey(1) command. - SSLCertificateFile /etc/pki/tls/certs/localhost.crt - - # Server Private Key: - # If the key is not combined with the certificate, use this - # directive to point at the key file. Keep in mind that if - # you've both a RSA and a DSA private key you can configure - # both in parallel (to also allow the use of DSA ciphers, etc.) - SSLCertificateKeyFile /etc/pki/tls/private/localhost.key - - # Certificate Authority (CA): - # Set the CA certificate verification path where to find CA - # certificates for client authentication or alternatively one - # huge file containing all of them (file must be PEM encoded) - SSLCACertificateFile /etc/pki/tls/certs/ca-bundle.crt - - # Certificate Revocation Lists (CRL): - # Such a file is simply the concatenation of the various - # PEM-encoded CRL files, in order of preference. - # These are used to revoke the client certificate on - # Client Authentication. + SSLCertificateFile conf/pkgs.fedoraproject.org_key_and_cert.pem + SSLCertificateKeyFile conf/pkgs.fedoraproject.org_key_and_cert.pem + SSLCACertificateFile conf/cacert.pem SSLCARevocationFile /etc/pki/tls/crl.pem - SSLCipherSuite RSA:!EXPORT:!DH:!LOW:!NULL:+MEDIUM:+HIGH + SSLProtocol {{ ssl_protocols }} + SSLCipherSuite {{ ssl_ciphers }} - # Must be 'optional' everywhere in order to have POST operations work to upload.cgi +# Must be 'optional' everywhere in order to have POST operations work to upload.cgi SSLVerifyClient optional - # Must be here for POST operations to upload.cgi +# Must be here for POST operations to upload.cgi SSLOptions +OptRenegotiate ErrorLog logs/ssl_error_log CustomLog logs/ssl_access_log \ "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%{SSL_CLIENT_S_DN_OU}x\" %{SSL_CLIENT_S_DN_CN}x %{SSL_CLIENT_S_DN_emailAddress}x \"%r\" %b" - - SSLVerifyClient optional - SSLVerifyDepth 1 - SSLOptions +StrictRequire +StdEnvVars +OptRenegotiate - # Access Control: - # With SSLRequire you can do per-directory access control based - # on arbitrary complex boolean expressions containing server - # variable checks and other lookup directives. The syntax is a - # mixture between C and Perl. See the mod_ssl documentation - # for more details. - # - # Following example means: - # require that the access comes from internal or that - # the client auth cert was created by us and signed by us - SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \ + + SSLVerifyClient optional + SSLVerifyDepth 1 + SSLOptions +StrictRequire +StdEnvVars +OptRenegotiate + # require that the client auth cert was created by us and signed by us + SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \ and %{SSL_CLIENT_S_DN_O} eq "Fedora Project" \ and %{SSL_CLIENT_S_DN_OU} eq "Fedora User Cert" \ and %{SSL_CLIENT_I_DN_O} eq "Fedora Project" \ and %{SSL_CLIENT_I_DN_OU} eq "Fedora Project CA" ) - + - - SSLRequireSSL + + SSLRequireSSL - Options +ExecCGI - Require all granted + Options +ExecCGI + Require all granted - SSLVerifyClient optional - SSLVerifyDepth 1 - SSLOptions +StrictRequire +StdEnvVars +OptRenegotiate - # Access Control: - # With SSLRequire you can do per-directory access control based - # on arbitrary complex boolean expressions containing server - # variable checks and other lookup directives. The syntax is a - # mixture between C and Perl. See the mod_ssl documentation - # for more details. - # - # Following example means: - # require that the access comes from internal or that - # the client auth cert was created by us and signed by us - SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \ + SSLVerifyClient optional + SSLVerifyDepth 1 + SSLOptions +StrictRequire +StdEnvVars +OptRenegotiate + # require that the access comes from internal or that + # the client auth cert was created by us and signed by us + SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \ and %{SSL_CLIENT_S_DN_O} eq "Fedora Project" \ and %{SSL_CLIENT_S_DN_OU} eq "Fedora User Cert" \ and %{SSL_CLIENT_I_DN_O} eq "Fedora Project" \ diff --git a/configs/httpd/dist-git/lookaside.conf b/configs/httpd/dist-git/lookaside.conf index b4e43ad..de5b441 100644 --- a/configs/httpd/dist-git/lookaside.conf +++ b/configs/httpd/dist-git/lookaside.conf @@ -1,5 +1,5 @@ -Alias /lookaside /var/lib/dist-git/cache/lookaside - +Alias /lookaside /srv/cache/lookaside + Options Indexes FollowSymLinks AllowOverride None Require all granted diff --git a/configs/systemd/dist-git@.service b/configs/systemd/dist-git@.service index 20305bd..3838250 100644 --- a/configs/systemd/dist-git@.service +++ b/configs/systemd/dist-git@.service @@ -5,5 +5,5 @@ Wants=dist-git.socket [Service] User=nobody -ExecStart=/usr/libexec/git-core/git-daemon --base-path=/var/lib/dist-git/git/rpms --export-all --user-path=public_git --syslog --inetd --verbose +ExecStart=/usr/libexec/git-core/git-daemon --base-path=/srv/git/repositories --export-all --user-path=public_git --syslog --inetd --verbose StandardInput=socket diff --git a/dist-git.spec b/dist-git.spec index 49ef0cd..9893d5a 100644 --- a/dist-git.spec +++ b/dist-git.spec @@ -1,7 +1,7 @@ %global selinux_variants mls targeted %global selinux_policyver %(%{__sed} -e 's,.*selinux-policy-\\([^/]*\\)/.*,\\1,' /usr/share/selinux/devel/policyhelp || echo 0.0.0) %global modulename dist_git - +%global installdir /srv Name: dist-git Version: 0.13 @@ -29,7 +29,6 @@ Requires: git-daemon Requires: python-requests Requires: mod_ssl Requires: fedmsg -Requires: cronie Requires(pre): shadow-utils %description @@ -90,65 +89,58 @@ getent group gen-acls > /dev/null || \ getent passwd gen-acls > /dev/null || \ useradd -r -g gen-acls -G packager -s /bin/bash \ - -d %{_sharedstatedir}/dist-git/git gen-acls + -d /srv/git gen-acls %install # ------------------------------------------------------------------------------ -# /usr/share/ .... static files +# /usr/local/bin ........... scripts # ------------------------------------------------------------------------------ -install -d %{buildroot}%{_datadir}/dist-git - -cp -a scripts/dist-git/* %{buildroot}%{_datadir}/dist-git/ +install -d %{buildroot}/usr/local/bin/ +cp -a scripts/dist-git/* %{buildroot}/usr/local/bin/ # ------------------------------------------------------------------------------ # /etc/ .......... config files # ------------------------------------------------------------------------------ install -d %{buildroot}%{_sysconfdir}/dist-git +install -d %{buildroot}%{_sysconfdir}/gitolite install -d %{buildroot}%{_sysconfdir}/httpd/conf.d/dist-git -install -d %{buildroot}%{_sysconfdir}/cron.d/dist-git mkdir -p %{buildroot}%{_unitdir} cp -a configs/dist-git/dist-git.conf %{buildroot}%{_sysconfdir}/dist-git/ -cp -a configs/gitolite/gitolite.rc %{buildroot}%{_sysconfdir}/dist-git/ cp -a configs/httpd/dist-git.conf %{buildroot}%{_sysconfdir}/httpd/conf.d/ +cp -a configs/gitolite/gitolite.rc %{buildroot}%{_sysconfdir}/gitolite/ cp -a configs/httpd/ssl.conf.example %{buildroot}%{_sysconfdir}/httpd/conf.d/ cp -a configs/httpd/dist-git/* %{buildroot}%{_sysconfdir}/httpd/conf.d/dist-git/ -cp -a configs/cron/* %{buildroot}%{_sysconfdir}/cron.d/dist-git/ cp -a configs/systemd/* %{buildroot}%{_unitdir}/ +install -d %{buildroot}%{_sysconfdir}/gitolite +install -d %{buildroot}%{_sysconfdir}/gitolite/conf +install -d %{buildroot}%{_sysconfdir}/gitolite/logs +install -d %{buildroot}%{_sysconfdir}/gitolite/local +install -d %{buildroot}%{_sysconfdir}/gitolite/local/VREF +install -d %{buildroot}%{_sysconfdir}/gitolite +install -d %{buildroot}%{_sysconfdir}/gitolite/hooks +install -d %{buildroot}%{_sysconfdir}/gitolite/hooks/common + # ------------------------------------------------------------------------------ # /var/lib/ ...... dynamic persistent files # ------------------------------------------------------------------------------ -install -d %{buildroot}%{_sharedstatedir}/dist-git -install -d %{buildroot}%{_sharedstatedir}/dist-git/git -install -d %{buildroot}%{_sharedstatedir}/dist-git/git/rpms -install -d %{buildroot}%{_sharedstatedir}/dist-git/gitolite -install -d %{buildroot}%{_sharedstatedir}/dist-git/gitolite/conf -install -d %{buildroot}%{_sharedstatedir}/dist-git/gitolite/logs -install -d %{buildroot}%{_sharedstatedir}/dist-git/gitolite/local -install -d %{buildroot}%{_sharedstatedir}/dist-git/gitolite/local/VREF -install -d %{buildroot}%{_sharedstatedir}/dist-git/cache -install -d %{buildroot}%{_sharedstatedir}/dist-git/cache/lookaside -install -d %{buildroot}%{_sharedstatedir}/dist-git/cache/lookaside/pkgs -install -d %{buildroot}%{_sharedstatedir}/dist-git/web -install -d %{buildroot}%{_sharedstatedir}/dist-git/gitolite -install -d %{buildroot}%{_sharedstatedir}/dist-git/gitolite/hooks -install -d %{buildroot}%{_sharedstatedir}/dist-git/gitolite/hooks/common +install -d %{buildroot}%{installdir} +install -d %{buildroot}%{installdir}/git +install -d %{buildroot}%{installdir}/git/repositories +install -d %{buildroot}%{installdir}/cache +install -d %{buildroot}%{installdir}/cache/lookaside +install -d %{buildroot}%{installdir}/cache/lookaside/pkgs +install -d %{buildroot}%{installdir}/web -cp -a scripts/httpd/upload.cgi %{buildroot}%{_sharedstatedir}/dist-git/web/ +cp -a scripts/httpd/upload.cgi %{buildroot}%{installdir}/web/ -cp -a scripts/git/hooks/update-block-push-origin \ - %{buildroot}%{_sharedstatedir}/dist-git/gitolite/local/VREF/update-block-push-origin +ln -f -s %{_sysconfdir}/gitolite/gitolite.rc \ + %{buildroot}%{installdir}/git/.gitolite.rc -ln -f -s %{_sysconfdir}/dist-git/gitolite.rc \ - %{buildroot}%{_sharedstatedir}/dist-git/git/.gitolite.rc - -ln -f -s %{_sharedstatedir}/dist-git/gitolite \ - %{buildroot}%{_sharedstatedir}/dist-git/git/.gitolite - -ln -f -s %{_sharedstatedir}/dist-git/git/rpms \ - %{buildroot}%{_sharedstatedir}/dist-git/git/repositories +ln -f -s %{_sysconfdir}/gitolite \ + %{buildroot}%{installdir}/git/.gitolite # ------------------------------------------------------------------------------ # SELinux @@ -171,7 +163,7 @@ do /usr/sbin/semodule -s ${selinuxvariant} -i \ %{_datadir}/selinux/${selinuxvariant}/%{modulename}.pp &> /dev/null || : done -%{_sbindir}/restorecon -Rv %{_sharedstatedir}/dist-git || : +%{_sbindir}/restorecon -Rv /srv/git/ || : %postun selinux @@ -190,67 +182,58 @@ fi %license LICENSE %doc README.md -# ------------------------------------------------------------------------------ -# /usr/share/ .... static files -# ------------------------------------------------------------------------------ -%dir %{_datadir}/dist-git -%attr (755, -, -) %{_datadir}/dist-git/* - # ------------------------------------------------------------------------------ # /etc/ .......... config files # ------------------------------------------------------------------------------ %dir %{_sysconfdir}/dist-git %config(noreplace) %{_sysconfdir}/dist-git/dist-git.conf -%config(noreplace) %{_sysconfdir}/dist-git/gitolite.rc %config(noreplace) %{_sysconfdir}/httpd/conf.d/dist-git.conf +%config(noreplace) %{_sysconfdir}/gitolite/gitolite.rc %config %{_sysconfdir}/httpd/conf.d/ssl.conf.example %dir %{_sysconfdir}/httpd/conf.d/dist-git %config(noreplace) %{_sysconfdir}/httpd/conf.d/dist-git/* -%dir %{_sysconfdir}/cron.d/dist-git -%config(noreplace) %{_sysconfdir}/cron.d/dist-git/cgit_pkg_list.cron -%config(noreplace) %{_sysconfdir}/cron.d/dist-git/dist_git_sync.cron + +# non-standard-dir-perm: +# - write access needed into log directory for gitolite +%attr (775, gen-acls, packager) %{_sysconfdir}/gitolite/logs +%dir %{_sysconfdir}/gitolite/local +# non-standard-dir-perm: +# - write access needed for gitolite admin groups +%attr (775, gen-acls, packager) %{_sysconfdir}/gitolite/local/VREF +# non-standard-dir-perm: +# - write access needed for gitolite admin groups +%attr (770, -, packager) %{_sysconfdir}/gitolite/hooks +# script-without-shebang: +# zero-length: +# - initial empty file required by gitolite with the correct perms +%dir %{_sysconfdir}/gitolite/hooks/common +%ghost %attr (775, gen-acls, packager) %{_sysconfdir}/gitolite/hooks/common/update + %{_unitdir}/dist-git@.service %{_unitdir}/dist-git.socket +%dir %{_sysconfdir}/gitolite +%attr (755, gen-acls, gen-acls) %{_sysconfdir}/gitolite/conf + # ------------------------------------------------------------------------------ # /var/lib/ ...... dynamic persistent files # ------------------------------------------------------------------------------ # non-standard-dir-perm: # - git repositories and their contents must have w permission for their creators -%dir %{_sharedstatedir}/dist-git -%dir %{_sharedstatedir}/dist-git/git -%attr (2775, -, packager) %{_sharedstatedir}/dist-git/git/rpms -%dir %{_sharedstatedir}/dist-git/gitolite -%attr (755, gen-acls, gen-acls) %{_sharedstatedir}/dist-git/gitolite/conf -# non-standard-dir-perm: -# - write access needed into log directory for gitolite -%attr (775, gen-acls, packager) %{_sharedstatedir}/dist-git/gitolite/logs -%dir %{_sharedstatedir}/dist-git/gitolite/local -# non-standard-dir-perm: -# - write access needed for gitolite admin groups -%attr (775, gen-acls, packager) %{_sharedstatedir}/dist-git/gitolite/local/VREF -# non-standard-executable-perm: -# - write access needed for gitolite admin groups -# - exec permission needed for execution by git (it's a git hook script) -%attr (775, gen-acls, packager) %{_sharedstatedir}/dist-git/gitolite/local/VREF/update-block-push-origin -# non-standard-dir-perm: -# - write access needed for gitolite admin groups -%attr (770, -, packager) %{_sharedstatedir}/dist-git/gitolite/hooks -# script-without-shebang: -# zero-length: -# - initial empty file required by gitolite with the correct perms -%dir %{_sharedstatedir}/dist-git/gitolite/hooks/common -%ghost %attr (775, gen-acls, packager) %{_sharedstatedir}/dist-git/gitolite/hooks/common/update -%dir %{_sharedstatedir}/dist-git/web -%attr (755, apache, apache) %{_sharedstatedir}/dist-git/web/upload.cgi -%dir %{_sharedstatedir}/dist-git/cache -%dir %{_sharedstatedir}/dist-git/cache/lookaside -%attr (775, apache, apache) %{_sharedstatedir}/dist-git/cache/lookaside/pkgs -%{_sharedstatedir}/dist-git/git/repositories -%{_sharedstatedir}/dist-git/git/.gitolite -%{_sharedstatedir}/dist-git/git/.gitolite.rc - +%dir %{installdir} +%dir %{installdir}/git +%attr (2775, -, packager) %{installdir}/git/repositories +%dir %{installdir}/web +%attr (755, apache, apache) %{installdir}/web/upload.cgi +%dir %{installdir}/cache +%dir %{installdir}/cache/lookaside +%attr (775, apache, apache) %{installdir}/cache/lookaside/pkgs +%{installdir}/git/.gitolite +%{installdir}/git/.gitolite.rc +%attr (755, root, root) /usr/local/bin/mkbranch +%attr (755, root, root) /usr/local/bin/mkbranch_branching +%attr (755, root, root) /usr/local/bin/setup_git_package %files selinux %defattr(-,root,root,0755) diff --git a/scripts/dist-git/cgit_pkg_list.sh b/scripts/dist-git/cgit_pkg_list.sh deleted file mode 100644 index 65d1321..0000000 --- a/scripts/dist-git/cgit_pkg_list.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -# -# This simple script lists out the current pkgs git repos to a file. -# This speeds up cgit as it doesn't have to recurse into all dirs -# Looking for git repos. -# - -destination=/var/lib/dist-git/git/pkgs-git-repos-list - -if [ -n "$1" ] -then - destination=$1 -fi - -newfile=`mktemp` - -cd /var/lib/dist-git/git/rpms -ls > $newfile -cp -fZ $newfile $destination -rm $newfile -#chown apache:apache $destination -chmod 644 $destination diff --git a/scripts/dist-git/dist_git_sync.sh b/scripts/dist-git/dist_git_sync.sh deleted file mode 100644 index 5a213d3..0000000 --- a/scripts/dist-git/dist_git_sync.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -python /usr/share/dist-git/pkgdb_sync_git_branches.py - -TEMPDIR=`mktemp -d -p /var/tmp genacls.XXXXX` -export GL_BINDIR=/usr/bin - -cd $TEMPDIR -# Only replace the acls if genacls completes successfully -if /usr/share/dist-git/pkgdb_gen_gitolite_conf.py > gitolite.conf ; then - mv gitolite.conf /var/lib/dist-git/gitolite/conf/ - chown gen-acls:gen-acls -R /var/lib/dist-git/gitolite/conf/ - HOME=/var/lib/dist-git/git /usr/bin/gitolite compile -fi -cd / -rm -rf $TEMPDIR -chown root:packager /var/lib/dist-git/gitolite/conf/gitolite.conf-compiled.pm -chmod g+r /var/lib/dist-git/gitolite/conf/gitolite.conf-compiled.pm diff --git a/scripts/dist-git/git_branch.sh b/scripts/dist-git/mkbranch similarity index 91% rename from scripts/dist-git/git_branch.sh rename to scripts/dist-git/mkbranch index 46965a1..bde1f1e 100644 --- a/scripts/dist-git/git_branch.sh +++ b/scripts/dist-git/mkbranch @@ -10,7 +10,7 @@ # Figure out the environment we're running in RUNDIR=$(cd $(dirname $0) && pwd) -GITROOT=/var/lib/dist-git/git/rpms +GITROOT=/srv/git/repositories # check if a moron is driving me if [ ! -d $GITROOT ] ; then @@ -30,6 +30,7 @@ IGNORE= BRANCH="" PACKAGES="" SRC_BRANCH="master" +AUTHOR="Fedora Release Engineering " Usage() { cat <] ... + + Creates a new branch for the list of s. + The /master suffix on branch names is assumed. + +Options: + -n,--test Don't do nothing, only test + -i,--ignore Ignore erroneous modules + -h,--help This help message + -v,--verbose Increase verbosity +EOF +} + +# parse the arguments +while [ -n "$1" ] ; do + case "$1" in + -h | --help ) + Usage + exit 0 + ;; + + -v | --verbose ) + VERBOSE=$(($VERBOSE + 1)) + ;; + + -i | --ignore ) + IGNORE="yes" + ;; + + -n | --test ) + TEST="yes" + ;; + + + -b | --branch ) + shift + BRANCH=$1/master + ;; + + * ) + if [ -z "$BRANCH" ] ; then + BRANCH="$1" + else + PACKAGES="$PACKAGES $1" + fi + ;; + esac + shift +done + +# check the arguments +if [ -z "$BRANCH" -o -z "$PACKAGES" ] ; then + Usage + exit -1 +fi + + +# Sanity checks before we start doing damage +NEWP= +for p in $PACKAGES ; do + [ $VERBOSE -gt 1 ] && echo "Checking package $p..." + if [ ! -d $GITROOT/$p.git ] ; then + echo "ERROR: Package module $p is invalid" >&2 + [ "$IGNORE" = "yes" ] && continue || exit -1 + fi + $(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \ + $BRANCH >/dev/null) && \ + (echo "IGNORING: Package module $p already has a branch $BRANCH" >&2; \ + [ "$IGNORE" = "yes" ] && continue || exit -1) + NEWP="$NEWP $p" +done +PACKAGES="$(echo $NEWP)" +if [ -z "$PACKAGES" ] ; then + echo "NOOP: no valid packages found to process" + exit -1 +fi + +if [ -n "$TEST" ] ; then + echo "Branch $BRANCH valid for $PACKAGES" + exit 0 +fi + +# "global" permissions check +if [ ! -w $GITROOT ] ; then + echo "ERROR: You can not write to $GITROOT" + echo "ERROR: You can not perform branching operations" + exit -1 +fi + +# Now start working on creating those branches + +# For every module, "create" the branch +for NAME in $PACKAGES ; do + echo + echo "Creating new module branch '$BRANCH' for '$NAME'..." + + # permissions checks for this particular module + if [ ! -w $GITROOT/$NAME.git/refs/heads/ ] ; then + echo "ERROR: You can not write to $d" + echo "ERROR: $NAME can not be branched by you" + continue + fi + #### Replace the above with a gitolite permission check + #[ $VERBOSE -gt 0 ] && echo "Creating $BRANCH-split tag for $NAME/$SRC_BRANCH..." + # Is the above needed? + #cvs -Q rtag -f "$BRANCH-split" $TOPLEVEL/$NAME/$SRC_BRANCH || { + #echo "ERROR: Branch split tag for $NAME/$SRC_BRANCH could not be created" >&2 + #exit -2 + #} + [ $VERBOSE -gt 0 ] && echo "Creating $NAME $BRANCH from $NAME ..." + $(pushd $GITROOT/$NAME.git >/dev/null && \ + git branch --no-track $BRANCH `git rev-list master | head -1` && \ + popd >/dev/null) || { + echo "ERROR: Branch $NAME $BRANCH could not be created" >&2 + popd >/dev/null + exit -2 + } +done + +echo +echo "Done." diff --git a/scripts/dist-git/pkgdb_gen_gitolite_conf.py b/scripts/dist-git/pkgdb_gen_gitolite_conf.py deleted file mode 100644 index 2476021..0000000 --- a/scripts/dist-git/pkgdb_gen_gitolite_conf.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/python -t -# -# Create an /etc/gitolog/conf/getolog.conf file with acls for dist-git -# -# Takes no arguments! -# - -import grp -import sys - -import requests -from ConfigParser import ConfigParser - -def _get_conf(cp, section, option, default): - if cp.has_section(section) and cp.has_option(section, option): - return cp.get(section, option) - return default - - -if __name__ == '__main__': - config = ConfigParser() - config.read("/etc/dist-git/dist-git.conf") - - user_groups = _get_conf(config, "acls", "user_groups", "").split(",") - admin_groups = _get_conf(config, "acls", "admin_groups", "").split(",") - ACTIVE = _get_conf(config, "acls", "active_branches", "").split(",") - RESERVED = _get_conf(config, "acls", "reserved_branches", "").split(",") - pkgdb_acls_url = _get_conf(config, "acls", "pkgdb_acls_url", "") - pkgdb_groups_url = _get_conf(config, "acls", "pkgdb_groups_url", "") - - - # Read the ACL information from the packageDB - data = {"packageAcls":{}} - if pkgdb_acls_url: - data = requests.get(pkgdb_acls_url).json() - - # Get a list of all the packages - acls = data['packageAcls'] - pkglist = data['packageAcls'].keys() - pkglist.sort() - - # sanity check - #if len(pkglist) < 2500: - # sys.exit(1) - - # get the list of all groups - pkgdb_groups = {"groups":[]} - if pkgdb_groups_url: - pkgdb_groups = requests.get(pkgdb_groups_url).json() - - # print out our user groups - for group in user_groups + pkgdb_groups["groups"]: - print "@{0} = {1}".format(group, " ".join(grp.getgrnam(group)[3])) - - - # Give a little space before moving onto the permissions - print '' - # print our default permissions - print 'repo @all' - print ' - VREF/update-block-push-origin = @all' - if admin_groups: - print ' RWC = @{}'.format(" @".join(admin_groups)) - print ' R = @all' - #print ' RW private- = @all' - # dont' enable the above until we prevent building for real from private- - - for pkg in pkglist: - branchAcls = {} # Check whether we need to set separate per branch acls - buffer = [] # Buffer the output per package - masters = [] # Folks that have commit to master - writers = [] # Anybody that has write access - - # Examine each branch in the package - branches = acls[pkg].keys() - branches.sort() - for branch in branches: - if not branch in ACTIVE: - continue - if 'packager' in acls[pkg][branch]['commit']['groups']: - # If the packager group is defined, everyone has access - buffer.append(' RWC %s = @all' % (branch)) - branchAcls.setdefault('@all', []).append((pkg, branch)) - if branch == 'master': - masters.append('@all') - if '@all' not in writers: - writers.append('@all') - else: - # Extract the owners - committers = [] - owners = acls[pkg][branch]['commit']['people'] - owners.sort() - for owner in owners: - committers.append(owner) - for group in acls[pkg][branch]['commit']['groups']: - committers.append('@%s' % group) - if branch == 'master': - masters.extend(committers) - - # add all the committers to the top writers list - for committer in committers: - if not committer in writers: - writers.append(committer) - - # Print the committers to the acl for this package-branch - committers = ' '.join(committers) - buffer.append(' RWC %s = %s' % - (branch, committers)) - branchAcls.setdefault(committers, []).append((pkg, branch)) - - print - print 'repo %s' % pkg - #if len(branchAcls.keys()) == 1: - # acl = branchAcls.keys()[0] - # print ' RW = %s' % acl - #else: - print '\n'.join(buffer) - for reserved in RESERVED: - print ' - %s = @all' % reserved - print ' RWC refs/tags/ = %s' % ' '.join(writers) - if masters: - print ' RWC = %s' % ' '.join(masters) - sys.exit(0) diff --git a/scripts/dist-git/pkgdb_sync_git_branches.py b/scripts/dist-git/pkgdb_sync_git_branches.py deleted file mode 100644 index 6defdfb..0000000 --- a/scripts/dist-git/pkgdb_sync_git_branches.py +++ /dev/null @@ -1,269 +0,0 @@ -#!/usr/bin/python -tt -# -*- coding: utf-8 -*- - -""" -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along -with this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -This script is able to query pkgdb and retrieve for all packages which active -branches should be there, browse all the git repos and find out which active -branches are missing. - -It even goes one step further but actually adjusting the git repo by adding -the missing branches (or even the missing repo) - -""" - -import multiprocessing.pool -import os -import subprocess -import time -import pipes - -import requests -from ConfigParser import ConfigParser - -# Do some off-the-bat configuration of fedmsg. -# 1) since this is a one-off script and not a daemon, it needs to connect -# to the fedmsg-relay process running on another node (or noone will -# hear it) -# 2) its going to use the 'shell' certificate which only 'sysadmin' has -# read access to. Contrast that with the 'scm' certificate which -# everyone in the 'packager' group has access to. - -def _get_conf(cp, section, option, default): - if cp.has_section(section) and cp.has_option(section, option): - return cp.get(section, option) - return default - -config = ConfigParser() -config.read("/etc/dist-git/dist-git.conf") -PKGDB_URL = _get_conf(config, "acls", "pkgdb_acls_url", "") -EMAIL_DOMAIN = _get_conf(config, "notifications", "email_domain", "fedoraproject.org") -PKG_OWNER_EMAILS = _get_conf(config, "notifications", "pkg_owner_emails", - "$PACKAGE-owner@fedoraproject.org,scm-commits@lists.fedoraproject.org") -DEFAULT_BRANCH_AUTHOR = _get_conf(config, "git", "default_branch_author", - "Fedora Release Engineering ") - - -GIT_FOLDER = '/var/lib/dist-git/git/rpms/' -MKBRANCH = '/usr/share/dist-git/git_branch.sh' -SETUP_PACKAGE = '/usr/share/dist-git/git_package.sh' - -THREADS = 20 -VERBOSE = False - - -class InternalError(Exception): - pass - - -class ProcessError(InternalError): - pass - - -def _invoke(program, args, cwd=None): - '''Run a command and raise an exception if an error occurred. - - :arg program: The program to invoke - :args: List of arguments to pass to the program - - raises ProcessError if there's a problem. - ''' - cmdLine = [program] - cmdLine.extend(args) - if VERBOSE: - print ' '.join(cmdLine) - print ' in', cwd - - program = subprocess.Popen( - cmdLine, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd) - - stdout, stderr = program.communicate() - - if program.returncode != 0: - e = ProcessError() - e.returnCode = program.returncode - e.cmd = ' '.join(cmdLine) - e.cwd = cwd - e.message = 'Error, "%s" (in %r) returned %s\n stdout: %s\n stderr: %s' % ( - e.cmd, e.cwd, e.returnCode, stdout, stderr) - print e.message - raise e - - return stdout.strip() - - -def _create_branch(pkgname, branch, existing_branches): - '''Create a specific branch for a package. - - :arg pkgname: Name of the package to branch - :arg branch: Name of the branch to create - :arg existing_branches: A list of the branches that already exist locally. - - ''' - branch = branch.replace('*', '').strip() - if branch == 'master': - print 'ERROR: Proudly refusing to create master branch. Invalid repo?' - print 'INFO: Please check %s repo' % pkgname - return - - if branch in existing_branches: - print 'ERROR: Refusing to create a branch %s that exists' % branch - return - - try: - _invoke(MKBRANCH, ["--default-branch-author", pipes.quote(DEFAULT_BRANCH_AUTHOR), - branch, pkgname]) - except ProcessError, e: - if e.returnCode == 255: - # This is a warning, not an error - return - raise - - -def pkgdb_pkg_branch(): - """ Queries pkgdb information about VCS and return a dictionnary of - which branches are available for which packages. - - :return: a dict[pkg_name] = [pkg_branches] - :rtype: dict - """ - data = {"packageAcls": {}} - if PKGDB_URL: - data = requests.get(PKGDB_URL).json() - - output = {} - for pkg in data['packageAcls']: - if pkg in output: - if VERBOSE: - print 'Strange package: %s, it is present twice in the ' \ - 'pkgdb output' % pkg - output[pkg].updated(data['packageAcls'][pkg].keys()) - else: - output[pkg] = set(data['packageAcls'][pkg].keys()) - - return output - - -def get_git_branch(pkg): - """ For the specified package name, check the local git and return the - list of branches found. - """ - git_folder = os.path.join(GIT_FOLDER, '%s.git' % pkg) - if not os.path.exists(git_folder): - if VERBOSE: - print 'Could not find %s' % git_folder - return set() - - branches = [ - lclbranch.replace('*', '').strip() - for lclbranch in _invoke('git', ['branch'], cwd=git_folder).split('\n') - ] - return set(branches) - - -def branch_package(pkgname, requested_branches, existing_branches): - '''Create all the branches that are listed in the pkgdb for a package. - - :arg pkgname: The package to create branches for - :arg requested_branches: The branches to creates - :arg existing_branches: A list of existing local branches - - ''' - if VERBOSE: - print 'Fixing package %s for branches %s' % (pkgname, requested_branches) - - # Create the devel branch if necessary - exists = os.path.exists(os.path.join(GIT_FOLDER, '%s.git' % pkgname)) - if not exists or 'master' not in existing_branches: - emails = PKG_OWNER_EMAILS.replace("$PACKAGE", pkgname) - _invoke(SETUP_PACKAGE, ["--pkg-owner-emails", pipes.quote(emails), - "--email-domain", pipes.quote(EMAIL_DOMAIN), - "--default-branch-author", pipes.quote(DEFAULT_BRANCH_AUTHOR), - pkgname]) - if 'master' in requested_branches: - requested_branches.remove('master') # SETUP_PACKAGE creates master - - # Create all the required branches for the package - # Use the translated branch name until pkgdb falls inline - for branch in requested_branches: - _create_branch(pkgname, branch, existing_branches) - - -def main(): - """ For each package found via pkgdb, check the local git for its - branches and fix inconsistencies. - """ - - local_pkgs = set(os.listdir(GIT_FOLDER)) - local_pkgs = set([it.replace('.git', '') for it in local_pkgs]) - if VERBOSE: - print "Found %i local packages" % len(local_pkgs) - - pkgdb_info = pkgdb_pkg_branch() - - pkgdb_pkgs = set(pkgdb_info.keys()) - if VERBOSE: - print "Found %i pkgdb packages" % len(pkgdb_pkgs) - - ## Commented out as we keep the git of retired packages while they won't - ## show up in the information retrieved from pkgdb. - - #if (local_pkgs - pkgdb_pkgs): - #print 'Some packages are present locally but not on pkgdb:' - #print ', '.join(sorted(local_pkgs - pkgdb_pkgs)) - - if (pkgdb_pkgs - local_pkgs): - print 'Some packages are present in pkgdb but not locally:' - print ', '.join(sorted(pkgdb_pkgs - local_pkgs)) - - - if VERBOSE: - print "Finding the lists of local branches for local repos." - start = time.time() - if THREADS == 1: - git_branch_lookup = map(get_git_branch, sorted(pkgdb_info)) - else: - threadpool = multiprocessing.pool.ThreadPool(processes=THREADS) - git_branch_lookup = threadpool.map(get_git_branch, sorted(pkgdb_info)) - - # Zip that list of results up into a lookup dict. - git_branch_lookup = dict(zip(sorted(pkgdb_info), git_branch_lookup)) - - if VERBOSE: - print "Found all local git branches in %0.2fs" % (time.time() - start) - - tofix = set() - for pkg in sorted(pkgdb_info): - pkgdb_branches = pkgdb_info[pkg] - git_branches = git_branch_lookup[pkg] - diff = (pkgdb_branches - git_branches) - if diff: - print '%s missing: %s' % (pkg, ','.join(sorted(diff))) - tofix.add(pkg) - branch_package(pkg, diff, git_branches) - - if tofix: - print 'Packages fixed (%s): %s' % ( - len(tofix), ', '.join(sorted(tofix))) - else: - if VERBOSE: - print 'Didn\'t find any packages to fix.' - - -if __name__ == '__main__': - import sys - sys.exit(main()) diff --git a/scripts/dist-git/git_package.sh b/scripts/dist-git/setup_git_package similarity index 67% rename from scripts/dist-git/git_package.sh rename to scripts/dist-git/setup_git_package index 2750d83..3737e06 100644 --- a/scripts/dist-git/git_package.sh +++ b/scripts/dist-git/setup_git_package @@ -8,9 +8,8 @@ # All local changes will be lost. - # Figure out the environment we're running in -GITROOT=/var/lib/dist-git/git/rpms +GITROOT=/srv/git/repositories # check if a moron is driving me if [ ! -d $GITROOT ] ; then @@ -24,67 +23,49 @@ fi VERBOSE=0 TEST= IGNORE= +AUTHOR="Fedora Release Engineering " GIT_SSH_URL="ssh://localhost" Usage() { cat < + $0 Creates a new repo for Options: -h,--help This help message - --email-domain DOMAIN Email domain for git hooks.maildomain - --pkg-owner-emails EMAILS Comma separated list of emails for - git hooks.mailinglist EOF } -# fail with no arguments -if [ $# -eq 0 ]; then +if [ $# -gt 2 ]; then Usage exit -1 fi -OPTS=$(getopt -o h -l help -l email-domain: -l pkg-owner-emails: -l default-branch-author: -- "$@") -if [ $? != 0 ] -then - exit 1 -fi - -eval set -- "$OPTS" - -while true ; do +# parse the arguments +while [ -n "$1" ] ; do case "$1" in - -h | --help) Usage; exit 0;; - --email-domain) EMAIL_DOMAIN=$2; shift 2;; - --pkg-owner-emails) PKG_OWNER_EMAILS=$2; shift 2;; - --default-branch-author) AUTHOR=$2; shift 2;; - --) shift; break;; + -h | --help ) + Usage + exit 0 + ;; + + * ) + PACKAGE="$1" + ;; esac + shift done -# fail when more or none packages are specified -if ! [ $# -eq 1 ]; then +# I hate shell scripting. I'm sure the above is totally wrong + +# check the arguments +if [ -z "$PACKAGE" ] ; then Usage exit -1 fi -PACKAGE=$1 - -if [ -z $EMAIL_DOMAIN ]; then - EMAIL_DOMAIN=fedoraproject.org -fi - -if [ -z $PKG_OWNER_EMAILS ]; then - PKG_OWNER_EMAILS=$PACKAGE-owner@fedoraproject.org,scm-commits@lists.fedoraproject.org -fi - -if [ -z $AUTHOR ]; then - AUTHOR="Fedora Release Engineering " -fi - # Sanity checks before we start doing damage [ $VERBOSE -gt 1 ] && echo "Checking package $PACKAGE..." if [ -f $GITROOT/$PACKAGE.git/refs/heads/master ] ; then @@ -114,8 +95,8 @@ mkdir -p $GITROOT/$PACKAGE.git pushd $GITROOT/$PACKAGE.git >/dev/null git init -q --shared --bare echo "$PACKAGE" > description # This is used to figure out who to send mail to. -git config --add hooks.mailinglist $PKG_OWNER_EMAILS -git config --add hooks.maildomain $EMAIL_DOMAIN +git config --add hooks.mailinglist "$PACKAGE-owner@fedoraproject.org,scm-commits@lists.fedoraproject.org" +git config --add hooks.maildomain fedoraproject.org popd >/dev/null # Now clone that repo and create the .gitignore and sources file @@ -129,7 +110,7 @@ git push -q origin master popd >/dev/null # Place the gitolite update hook in place since we're not using our own -ln -s /var/lib/dist-git/gitolite/hooks/common/update $GITROOT/$PACKAGE.git/hooks/update +ln -s /etc/gitolite/hooks/common/update $GITROOT/$PACKAGE.git/hooks/update # Setup our post-receive hooks diff --git a/scripts/git/hooks/update-block-push-origin b/scripts/git/hooks/update-block-push-origin deleted file mode 100644 index 8b427a8..0000000 --- a/scripts/git/hooks/update-block-push-origin +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -# -# Block pushes to branches if their name starts with `origin/` -# https://fedorahosted.org/rel-eng/ticket/4071 - -refname="${1}" - -echo "${refname}" | grep -qE '^refs/heads/origin/' - -if [ $? == 0 ]; then - echo "Can't push a branch named $refname" - exit 1 -fi - -exit 0 diff --git a/scripts/httpd/upload.cgi b/scripts/httpd/upload.cgi index bb3547f..2b000a6 100644 --- a/scripts/httpd/upload.cgi +++ b/scripts/httpd/upload.cgi @@ -6,56 +6,59 @@ # # License: GPL +import cgi +import errno +import grp +import hashlib import os import sys -import cgi import tempfile -import grp -import pwd -import syslog -import smtplib -import errno import fedmsg import fedmsg.config -from ConfigParser import ConfigParser - -from email import Header, Utils -try: - from email.mime.text import MIMEText -except ImportError: - from email.MIMEText import MIMEText - -import hashlib # Reading buffer size BUFFER_SIZE = 4096 # We check modules exist from this dircetory -GITREPO = '/var/lib/dist-git/git/rpms' +GITREPO = '/srv/git/repositories' # Lookaside cache directory -CACHE_DIR = '/var/lib/dist-git/cache/lookaside/pkgs' +CACHE_DIR = '/srv/cache/lookaside/pkgs' # Fedora Packager Group PACKAGER_GROUP = 'packager' -# dist git configuration file -CONFIG_FILE = "/etc/dist-git/dist-git.conf" -def send_error(text): +def send_error(text, status='500 Internal Server Error'): + """Send an error back to the client + + This ensures that the client will get a proper error, including the HTTP + status code, so that it can handle problems appropriately. + + Args: + text (str): The error message to send the client + status (str, optional): The HTTP status code to return to the client. + """ + print 'Status: %s' % status + print 'Content-type: text/plain' + print print text - sys.exit(1) + sys.exit(0) + def check_form(form, var): ret = form.getvalue(var, None) if ret is None: - send_error('Required field "%s" is not present.' % var) + send_error('Required field "%s" is not present.' % var, + status='400 Bad Request') if isinstance(ret, list): - send_error('Multiple values given for "%s". Aborting.' % var) + send_error('Multiple values given for "%s". Aborting.' % var, + status='400 Bad Request') return ret + def check_auth(username): authenticated = False try: @@ -65,22 +68,42 @@ def check_auth(username): pass return authenticated + +def hardlink(src, dest, username): + makedirs(os.path.dirname(dest), username) + + try: + os.link(src, dest) + + except OSError as e: + if e.errno != errno.EEXIST: + send_error(str(e)) + + # The file already existed at the dest path, hardlink over it + os.unlink(dest) + os.link(src, dest) + + sys.stderr.write("[username=%s] ln %s %s\n" % (username, src, dest)) + + +def makedirs(dir_, username, mode=02755): + try: + os.makedirs(dir_, mode=mode) + sys.stderr.write('[username=%s] mkdir %s\n' % (username, dir_)) + + except OSError as e: + if e.errno != errno.EEXIST: + send_error(str(e)) + + def main(): - config = ConfigParser() - config.read(CONFIG_FILE) - - EMAIL_DOMAIN = _get_conf(config, "notifications", "email_domain", "") - PKG_OWNER_EMAILS = _get_conf(config, "notifications", "pkg_owner_emails", "") - os.umask(002) username = os.environ.get('SSL_CLIENT_S_DN_CN', None) if not check_auth(username): - print 'Status: 403 Forbidden' - print 'Content-type: text/plain' - print - print 'You must connect with a valid certificate and be in the %s group to upload.' % PACKAGER_GROUP - sys.exit(0) + send_error('You must connect with a valid certificate and be in the ' + '%s group to upload.' % PACKAGER_GROUP, + status='403 Forbidden') print 'Content-Type: text/plain' print @@ -91,17 +114,18 @@ def main(): name = check_form(form, 'name') # Search for the file hash, start with stronger hash functions - if form.has_key('sha512sum'): + if 'sha512sum' in form: checksum = check_form(form, 'sha512sum') hash_type = "sha512" - elif form.has_key('md5sum'): + elif 'md5sum' in form: # Fallback on md5, as it's what we currently use checksum = check_form(form, 'md5sum') hash_type = "md5" else: - send_error('Required checksum is not present.') + send_error('Required checksum is not present.', + status='400 Bad Request') action = None upload_file = None @@ -110,38 +134,47 @@ def main(): # Is this a submission or a test? # in a test, we don't get a file, just a filename. # In a submission, we don;t get a filename, just the file. - if form.has_key('filename'): + if 'filename' in form: action = 'check' filename = check_form(form, 'filename') filename = os.path.basename(filename) - print >> sys.stderr, '[username=%s] Checking file status: NAME=%s FILENAME=%s %sSUM=%s' % (username, name, filename, hash_type.upper(), checksum) + sys.stderr.write('[username=%s] Checking file status: NAME=%s ' + 'FILENAME=%s %sSUM=%s\n' % (username, name, filename, + hash_type.upper(), + checksum)) else: action = 'upload' - if form.has_key('file'): + if 'file' in form: upload_file = form['file'] if not upload_file.file: - send_error('No file given for upload. Aborting.') + send_error('No file given for upload. Aborting.', + status='400 Bad Request') filename = os.path.basename(upload_file.filename) else: - send_error('Required field "file" is not present.') - print >> sys.stderr, '[username=%s] Processing upload request: NAME=%s FILENAME=%s %sSUM=%s' % (username, name, filename, hash_type.upper(), checksum) + send_error('Required field "file" is not present.', + status='400 Bad Request') + + sys.stderr.write('[username=%s] Processing upload request: ' + 'NAME=%s FILENAME=%s %sSUM=%s\n' % ( + username, name, filename, hash_type.upper(), + checksum)) module_dir = os.path.join(CACHE_DIR, name) hash_dir = os.path.join(module_dir, filename, hash_type, checksum) - msgpath = os.path.join(name, module_dir, filename, hash_type, checksum, filename) - - unwanted_prefix = '/var/lib/dist-git/cache/lookaside/pkgs/' - if msgpath.startswith(unwanted_prefix): - msgpath = msgpath[len(unwanted_prefix):] + msgpath = os.path.join(name, filename, hash_type, checksum, filename) # first test if the module really exists - git_dir = os.path.join(GITREPO, '%s.git' % name) + git_dir = os.path.join(GITREPO, '%s.git' % name) if not os.path.isdir(git_dir): - print >> sys.stderr, '[username=%s] Unknown module: %s' % (username, name) - send_error('Module "%s" does not exist!' % name) + sys.stderr.write('[username=%s] Unknown module: %s' % (username, name)) + send_error('Module "%s" does not exist!' % name, + status='404 Not Found') # try to see if we already have this file... dest_file = os.path.join(hash_dir, filename) + old_dir = os.path.join(module_dir, filename, checksum) + old_path = os.path.join(old_dir, filename) + if os.path.exists(dest_file): if action == 'check': print 'Available' @@ -152,12 +185,17 @@ def main(): print 'File: %s Size: %d' % (dest_file, dest_file_stat.st_size) sys.exit(0) elif action == 'check': - print 'Missing' + if os.path.exists(old_path): + # The file had been uploaded at the old path + hardlink(old_path, dest_file, username) + print 'Available' + else: + print 'Missing' + sys.exit(0) # check that all directories are in place - if not os.path.isdir(module_dir): - os.makedirs(module_dir, 02775) + makedirs(module_dir, username) # grab a temporary filename and dump our file in there tempfile.tempdir = module_dir @@ -180,41 +218,24 @@ def main(): check_checksum = m.hexdigest() if checksum != check_checksum: os.unlink(tmpfile) - send_error("%s check failed. Received %s instead of %s." % (hash_type.upper(), check_checksum, checksum)) + send_error("%s check failed. Received %s instead of %s." % + (hash_type.upper(), check_checksum, checksum), + status='400 Bad Request') # wow, even the checksum matches. make sure full path is valid now - if not os.path.isdir(hash_dir): - os.makedirs(hash_dir, 02775) - print >> sys.stderr, '[username=%s] mkdir %s' % (username, hash_dir) - + makedirs(hash_dir, username) os.rename(tmpfile, dest_file) os.chmod(dest_file, 0644) - print >> sys.stderr, '[username=%s] Stored %s (%d bytes)' % (username, dest_file, filesize) - print 'File %s size %d %s %s stored OK' % (filename, filesize, hash_type.upper(), checksum) + sys.stderr.write('[username=%s] Stored %s (%d bytes)' % (username, + dest_file, + filesize)) + print 'File %s size %d %s %s stored OK' % (filename, filesize, + hash_type.upper(), checksum) # Add the file to the old path, where fedpkg is currently looking for it if hash_type == "md5": - old_dir = os.path.join(module_dir, filename, checksum) - old_path = os.path.join(old_dir, filename) - - try: - os.makedirs(old_dir) - - except OSError as e: - if e.errno != errno.EEXIST: - raise e - - try: - os.link(dest_file, old_path) - - except OSError as e: - if e.errno != errno.EEXIST: - raise e - - # The file already existed at the old path, hardlink over it - os.unlink(old_path) - os.link(old_path) + hardlink(dest_file, old_path, username) # Emit a fedmsg message. Load the config to talk to the fedmsg-relay. try: @@ -224,11 +245,18 @@ def main(): fedmsg.init(name="relay_inbound", cert_prefix="lookaside", **config) topic = "lookaside.new" - msg = dict(name=name, md5sum=checksum, filename=filename.split('/')[-1], - agent=username, path=msgpath) + msg = dict(name=name, md5sum=checksum, + filename=filename.split('/')[-1], agent=username, + path=msgpath) fedmsg.publish(modname="git", topic=topic, msg=msg) except Exception as e: print "Error with fedmsg", str(e) if __name__ == '__main__': - main() + try: + main() + + except Exception as e: + import traceback + sys.stderr.write('%s\n' % traceback.format_exc()) + send_error(str(e)) diff --git a/selinux/dist_git.fc b/selinux/dist_git.fc index 2b32477..3900974 100644 --- a/selinux/dist_git.fc +++ b/selinux/dist_git.fc @@ -1,4 +1,4 @@ -/var/lib/dist-git/git(/.*)? gen_context(system_u:object_r:git_user_content_t,s0) -/var/lib/dist-git/cache(/.*)? gen_context(system_u:object_r:git_rw_content_t,s0) -/var/lib/dist-git/web/upload.cgi gen_context(system_u:object_r:git_script_exec_t,s0) +/srv/git(/.*)? gen_context(system_u:object_r:git_content_t,s0) +/srv/cache(/.*)? gen_context(system_u:object_r:git_rw_content_t,s0) +/srv/web/upload.cgi gen_context(system_u:object_r:git_script_exec_t,s0)