URPM-repoclosure 1.5: support of XML profiles and index generation

This commit is contained in:
Andrey Ponomarenko 2012-09-19 16:13:26 +04:00
parent 2c188d62cc
commit fb6d8d3561

View file

@ -1,6 +1,6 @@
#!/usr/bin/perl
########################################################
# URPM Repo Closure Checker 1.4.5 for Linux
# URPM Repo Closure Checker 1.5 for Linux
# A tool for checking closure of a set of RPM packages
#
# Copyright (C) 2011-2012 ROSA Laboratory
@ -38,12 +38,13 @@ use File::Copy qw(copy move);
use Data::Dumper;
use strict;
my $TOOL_VERSION = "1.4.5";
my $TOOL_VERSION = "1.5";
my $CmdName = get_filename($0);
my ($Help, $ShowVersion, $RPMlist, $RPMdir, $StaticMode,
$DynamicMode, $NoClean, $HDlist, $FileDeps, $ReportDir,
$AddRPMs, $RTitle, $DepHDlists, $UpdateHDlists);
$AddRPMs, $RTitle, $DepHDlists, $UpdateHDlists, $Profile,
$Target, $ExtInfo);
my $ShortUsage = "URPM Repo Closure Checker $TOOL_VERSION
A tool for checking closure of a set of RPM packages
@ -73,7 +74,10 @@ GetOptions("h|help!" => \$Help,
"report-dir=s" => \$ReportDir,
"title=s" => \$RTitle,
"dep-hdlists=s" => \$DepHDlists,
"update-hdlists=s" => \$UpdateHDlists
"update-hdlists=s" => \$UpdateHDlists,
"profile=s" => \$Profile,
"target=s" => \$Target,
"info=s" => \$ExtInfo
) or ERR_MESSAGE();
my %EXIT_CODES = (
@ -140,6 +144,58 @@ OPTIONS:
-update-hdlists <path>
The list of HDlists from update repositories.
-profile <path>
Profile of the test run.
-target <name>
Run particular test described in the profile.
-info <path>
The list of paths to info.xml.lzma files.
PROFILE FORMAT:
<repos>
<distr>
2012lts
</distr>
<name>
ROSA 2012 LTS
</name>
<arch>
i586
</arch>
<section>
contrib
</section>
<hdlist>
http://abf.rosalinux.ru/downloads/rosa2012lts/repository/i586/contrib/release/media_info/synthesis.hdlist.cz
</hdlist>
<updates>
http://abf.rosalinux.ru/downloads/rosa2012lts/repository/i586/contrib/updates/media_info/synthesis.hdlist.cz
</updates>
<deps>
http://abf.rosalinux.ru/downloads/rosa2012lts/repository/i586/main/release/media_info/synthesis.hdlist.cz
http://abf.rosalinux.ru/downloads/rosa2012lts/repository/i586/main/updates/media_info/synthesis.hdlist.cz
</deps>
<info>
http://abf.rosalinux.ru/downloads/rosa2012lts/repository/i586/contrib/release/media_info/info.xml.lzma
http://abf.rosalinux.ru/downloads/rosa2012lts/repository/i586/contrib/updates/media_info/info.xml.lzma
</info>
</repos>
<repos>
...
</repos>
EXIT CODES:
0 - Suceess. The tool has run without any errors
non-zero - Failed or the tool has run with errors. In particular:
@ -168,6 +224,38 @@ my %BrokenSignature;
my %InstallFailed;
my $RESULTS_DIR = "repoclosure_reports";
sub parseTag(@)
{
my $CodeRef = shift(@_);
my $Tag = shift(@_);
if(not $Tag or not $CodeRef) {
return undef;
}
my $Sp = 0;
if(@_) {
$Sp = shift(@_);
}
my $Start = index(${$CodeRef}, "<$Tag>");
if($Start!=-1)
{
my $End = index(${$CodeRef}, "</$Tag>");
if($End!=-1)
{
my $TS = length($Tag)+3;
my $Content = substr(${$CodeRef}, $Start, $End-$Start+$TS, "");
substr($Content, 0, $TS-1, ""); # cut start tag
substr($Content, -$TS, $TS, ""); # cut end tag
if(not $Sp)
{
$Content=~s/\A\s+//g;
$Content=~s/\s+\Z//g;
}
return $Content;
}
}
return undef;
}
sub appendFile($$)
{
my ($Path, $Content) = @_;
@ -661,6 +749,96 @@ sub readDeps($$$)
}
}
sub downloadFile($)
{
my $Path = $_[0];
my $DownloadTo = $TMP_DIR."/extract/".get_filename($Path);
$DownloadTo=~s/\.cz/\.gz/g; # cz == gz
my $Dir = get_dirname($DownloadTo);
mkdir($Dir);
system("wget -U '' --no-check-certificate \"$Path\" --connect-timeout=5 --tries=1 --output-document=\"$DownloadTo\" >/dev/null 2>&1");
if(not -f $DownloadTo
or not -s $DownloadTo)
{
print STDERR "ERROR: cannot access \'$Path\'\n";
exit(1);
}
my %Extract = (
"xz"=>"unxz",
"lzma"=>"unlzma",
"gz"=>"gunzip"
);
if($DownloadTo=~/\.(gz|xz|lzma)\Z/)
{
my ($Format, $Cmd) = ($1, $Extract{$1});
if($Cmd) {
system("cd $Dir && $Cmd $DownloadTo");
}
my @Files = cmd_find($Dir, "f", "", "");
if(not @Files)
{
print STDERR "ERROR: cannot extract \'$Path\'\n";
exit(1);
}
unlink($DownloadTo);
$DownloadTo = $Files[0];
}
return $DownloadTo;
}
sub readInfo($$)
{
my ($Path, $Info) = @_;
my $Content = "";
if($Path=~/(http|https|ftp):\/\//)
{
print "Downloading Info ...\n";
$Path = downloadFile($Path);
}
else
{
if(not -f $Path)
{
print STDERR "ERROR: cannot access \'$Path\'\n";
exit(1);
}
}
my $RPM = "";
open(INFO, $Path) || die "can't open file \'$Path\': $!\n";
while(<INFO>)
{
if(index($_, "fn=")!=-1)
{
if(/fn=\"(.+?)\"/) {
$RPM = $1;
}
else {
$RPM = "";
}
}
if($RPM)
{
if(index($_, "sourcerpm=")!=-1)
{
if(/sourcerpm=\'(.+?)\'/)
{
my $SRPM = $1;
$SRPM=~s/\.src\.rpm//g;
$SRPM=~s/\.srpm//g;
$Info->{$RPM}{"SRPM"} = $SRPM;
}
else {
$RPM = "";
}
}
}
}
close(INFO);
rmtree($TMP_DIR."/extract/");
}
sub readHDlist($$$$$)
{
my ($Path, $Dep, $RPMdep, $Registered, $Kind) = @_;
@ -669,38 +847,7 @@ sub readHDlist($$$$$)
if($Path=~/(http|https|ftp):\/\//)
{
print "Downloading $Kind HDlist ...\n";
my $DownloadTo = $TMP_DIR."/extract/".get_filename($Path);
$DownloadTo=~s/\.cz/\.gz/g; # cz == gz
my $Dir = get_dirname($DownloadTo);
mkdir($Dir);
system("wget -U '' --no-check-certificate \"$Path\" --connect-timeout=5 --tries=1 --output-document=\"$DownloadTo\" >/dev/null 2>&1");
if(not -f $DownloadTo
or not -s $DownloadTo)
{
print STDERR "ERROR: cannot access \'$Path\'\n";
exit(1);
}
my %Extract = (
"xz"=>"unxz",
"lzma"=>"unlzma",
"gz"=>"gunzip"
);
if($DownloadTo=~/\.(gz|xz|lzma)\Z/)
{
my ($Format, $Cmd) = ($1, $Extract{$1});
if($Cmd) {
system("cd $Dir && $Cmd $DownloadTo");
}
my @Files = cmd_find($Dir, "f", "", "");
if(not @Files)
{
print STDERR "ERROR: cannot extract \'$Path\'\n";
exit(1);
}
unlink($DownloadTo);
$DownloadTo = $Files[0];
}
my $DownloadTo = downloadFile($Path);
if(my $Line = readLineNum($DownloadTo, 1))
{
if($Line!~/\A\@\w+\@/)
@ -710,7 +857,6 @@ sub readHDlist($$$$$)
}
}
$Content = readFile($DownloadTo);
unlink($DownloadTo);
}
else
{
@ -777,6 +923,7 @@ sub readHDlist($$$$$)
$Registered->{$TKind}{$PkgName{$Name}} = 1;
}
}
rmtree($TMP_DIR."/extract/");
}
sub staticCheck()
@ -786,6 +933,16 @@ sub staticCheck()
print STDERR "ERROR: --hdlist, --dir or --list option should be specified\n";
exit(1);
}
my %AddInfo = ();
if($ExtInfo)
{
foreach my $Url (split(/\s+/, readFile($ExtInfo))) {
readInfo($Url, \%AddInfo);
}
}
my (%Dep, %RPMdep, %Registered) = ();
my (%Dep_D, %RPMdep_D) = ();
if($AddRPMs)
@ -953,22 +1110,34 @@ sub staticCheck()
$VReport .= "<a name='Unresolved'></a>\n";
$VReport .= "<table class='report'>\n";
$VReport .= "<tr><th>Dependency Name (".($#Ns+1).")</th><th>Required by Package</th></tr>\n";
$VReport .= "<tr>";
$VReport .= "<th onclick='javascript:sort(this)'>Dependency (".($#Ns+1).")</th><th onclick=\"sort(this)\">RPM</th>";
if($ExtInfo) {
$VReport .= "<th onclick=\"sort(this)\">SRPM</th>";
}
$VReport .= "</tr>\n";
my $Num = 1;
foreach my $N (@Ns)
{
foreach my $O (keys(%{$Unresolved{$N}}))
foreach my $O (sort keys(%{$Unresolved{$N}}))
{
foreach my $V (keys(%{$Unresolved{$N}{$O}}))
foreach my $V (sort keys(%{$Unresolved{$N}{$O}}))
{
my $Dep = showDep($N, $O, $V);
my $Pkg = $Unresolved{$N}{$O}{$V};
$Report .= $Dep." (required by $Pkg)\n";
my $Class = " class='even'";
$Class = "" if($Num++ % 2 != 0);
$VReport .= "<tr$Class><td>".htmlSpecChars($Dep)."</td><td>$Pkg</td></tr>\n";
$VReport .= "<tr$Class>";
$VReport .= "<td>".htmlSpecChars($Dep)."</td><td>$Pkg</td>";
if($ExtInfo) {
$VReport .= "<td>".$AddInfo{$Pkg}{"SRPM"}."</td>";
}
$VReport .= "</tr>\n";
}
}
}
@ -977,30 +1146,44 @@ sub staticCheck()
}
if(my @Ns = sort {lc($a) cmp lc($b)} keys(%UnresolvedSuggested))
{
if($Report) {
if($Report)
{
$Report .= "\n";
$VReport .= "<br/>\n";
}
my $Title = "Unresolved Suggests (".($#Ns+1).")";
$Report .= "\n$Title:\n\n";
$VReport .= "<table class='report'>\n";
$VReport .= "<tr><th>Dependency Name (".($#Ns+1).")</th><th>Suggested by Package</th></tr>\n";
$VReport .= "<tr>";
$VReport .= "<th onclick='javascript:sort(this)'>Suggestion (".($#Ns+1).")</th><th onclick=\"sort(this)\">RPM</th>";
if($ExtInfo) {
$VReport .= "<th onclick=\"sort(this)\">SRPM</th>";
}
$VReport .= "</tr>\n";
my $Num = 1;
foreach my $N (@Ns)
{
foreach my $O (keys(%{$UnresolvedSuggested{$N}}))
foreach my $O (sort keys(%{$UnresolvedSuggested{$N}}))
{
foreach my $V (keys(%{$UnresolvedSuggested{$N}{$O}}))
foreach my $V (sort keys(%{$UnresolvedSuggested{$N}{$O}}))
{
my $Dep = showDep($N, $O, $V);
my $Pkg = $UnresolvedSuggested{$N}{$O}{$V};
$Report .= $Dep." (required by $Pkg)\n";
my $Class = " class='even'";
$Class = "" if($Num++ % 2 != 0);
$VReport .= "<tr$Class><td>".htmlSpecChars($Dep)."</td><td>$Pkg</td></tr>\n";
$VReport .= "<tr$Class>";
$VReport .= "<td>".htmlSpecChars($Dep)."</td><td>$Pkg</td>";
if($ExtInfo) {
$VReport .= "<td>".$AddInfo{$Pkg}{"SRPM"}."</td>";
}
$VReport .= "</tr>\n";
}
}
}
@ -1021,7 +1204,7 @@ sub staticCheck()
$VReport .= "<a name='Broken'></a>\n";
$VReport .= "<table class='report'>\n";
$VReport .= "<tr><th>Broken Packages (".($#Ns+1).")</th></tr>\n";
$VReport .= "<tr><th onclick=\"sort(this)\">Broken Packages (".($#Ns+1).")</th></tr>\n";
my $Num = 1;
@ -1038,69 +1221,139 @@ sub staticCheck()
}
my $Styles = "
body {
margin: 1.5em;
color:Black;
}
h1 {
font-size: 2em;
margin-bottom:5px;
}
h2 {
font-size: 1.5em;
margin-bottom:0px;
}
hr {
color:Black;
background-color:Black;
height:1px;
border:0;
}
table.summary {
border-collapse:collapse;
border:1px outset black;
}
table.summary th {
background-color:#EEEEEE;
font-weight:100;
text-align:left;
border:1px inset gray;
padding: 3px;
}
table.summary td {
text-align:right;
border:1px inset gray;
padding: 3px 5px 3px 10px;
}
tr.even {
background-color:#CCCCCC;
}
table.report th {
border-bottom-style:double;
font-weight:bold;
text-align:center;
font-size: 1.3em;
padding:3px;
}
table.report td {
text-align:left;
padding-right:15px;
}";
body {
margin: 1.5em;
color:Black;
}
h1 {
font-size: 2em;
margin-bottom:5px;
}
h2 {
font-size: 1.5em;
margin-bottom:0px;
}
hr {
color:Black;
background-color:Black;
height:1px;
border:0;
}
table.summary {
border-collapse:collapse;
border:1px outset black;
}
table.summary th {
background-color:#EEEEEE;
font-weight:100;
text-align:left;
border:1px inset gray;
padding: 3px;
}
table.summary td {
text-align:right;
border:1px inset gray;
padding: 3px 5px 3px 10px;
}
tr.even {
background-color:#CCCCCC;
}
table.report th {
border-bottom-style:double;
font-weight:bold;
text-align:center;
font-size: 1.3em;
padding:3px;
cursor:pointer;
}
table.report td {
text-align:left;
padding-right:15px;
}";
my $Footer = "<hr/><div style='width:100%;font-size:11px;font-family:Arial;' align='right'><i>Generated on ".(localtime time);
$Footer .= " by <a href='https://abf.rosalinux.ru/import/urpm-tools/'>URPM-repoclosure</a> $TOOL_VERSION &#160;</i></div><div style='height:50px;'></div>";
$VReport = "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">
<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">
<head>
<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />
<title>
URPM-repoclosure report
</title>
<style type=\"text/css\">
$Styles
</style></head><body>".$VReport."\n".$Footer."\n</body></html>\n";
my $Scripts = "
function sort(el)
{
var col_sort = el.innerHTML;
var tr = el.parentNode;
var table = tr.parentNode;
var td, col_sort_num;
for (var i=0; (td = tr.getElementsByTagName('th').item(i)); i++)
{
if(td.innerHTML == col_sort)
{
col_sort_num = i;
if(td.prevsort == 'y') {
el.up = Number(!el.up);
}
else if(td.prevsort == 'n') {
td.prevsort = 'y';
el.up = 0;
}
else
{
if(col_sort_num==0)
{ // already sorted
td.prevsort = 'n';
el.up = 1;
}
else
{
td.prevsort = 'y';
el.up = 0;
}
}
}
else
{
if(td.prevsort == 'y') {
td.prevsort = 'n';
}
}
}
var a = new Array();
for(i=1; i < table.rows.length; i++)
{
a[i-1] = new Array();
a[i-1][0] = table.rows[i].getElementsByTagName('td').item(col_sort_num).innerHTML;
a[i-1][1] = table.rows[i];
}
a.sort(sort_array);
if(el.up) a.reverse();
for(i=0; i < a.length; i++)
table.appendChild(a[i][1]);
for (var i = 1, row; row = table.rows[i]; i++)
{
if(i % 2 == 0) {
row.className = 'even';
}
else {
row.className = '';
}
}
}
function sort_array(a,b)
{
a = a[0].toLowerCase();
b = b[0].toLowerCase();
if( a == b) return 0;
if( a > b) return 1;
return -1;
}";
my $Title = "URPM-repoclosure report";
my $Keywords = "URPM, repository, closure, report";
my $Description = "URPM-repoclosure report";
$VReport = composeHTML_Head($Title, $Keywords, $Description, $Styles, $Scripts)."<body>".$VReport."\n".$Footer."\n</body></html>\n";
# report
if($Report or $VReport)
@ -1134,6 +1387,29 @@ table.report td {
}
}
sub composeHTML_Head($$$$$)
{
my ($Title, $Keywords, $Description, $Styles, $Scripts) = @_;
return "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">
<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">
<head>
<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />
<meta name=\"keywords\" content=\"$Keywords\" />
<meta name=\"description\" content=\"$Description\" />
<title>
$Title
</title>
<style type=\"text/css\">
$Styles
</style>
<script type=\"text/javascript\" language=\"JavaScript\">
<!--
$Scripts
-->
</script>
</head>";
}
sub htmlSpecChars($)
{
my $Str = $_[0];
@ -1218,6 +1494,174 @@ sub clearCache()
}
}
sub checkProfile()
{
if(not -f $Profile)
{
print STDERR "ERROR: can't access \'$Profile\'\n";
exit(1);
}
my $Content = readFile($Profile);
my %Index = ();
my %Order = ();
my $Num = 0;
while(my $Repos = parseTag(\$Content, "repos"))
{
my %Info = ();
foreach my $Tag ("distr", "name", "arch", "section", "hdlist", "updates", "deps", "info") {
$Info{$Tag} = parseTag(\$Repos, $Tag);
}
my $Dir = $Info{"distr"}."/".$Info{"arch"}."/".$Info{"section"};
if(defined $Target and $Target ne $Dir) {
next;
}
writeFile("dep.hdlists", $Info{"deps"});
writeFile("update.hdlists", $Info{"updates"});
writeFile("info.hdlists", $Info{"info"});
my $Cmd = "perl urpm-repoclosure.pl";
$Cmd .= " --hdlist=".$Info{"hdlist"};
if($Info{"name"}) {
$Cmd .= " --title=\"".$Info{"name"}."/".$Info{"arch"}."-".$Info{"section"}."\"";
}
if($Info{"deps"}) {
$Cmd .= " --dep-hdlists=dep.hdlists";
}
if($Info{"updates"}) {
$Cmd .= " --update-hdlists=update.hdlists";
}
if($Info{"info"}) {
$Cmd .= " --info=info.hdlists";
}
$Cmd .= " --file-deps=file-deps.txt";
print "Running $Cmd\n";
system($Cmd." >/dev/null");
$Info{"HTML"} = $Dir."/report.html";
$Info{"TXT"} = $Dir."/report.txt";
mkpath($Dir);
move($RESULTS_DIR."/static/report.html", $Info{"HTML"});
move($RESULTS_DIR."/static/report.txt", $Info{"TXT"});
if(not defined $Target) {
rmtree($RESULTS_DIR);
}
unlink("dep.hdlists");
unlink("update.hdlists");
$Index{$Info{"name"}}{$Info{"arch"}}{$Info{"section"}} = \%Info;
$Order{$Info{"name"}} = $Num++;
$Order{$Info{"section"}} = $Num++;
$Order{$Info{"arch"}} = $Num++;
}
my $Styles = "
body {
margin: 1.5em;
color:Black;
}
h1 {
font-size: 2em;
margin-bottom:5px;
}
h2 {
font-size: 1.5em;
margin-bottom:10px;
}
table.contents {
border-collapse:collapse;
border:1px Black solid;
margin-left:15px;
font-family:\"Times New Roman\";
}
table.contents td {
padding:5px;
font-size:16px;
}
table.summary {
border-collapse:collapse;
border:1px Black solid;
margin-left:20px;
}
table.summary th {
background-color:#F2F2F2;
text-align:center;
font-weight:100;
padding: 3px;
border:1px Black solid;
}
table.summary td {
text-align:center;
padding: 3px;
border:1px Black solid;
}";
my $INDEX = "<h1>URPM-repoclosure report</h1>";
$INDEX .= "This report has been updated on ".strftime("%b %e %H:%M %Y", localtime(time)).".\n";
$INDEX .= "<br/>\n";
# contents
my $Contents = "<br/>\n<table class='contents'>\n";
$Contents .= "<tr><td><b>Contents</b></td></tr>\n";
foreach my $Name (keys(%Index))
{
my $Anchor = $Name;
$Anchor=~s/\s+/_/g;
$Contents .= "<tr><td><a href=\'#$Anchor\'>$Name</a></td></tr>\n";
}
$Contents .= "</table>\n";
#$INDEX .= $Contents;
foreach my $Name (sort {$Order{$a}<=>$Order{$b}} keys(%Index))
{
my $Anchor = $Name;
$Anchor=~s/\s+/_/g;
$INDEX .= "<a name=\'$Anchor\'></a>\n";
$INDEX .= "<h2>$Name</h2>\n";
$INDEX .= "<table class='summary'>\n";
$INDEX .= "<tr>\n";
$INDEX .= "<td></td>\n";
foreach my $Arch (sort {$Order{$a}<=>$Order{$b}} keys(%{$Index{$Name}}))
{
foreach my $Section (sort {$Order{$a}<=>$Order{$b}} keys(%{$Index{$Name}{$Arch}}))
{
$INDEX .= "<th>$Section</th>\n";
}
last;
}
$INDEX .= "</tr>\n";
foreach my $Arch (sort {$Order{$a}<=>$Order{$b}} keys(%{$Index{$Name}}))
{
$INDEX .= "<tr>\n";
$INDEX .= "<th>$Arch</th>\n";
foreach my $Section (sort {$Order{$a}<=>$Order{$b}} keys(%{$Index{$Name}{$Arch}}))
{
my %Info = %{$Index{$Name}{$Arch}{$Section}};
$INDEX .= "<td><a href=\'".$Info{"HTML"}."\'>Report</a> (<a href=\'".$Info{"TXT"}."\'>txt</a>)</td>\n"; # (<a href=\'".$Info{"TXT"}."\'>txt</a>)
}
$INDEX .= "</tr>\n";
}
$INDEX .= "</table>\n";
}
my $Title = "URPM-repoclosure report";
my $Keywords = "URPM, repository, closure, report";
my $Description = "URPM-repoclosure report";
$INDEX = composeHTML_Head($Title, $Keywords, $Description, $Styles, "")."<body>".$INDEX."\n</body></html>\n";
writeFile("index.html", $INDEX);
print "index.html has been created\n";
}
sub scenario()
{
if($Help)
@ -1230,6 +1674,13 @@ sub scenario()
print "URPM Repo Closure Checker $TOOL_VERSION\nCopyright (C) 2012 ROSA Laboratory\nLicense: GPL <http://www.gnu.org/licenses/>\nThis program is free software: you can redistribute it and/or modify it.\n\nWritten by Andrey Ponomarenko.\n";
exit(0);
}
if($Profile)
{
checkProfile();
exit(0);
}
if($HDlist) {
$StaticMode = 1;
}