diff -Naur rpmtools-6.1.orig/genhdlist2 rpmtools-6.1/genhdlist2 --- rpmtools-6.1.orig/genhdlist2 2014-09-25 04:42:56.529840196 -0400 +++ rpmtools-6.1/genhdlist2 2014-10-01 06:59:19.830021324 -0400 @@ -1,10 +1,14 @@ #!/usr/bin/perl +eval 'exec /usr/bin/perl -S $0 ${1+"$@"}' + if 0; # not running under some shell + (our $VERSION) = q(Id: genhdlist2 20460 2006-11-23 13:19:11Z pixel ) =~ /(\d+\.\d+)/; use URPM; use MDV::Packdrakeng; use Getopt::Long; +use XML::LibXML qw( ); main(); @@ -26,6 +30,7 @@ 'no-bad-rpm' => \$options{no_bad_rpm}, 'no-md5sum' => \$options{no_md5sum}, 'no-clean-old-rpms' => \$options{no_clean_old_rpms}, + 'merge' => \$options{merge}, 'only-clean-old-rpms' => \$options{only_clean_old_rpms}, 'nolock' => \$options{nolock}, 'no-hdlist' => \$options{no_hdlist}, @@ -45,6 +50,11 @@ @ARGV == 1 or usage(); my $rpms_dir = $ARGV[0]; + if ($options{merge} && !$options{no_hdlist}) { + print "--merge implies --no-hdlist!\n"; + $options{no_hdlist} = 1; + } + $options{no_incremental} ||= $options{no_hdlist}; # Force locale to be C @@ -83,13 +93,12 @@ my $lock = !$options{nolock} && global_lock($lock_file); $SIG{INT} = sub { - unlink "$media_info_dir/hdlist.cz.tmp", "$synthesis.tmp", $tmp_header; + unlink "$media_info_dir/hdlist.cz.tmp", "$synthesis.tmp", $tmp_header, "files.xml.tmp", "changelog.xml.tmp", "info.xml.tmp"; unlink $lock_file if $lock; exit 1; }; END { unlink $lock_file if $lock } - #- handle old-rpms.lst my $old_rpms_file = "$media_info_dir/old-rpms.lst"; my $old_rpms = read_old_rpms_lst($old_rpms_file, $options{nolock}); @@ -102,8 +111,21 @@ } $options{only_clean_old_rpms} and return; - - my %rpms_todo = map { /(.*)\.rpm/ => 1 } @rpms; + my %rpms_todo = (); + if ($options{merge} && -e "$media_info_dir/new-metadata.lst") { + my $new_metadata_file = "$media_info_dir/new-metadata.lst"; + open(N, "$media_info_dir/new-metadata.lst"); + while() { + chomp; + my $pkg = $_; + $pkg =~ s/\.rpm$//; + $rpms_todo{$pkg} = 1; + } + close(N); + } + elsif (!$options{merge}) { + %rpms_todo = map { /(.*)\.rpm/ => 1 } @rpms; + } my $urpm = new URPM; @@ -114,16 +136,69 @@ } my @xml_media_info = $options{xml_info} ? ('info', 'files', 'changelog') : (); - build($urpm, \%rpms_todo, $media_info_dir, $rpms_dir, \@xml_media_info, $options{no_incremental}, $options{no_hdlist}, $xml_info_suffix, $xml_info_filter); + #- handle old-metadata.lst - drop packages listed there from metadata + if ($options{merge} ) { + # Unpack all files to be processed + my @files_to_clean = ("synthesis.hdlist$synthesis_suffix", map { "$_.xml$xml_info_suffix" } @xml_media_info); + foreach my $name (@xml_media_info) { + system("rm -f $media_info_dir/$name.xml"); + system("$xml_info_filter -d -S $xml_info_suffix $media_info_dir/$name.xml$xml_info_suffix || echo -e '\n' > $media_info_dir/$name.xml") == 0 or die("Failed to unpack xml metadata files"); + } + system("rm -f $media_info_dir/synthesis.hdlist"); + system("$synthesis_filter -d -S $synthesis_suffix $media_info_dir/synthesis.hdlist$synthesis_suffix || touch $media_info_dir/synthesis.hdlist") == 0 or die("Failed to unpack synthesis file"); + + my $old_metadata_file = "$media_info_dir/old-metadata.lst"; + if (-e $old_metadata_file) { + print "removing obsolete metadata...\n" if $verbose >= 0; + filter_out_old_metadata($media_info_dir, $old_metadata_file, "$media_info_dir/synthesis.hdlist"); + + # Rename the temporary metadata files + foreach my $name (@xml_media_info) { + rename "$media_info_dir/$name.xml.tmp", "$media_info_dir/$name.xml" or die "rename $media_info_dir/$name.xml.tmp failed: $!\n"; + } + rename "$media_info_dir/synthesis.hdlist.tmp", "$media_info_dir/synthesis.hdlist" or die "rename $media_info_dir/$name.xml.tmp failed: $!\n"; + } + } + + build($urpm, \%rpms_todo, $media_info_dir, $rpms_dir, \@xml_media_info, $options{no_incremental}, $options{no_hdlist}, $xml_info_suffix, $xml_info_filter, $options{merge}); build_synthesis($urpm, "$synthesis.tmp", $synthesis_filter); if (1) { my @media_info_files = ($options{no_hdlist} ? () : 'hdlist.cz', "synthesis.hdlist$synthesis_suffix", map { "$_.xml$xml_info_suffix" } @xml_media_info); foreach my $name (@media_info_files) { - print "replacing $media_info_dir/$name with $name.tmp\n" if $verbose >= 0; - rename "$media_info_dir/$name.tmp", "$media_info_dir/$name" or die "rename $media_info_dir/$name failed: $!\n"; + if ($options{merge} ) { + print "Merging metadata ($name)...\n"; + if ($name =~/^synthesis.hdlist/) { + os.system("$synthesis_filter -dc -S $synthesis_suffix $media_info_dir/$name.tmp >> $media_info_dir/synthesis.hdlist" ); + } + else { + my $clear_name = $name; + $clear_name =~ s/$xml_info_suffix//; + os.system("sed -i 's/<\\/media_info>//' $media_info_dir/$clear_name") == 0 or die("Failed to drop leading tag from $clear_name"); + # If we have an empty tag, replace it with opening tag + os.system("sed -i 's///' $media_info_dir/$clear_name") == 0 or die("Failed to drop leading tag from $clear_name"); + # A hack - remove last EOL at EOF + os.system("perl -0 -pe 's/\\n\\Z//' -i $media_info_dir/$clear_name"); + os.system("$xml_info_filter -dc -S $xml_info_suffix $media_info_dir/$name.tmp >> $media_info_dir/$clear_name" ) == 0 or die("Failed to concat $name.tmp with $clear_name"); + } + unlink "$media_info_dir/$name.tmp"; + } + else { + print "replacing $media_info_dir/$name with $name.tmp\n" if $verbose >= 0; + rename "$media_info_dir/$name.tmp", "$media_info_dir/$name" or die "rename $media_info_dir/$name failed: $!\n"; + } } + + if ($options{merge} ) { + filter_out_duplicates($media_info_dir, "$media_info_dir/synthesis.hdlist"); + # Pack the metadata files + foreach my $name (@xml_media_info) { + system("$xml_info_filter -S $xml_info_suffix $media_info_dir/$name.xml") == 0 or die("Failed to pack xml metadata files"); + } + system("$synthesis_filter -S $synthesis_suffix $media_info_dir/synthesis.hdlist") == 0 or die("Failed to pack synthesis file"); + } + my $existed = remove_versioned_media_info($media_info_dir); if ($options{versioned} && ($options{versioned} ne 'auto' || $existed)) { push @media_info_files, generate_versioned_media_info($media_info_dir, \@media_info_files); @@ -161,9 +236,10 @@ } sub build { - my ($urpm, $rpms_todo, $media_info_dir, $rpms_dir, $xml_media_info, $b_no_incremental, $b_no_hdlist, $xml_info_suffix, $xml_info_filter) = @_; + my ($urpm, $rpms_todo, $media_info_dir, $rpms_dir, $xml_media_info, $b_no_incremental, $b_no_hdlist, $xml_info_suffix, $xml_info_filter, $b_merge) = @_; my $hdlist = "$media_info_dir/hdlist.cz"; + my $dump_header = $b_merge ? 0 : 1; my $out_hdlist; if (!$b_no_hdlist) { @@ -177,7 +253,7 @@ my $out = { hdlist => $out_hdlist, - map { $_ => open_xml_filter("$media_info_dir/$_.xml${xml_info_suffix}.tmp", $xml_info_filter) } @$xml_media_info + map { $_ => open_xml_filter("$media_info_dir/$_.xml${xml_info_suffix}.tmp", $xml_info_filter, $dump_header) } @$xml_media_info }; if (-e $hdlist && !$b_no_incremental) { @@ -185,7 +261,7 @@ filter_existing_hdlist($urpm, $rpms_todo, $hdlist, $out); } - add_new_rpms_to_hdlist($urpm, $rpms_todo, $out, $rpms_dir); + add_new_rpms_to_hdlist($urpm, $rpms_todo, $out, $rpms_dir, $b_merge); close_xml($out->{$_}) foreach @$xml_media_info; } @@ -245,12 +321,14 @@ } sub open_xml_filter { - my ($file, $xml_info_filter) = @_; + my ($file, $xml_info_filter, $dump_header) = @_; open(my $F, "| $xml_info_filter > $file") or die "can't open $file\n"; binmode $F, ':utf8'; - print $F qq(\n); - print $F ""; + if ($dump_header) { + print $F qq(\n); + print $F ""; + } $F; } @@ -503,6 +581,138 @@ @$rpms_list = grep { !$old{$_} } @$rpms_list; } + +sub filter_out_old_metadata { + my ($media_info_dir, $old_metadata_file, $synthesis_name) = @_; + + open(M, "cat $old_metadata_file | sed s/\.rpm\$// |"); + my @to_remove = ; + close M or die "Cannot close $old_metadata_file: $!"; + chomp(@to_remove); + + my %tags = ( "$media_info_dir/files.xml" => "files", + "$media_info_dir/changelog.xml" => "changelogs", + "$media_info_dir/info.xml" => "info" + ); + + for my $file ("$media_info_dir/files.xml", "$media_info_dir/changelog.xml", "$media_info_dir/info.xml") { + my %processed_entries = {}; + my $parser = XML::LibXML->new(); + my $doc = $parser->parse_file($file); + my $top_tag = $tags{$file}; + + for my $node ($doc->findnodes("/media_info/$top_tag")) { + if ($processed_entries{$node->getAttribute("fn")}) { + my $parent = $node->parentNode; + $parent->removeChild($node); + next; + } + $processed_entries{$node->getAttribute("fn")} = 1; + if (grep {$_ eq $node->getAttribute("fn")} @to_remove) { + my $parent = $node->parentNode; + $parent->removeChild($node); + } + } + + if (-e "$file.tmp") { + unlink "$file.tmp"; + } + $doc->toFile($file.".tmp"); + } + + open(S, "< $synthesis_name"); + open(NS, "> $synthesis_name.tmp"); + + my $buff = ""; + my %processed = {}; + while() { + my $line = $_; + if (!/^\@info/ ) { + $buff .= $line; + next; + } + if (/\@info\@([^@]+)\@/) { + # Remove possible duplicates + if ($processed{$_}) { + $buff = ""; + next; + } + $processed{$_} = 1; + my $name = $1; + print $name."\n"; + if (! grep(/^$name$/, @to_remove) ) { + print NS $buff; + print NS $line; + } + $buff = ""; + } + } + close S or die "Cannot close $synthesis_name: $!"; + close NS or die "Cannot close $synthesis_name.tmp: $!"; +} + +# With --merge option, duplicated records can occur in case of interrupts. +# Let's remove them for safety. +sub filter_out_duplicates { + my ($media_info_dir, $synthesis_name) = @_; + + my %tags = ( "$media_info_dir/files.xml" => "files", + "$media_info_dir/changelog.xml" => "changelogs", + "$media_info_dir/info.xml" => "info" + ); + + for my $file ("$media_info_dir/files.xml", "$media_info_dir/changelog.xml", "$media_info_dir/info.xml") { + my %processed_entries = {}; + my $parser = XML::LibXML->new(); + my $doc = $parser->parse_file($file); + my $top_tag = $tags{$file}; + + for my $node ($doc->findnodes("/media_info/$top_tag")) { + if ($processed_entries{$node->getAttribute("fn")}) { + my $parent = $node->parentNode; + $parent->removeChild($node); + next; + } + $processed_entries{$node->getAttribute("fn")} = 1; + } + + if (-e "$file.tmp") { + unlink "$file.tmp"; + } + $doc->toFile($file.".tmp"); + + rename "$file.tmp", "$file" or die "rename $file.tmp failed: $!\n"; + } + + open(S, "< $synthesis_name"); + open(NS, "> $synthesis_name.tmp"); + + my $buff = ""; + my %processed = {}; + while() { + my $line = $_; + if (!/^\@info/ ) { + $buff .= $line; + next; + } + if (/\@info\@([^@]+)\@/) { + # Remove possible duplicates + if ($processed{$_}) { + $buff = ""; + next; + } + $processed{$_} = 1; + print NS $buff; + print NS $line; + $buff = ""; + } + } + close S or die "Cannot close $synthesis_name: $!"; + close NS or die "Cannot close $synthesis_name.tmp: $!"; + + rename "$synthesis_name.tmp", "$synthesis_name" or die "rename $synthesis_name.tmp failed: $!\n"; +} + ################################################################################