Restructure repo.db; Support multiple dependencies for a requirement

This commit is contained in:
Alexander Lakhin 2014-02-10 14:16:01 +04:00
parent 3d3f873e57
commit 38966cb244
4 changed files with 171 additions and 147 deletions

View file

@ -91,10 +91,10 @@ SELECT id, nvra, version, release FROM packages WHERE repodir_id IN (%s) AND nam
in_packages = ','.join(str(id) for id in package_ids) in_packages = ','.join(str(id) for id in package_ids)
for required_package in c.execute(""" for required_package in c.execute("""
SELECT packages.id, packages.name, nvra SELECT packages.id, packages.name, nvra
FROM packages, requires FROM packages, package_depend_res
WHERE packages.id = requires.dep_package_id AND WHERE packages.id = package_depend_res.dep_package_id AND
package_depend_res.package_id IN (%s) AND
packages.repodir_id IN (%s) AND packages.repodir_id IN (%s) AND
requires.package_id IN (%s) AND
packages.id NOT IN (%s) packages.id NOT IN (%s)
ORDER BY packages.name ORDER BY packages.name
""" % (in_repodirs, in_packages, in_packages)): """ % (in_repodirs, in_packages, in_packages)):

View file

@ -29,10 +29,14 @@ def detect_broken_dependencies(dbc):
repodirs_analyzed = [] repodirs_analyzed = []
broken = {} broken = {}
broken_level0 = dbc.execute(""" broken_level0 = dbc.execute("""
SELECT packages.id, nvra, repodir_id, repodirs.name, requires.name AS req_name, requires.build_arch AS req_arch SELECT packages.id, nvra, repodir_id, repodirs.name,
FROM packages, repodirs, requires rpm_requires.name AS req_name, rpm_requires.build_arch AS req_arch
WHERE packages.repodir_id = repodirs.id AND packages.id=requires.package_id AND FROM packages, repodirs, rpm_requires
dep_package_id IS NULL WHERE packages.repodir_id = repodirs.id AND
packages.id=rpm_requires.package_id AND
NOT EXISTS (SELECT 1 FROM package_depend_res pdr
WHERE pdr.package_id = packages.id AND
pdr.requires_id = rpm_requires.id)
ORDER BY repodir_id, nvra""").fetchall() ORDER BY repodir_id, nvra""").fetchall()
all_broken = {} all_broken = {}
@ -75,16 +79,22 @@ SELECT packages.id, nvra, repodir_id, repodirs.name, requires.name AS req_name,
while all_broken_cnt < len(all_broken): while all_broken_cnt < len(all_broken):
all_broken_cnt = len(all_broken) all_broken_cnt = len(all_broken)
pids = ','.join(str(id) for id in all_broken.keys()) pids = ','.join(str(id) for id in all_broken.keys())
#print pids
packages_broken_recurs = dbc.execute(""" packages_broken_recurs = dbc.execute("""
SELECT packages.id, nvra, repodir_id, repodirs.name, requires.name AS req_name, build_arch, dep_package_id SELECT packages.id, nvra, repodir_id, repodirs.name,
FROM packages, repodirs, requires rpm_requires.name AS req_name, build_arch, dep_package_id
WHERE packages.repodir_id = repodirs.id AND packages.id=requires.package_id AND FROM packages, repodirs, rpm_requires, package_depend_res
dep_package_id IN (%(pids)s) AND packages.id NOT IN (%(pids)s) WHERE packages.repodir_id = repodirs.id AND
packages.id = rpm_requires.package_id AND
packages.id = package_depend_res.package_id AND
rpm_requires.id = package_depend_res.requires_id AND
dep_package_id IN (%(pids)s) AND
packages.id NOT IN (%(pids)s)
ORDER BY repodir_id, nvra""" % {'pids': pids}).fetchall() ORDER BY repodir_id, nvra""" % {'pids': pids}).fetchall()
# print len(packages_broken_recurs) # print len(packages_broken_recurs)
for packb in packages_broken_recurs: for packb in packages_broken_recurs:
all_broken[packb[0]] = {'repo': packb[2], 'nvra': packb[1], 'reqname': packb[4], 'build_arch': packb[5], 'depid': packb[6]} all_broken[packb[0]] = {'repo': packb[2], 'nvra': packb[1],
'reqname': packb[4], 'build_arch': packb[5],
'depid': packb[6]}
broken_recursive.append(packb[0]) broken_recursive.append(packb[0])
#print len(all_broken.keys()) #print len(all_broken.keys())
@ -96,7 +106,8 @@ SELECT id, name, sources FROM repodirs ORDER BY id""").fetchall()
(rd_id, rd_name, rd_sources) = (rd[0], rd[1], rd[2]) (rd_id, rd_name, rd_sources) = (rd[0], rd[1], rd[2])
if rd_sources == '.': if rd_sources == '.':
archs = dbc.execute(""" archs = dbc.execute("""
SELECT DISTINCT build_arch FROM requires WHERE package_id IN (SELECT id FROM packages WHERE repodir_id = ?) SELECT DISTINCT build_arch FROM rpm_requires
WHERE package_id IN (SELECT id FROM packages WHERE repodir_id = ?)
""", [rd_id]).fetchall() """, [rd_id]).fetchall()
for arch_rec in archs: for arch_rec in archs:
repo_packages_broken = {all_broken[id]['nvra']: id for id in broken_recursive \ repo_packages_broken = {all_broken[id]['nvra']: id for id in broken_recursive \
@ -110,18 +121,6 @@ SELECT DISTINCT build_arch FROM requires WHERE package_id IN (SELECT id FROM pac
print '%d) %s' % (rd_id, rd_name) print '%d) %s' % (rd_id, rd_name)
print_broken_packages() print_broken_packages()
#all_repodirs = dbc.execute("""
#SELECT id, name FROM repodirs ORDER BY id""").fetchall()
#for rd in all_repodirs:
#(rd_id, rd_name) = (rd[0], rd[1])
#repo_broken_recurs = dbc.execute("""
#SELECT packages.id, nvra, requires.name AS req_name
#FROM packages, requires
#WHERE packages.repodir_id = ? AND packages.id=requires.package_id AND
#dep_package_id IS NULL AND packages.id NOT IN()
#""").fetchall()
#print rd_name
def detect_lost_sources(dbc): def detect_lost_sources(dbc):
print '===' print '==='
@ -180,20 +179,20 @@ SELECT id, name, nvra
part_packages = ','.join(str(id) for id in partition_ids) part_packages = ','.join(str(id) for id in partition_ids)
upper_packages = dbc.execute(""" upper_packages = dbc.execute("""
SELECT packages.id, packages.name, nvra SELECT packages.id, packages.name, nvra
FROM packages, requires FROM packages, package_depend_res
WHERE packages.id = requires.package_id AND WHERE packages.id = package_depend_res.package_id AND
packages.repodir_id = ? AND packages.repodir_id = ? AND
requires.dep_package_id IN (%s) AND package_depend_res.dep_package_id IN (%s) AND
packages.id NOT IN (%s) packages.id NOT IN (%s)
ORDER BY packages.name ORDER BY packages.name
""" % (cl_packages, part_packages), [rd_id]).fetchall() """ % (cl_packages, part_packages), [rd_id]).fetchall()
lower_packages = dbc.execute(""" lower_packages = dbc.execute("""
SELECT packages.id, packages.name, nvra SELECT packages.id, packages.name, nvra
FROM packages, requires FROM packages, package_depend_res
WHERE packages.id = requires.dep_package_id AND WHERE packages.id = package_depend_res.dep_package_id AND
packages.repodir_id = ? AND packages.repodir_id = ? AND
requires.package_id IN (%s) AND package_depend_res.package_id IN (%s) AND
packages.id NOT IN (%s) packages.id NOT IN (%s)
ORDER BY packages.name ORDER BY packages.name
""" % (cl_packages, part_packages), [rd_id]).fetchall() """ % (cl_packages, part_packages), [rd_id]).fetchall()

View file

@ -69,9 +69,14 @@ CREATE TABLE repodir_depends(id INTEGER PRIMARY KEY NOT NULL,
repodir_id INTEGER, depend_repodir_name TEXT)""") repodir_id INTEGER, depend_repodir_name TEXT)""")
conn.execute(""" conn.execute("""
CREATE TABLE IF NOT EXISTS package_files(id INTEGER PRIMARY KEY NOT NULL, CREATE TABLE IF NOT EXISTS package_files(id INTEGER PRIMARY KEY NOT NULL,
package_id INTEGER NOT NULL, basename TEXT, path TEXT, size INTEGER, mode INTEGER, package_id INTEGER NOT NULL, basename TEXT, path TEXT,
size INTEGER, mode INTEGER,
link_to_file_id INTEGER, link_to_path TEXT, mark TEXT)""") link_to_file_id INTEGER, link_to_path TEXT, mark TEXT)""")
conn.execute(""" conn.execute("""
CREATE TABLE package_depend_res(id INTEGER PRIMARY KEY NOT NULL,
package_id INTEGER, requires_id INTEGER,
provides_id INTEGER, dep_package_id INTEGER)""")
conn.execute("""
CREATE TABLE so_needed(id INTEGER PRIMARY KEY NOT NULL, CREATE TABLE so_needed(id INTEGER PRIMARY KEY NOT NULL,
obj_file_id INTEGER, name TEXT)""") obj_file_id INTEGER, name TEXT)""")
conn.execute(""" conn.execute("""
@ -88,37 +93,44 @@ CREATE TABLE obj_symbols_res(id INTEGER PRIMARY KEY NOT NULL,
def index_database(conn): def index_database(conn):
print 'Indexing the database...' print 'Indexing the database...'
conn.execute('CREATE INDEX IF NOT EXISTS rd_name ON repodirs(name)') conn.executescript("""
conn.execute('CREATE INDEX IF NOT EXISTS pkg_name ON packages(name)') CREATE INDEX rd_name ON repodirs(name);
conn.execute('CREATE INDEX IF NOT EXISTS pkg_arch ON packages(nvra)') CREATE INDEX pkg_name ON packages(name);
conn.execute('CREATE INDEX IF NOT EXISTS pkg_arch ON packages(arch)') CREATE INDEX pkg_nvra ON packages(nvra);
conn.execute('CREATE INDEX IF NOT EXISTS pkg_group ON packages(rpm_group)') CREATE INDEX pkg_arch ON packages(arch);
conn.execute('CREATE INDEX IF NOT EXISTS pkg_repodir ON packages(repodir_id)') CREATE INDEX pkg_group ON packages(rpm_group);
conn.execute('CREATE INDEX IF NOT EXISTS pkg_file_pkg_id ON package_files(package_id)') CREATE INDEX pkg_repodir ON packages(repodir_id);
conn.execute('CREATE INDEX IF NOT EXISTS pkg_file_name ON package_files(basename)') CREATE INDEX pkg_dr_pkg_req ON package_depend_res(package_id, requires_id);
conn.execute('CREATE INDEX IF NOT EXISTS pkg_file_path ON package_files(path)') CREATE INDEX pkg_dr_pkg_prov ON package_depend_res(dep_package_id, provides_id);
conn.execute('CREATE INDEX IF NOT EXISTS pkg_file_mark ON package_files(mark)') CREATE INDEX pkg_file_pkg_id ON package_files(package_id);
conn.execute('CREATE INDEX IF NOT EXISTS so_needed_obj_id ON so_needed(obj_file_id)') CREATE INDEX pkg_file_name ON package_files(basename);
conn.execute('CREATE INDEX IF NOT EXISTS so_needed_res_sn ON so_needed_res(so_needed_id)') CREATE INDEX pkg_file_path ON package_files(path);
conn.execute('CREATE INDEX IF NOT EXISTS symbols_obj_name_type ON obj_symbols(obj_file_id, name, sym_type)') CREATE INDEX pkg_file_mark ON package_files(mark);
conn.execute('CREATE INDEX IF NOT EXISTS symbols_name_type ON obj_symbols(name, sym_type)') CREATE INDEX so_needed_obj_id ON so_needed(obj_file_id);
conn.execute('CREATE INDEX IF NOT EXISTS symbols_res_sym ON obj_symbols_res(obj_sym_id)') CREATE INDEX so_needed_res_sn ON so_needed_res(so_needed_id);
dep_tables = ['requires', 'provides', 'conflicts', 'obsoletes'] CREATE INDEX symbols_obj_name_type ON obj_symbols(obj_file_id, name, sym_type);
CREATE INDEX symbols_name_type ON obj_symbols(name, sym_type);
CREATE INDEX symbols_res_sym ON obj_symbols_res(obj_sym_id);
""")
dep_tables = ['rpm_requires', 'rpm_provides',
'rpm_conflicts', 'rpm_obsoletes']
for table in dep_tables: for table in dep_tables:
conn.execute('CREATE INDEX IF NOT EXISTS %(tbl)s_pkg ON %(tbl)s(package_id)' % {'tbl': table}) conn.execute('CREATE INDEX %(tbl)s_pkg ON %(tbl)s(package_id)' %
conn.execute('CREATE INDEX IF NOT EXISTS %(tbl)s_name ON %(tbl)s(name)' % {'tbl': table}) {'tbl': table})
conn.execute('CREATE INDEX IF NOT EXISTS %(tbl)s_dep_pkg ON %(tbl)s(dep_package_id)' % {'tbl': table}) conn.execute('CREATE INDEX %(tbl)s_name ON %(tbl)s(name)' %
{'tbl': table})
conn.commit() conn.commit()
def add_repodir(xrepodir, conn): def add_repodir(xrepodir, conn):
dbc = conn.cursor() dbc = conn.cursor()
dbc.execute("""INSERT INTO repodirs (name, path, sources) VALUES (?, ?, ?)""", dbc.execute("""
[xrepodir.get('name'), xrepodir.get('path'), xrepodir.get('sources')]) INSERT INTO repodirs (name, path, sources) VALUES (?, ?, ?)
""", [xrepodir.get('name'), xrepodir.get('path'), xrepodir.get('sources')])
repodir_id = dbc.lastrowid repodir_id = dbc.lastrowid
for depend in xrepodir.findall('dependency'): for depend in xrepodir.findall('dependency'):
dbc.execute("""INSERT INTO repodir_depends(repodir_id, depend_repodir_name) dbc.execute("""
VALUES (?, ?)""", INSERT INTO repodir_depends(repodir_id, depend_repodir_name) VALUES (?, ?)
[repodir_id, depend.text.strip()]) """, [repodir_id, depend.text.strip()])
conn.commit() conn.commit()
return repodir_id return repodir_id
@ -307,11 +319,10 @@ def process_package_worker(num, queue_in, generator, gen_lock, db_struct,
data['packages'].append(package_values) data['packages'].append(package_values)
for table in db_struct['dep_tables']: for table in db_struct['dep_tables']:
table_data = data[table] table_data = data[table]
rpref = 'RPMTAG_' + table[:-1].upper() rpref = 'RPMTAG_' + table[4 : -1].upper() # rpm_requires
(dep_name, dep_flags, dep_version) = \ (dep_name, dep_flags, dep_version) = \
(hdr[rpref + 'NAME'], hdr[rpref + 'FLAGS'], hdr[rpref + 'VERSION']) (hdr[rpref + 'NAME'], hdr[rpref + 'FLAGS'], hdr[rpref + 'VERSION'])
for i in xrange(0, len(hdr[rpref + 'NAME'])): for i in xrange(0, len(hdr[rpref + 'NAME'])):
#print pkg, table, hdr[rpref + 'NAME'][i]
for build_arch in build_archs: for build_arch in build_archs:
table_data.append([dep_name[i].decode('utf-8'), table_data.append([dep_name[i].decode('utf-8'),
dep_flags[i], dep_flags[i],
@ -448,8 +459,8 @@ def process_repodir(repodir_path, repodir_id, build_archs, conn, db_struct,
dep_tags_re = r'^RPMTAG_(CONFLICT|OBSOLETE|PROVIDE|REQUIRE)\w+' dep_tags_re = r'^RPMTAG_(CONFLICT|OBSOLETE|PROVIDE|REQUIRE)\w+'
db_struct['dep_tables'] = ['requires', 'provides', db_struct['dep_tables'] = ['rpm_requires', 'rpm_provides',
'conflicts', 'obsoletes'] 'rpm_conflicts', 'rpm_obsoletes']
packages_field_names = 'id, ' packages_field_names = 'id, '
packages_values_template = '?,' packages_values_template = '?,'
@ -487,7 +498,7 @@ CREATE TABLE IF NOT EXISTS packages(id INTEGER PRIMARY KEY NOT NULL, %s)
conn.execute(""" conn.execute("""
CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY NOT NULL, CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY NOT NULL,
name TEXT, flags INTEGER, version TEXT, build_arch TEXT, name TEXT, flags INTEGER, version TEXT, build_arch TEXT,
package_id INTEGER NOT NULL, dep_package_id INTEGER)""" % (table)) package_id INTEGER NOT NULL)""" % (table))
conn.commit() conn.commit()
db_struct['packages_tags'] = packages_tags db_struct['packages_tags'] = packages_tags
db_struct['packages_field_names'] = packages_field_names db_struct['packages_field_names'] = packages_field_names

View file

@ -6,7 +6,6 @@ import sys
import gettext import gettext
import argparse import argparse
import sqlite3 import sqlite3
import string
import re import re
from versutils import * from versutils import *
@ -18,10 +17,10 @@ def process_repodir_requires(dbc, repodir_id, repodir_name, repodir_depends, req
print 'Processing repo %d: %s (with depends: %s)' % (repodir_id, repodir_name, str(repodir_depends)) print 'Processing repo %d: %s (with depends: %s)' % (repodir_id, repodir_name, str(repodir_depends))
package_requires = dbc.execute(""" package_requires = dbc.execute("""
SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra, SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra,
requires.id, requires.name, flags, requires.version req.id, req.name, flags, req.version
FROM packages, requires FROM packages, rpm_requires req
WHERE repodir_id = ? AND requires.package_id = packages.id %s WHERE repodir_id = ? AND req.package_id = packages.id %s
ORDER BY packages.name, requires.name ORDER BY packages.name, req.name
""" % ((" AND build_arch = '%s'" % requires_build_arch) """ % ((" AND build_arch = '%s'" % requires_build_arch)
if requires_build_arch is not None else ""), if requires_build_arch is not None else ""),
[repodir_id]).fetchall() [repodir_id]).fetchall()
@ -37,84 +36,90 @@ SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra,
for packreq in package_requires: for packreq in package_requires:
(cpackage_id, package_nvra, requires_id, requires_name, requires_flags, requires_version) = \ (cpackage_id, package_nvra, requires_id, requires_name, requires_flags, requires_version) = \
(packreq[0], packreq[2], packreq[3], packreq[4], packreq[5], packreq[6]) (packreq[0], packreq[2], packreq[3], packreq[4], packreq[5], packreq[6])
# if requires_name != '/usr/sbin/glibc-post-wrapper':
# continue
# print packreq
requirement_uid = requires_name + '\0' + str(requires_flags) + '\0' + requires_version requirement_uid = requires_name + '\0' + str(requires_flags) + '\0' + requires_version
pkg_id = requires_cache.get(requirement_uid) dep_res = requires_cache.get(requirement_uid, None)
if pkg_id is None: if dep_res is None:
dep_res = []
if (re.match(r'\A(rpmlib|executable)\(.+\)\Z', requires_name)): if (re.match(r'\A(rpmlib|executable)\(.+\)\Z', requires_name)):
# see if($N=~/\A(rpmlib|executable)\(.+\)\Z/) in urpm_repoclosure.pl # see if($N=~/\A(rpmlib|executable)\(.+\)\Z/) in urpm_repoclosure.pl
pkg_id = -1 dep_res.append({})
if pkg_id is None:
depend_candidates = dbc.execute("""
SELECT packages.id AS package_id, packages.name AS package_name,
provides.id, provides.name, flags, provides.version
FROM packages, provides
WHERE provides.package_id = packages.id AND repodir_id IN (%s) AND provides.name = ?
""" % in_repodirs, [requires_name]).fetchall()
provides_found_pkg_id = None
provides_found_id = None
provides_found_version = None
for dep_cand in depend_candidates:
(provides_pkg_id, provides_id, provides_flags, provides_version) = \
(dep_cand[0], dep_cand[2], dep_cand[4], dep_cand[5])
#print 'provides_version: ', provides_flags, ' ', provides_version
if provides_flags & RPMSENSE_SENSEMASK == 0:
if not provides_version:
provides_version = '*'
else:
raise Exception('Invalid provides version (flags = %d, version = %s)!' %
(provides_flags, provides_version))
if version_ok(requires_version,
requires_flags & RPMSENSE_SENSEMASK,
provides_version):
better_version = provides_found_version is None or \
provides_version == '*'
if not better_version:
better_version = version_ok(provides_version,
RPMSENSE_GREATER,
provides_found_version)
if better_version:
(provides_found_pkg_id,
provides_found_id,
provides_found_version) = \
(provides_pkg_id,
provides_id,
provides_version)
pkg_id = provides_found_pkg_id
#print "->", provides_found_version, ' ->', provides_found_pkg_id
if pkg_id is None and (requires_flags & RPMSENSE_MISSINGOK) != 0:
pkg_id = -1 # TODO: Fix for valid foreign key
if pkg_id is None and requires_name.startswith('/'): # file dependency
if (requires_flags & (RPMSENSE_SCRIPT_POST |
RPMSENSE_SCRIPT_PREUN |
RPMSENSE_SCRIPT_POSTUN)) != 0:
internal_files = dbc.execute("""
SELECT 1 FROM package_files WHERE package_id = ? AND path = ?
""", [cpackage_id, requires_name]).fetchall()
if len(internal_files) > 0:
pkg_id = cpackage_id
else: else:
depend_candidates = dbc.execute("""
SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra,
prov.id, prov.name, flags, prov.version
FROM packages, rpm_provides AS prov
WHERE prov.package_id = packages.id AND repodir_id IN (%s) AND prov.name = ?
ORDER by packages.name, packages.nvra
""" % in_repodirs, [requires_name]).fetchall()
preferred_version = None
for dep_cand in depend_candidates:
(provides_flags, provides_version) = \
(dep_cand[5], dep_cand[6])
#print 'provides_version: ', provides_flags, ' ', provides_version
if provides_flags & RPMSENSE_SENSEMASK == 0:
if not provides_version:
provides_version = '*'
else:
raise Exception('Invalid provides version '
'(flags = %d, version = %s)!' %
(provides_flags, provides_version))
if version_ok(requires_version,
requires_flags & RPMSENSE_SENSEMASK,
provides_version):
better_version = preferred_version is None or \
provides_version == '*'
if not better_version:
better_version = version_ok(provides_version,
RPMSENSE_GREATER,
preferred_version)
if better_version:
preferred_version = provides_version
if preferred_version is not None:
for dep_cand in depend_candidates:
(pkg_id, provides_id, provides_version) = \
(dep_cand[0], dep_cand[3], dep_cand[6])
if provides_version == preferred_version or \
version_ok(provides_version, RPMSENSE_EQUAL,
preferred_version):
dep_res.append({'prov_id': provides_id,
'pkg_id': pkg_id})
if len(dep_res) == 0 and requires_name.startswith('/'): # file dependency
if (requires_flags & (RPMSENSE_SCRIPT_POST |
RPMSENSE_SCRIPT_PREUN |
RPMSENSE_SCRIPT_POSTUN)) != 0:
int_files_cnt = dbc.execute("""
SELECT COUNT(1) FROM package_files WHERE package_id = ? AND path = ?
""", [cpackage_id, requires_name]).fetchone()
if int_files_cnt[0] > 0:
dep_res.append({})
else:
#TODO: Check file dependencies (/usr/bin/python (required by ant-scripts-1.7.1-7.0.6.noarch), /usr/sbin/useradd (required by tomcat5-5.5.28-0.5.2.noarch))? #TODO: Check file dependencies (/usr/bin/python (required by ant-scripts-1.7.1-7.0.6.noarch), /usr/sbin/useradd (required by tomcat5-5.5.28-0.5.2.noarch))?
files_dependency = dbc.execute(""" files_deps = dbc.execute("""
SELECT package_id FROM package_files SELECT package_id FROM package_files
WHERE path = ? AND WHERE path = ? AND
package_id in (SELECT id FROM packages WHERE repodir_id IN (%s)) package_id in (SELECT id FROM packages WHERE repodir_id IN (%s))
""" % in_repodirs, [requires_name]).fetchall() """ % in_repodirs, [requires_name]).fetchall()
if len(files_dependency) > 0: for file_dep in files_deps:
if len(files_dependency) == 1: dep_res.append({'pkg_id': file_dep[0]})
pkg_id = files_dependency[0][0]
else: if len(dep_res) == 0 and (requires_flags & RPMSENSE_MISSINGOK) != 0:
print "File dependency (%s) has multiple resolutions (%d)." % (requires_name, len(files_dependency)) dep_res.append({})
if pkg_id is not None:
dbc.execute(""" if len(dep_res) > 0:
UPDATE requires SET dep_package_id = ? WHERE id = ? for res_rec in dep_res:
""", [pkg_id, requires_id]) dbc.execute("""
requires_cache[requirement_uid] = pkg_id INSERT INTO package_depend_res(package_id, requires_id,
provides_id, dep_package_id)
VALUES (?, ?, ?, ?)
""", [cpackage_id, requires_id, res_rec.get('prov_id'), res_rec.get('pkg_id')])
else: else:
print requires_name, ' ', requires_version, ' (required by %s)' % package_nvra, ' not found!!!' print requires_name, ' ', requires_version, ' (required by %s)' % package_nvra, ' not found!!!'
broken_dep += 1 broken_dep += 1
requires_cache[requirement_uid] = dep_res
n = n + 1 n = n + 1
#print "n = ", n #print "n = ", n
# if n == 60000: # if n == 60000:
@ -126,12 +131,12 @@ def extract_arch(arch_template, repo_name):
arch_sign = '$arch' arch_sign = '$arch'
spos = arch_template.find(arch_sign) spos = arch_template.find(arch_sign)
if spos >= 0: if spos >= 0:
repo_prefix= arch_template[:spos] repo_prefix = arch_template[:spos]
repo_postfix = arch_template[spos + len(arch_sign):] repo_postfix = arch_template[spos + len(arch_sign):]
if repo_name.startswith(repo_prefix) and \ if repo_name.startswith(repo_prefix) and \
repo_name.endswith(repo_postfix): repo_name.endswith(repo_postfix):
return repo_name[len(repo_prefix) : return repo_name[len(repo_prefix) :
len(repo_name) - len(repo_postfix)] len(repo_name) - len(repo_postfix)]
return None return None
def process_repodir_file_links(dbc, repodir_id, repodir_name, repodir_depends): def process_repodir_file_links(dbc, repodir_id, repodir_name, repodir_depends):
@ -164,9 +169,10 @@ SELECT id, link_to_path FROM package_files WHERE path = ? AND package_id = ?
# Just two level of dependency recursion - TODO: Full depth recursion? # Just two level of dependency recursion - TODO: Full depth recursion?
tofile = dbc.execute(""" tofile = dbc.execute("""
SELECT id, link_to_path FROM package_files WHERE path = ? AND package_id IN ( SELECT id, link_to_path FROM package_files WHERE path = ? AND package_id IN (
SELECT dep_package_id FROM requires WHERE package_id = ? SELECT dep_package_id FROM package_depend_res WHERE package_id = ?
UNION UNION
SELECT dep_package_id FROM requires WHERE package_id IN (SELECT dep_package_id FROM requires WHERE package_id = ?) SELECT dep_package_id FROM package_depend_res WHERE package_id IN
(SELECT dep_package_id FROM package_depend_res WHERE package_id = ?)
) )
""", [target_path, pkg_id, pkg_id]).fetchone() """, [target_path, pkg_id, pkg_id]).fetchone()
if tofile: if tofile:
@ -194,11 +200,16 @@ def process_repodir_so_needed(dbc, repodir_id, repodir_name, repodir_depends):
dbc.execute(""" dbc.execute("""
INSERT INTO so_needed_res(so_needed_id, dep_obj_file_id, res_type) INSERT INTO so_needed_res(so_needed_id, dep_obj_file_id, res_type)
SELECT so_needed.id, tpf.id, 1 FROM packages SELECT so_needed.id, tpf.id, 1 FROM packages
CROSS JOIN package_files spf CROSS JOIN so_needed CROSS JOIN requires CROSS JOIN package_files tpf CROSS JOIN package_files spf CROSS JOIN so_needed CROSS JOIN rpm_requires
WHERE so_needed.obj_file_id = spf.id AND spf.package_id = packages.id AND packages.repodir_id = ? AND CROSS JOIN package_depend_res dep_res CROSS JOIN package_files tpf
spf.package_id = requires.package_id AND WHERE so_needed.obj_file_id = spf.id AND spf.package_id = packages.id AND
(so_needed.name = requires.name OR so_needed.name || '()(64bit)' = requires.name) AND packages.repodir_id = ? AND spf.package_id = rpm_requires.package_id AND
requires.dep_package_id = tpf.package_id AND so_needed.name = tpf.basename (so_needed.name = rpm_requires.name OR
so_needed.name || '()(64bit)' = rpm_requires.name) AND
packages.id = dep_res.package_id AND
rpm_requires.id = dep_res.requires_id AND
dep_res.dep_package_id = tpf.package_id AND
so_needed.name = tpf.basename
""", [repodir_id]) """, [repodir_id])
search_repodirs = [repodir_id] search_repodirs = [repodir_id]
@ -212,7 +223,8 @@ SELECT packages.id AS package_id, packages.nvra,
FROM packages CROSS JOIN package_files CROSS JOIN so_needed FROM packages CROSS JOIN package_files CROSS JOIN so_needed
WHERE repodir_id = ? AND package_files.package_id = packages.id AND WHERE repodir_id = ? AND package_files.package_id = packages.id AND
so_needed.obj_file_id = package_files.id AND so_needed.obj_file_id = package_files.id AND
NOT EXISTS (SELECT 1 FROM so_needed_res WHERE so_needed_res.so_needed_id = so_needed.id) NOT EXISTS (SELECT 1 FROM so_needed_res
WHERE so_needed_res.so_needed_id = so_needed.id)
ORDER BY packages.nvra, package_files.basename, so_needed.name ORDER BY packages.nvra, package_files.basename, so_needed.name
""", [repodir_id]).fetchall() """, [repodir_id]).fetchall()
print 'Object files not resolved by rpm requires-provides: ', len(objects_not_resolved1) print 'Object files not resolved by rpm requires-provides: ', len(objects_not_resolved1)
@ -330,6 +342,8 @@ DELETE FROM so_needed_res""")
dbc.execute(""" dbc.execute("""
DELETE FROM obj_symbols_res""") DELETE FROM obj_symbols_res""")
dbc.execute(""" dbc.execute("""
DELETE FROM package_depend_res""")
dbc.execute("""
ANALYZE""") ANALYZE""")
repodirs_processed = [] repodirs_processed = []
#Process binary rpms #Process binary rpms