Djam/repo-analyzer
1
0
Fork 0
mirror of https://abf.rosa.ru/djam/repo-analyzer.git synced 2025-02-23 10:02:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Restructure repo.db; Support multiple dependencies for a requirement

Browse source
This commit is contained in:
Alexander Lakhin 2014-02-10 14:16:01 +04:00
parent 3d3f873e57
commit 38966cb244
4 changed files with 171 additions and 147 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
analyze-repo-redundancy.py
View file

@ -91,10 +91,10 @@ SELECT id, nvra, version, release FROM packages WHERE repodir_id IN (%s) AND nam
in_packages = ','.join(str(id) for id in package_ids)
for required_package in c.execute("""
SELECT packages.id, packages.name, nvra
FROM packages, requires
WHERE packages.id = requires.dep_package_id AND
FROM packages, package_depend_res
WHERE packages.id = package_depend_res.dep_package_id AND
package_depend_res.package_id IN (%s) AND
packages.repodir_id IN (%s) AND
requires.package_id IN (%s) AND
packages.id NOT IN (%s)
ORDER BY packages.name
""" % (in_repodirs, in_packages, in_packages)):

57
analyze-repodb.py
View file

@ -29,10 +29,14 @@ def detect_broken_dependencies(dbc):
repodirs_analyzed = []
broken = {}
broken_level0 = dbc.execute("""
SELECT packages.id, nvra, repodir_id, repodirs.name, requires.name AS req_name, requires.build_arch AS req_arch
FROM packages, repodirs, requires
WHERE packages.repodir_id = repodirs.id AND packages.id=requires.package_id AND
dep_package_id IS NULL
SELECT packages.id, nvra, repodir_id, repodirs.name,
rpm_requires.name AS req_name, rpm_requires.build_arch AS req_arch
FROM packages, repodirs, rpm_requires
WHERE packages.repodir_id = repodirs.id AND
packages.id=rpm_requires.package_id AND
NOT EXISTS (SELECT 1 FROM package_depend_res pdr
WHERE pdr.package_id = packages.id AND
pdr.requires_id = rpm_requires.id)
ORDER BY repodir_id, nvra""").fetchall()
all_broken = {}
@ -75,16 +79,22 @@ SELECT packages.id, nvra, repodir_id, repodirs.name, requires.name AS req_name,
while all_broken_cnt < len(all_broken):
all_broken_cnt = len(all_broken)
pids = ','.join(str(id) for id in all_broken.keys())
#print pids
packages_broken_recurs = dbc.execute("""
SELECT packages.id, nvra, repodir_id, repodirs.name, requires.name AS req_name, build_arch, dep_package_id
FROM packages, repodirs, requires
WHERE packages.repodir_id = repodirs.id AND packages.id=requires.package_id AND
dep_package_id IN (%(pids)s) AND packages.id NOT IN (%(pids)s)
SELECT packages.id, nvra, repodir_id, repodirs.name,
rpm_requires.name AS req_name, build_arch, dep_package_id
FROM packages, repodirs, rpm_requires, package_depend_res
WHERE packages.repodir_id = repodirs.id AND
packages.id = rpm_requires.package_id AND
packages.id = package_depend_res.package_id AND
rpm_requires.id = package_depend_res.requires_id AND
dep_package_id IN (%(pids)s) AND
packages.id NOT IN (%(pids)s)
ORDER BY repodir_id, nvra""" % {'pids': pids}).fetchall()
# print len(packages_broken_recurs)
for packb in packages_broken_recurs:
all_broken[packb[0]] = {'repo': packb[2], 'nvra': packb[1], 'reqname': packb[4], 'build_arch': packb[5], 'depid': packb[6]}
all_broken[packb[0]] = {'repo': packb[2], 'nvra': packb[1],
'reqname': packb[4], 'build_arch': packb[5],
'depid': packb[6]}
broken_recursive.append(packb[0])
#print len(all_broken.keys())
@ -96,7 +106,8 @@ SELECT id, name, sources FROM repodirs ORDER BY id""").fetchall()
(rd_id, rd_name, rd_sources) = (rd[0], rd[1], rd[2])
if rd_sources == '.':
archs = dbc.execute("""
SELECT DISTINCT build_arch FROM requires WHERE package_id IN (SELECT id FROM packages WHERE repodir_id = ?)
SELECT DISTINCT build_arch FROM rpm_requires
WHERE package_id IN (SELECT id FROM packages WHERE repodir_id = ?)
""", [rd_id]).fetchall()
for arch_rec in archs:
repo_packages_broken = {all_broken[id]['nvra']: id for id in broken_recursive \
@ -110,18 +121,6 @@ SELECT DISTINCT build_arch FROM requires WHERE package_id IN (SELECT id FROM pac
print '%d) %s' % (rd_id, rd_name)
print_broken_packages()
#all_repodirs = dbc.execute("""
#SELECT id, name FROM repodirs ORDER BY id""").fetchall()
#for rd in all_repodirs:
#(rd_id, rd_name) = (rd[0], rd[1])
#repo_broken_recurs = dbc.execute("""
#SELECT packages.id, nvra, requires.name AS req_name
#FROM packages, requires
#WHERE packages.repodir_id = ? AND packages.id=requires.package_id AND
#dep_package_id IS NULL AND packages.id NOT IN()
#""").fetchall()
#print rd_name
def detect_lost_sources(dbc):
print '==='
@ -180,20 +179,20 @@ SELECT id, name, nvra
part_packages = ','.join(str(id) for id in partition_ids)
upper_packages = dbc.execute("""
SELECT packages.id, packages.name, nvra
FROM packages, requires
WHERE packages.id = requires.package_id AND
FROM packages, package_depend_res
WHERE packages.id = package_depend_res.package_id AND
packages.repodir_id = ? AND
requires.dep_package_id IN (%s) AND
package_depend_res.dep_package_id IN (%s) AND
packages.id NOT IN (%s)
ORDER BY packages.name
""" % (cl_packages, part_packages), [rd_id]).fetchall()
lower_packages = dbc.execute("""
SELECT packages.id, packages.name, nvra
FROM packages, requires
WHERE packages.id = requires.dep_package_id AND
FROM packages, package_depend_res
WHERE packages.id = package_depend_res.dep_package_id AND
packages.repodir_id = ? AND
requires.package_id IN (%s) AND
package_depend_res.package_id IN (%s) AND
packages.id NOT IN (%s)
ORDER BY packages.name
""" % (cl_packages, part_packages), [rd_id]).fetchall()

71
fill-repodb.py
View file

@ -69,9 +69,14 @@ CREATE TABLE repodir_depends(id INTEGER PRIMARY KEY NOT NULL,
repodir_id INTEGER, depend_repodir_name TEXT)""")
conn.execute("""
CREATE TABLE IF NOT EXISTS package_files(id INTEGER PRIMARY KEY NOT NULL,
package_id INTEGER NOT NULL, basename TEXT, path TEXT, size INTEGER, mode INTEGER,
package_id INTEGER NOT NULL, basename TEXT, path TEXT,
size INTEGER, mode INTEGER,
link_to_file_id INTEGER, link_to_path TEXT, mark TEXT)""")
conn.execute("""
CREATE TABLE package_depend_res(id INTEGER PRIMARY KEY NOT NULL,
package_id INTEGER, requires_id INTEGER,
provides_id INTEGER, dep_package_id INTEGER)""")
conn.execute("""
CREATE TABLE so_needed(id INTEGER PRIMARY KEY NOT NULL,
obj_file_id INTEGER, name TEXT)""")
conn.execute("""
@ -88,37 +93,44 @@ CREATE TABLE obj_symbols_res(id INTEGER PRIMARY KEY NOT NULL,
def index_database(conn):
print 'Indexing the database...'
conn.execute('CREATE INDEX IF NOT EXISTS rd_name ON repodirs(name)')
conn.execute('CREATE INDEX IF NOT EXISTS pkg_name ON packages(name)')
conn.execute('CREATE INDEX IF NOT EXISTS pkg_arch ON packages(nvra)')
conn.execute('CREATE INDEX IF NOT EXISTS pkg_arch ON packages(arch)')
conn.execute('CREATE INDEX IF NOT EXISTS pkg_group ON packages(rpm_group)')
conn.execute('CREATE INDEX IF NOT EXISTS pkg_repodir ON packages(repodir_id)')
conn.execute('CREATE INDEX IF NOT EXISTS pkg_file_pkg_id ON package_files(package_id)')
conn.execute('CREATE INDEX IF NOT EXISTS pkg_file_name ON package_files(basename)')
conn.execute('CREATE INDEX IF NOT EXISTS pkg_file_path ON package_files(path)')
conn.execute('CREATE INDEX IF NOT EXISTS pkg_file_mark ON package_files(mark)')
conn.execute('CREATE INDEX IF NOT EXISTS so_needed_obj_id ON so_needed(obj_file_id)')
conn.execute('CREATE INDEX IF NOT EXISTS so_needed_res_sn ON so_needed_res(so_needed_id)')
conn.execute('CREATE INDEX IF NOT EXISTS symbols_obj_name_type ON obj_symbols(obj_file_id, name, sym_type)')
conn.execute('CREATE INDEX IF NOT EXISTS symbols_name_type ON obj_symbols(name, sym_type)')
conn.execute('CREATE INDEX IF NOT EXISTS symbols_res_sym ON obj_symbols_res(obj_sym_id)')
dep_tables = ['requires', 'provides', 'conflicts', 'obsoletes']
conn.executescript("""
CREATE INDEX rd_name ON repodirs(name);
CREATE INDEX pkg_name ON packages(name);
CREATE INDEX pkg_nvra ON packages(nvra);
CREATE INDEX pkg_arch ON packages(arch);
CREATE INDEX pkg_group ON packages(rpm_group);
CREATE INDEX pkg_repodir ON packages(repodir_id);
CREATE INDEX pkg_dr_pkg_req ON package_depend_res(package_id, requires_id);
CREATE INDEX pkg_dr_pkg_prov ON package_depend_res(dep_package_id, provides_id);
CREATE INDEX pkg_file_pkg_id ON package_files(package_id);
CREATE INDEX pkg_file_name ON package_files(basename);
CREATE INDEX pkg_file_path ON package_files(path);
CREATE INDEX pkg_file_mark ON package_files(mark);
CREATE INDEX so_needed_obj_id ON so_needed(obj_file_id);
CREATE INDEX so_needed_res_sn ON so_needed_res(so_needed_id);
CREATE INDEX symbols_obj_name_type ON obj_symbols(obj_file_id, name, sym_type);
CREATE INDEX symbols_name_type ON obj_symbols(name, sym_type);
CREATE INDEX symbols_res_sym ON obj_symbols_res(obj_sym_id);
""")
dep_tables = ['rpm_requires', 'rpm_provides',
'rpm_conflicts', 'rpm_obsoletes']
for table in dep_tables:
conn.execute('CREATE INDEX IF NOT EXISTS %(tbl)s_pkg ON %(tbl)s(package_id)' % {'tbl': table})
conn.execute('CREATE INDEX IF NOT EXISTS %(tbl)s_name ON %(tbl)s(name)' % {'tbl': table})
conn.execute('CREATE INDEX IF NOT EXISTS %(tbl)s_dep_pkg ON %(tbl)s(dep_package_id)' % {'tbl': table})
conn.execute('CREATE INDEX %(tbl)s_pkg ON %(tbl)s(package_id)' %
{'tbl': table})
conn.execute('CREATE INDEX %(tbl)s_name ON %(tbl)s(name)' %
{'tbl': table})
conn.commit()
def add_repodir(xrepodir, conn):
dbc = conn.cursor()
dbc.execute("""INSERT INTO repodirs (name, path, sources) VALUES (?, ?, ?)""",
[xrepodir.get('name'), xrepodir.get('path'), xrepodir.get('sources')])
dbc.execute("""
INSERT INTO repodirs (name, path, sources) VALUES (?, ?, ?)
""", [xrepodir.get('name'), xrepodir.get('path'), xrepodir.get('sources')])
repodir_id = dbc.lastrowid
for depend in xrepodir.findall('dependency'):
dbc.execute("""INSERT INTO repodir_depends(repodir_id, depend_repodir_name)
VALUES (?, ?)""",
[repodir_id, depend.text.strip()])
dbc.execute("""
INSERT INTO repodir_depends(repodir_id, depend_repodir_name) VALUES (?, ?)
""", [repodir_id, depend.text.strip()])
conn.commit()
return repodir_id
@ -307,11 +319,10 @@ def process_package_worker(num, queue_in, generator, gen_lock, db_struct,
data['packages'].append(package_values)
for table in db_struct['dep_tables']:
table_data = data[table]
rpref = 'RPMTAG_' + table[:-1].upper()
rpref = 'RPMTAG_' + table[4 : -1].upper() # rpm_requires
(dep_name, dep_flags, dep_version) = \
(hdr[rpref + 'NAME'], hdr[rpref + 'FLAGS'], hdr[rpref + 'VERSION'])
for i in xrange(0, len(hdr[rpref + 'NAME'])):
#print pkg, table, hdr[rpref + 'NAME'][i]
for build_arch in build_archs:
table_data.append([dep_name[i].decode('utf-8'),
dep_flags[i],
@ -448,8 +459,8 @@ def process_repodir(repodir_path, repodir_id, build_archs, conn, db_struct,
dep_tags_re = r'^RPMTAG_(CONFLICT|OBSOLETE|PROVIDE|REQUIRE)\w+'
db_struct['dep_tables'] = ['requires', 'provides',
'conflicts', 'obsoletes']
db_struct['dep_tables'] = ['rpm_requires', 'rpm_provides',
'rpm_conflicts', 'rpm_obsoletes']
packages_field_names = 'id, '
packages_values_template = '?,'
@ -487,7 +498,7 @@ CREATE TABLE IF NOT EXISTS packages(id INTEGER PRIMARY KEY NOT NULL, %s)
conn.execute("""
CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY NOT NULL,
name TEXT, flags INTEGER, version TEXT, build_arch TEXT,
package_id INTEGER NOT NULL, dep_package_id INTEGER)""" % (table))
package_id INTEGER NOT NULL)""" % (table))
conn.commit()
db_struct['packages_tags'] = packages_tags
db_struct['packages_field_names'] = packages_field_names

130
prepare-repodb.py
View file

@ -6,7 +6,6 @@ import sys
import gettext
import argparse
import sqlite3
import string
import re
from versutils import *
@ -18,10 +17,10 @@ def process_repodir_requires(dbc, repodir_id, repodir_name, repodir_depends, req
print 'Processing repo %d: %s (with depends: %s)' % (repodir_id, repodir_name, str(repodir_depends))
package_requires = dbc.execute("""
SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra,
requires.id, requires.name, flags, requires.version
FROM packages, requires
WHERE repodir_id = ? AND requires.package_id = packages.id %s
ORDER BY packages.name, requires.name
req.id, req.name, flags, req.version
FROM packages, rpm_requires req
WHERE repodir_id = ? AND req.package_id = packages.id %s
ORDER BY packages.name, req.name
""" % ((" AND build_arch = '%s'" % requires_build_arch)
if requires_build_arch is not None else ""),
[repodir_id]).fetchall()
@ -37,84 +36,90 @@ SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra,
for packreq in package_requires:
(cpackage_id, package_nvra, requires_id, requires_name, requires_flags, requires_version) = \
(packreq[0], packreq[2], packreq[3], packreq[4], packreq[5], packreq[6])
# if requires_name != '/usr/sbin/glibc-post-wrapper':
# continue
# print packreq
requirement_uid = requires_name + '\0' + str(requires_flags) + '\0' + requires_version
pkg_id = requires_cache.get(requirement_uid)
if pkg_id is None:
dep_res = requires_cache.get(requirement_uid, None)
if dep_res is None:
dep_res = []
if (re.match(r'\A(rpmlib|executable)\(.+\)\Z', requires_name)):
# see if($N=~/\A(rpmlib|executable)\(.+\)\Z/) in urpm_repoclosure.pl
pkg_id = -1
if pkg_id is None:
dep_res.append({})
else:
depend_candidates = dbc.execute("""
SELECT packages.id AS package_id, packages.name AS package_name,
provides.id, provides.name, flags, provides.version
FROM packages, provides
WHERE provides.package_id = packages.id AND repodir_id IN (%s) AND provides.name = ?
SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra,
prov.id, prov.name, flags, prov.version
FROM packages, rpm_provides AS prov
WHERE prov.package_id = packages.id AND repodir_id IN (%s) AND prov.name = ?
ORDER by packages.name, packages.nvra
""" % in_repodirs, [requires_name]).fetchall()
provides_found_pkg_id = None
provides_found_id = None
provides_found_version = None
preferred_version = None
for dep_cand in depend_candidates:
(provides_pkg_id, provides_id, provides_flags, provides_version) = \
(dep_cand[0], dep_cand[2], dep_cand[4], dep_cand[5])
(provides_flags, provides_version) = \
(dep_cand[5], dep_cand[6])
#print 'provides_version: ', provides_flags, ' ', provides_version
if provides_flags & RPMSENSE_SENSEMASK == 0:
if not provides_version:
provides_version = '*'
else:
raise Exception('Invalid provides version (flags = %d, version = %s)!' %
raise Exception('Invalid provides version '
'(flags = %d, version = %s)!' %
(provides_flags, provides_version))
if version_ok(requires_version,
requires_flags & RPMSENSE_SENSEMASK,
provides_version):
better_version = provides_found_version is None or \
better_version = preferred_version is None or \
provides_version == '*'
if not better_version:
better_version = version_ok(provides_version,
RPMSENSE_GREATER,
provides_found_version)
preferred_version)
if better_version:
(provides_found_pkg_id,
provides_found_id,
provides_found_version) = \
(provides_pkg_id,
provides_id,
provides_version)
pkg_id = provides_found_pkg_id
#print "->", provides_found_version, ' ->', provides_found_pkg_id
if pkg_id is None and (requires_flags & RPMSENSE_MISSINGOK) != 0:
pkg_id = -1 # TODO: Fix for valid foreign key
if pkg_id is None and requires_name.startswith('/'): # file dependency
preferred_version = provides_version
if preferred_version is not None:
for dep_cand in depend_candidates:
(pkg_id, provides_id, provides_version) = \
(dep_cand[0], dep_cand[3], dep_cand[6])
if provides_version == preferred_version or \
version_ok(provides_version, RPMSENSE_EQUAL,
preferred_version):
dep_res.append({'prov_id': provides_id,
'pkg_id': pkg_id})
if len(dep_res) == 0 and requires_name.startswith('/'): # file dependency
if (requires_flags & (RPMSENSE_SCRIPT_POST |
RPMSENSE_SCRIPT_PREUN |
RPMSENSE_SCRIPT_POSTUN)) != 0:
internal_files = dbc.execute("""
SELECT 1 FROM package_files WHERE package_id = ? AND path = ?
""", [cpackage_id, requires_name]).fetchall()
if len(internal_files) > 0:
pkg_id = cpackage_id
int_files_cnt = dbc.execute("""
SELECT COUNT(1) FROM package_files WHERE package_id = ? AND path = ?
""", [cpackage_id, requires_name]).fetchone()
if int_files_cnt[0] > 0:
dep_res.append({})
else:
#TODO: Check file dependencies (/usr/bin/python (required by ant-scripts-1.7.1-7.0.6.noarch), /usr/sbin/useradd (required by tomcat5-5.5.28-0.5.2.noarch))?
files_dependency = dbc.execute("""
files_deps = dbc.execute("""
SELECT package_id FROM package_files
WHERE path = ? AND
package_id in (SELECT id FROM packages WHERE repodir_id IN (%s))
""" % in_repodirs, [requires_name]).fetchall()
if len(files_dependency) > 0:
if len(files_dependency) == 1:
pkg_id = files_dependency[0][0]
else:
print "File dependency (%s) has multiple resolutions (%d)." % (requires_name, len(files_dependency))
if pkg_id is not None:
for file_dep in files_deps:
dep_res.append({'pkg_id': file_dep[0]})
if len(dep_res) == 0 and (requires_flags & RPMSENSE_MISSINGOK) != 0:
dep_res.append({})
if len(dep_res) > 0:
for res_rec in dep_res:
dbc.execute("""
UPDATE requires SET dep_package_id = ? WHERE id = ?
""", [pkg_id, requires_id])
requires_cache[requirement_uid] = pkg_id
INSERT INTO package_depend_res(package_id, requires_id,
provides_id, dep_package_id)
VALUES (?, ?, ?, ?)
""", [cpackage_id, requires_id, res_rec.get('prov_id'), res_rec.get('pkg_id')])
else:
print requires_name, ' ', requires_version, ' (required by %s)' % package_nvra, ' not found!!!'
broken_dep += 1
requires_cache[requirement_uid] = dep_res
n = n + 1
#print "n = ", n
# if n == 60000:
@ -164,9 +169,10 @@ SELECT id, link_to_path FROM package_files WHERE path = ? AND package_id = ?
# Just two level of dependency recursion - TODO: Full depth recursion?
tofile = dbc.execute("""
SELECT id, link_to_path FROM package_files WHERE path = ? AND package_id IN (
SELECT dep_package_id FROM requires WHERE package_id = ?
SELECT dep_package_id FROM package_depend_res WHERE package_id = ?
UNION
SELECT dep_package_id FROM requires WHERE package_id IN (SELECT dep_package_id FROM requires WHERE package_id = ?)
SELECT dep_package_id FROM package_depend_res WHERE package_id IN
(SELECT dep_package_id FROM package_depend_res WHERE package_id = ?)
)
""", [target_path, pkg_id, pkg_id]).fetchone()
if tofile:
@ -194,11 +200,16 @@ def process_repodir_so_needed(dbc, repodir_id, repodir_name, repodir_depends):
dbc.execute("""
INSERT INTO so_needed_res(so_needed_id, dep_obj_file_id, res_type)
SELECT so_needed.id, tpf.id, 1 FROM packages
CROSS JOIN package_files spf CROSS JOIN so_needed CROSS JOIN requires CROSS JOIN package_files tpf
WHERE so_needed.obj_file_id = spf.id AND spf.package_id = packages.id AND packages.repodir_id = ? AND
spf.package_id = requires.package_id AND
(so_needed.name = requires.name OR so_needed.name || '()(64bit)' = requires.name) AND
requires.dep_package_id = tpf.package_id AND so_needed.name = tpf.basename
CROSS JOIN package_files spf CROSS JOIN so_needed CROSS JOIN rpm_requires
CROSS JOIN package_depend_res dep_res CROSS JOIN package_files tpf
WHERE so_needed.obj_file_id = spf.id AND spf.package_id = packages.id AND
packages.repodir_id = ? AND spf.package_id = rpm_requires.package_id AND
(so_needed.name = rpm_requires.name OR
so_needed.name || '()(64bit)' = rpm_requires.name) AND
packages.id = dep_res.package_id AND
rpm_requires.id = dep_res.requires_id AND
dep_res.dep_package_id = tpf.package_id AND
so_needed.name = tpf.basename
""", [repodir_id])
search_repodirs = [repodir_id]
@ -212,7 +223,8 @@ SELECT packages.id AS package_id, packages.nvra,
FROM packages CROSS JOIN package_files CROSS JOIN so_needed
WHERE repodir_id = ? AND package_files.package_id = packages.id AND
so_needed.obj_file_id = package_files.id AND
NOT EXISTS (SELECT 1 FROM so_needed_res WHERE so_needed_res.so_needed_id = so_needed.id)
NOT EXISTS (SELECT 1 FROM so_needed_res
WHERE so_needed_res.so_needed_id = so_needed.id)
ORDER BY packages.nvra, package_files.basename, so_needed.name
""", [repodir_id]).fetchall()
print 'Object files not resolved by rpm requires-provides: ', len(objects_not_resolved1)
@ -330,6 +342,8 @@ DELETE FROM so_needed_res""")
dbc.execute("""
DELETE FROM obj_symbols_res""")
dbc.execute("""
DELETE FROM package_depend_res""")
dbc.execute("""
ANALYZE""")
repodirs_processed = []
#Process binary rpms