diff --git a/prepare-repodb.py b/prepare-repodb.py index 99bc719..952645a 100755 --- a/prepare-repodb.py +++ b/prepare-repodb.py @@ -372,12 +372,30 @@ def process_package_worker(num, queue_in, generator, gen_lock, db_struct, remove_cached_file(pkg) queue_in.task_done() - conn = sqlite3.connect(DB, timeout=30) - conn.executemany(""" + conn = None + retryCnt = 0 + retry = True + while retry: + retry = False + try: + conn = sqlite3.connect(DB, timeout=30) + conn.executemany(""" INSERT INTO packages (%s) VALUES (%s)""" % (db_struct['packages_field_names'], db_struct['packages_values_template']), - data['packages']) + data['packages']) + except sqlite3.OperationalError as ex: + if (str(ex) == 'database is locked'): + print 'Database is locked. Retrying...' + conn.close() + time.sleep(10) + retryCnt += 1 + if retryCnt < 3: + retry = True + else: + raise ex + else: + raise ex for table in db_struct['dep_tables']: conn.executemany(""" @@ -793,7 +811,7 @@ def import_repositories(options, conn): ### Post-process repo.db after data import ################################################ -def version_ok(required_version, compare_flag, candidate_version): +def version_ok(required_version, compare_flag, candidate_version, context=''): def sep_version(version): vrem = re.match(r'\A(.+)(\-[^\-\:]+)(\:[^\:]+|)\Z', version) if vrem: @@ -878,6 +896,10 @@ def version_ok(required_version, compare_flag, candidate_version): epoch = None if e1_m: (epoch, verstring) = (e1_m.group(1), e1_m.group(2)) + j = verstring.find(':') + if j != -1: + # ignore ":2012.0" in 2014-10:2012.0 + verstring = verstring[:j] j = verstring.find('-') if j != -1: if verstring[:j] == '': @@ -911,14 +933,14 @@ def version_ok(required_version, compare_flag, candidate_version): candidate_version == required_version: return True - cmp_res = None - try: - cmp_res = cmp_versions(candidate_version, required_version) - except ValueError as ex: - print ('Error when comparing versions: "%s" and "%s"\n%s' % - (candidate_version, required_version, str(ex))) - return False - rpm_cmp_res = rpm_cmp_versions(candidate_version, required_version) + #cmp_res = None + #try: + # cmp_res = cmp_versions(candidate_version, required_version) + #except ValueError as ex: + # print ('Error when comparing versions (%s): "%s" and "%s"\n%s' % + # (context, candidate_version, required_version, str(ex))) + # return False + cmp_res = rpm_cmp_versions(candidate_version, required_version) #if (cmp_res != rpm_cmp_res): # print >> sys.stderr, (('Invalid compare: "%s" vs "%s"! ' # 'Results: rc: %d, rpm: %d.') % @@ -951,8 +973,8 @@ SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra, in_repodirs = ','.join(str(id) for id in search_repodirs) dependency_cache = {} for packdep in package_depends: - (cpackage_id, package_nvra, dep_id, dep_name, dep_flags, dep_version) = \ - (packdep[0], packdep[2], packdep[3], packdep[4], packdep[5], packdep[6]) + (cpackage_id, package_nvra, dep_id, dep_name, dep_flags, dep_version) = \ + (packdep[0], packdep[2], packdep[3], packdep[4], packdep[5], packdep[6]) dep_res = [] depend_candidates = conn.execute(""" SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra, @@ -972,7 +994,8 @@ SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra, '(flags = %d, version = %s)!' % (provides_flags, provides_version)) if version_ok(dep_version, dep_flags & RPMSENSE_SENSEMASK, - provides_version): + provides_version, '%s %s %s' % + (package_nvra, dep_type, dep_name)): dep_res.append({'prov_id': provides_id, 'pkg_id': pkg_id}) if len(dep_res) > 0: for res_rec in dep_res: @@ -1004,7 +1027,7 @@ SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra, for packreq in package_requires: (cpackage_id, package_nvra, requires_id, requires_name, requires_flags, requires_version) = \ - (packreq[0], packreq[2], packreq[3], packreq[4], packreq[5], packreq[6]) + (packreq[0], packreq[2], packreq[3], packreq[4], packreq[5], packreq[6]) req_res = [] if (re.match(r'\A(rpmlib|executable)\(.+\)\Z', requires_name)): @@ -1033,7 +1056,8 @@ SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra, (provides_flags, provides_version)) if version_ok(requires_version, requires_flags & RPMSENSE_SENSEMASK, - provides_version): + provides_version, + '%s requires %s' % (package_nvra, requires_name)): if pkg_id == cpackage_id: # the same package is preferred over any other req_res.append({'prov_id': provides_id, @@ -1045,7 +1069,8 @@ SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra, if not better_version: better_version = version_ok(provides_version, RPMSENSE_GREATER, - preferred_version) + preferred_version, + '%s requires %s' % (package_nvra, requires_name)) if better_version: preferred_version = provides_version @@ -1055,7 +1080,8 @@ SELECT packages.id AS package_id, packages.name AS package_name, packages.nvra, (dep_cand[0], dep_cand[3], dep_cand[6]) if provides_version == preferred_version or \ version_ok(provides_version, RPMSENSE_EQUAL, - preferred_version): + preferred_version, + '%s requires %s' % (package_nvra, requires_name)): req_res.append({'prov_id': provides_id, 'pkg_id': pkg_id}) @@ -1089,7 +1115,7 @@ INSERT INTO package_requires_res(package_id, requires_id, VALUES (?, ?, ?, ?) """, [cpackage_id, requires_id, res_rec.get('prov_id'), res_rec.get('pkg_id')]) else: - print requires_name, ' ', requires_version, ' (required by %s)' % package_nvra, ' not found!!!' + print requires_name, requires_version, '(required by %s)' % package_nvra, 'not found!!!' broken_dep += 1 print 'broken_deps: ', broken_dep print ''