"""MOD-EJEC · 52 · cotejo del PREREG_2 §5 (md5 de3e22b1…) de la PLACSP contra el ATOM/CODICE original (ZIP de la sindicación en
disco, sin red; lectura como ej1/py/51_atom.py): de cada licitación de la muestra, su última versión; en ella, cada
ContractModification (ID, ContractID, importe sin IVA) y el importe adjudicado sin IVA de cada TenderResult; y el órgano
(LocatedContractingParty). Confirmada = todas nuestras M están con su importe a ± 1 € (y ninguna otra con importe que no sea de
las nuestras salvo prórrogas o plazo, que se cuentan aparte), Σ adjudicado a ± 1 € de nuestra A, s > 20 % con los importes del
ATOM y el mismo órgano. -> priv/sob_atom.json, salidas/52_cotejo_atom.json"""
import json, os, re, time, zipfile
from multiprocessing import Pool
from collections import defaultdict
import duckdb
T0 = time.time()
M = json.load(open('/w/priv/sob_muestra.json'))['PLACSP']
def corto(lid): return lid.split(':', 1)[1]
obj = sorted({corto(m['id']) for m in M})
c = duckdb.connect(); c.execute("SET memory_limit='1500MB'; SET threads=3")
arch = c.execute("""SELECT regexp_extract(id, '[^/]+/[^/]+$') k, archivo_origen, count(*) FROM '/n/salida/licitaciones_completo_2012_2026.parquet'
WHERE conjunto='licitaciones' AND regexp_extract(id, '[^/]+/[^/]+$') IN (SELECT unnest(?)) GROUP BY 1, 2""", [obj]).fetchall()
por_zip = defaultdict(set)
for k, z, n in arch:
p = '/n/zips/licitaciones/' + z
if not os.path.exists(p): p = '/n/zips/licitaciones/_historico/' + z
por_zip[p].add(k)
print('versiones según el scraper', sum(a[2] for a in arch), 'zips', len(por_zip), flush=True)
RE_ID = re.compile(rb'https://contrataciondelestado\.es/sindicacion/([^<]+)')
RE_UPD = re.compile(rb'([^<]+)')
def buscar(a):
zp, mi, claves = a; out = []
with zipfile.ZipFile(zp) as z: data = z.read(mi)
for m in RE_ID.finditer(data):
k = m.group(1).decode()
if k not in claves: continue
i0 = data.rfind(b'', m.end())
if i0 < 0 or i1 < 0: continue
ent = data[i0:i1 + 8]; u = RE_UPD.search(ent)
out.append((k, u.group(1).decode() if u else '', ent.decode('utf-8', 'replace')))
return out
tareas = []
for zp, cl in por_zip.items():
with zipfile.ZipFile(zp) as z:
tareas += [(zp, n, cl) for n in z.namelist() if n.endswith('.atom')]
ver = defaultdict(dict)
with Pool(3) as p:
for res in p.imap_unordered(buscar, tareas, chunksize=8):
for k, upd, ent in res: ver[k][upd] = ent
print('licitaciones encontradas', len(ver), round(time.time() - T0), 's', flush=True)
RE_CM = re.compile(r'<[a-z\-]+:ContractModification>(.*?)[a-z\-]+:ContractModification>', re.S)
RE_TR = re.compile(r'<[a-z\-]+:TenderResult>(.*?)[a-z\-]+:TenderResult>', re.S)
def campo(b, tag):
m = re.search(r'<[a-z\-]+:' + tag + r'[^>]*>([^<]*)', b); return m.group(1).strip() if m else None
def imp_cm(b):
m = re.search(r'ContractModificationLegalMonetaryTotal>.*?TaxExclusiveAmount[^>]*>([^<]*)<', b, re.S); return float(m.group(1)) if m and m.group(1).strip() else None
def imp_tr(b):
m = re.search(r'AwardedTenderedProject>.*?LegalMonetaryTotal>.*?TaxExclusiveAmount[^>]*>([^<]*)<', b, re.S); return float(m.group(1)) if m and m.group(1).strip() else None
norm = lambda t: re.sub(r'[^A-Z0-9]', '', (t or '').upper().translate(str.maketrans('ÁÉÍÓÚÜÑÀÈÌÒÙÇ', 'AEIOUUNAEIOUC')))
R = []
for i, m in enumerate(M):
k = corto(m['id']); vs = sorted(ver.get(k, {}).items())
r = {'caso': f'l{i+1:02d}', 'anio': m['anio'], 'reserva': m['reserva'], 'versiones': len(vs), 'A_nuestra': m['A'], 'n_M_nuestras': len(m['mods'])}
if not vs:
r.update(confirmada=False, estado='no encontrada'); R.append(r); continue
ult = vs[-1][1]
cms = [(campo(b, 'ID'), campo(b, 'ContractID'), imp_cm(b)) for b in RE_CM.findall(ult)]
trs = [(campo(b, 'ResultCode'), imp_tr(b)) for b in RE_TR.findall(ult)]
ok_m = []
for idm, idc, imp in m['mods']:
hit = [cm for cm in cms if cm[0] == idm and cm[1] == idc]
ok_m.append(bool(hit) and any(h[2] is not None and abs(h[2] - imp) <= 1 for h in hit))
nuestros = {(a, b) for a, b, _ in m['mods']}
otras = [cm for cm in cms if (cm[0], cm[1]) not in nuestros and cm[2] not in (None, 0.0)]
A_atom = sum(v for _, v in trs if v is not None)
s_atom = (sum(next(h[2] for h in cms if h[0] == a and h[1] == b) for a, b, _ in m['mods'] if any(h[0] == a and h[1] == b for h in cms)) / A_atom) if A_atom else None
lcp = re.search(r'LocatedContractingParty>(.*?)[a-z\-]+:LocatedContractingParty>', ult, re.S); lcp = lcp.group(1) if lcp else ''
ids = {v.strip().upper() for _, v in re.findall(r'([^<]+)', lcp)}
nom = re.search(r'\s*([^<]+)', lcp)
r.update(M_ok=all(ok_m), M_encontradas=sum(ok_m), cm_en_atom=len(cms), otras_cm_con_importe=len(otras), tender_results=len(trs), A_atom=A_atom,
A_ok=abs(A_atom - m['A']) <= 1, s_atom=s_atom, organo_ok=bool((m['organo_nif'] and m['organo_nif'].upper() in ids) or (nom and norm(nom.group(1)) == norm(m['organo_nombre']))))
r['confirmada'] = r['M_ok'] and r['A_ok'] and r['organo_ok'] and s_atom is not None and s_atom * A_atom - 0.2 * A_atom > 0.01
R.append(r)
json.dump(R, open('/w/priv/sob_atom.json', 'w'), ensure_ascii=False, indent=1)
S = {'principal': [sum(r['confirmada'] for r in R if not r['reserva']), sum(not r['reserva'] for r in R)],
'reserva': [sum(r['confirmada'] for r in R if r['reserva']), sum(r['reserva'] for r in R)],
'fallos': [{k: v for k, v in r.items() if k != 'A_nuestra'} for r in R if not r['confirmada']], 'segundos': round(time.time() - T0)}
json.dump(S, open('/w/salidas/52_cotejo_atom.json', 'w'), ensure_ascii=False, indent=1)
print(json.dumps(S, ensure_ascii=False)[:3000])