"""MOD-EJEC · 52 · cotejo del PREREG_2 §5 (md5 de3e22b1…) de la PLACSP contra el ATOM/CODICE original (ZIP de la sindicación en disco, sin red; lectura como ej1/py/51_atom.py): de cada licitación de la muestra, su última versión; en ella, cada ContractModification (ID, ContractID, importe sin IVA) y el importe adjudicado sin IVA de cada TenderResult; y el órgano (LocatedContractingParty). Confirmada = todas nuestras M están con su importe a ± 1 € (y ninguna otra con importe que no sea de las nuestras salvo prórrogas o plazo, que se cuentan aparte), Σ adjudicado a ± 1 € de nuestra A, s > 20 % con los importes del ATOM y el mismo órgano. -> priv/sob_atom.json, salidas/52_cotejo_atom.json""" import json, os, re, time, zipfile from multiprocessing import Pool from collections import defaultdict import duckdb T0 = time.time() M = json.load(open('/w/priv/sob_muestra.json'))['PLACSP'] def corto(lid): return lid.split(':', 1)[1] obj = sorted({corto(m['id']) for m in M}) c = duckdb.connect(); c.execute("SET memory_limit='1500MB'; SET threads=3") arch = c.execute("""SELECT regexp_extract(id, '[^/]+/[^/]+$') k, archivo_origen, count(*) FROM '/n/salida/licitaciones_completo_2012_2026.parquet' WHERE conjunto='licitaciones' AND regexp_extract(id, '[^/]+/[^/]+$') IN (SELECT unnest(?)) GROUP BY 1, 2""", [obj]).fetchall() por_zip = defaultdict(set) for k, z, n in arch: p = '/n/zips/licitaciones/' + z if not os.path.exists(p): p = '/n/zips/licitaciones/_historico/' + z por_zip[p].add(k) print('versiones según el scraper', sum(a[2] for a in arch), 'zips', len(por_zip), flush=True) RE_ID = re.compile(rb'https://contrataciondelestado\.es/sindicacion/([^<]+)') RE_UPD = re.compile(rb'([^<]+)') def buscar(a): zp, mi, claves = a; out = [] with zipfile.ZipFile(zp) as z: data = z.read(mi) for m in RE_ID.finditer(data): k = m.group(1).decode() if k not in claves: continue i0 = data.rfind(b'', m.end()) if i0 < 0 or i1 < 0: continue ent = data[i0:i1 + 8]; u = RE_UPD.search(ent) out.append((k, u.group(1).decode() if u else '', ent.decode('utf-8', 'replace'))) return out tareas = [] for zp, cl in por_zip.items(): with zipfile.ZipFile(zp) as z: tareas += [(zp, n, cl) for n in z.namelist() if n.endswith('.atom')] ver = defaultdict(dict) with Pool(3) as p: for res in p.imap_unordered(buscar, tareas, chunksize=8): for k, upd, ent in res: ver[k][upd] = ent print('licitaciones encontradas', len(ver), round(time.time() - T0), 's', flush=True) RE_CM = re.compile(r'<[a-z\-]+:ContractModification>(.*?)', re.S) RE_TR = re.compile(r'<[a-z\-]+:TenderResult>(.*?)', re.S) def campo(b, tag): m = re.search(r'<[a-z\-]+:' + tag + r'[^>]*>([^<]*).*?TaxExclusiveAmount[^>]*>([^<]*)<', b, re.S); return float(m.group(1)) if m and m.group(1).strip() else None def imp_tr(b): m = re.search(r'AwardedTenderedProject>.*?LegalMonetaryTotal>.*?TaxExclusiveAmount[^>]*>([^<]*)<', b, re.S); return float(m.group(1)) if m and m.group(1).strip() else None norm = lambda t: re.sub(r'[^A-Z0-9]', '', (t or '').upper().translate(str.maketrans('ÁÉÍÓÚÜÑÀÈÌÒÙÇ', 'AEIOUUNAEIOUC'))) R = [] for i, m in enumerate(M): k = corto(m['id']); vs = sorted(ver.get(k, {}).items()) r = {'caso': f'l{i+1:02d}', 'anio': m['anio'], 'reserva': m['reserva'], 'versiones': len(vs), 'A_nuestra': m['A'], 'n_M_nuestras': len(m['mods'])} if not vs: r.update(confirmada=False, estado='no encontrada'); R.append(r); continue ult = vs[-1][1] cms = [(campo(b, 'ID'), campo(b, 'ContractID'), imp_cm(b)) for b in RE_CM.findall(ult)] trs = [(campo(b, 'ResultCode'), imp_tr(b)) for b in RE_TR.findall(ult)] ok_m = [] for idm, idc, imp in m['mods']: hit = [cm for cm in cms if cm[0] == idm and cm[1] == idc] ok_m.append(bool(hit) and any(h[2] is not None and abs(h[2] - imp) <= 1 for h in hit)) nuestros = {(a, b) for a, b, _ in m['mods']} otras = [cm for cm in cms if (cm[0], cm[1]) not in nuestros and cm[2] not in (None, 0.0)] A_atom = sum(v for _, v in trs if v is not None) s_atom = (sum(next(h[2] for h in cms if h[0] == a and h[1] == b) for a, b, _ in m['mods'] if any(h[0] == a and h[1] == b for h in cms)) / A_atom) if A_atom else None lcp = re.search(r'LocatedContractingParty>(.*?)', ult, re.S); lcp = lcp.group(1) if lcp else '' ids = {v.strip().upper() for _, v in re.findall(r'([^<]+)', lcp)} nom = re.search(r'\s*([^<]+)', lcp) r.update(M_ok=all(ok_m), M_encontradas=sum(ok_m), cm_en_atom=len(cms), otras_cm_con_importe=len(otras), tender_results=len(trs), A_atom=A_atom, A_ok=abs(A_atom - m['A']) <= 1, s_atom=s_atom, organo_ok=bool((m['organo_nif'] and m['organo_nif'].upper() in ids) or (nom and norm(nom.group(1)) == norm(m['organo_nombre'])))) r['confirmada'] = r['M_ok'] and r['A_ok'] and r['organo_ok'] and s_atom is not None and s_atom * A_atom - 0.2 * A_atom > 0.01 R.append(r) json.dump(R, open('/w/priv/sob_atom.json', 'w'), ensure_ascii=False, indent=1) S = {'principal': [sum(r['confirmada'] for r in R if not r['reserva']), sum(not r['reserva'] for r in R)], 'reserva': [sum(r['confirmada'] for r in R if r['reserva']), sum(r['reserva'] for r in R)], 'fallos': [{k: v for k, v in r.items() if k != 'A_nuestra'} for r in R if not r['confirmada']], 'segundos': round(time.time() - T0)} json.dump(S, open('/w/salidas/52_cotejo_atom.json', 'w'), ensure_ascii=False, indent=1) print(json.dumps(S, ensure_ascii=False)[:3000])