"""DAT-CRUCE · H-ENC · segunda implementación (PREREG_ENC §5c): xml.etree sobre los ZIP del ATOM de la PLACSP (sindicación 1383), sin DuckDB, sin el ETL ni el scraper. Versión más reciente de cada ; borrados (at:deleted-entry) posteriores a esa versión fuera; mismas reglas de §1. Salida: /dc/interno/enc_impl2_entidad.json y /dc/interno/enc_impl2_filas.json (700).""" import zipfile, glob, re, json, unicodedata, sys import xml.etree.ElementTree as ET NS = {'a': 'http://www.w3.org/2005/Atom', 'cbc': 'urn:dgpe:names:draft:codice:schema:xsd:CommonBasicComponents-2', 'cac': 'urn:dgpe:names:draft:codice:schema:xsd:CommonAggregateComponents-2', 'cacx': 'urn:dgpe:names:draft:codice-place-ext:schema:xsd:CommonAggregateComponents-2', 'cbcx': 'urn:dgpe:names:draft:codice-place-ext:schema:xsd:CommonBasicComponents-2', 'at': 'http://purl.org/atompub/tombstones/1.0'} A = '{http://www.w3.org/2005/Atom}' PF = re.compile(r'^([0-9]{8}[A-Z]|[XYZKLM][0-9]{7}[A-Z])$') MARC = re.compile(r'^(9{8,9}[A-Z]?|0{8,9}[A-Z]?)$') zips = sorted(glob.glob('/zips/encargos/*.zip')) + sorted(glob.glob('/zips/encargos/_historico/*.zip')) ult = {} # id -> (updated, campos) borr = {} # id -> fecha de borrado más reciente n_ent = 0 def txt(e, path): x = e.find(path, NS) return x.text.strip() if x is not None and x.text else None def ident(party, scheme): for pi in party.findall('cac:PartyIdentification', NS): i = pi.find('cbc:ID', NS) if i is not None and i.get('schemeName') == scheme and i.text: return i.text.strip() return None for zp in zips: z = zipfile.ZipFile(zp) for nm in z.namelist(): if not nm.endswith('.atom'): continue root = ET.fromstring(z.read(nm)) for d in root.findall('at:deleted-entry', NS): ref, when = d.get('ref'), d.get('when') if ref and when and (ref not in borr or when > borr[ref]): borr[ref] = when for en in root.findall(f'{A}entry'): n_ent += 1 eid = en.find(f'{A}id').text.strip() upd = en.find(f'{A}updated').text.strip() if eid in ult and ult[eid][0] >= upd: continue cfs = en.find('cacx:ContractFolderStatus', NS) lcp = cfs.find('cacx:LocatedContractingParty', NS) if cfs is not None else None party = lcp.find('cac:Party', NS) if lcp is not None else None res = cfs.findall('cac:TenderResult', NS) if cfs is not None else [] filas = [] for r in res: wp = r.find('cac:WinningParty', NS) filas.append({'mp_nif': (ident(wp, 'NIF') or '').upper() if wp is not None else '', 'mp_nombre': txt(wp, 'cac:PartyName/cbc:Name') if wp is not None else None, 'fecha': txt(r, 'cbc:AwardDate')}) ult[eid] = (upd, { 'organo_nif': (ident(party, 'NIF') or '').upper() if party is not None else '', 'organo_nombre': txt(party, 'cac:PartyName/cbc:Name') if party is not None else None, 'expediente': txt(cfs, 'cbc:ContractFolderID') if cfs is not None else None, 'objeto': txt(cfs, 'cac:ProcurementProject/cbc:Name') if cfs is not None else None, 'imp': txt(cfs, 'cac:ProcurementProject/cac:BudgetAmount/cbc:TaxExclusiveAmount') if cfs is not None else None, 'estado': txt(cfs, 'cbcx:ContractFolderStatusCode') if cfs is not None else None, 'filas': filas}) def norm(s): s = unicodedata.normalize('NFKD', (s or '').upper()) s = ''.join(c for c in s if not unicodedata.combining(c)) return re.sub(' +', ' ', re.sub('[^A-Z0-9 ]', ' ', s)).strip() R = {'zips': len(zips), 'entradas_leidas': n_ent, 'ids': len(ult)} cont = {'borrado': 0, 'mp_persona': 0, 'organo': 0, 'fecha': 0, 'sin_resultado': 0, 'repetidos': 0} filas = [] for eid, (upd, c) in ult.items(): if eid in borr and borr[eid] >= upd: cont['borrado'] += 1; continue if not c['filas']: cont['sin_resultado'] += 1; continue vistos = set() for f in c['filas']: k = (f['mp_nif'], f['fecha']) if k in vistos: # copia exacta dentro de la entrada continue vistos.add(k) if not f['mp_nif'] or PF.match(f['mp_nif']) or MARC.match(f['mp_nif']): cont['mp_persona'] += 1; continue o = c['organo_nif'] if not o or PF.match(o) or '*' in o: cont['organo'] += 1; continue if not f['fecha'] or not ('2021-01-01' <= f['fecha'][:10] <= '2026-11-01'): cont['fecha'] += 1; continue filas.append({'id': eid, 'upd': upd, 'organo_nif': o, 'mp_nif': f['mp_nif'], 'mp_nombre': f['mp_nombre'], 'organo_nombre': c['organo_nombre'], 'imp': float(c['imp']) if c['imp'] else 0.0, 'fecha': f['fecha'][:10], 'exp_n': norm(c['expediente']), 'obj_n': norm(c['objeto']), 'estado': c['estado']}) # repetidos: mismo (órgano, mp, importe, fecha) y mismo expediente u objeto → uno (el id menor, como texto) filas.sort(key=lambda f: f['id']) grupos = {} fuera = set() for f in filas: g = grupos.setdefault((f['organo_nif'], f['mp_nif'], f['imp'], f['fecha']), []) for h in g: if h['exp_n'] == f['exp_n'] or h['obj_n'] == f['obj_n']: fuera.add(f['id']); break g.append(f) cont['repetidos'] = len(fuera) filas = [f for f in filas if f['id'] not in fuera] R['fuera'] = cont ent = {} for f in filas: e = ent.setdefault(f['organo_nif'], {'k': 0, 'i': 0.0, 'mp': set()}) e['k'] += 1; e['i'] += f['imp']; e['mp'].add(f['mp_nif']) R['universo'] = {'encargos': len(filas), 'importe': sum(f['imp'] for f in filas), 'entidades': len(ent)} json.dump({o: {'k': v['k'], 'i': round(v['i'], 2), 'm': len(v['mp'])} for o, v in ent.items()}, open('/dc/interno/enc_impl2_entidad.json', 'w')) json.dump(filas, open('/dc/interno/enc_impl2_filas.json', 'w'), ensure_ascii=False) json.dump(R, open('/dc/salidas/enc_impl2.json', 'w'), ensure_ascii=False, indent=1) print(json.dumps(R, ensure_ascii=False, indent=1))