"""E3 · Barrido de los XML eForms guardados por el scraper TED (solo lectura, sin red). Para cada aviso: tipo, subtipo, SDK, comprador (país, tipo jurídico, perfil, NIF), expediente (BT-22). Para cada resultado de lote (efac:LotResult): estado (BT-142), TODAS las estadísticas de ofertas recibidas (BT-759 código / BT-760 número), ofertas descritas (OPT-320), contratos. Para cada oferta descrita (efac:LotTender citado por un resultado) × organización de su parte licitadora: importe (BT-720), rango (BT-171), si la cita un contrato (BT-3202), organización (nombre, NIF, país, tamaño). TED_EXPORT (esquema anterior): estadísticas NB_TENDERS_RECEIVED* por AWARD_CONTRACT y contratistas. Uso (dentro del contenedor): python3 -I barrido.py /xml /salida """ import gzip import os import sys import xml.etree.ElementTree as ET from multiprocessing import Pool import pyarrow as pa import pyarrow.parquet as pq NS = { 'cac': 'urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2', 'cbc': 'urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2', 'efac': 'http://data.europa.eu/p27/eforms-ubl-extension-aggregate-components/1', 'efbc': 'http://data.europa.eu/p27/eforms-ubl-extension-basic-components/1', 'efext': 'http://data.europa.eu/p27/eforms-ubl-extensions/1', 'ext': 'urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2', } EXT = 'ext:UBLExtensions/ext:UBLExtension/ext:ExtensionContent/efext:EformsExtension' def t(e, ruta): x = e.find(ruta, NS) if e is not None else None return x.text.strip() if x is not None and x.text else '' def ts(e, ruta): return [x.text.strip() for x in e.findall(ruta, NS) if x.text and x.text.strip()] if e is not None else [] def local(e): return e.tag.rsplit('}', 1)[-1] def hijo(e, *nombres): for n in nombres: if e is None: return None e = next((c for c in e if local(c) == n), None) return e def txt_te(e, *nombres): x = hijo(e, *nombres) return x.text.strip() if x is not None and x.text else '' def parse_eforms(raiz, numero, anio): av = {'numero': numero, 'anio': anio, 'esquema': 'eforms', 'raiz': local(raiz), 'notice_type': t(raiz, 'cbc:NoticeTypeCode'), 'subtype': t(raiz, EXT + '/efac:NoticeSubType/cbc:SubTypeCode'), 'sdk': t(raiz, 'cbc:CustomizationID'), 'version': t(raiz, 'cbc:VersionID'), 'issue_date': t(raiz, 'cbc:IssueDate'), 'pub_date': t(raiz, EXT + '/efac:Publication/efbc:PublicationDate'), 'procedure_code': t(raiz, 'cac:TenderingProcess/cbc:ProcedureCode'), 'internal_id_proc': t(raiz, 'cac:ProcurementProject/cbc:ID'), 'contract_nature': t(raiz, 'cac:ProcurementProject/cbc:ProcurementTypeCode'), 'buyer_profile': t(raiz, 'cac:ContractingParty/cbc:BuyerProfileURI'), 'buyer_legal_type': t(raiz, 'cac:ContractingParty/cac:ContractingPartyType/cbc:PartyTypeCode'), 'regulatory_domain': t(raiz, 'cbc:RegulatoryDomain')} orgs = {} for org in raiz.findall(EXT + '/efac:Organizations/efac:Organization', NS): emp = org.find('efac:Company', NS) if emp is None: continue ids = [(x.get('schemeName', ''), x.text.strip()) for x in emp.findall('cac:PartyLegalEntity/cbc:CompanyID', NS) if x.text and x.text.strip()] orgs[t(emp, 'cac:PartyIdentification/cbc:ID')] = { 'name': t(emp, 'cac:PartyName/cbc:Name'), 'ids': ';'.join(f'{a}={b}' for a, b in ids), 'country': t(emp, 'cac:PostalAddress/cac:Country/cbc:IdentificationCode'), 'size': t(emp, 'efbc:CompanySizeCode'), 'nuts': t(emp, 'cac:PostalAddress/cbc:CountrySubentityCode'), } comp = ts(raiz, 'cac:ContractingParty/cac:Party/cac:PartyIdentification/cbc:ID') b = orgs.get(comp[0], {}) if comp else {} av.update(buyer_country=b.get('country', ''), buyer_ids=b.get('ids', ''), buyer_name=b.get('name', ''), buyer_nuts=b.get('nuts', ''), n_compradores=len(comp)) av['n_lotes'] = len(raiz.findall('cac:ProcurementProjectLot', NS)) res, ofe = [], [] nr = raiz.find(EXT + '/efac:NoticeResult', NS) av['tiene_noticeresult'] = nr is not None if nr is None: return av, res, ofe ofertas = {} for x in nr.findall('efac:LotTender', NS): ofertas[t(x, 'cbc:ID')] = { 'valor': t(x, 'cac:LegalMonetaryTotal/cbc:PayableAmount'), 'moneda': (x.find('cac:LegalMonetaryTotal/cbc:PayableAmount', NS).get('currencyID', '') if x.find('cac:LegalMonetaryTotal/cbc:PayableAmount', NS) is not None else ''), 'rango': t(x, 'cbc:RankCode'), 'parte': t(x, 'efac:TenderingParty/cbc:ID'), 'lote': t(x, 'efac:TenderLot/cbc:ID'), 'variante': t(x, 'efbc:TenderVariantIndicator'), } partes = {} for p in nr.findall('efac:TenderingParty', NS): partes[t(p, 'cbc:ID')] = [(t(m, 'cbc:ID'), t(m, 'efbc:GroupLeadIndicator')) for m in p.findall('efac:Tenderer', NS)] citadas = set() for c in nr.findall('efac:SettledContract', NS): citadas.update(ts(c, 'efac:LotTender/cbc:ID')) av['n_lottender'] = len(ofertas) av['n_tenderingparty'] = len(partes) av['n_settledcontract'] = len(nr.findall('efac:SettledContract', NS)) descritas_todas = set() for r in nr.findall('efac:LotResult', NS): est = {} for s in r.findall('efac:ReceivedSubmissionsStatistics', NS): cod = t(s, 'efbc:StatisticsCode') est[cod] = t(s, 'efbc:StatisticsNumeric') descritas = list(dict.fromkeys(ts(r, 'efac:LotTender/cbc:ID'))) descritas_todas.update(descritas) rid = t(r, 'cbc:ID') fila = {'numero': numero, 'anio': anio, 'esquema': 'eforms', 'result_id': rid, 'lot_id': t(r, 'efac:TenderLot/cbc:ID'), 'estado': t(r, 'cbc:TenderResultCode'), 'stats': ';'.join(f'{k}={v}' for k, v in est.items()), 'n_descritas': len(descritas), 'n_citadas': sum(1 for d in descritas if d in citadas), 'n_contratos_res': len(ts(r, 'efac:SettledContract/cbc:ID')), 'lowest': t(r, 'cbc:LowerTenderAmount'), 'highest': t(r, 'cbc:HigherTenderAmount')} for k, v in est.items(): fila['st_' + k] = v res.append(fila) for d in descritas: o = ofertas.get(d, {}) miembros = partes.get(o.get('parte', ''), []) base = {'numero': numero, 'anio': anio, 'result_id': rid, 'lot_id': fila['lot_id'], 'estado': fila['estado'], 'tender_id': d, 'valor': o.get('valor', ''), 'moneda': o.get('moneda', ''), 'rango': o.get('rango', ''), 'parte': o.get('parte', ''), 'lote_oferta': o.get('lote', ''), 'citada': d in citadas, 'n_miembros': len(miembros), 'existe_oferta': d in ofertas} if not miembros: ofe.append(dict(base, org='', lider='', name='', ids='', country='', size='')) for org, lider in miembros: g = orgs.get(org, {}) ofe.append(dict(base, org=org, lider=lider, name=g.get('name', ''), ids=g.get('ids', ''), country=g.get('country', ''), size=g.get('size', ''))) av['n_lottender_no_descritas'] = len(set(ofertas) - descritas_todas) return av, res, ofe TE_STATS = ('NB_TENDERS_RECEIVED', 'NB_TENDERS_RECEIVED_SME', 'NB_TENDERS_RECEIVED_OTHER_EU', 'NB_TENDERS_RECEIVED_NON_EU', 'NB_TENDERS_RECEIVED_EMEANS') def parse_te(raiz, numero, anio): form = None for f in raiz.iter(): if local(f) == 'FORM_SECTION': form = next(iter(f), None) break av = {'numero': numero, 'anio': anio, 'esquema': 'ted_export', 'raiz': local(raiz), 'notice_type': local(form) if form is not None else '', 'subtype': '', 'sdk': '', 'version': ''} res, ofe = [], [] if form is None: return av, res, ofe for i, ac in enumerate(x for x in form if local(x) == 'AWARD_CONTRACT'): adj = hijo(ac, 'AWARDED_CONTRACT') fila = {'numero': numero, 'anio': anio, 'esquema': 'ted_export', 'result_id': f'AC-{i + 1}', 'lot_id': txt_te(ac, 'LOT_NO'), 'estado': 'adjudicado' if adj is not None else 'no_adjudicado', 'n_descritas': 0, 'n_citadas': 0} st = [] for k in TE_STATS: v = txt_te(adj, 'TENDERS', k) if adj is not None else '' if v: fila['te_' + k] = v st.append(f'{k}={v}') fila['stats'] = ';'.join(st) res.append(fila) if adj is not None: cons = hijo(adj, 'CONTRACTORS') cs = [c for c in cons if local(c) == 'CONTRACTOR'] if cons is not None else [] for c in cs: ofe.append({'numero': numero, 'anio': anio, 'result_id': fila['result_id'], 'lot_id': fila['lot_id'], 'estado': fila['estado'], 'tender_id': 'TE', 'citada': True, 'name': txt_te(c, 'ADDRESS_CONTRACTOR', 'OFFICIALNAME'), 'ids': 'NATIONALID=' + txt_te(c, 'ADDRESS_CONTRACTOR', 'NATIONALID'), 'country': (hijo(c, 'ADDRESS_CONTRACTOR', 'COUNTRY').get('VALUE', '') if hijo(c, 'ADDRESS_CONTRACTOR', 'COUNTRY') is not None else ''), 'size': 'sme' if hijo(c, 'SME') is not None else ('large' if hijo(c, 'NO_SME') is not None else ''), 'n_miembros': len(cs)}) return av, res, ofe def uno(ruta): anio = os.path.basename(os.path.dirname(ruta)) numero = os.path.basename(ruta).replace('.xml.gz', '') try: with gzip.open(ruta, 'rb') as fh: raiz = ET.fromstring(fh.read()) except Exception as e: # noqa: BLE001 return {'numero': numero, 'anio': anio, 'esquema': 'error', 'error': repr(e)[:200]}, [], [] if raiz.tag.startswith('{urn:oasis:names:specification:ubl:schema:xsd:'): return parse_eforms(raiz, numero, anio) return parse_te(raiz, numero, anio) def guardar(filas, ruta): cols = sorted({k for f in filas for k in f}) datos = {c: [None if f.get(c) is None else (f[c] if isinstance(f.get(c), (bool, int)) else str(f[c])) for f in filas] for c in cols} pq.write_table(pa.table(datos), ruta) def main(xml_dir, salida): rutas = sorted(os.path.join(xml_dir, a, f) for a in os.listdir(xml_dir) for f in os.listdir(os.path.join(xml_dir, a)) if f.endswith('.xml.gz')) print('xml', len(rutas), flush=True) avs, ress, ofes = [], [], [] with Pool(4) as p: for i, (a, r, o) in enumerate(p.imap_unordered(uno, rutas, chunksize=200)): avs.append(a) ress.extend(r) ofes.extend(o) if i % 10000 == 0: print(i, flush=True) os.makedirs(salida, exist_ok=True) guardar(avs, os.path.join(salida, 'avisos.parquet')) guardar(ress, os.path.join(salida, 'resultados.parquet')) guardar(ofes, os.path.join(salida, 'ofertas.parquet')) print('avisos', len(avs), 'resultados', len(ress), 'ofertas', len(ofes), flush=True) if __name__ == '__main__': main(sys.argv[1], sys.argv[2])