"""DOUE-PLAZO · segunda implementación (pyarrow + Python, sin DuckDB, sin SQL, sin el calendario de 10_medida salvo por ser independiente: reimplementa el calendario con su propia lista). Uso: 20_impl2.py DESDE HASTA. Compara, por licitación, con /o/priv/medida_DESDE_HASTA.parquet: población, tarde, fechas.""" import sys, json, time, datetime as dt, collections, re import pyarrow.parquet as pq, pyarrow.dataset as ds, pyarrow.compute as pc D, Hh = int(sys.argv[1]), int(sys.argv[2]) T0 = time.time() H = '/h/nacional/salida/licitaciones_completo_2012_2026' UNA = dt.timedelta(days=1) # Calendario propio (lista de Viernes Santo escrita a mano; 10_medida usa computus): sábados, domingos y fiestas nacionales VS = {2012: (4, 6), 2013: (3, 29), 2014: (4, 18), 2015: (4, 3), 2016: (3, 25), 2017: (4, 14), 2018: (3, 30), 2019: (4, 19), 2020: (4, 10), 2021: (4, 2), 2022: (4, 15), 2023: (4, 7), 2024: (3, 29), 2025: (4, 18), 2026: (4, 3), 2027: (3, 26), 2028: (4, 14)} FIJOS = [(1, 1), (1, 6), (5, 1), (8, 15), (10, 12), (11, 1), (12, 6), (12, 8), (12, 25)] FEST = set() for a, (m, d_) in VS.items(): FEST.add(dt.date(a, m, d_)); FEST.update(dt.date(a, x, y) for x, y in FIJOS) def limite(ff): x = ff + dt.timedelta(days=10) while x.weekday() > 4 or x in FEST: x += UNA return x def fecha(s): if s is None: return None if isinstance(s, dt.datetime): return s.date() if isinstance(s, dt.date): return s s = str(s).strip()[:10] if not re.fullmatch(r'\d{4}-\d{2}-\d{2}', s): return None try: return dt.date.fromisoformat(s) except ValueError: return None # 1) Anuncios, en dos pasadas para no guardar millones de filas: primero los id con anuncio de formalización enviado al DOUE; luego, sus anuncios F = ds.field a = ds.dataset(f'{H}_anuncios.parquet') ids = set() f0 = (F('tipo_anuncio_code') == 'DOC_FORM') & (F('es_ultima_version') == True) & (F('medio_publicacion') == 'DOUE') & F('envio_fecha').is_valid() for batch in a.to_batches(columns=['id', 'entrada_repetida'], filter=f0, batch_size=200_000, batch_readahead=1, fragment_readahead=1, use_threads=False): for i, rep in zip(batch.column('id').to_pylist(), batch.column('entrada_repetida').to_pylist()): if not rep: ids.add(i) print('ids', len(ids), round(time.time() - T0), flush=True) filt = (F('tipo_anuncio_code').isin(['DOC_CN', 'DOC_FORM'])) & (F('es_ultima_version') == True) rows = collections.defaultdict(list) # id -> [(tipo, medio, envio_fecha, fp)] n = 0 for batch in a.to_batches(columns=['id', 'tipo_anuncio_code', 'medio_publicacion', 'envio_fecha', 'fecha_publicacion', 'entrada_repetida'], filter=filt, batch_size=200_000, batch_readahead=1, fragment_readahead=1, use_threads=False): for i, tipo, medio, env, fp, rep in zip(*[batch.column(k).to_pylist() for k in ['id', 'tipo_anuncio_code', 'medio_publicacion', 'envio_fecha', 'fecha_publicacion', 'entrada_repetida']]): if rep or i not in ids: continue m = (medio or '') if not (m == 'DOUE' or m.lower().startswith('perfil')): continue rows[i].append((tipo, m, env, fecha(fp))) n += 1 print('filas anuncios', n, 'ids con anuncios', len(rows), round(time.time() - T0), flush=True) # 2) Resultados r = ds.dataset(f'{H}_resultados.parquet') res = {} filt2 = (F('es_ultima_version') == True) & F('fecha_formalizacion').is_valid() for batch in r.to_batches(columns=['id', 'fecha_formalizacion', 'fecha_adjudicacion', 'entrada_repetida'], filter=filt2, batch_size=300_000, batch_readahead=1, fragment_readahead=1, use_threads=False): for i, ff, fa, rep in zip(*[batch.column(k).to_pylist() for k in ['id', 'fecha_formalizacion', 'fecha_adjudicacion', 'entrada_repetida']]): if rep or i not in ids: continue ff = fecha(ff); fa = fecha(fa) v = res.setdefault(i, [set(), None]) v[0].add(ff) if fa is not None: v[1] = fa if v[1] is None or fa > v[1] else v[1] print('resultados', len(res), round(time.time() - T0), flush=True) # 3) Licitaciones l = ds.dataset(f'{H}.parquet') lic = {} filt3 = (F('es_ultima_version') == True) & (F('conjunto') == 'licitaciones') cols = ['id', 'expediente', 'nif_organo', 'tipo_contrato_code', 'procedimiento_code', 'sara', 'valor_estimado_contrato', 'fecha_publicacion', 'entrada_repetida'] for batch in l.to_batches(columns=cols, filter=filt3, batch_size=300_000, batch_readahead=1, fragment_readahead=1, use_threads=False): for vals in zip(*[batch.column(k).to_pylist() for k in cols]): i = vals[0] if vals[8] or i not in ids: continue lic[i] = vals[1:8] print('lic', len(lic), round(time.time() - T0), flush=True) def umbral(tc, anio): if tc == '3': return 5225000 if anio <= 2017 else 5548000 if anio <= 2019 else 5350000 if anio <= 2021 else 5382000 if anio <= 2023 else 5538000 if anio <= 2025 else 5404000 return 209000 if anio <= 2017 else 221000 if anio <= 2019 else 214000 if anio <= 2021 else 215000 if anio <= 2023 else 221000 if anio <= 2025 else 216000 P = {} for i, an in rows.items(): if i not in lic or i not in res: continue exp, nif, tc, proc, sara, ve, fpub = lic[i] ff_set, fa_max = res[i] if len(ff_set) != 1 or None in ff_set: continue ff = next(iter(ff_set)) if not (D <= ff.year <= Hh): continue if tc not in ('1', '2', '3') or (proc or '') in ('7', '12'): continue if fa_max is None or not ff > fa_max: continue envs, cn_env = set(), [] form_pub = []; cn_pub_doue = []; cn_perfil = [] for tipo, m, env, fp in an: doue = m == 'DOUE' if doue and env is not None: for e in str(env).split('|'): d_ = fecha(e) if d_ is None: continue if tipo == 'DOC_FORM': envs.add(d_) else: cn_env.append(d_) if fp is not None: if tipo == 'DOC_FORM' and doue: form_pub.append(fp) if tipo == 'DOC_CN' and doue: cn_pub_doue.append(fp) if tipo == 'DOC_CN' and not doue: cn_perfil.append(fp) if len(envs) != 1: continue env = next(iter(envs)) anio_lic = (min(cn_env) if cn_env else (min(cn_pub_doue) if cn_pub_doue else (min(cn_perfil) if cn_perfil else fecha(fpub)))).year if not (sara is True or (ve is not None and ve >= umbral(tc, anio_lic))): continue if form_pub and min(form_pub) < env: continue tp = limite(ff) P[i] = dict(anio=ff.year, ff=str(ff), env=str(env), tope=str(tp), tarde=env > tp, sara=sara is True, d=(env - ff).days) # Regla de las versiones (pre-registro): cualquier versión guardada con un envío del DOC_FORM al DOUE dentro de plazo saca el contrato de los tardíos tard = {i for i, v in P.items() if v['tarde']} prev = collections.defaultdict(list) fv = (F('tipo_anuncio_code') == 'DOC_FORM') & (F('medio_publicacion') == 'DOUE') & F('envio_fecha').is_valid() for batch in a.to_batches(columns=['id', 'envio_fecha', 'entrada_repetida'], filter=fv, batch_size=200_000, batch_readahead=1, fragment_readahead=1, use_threads=False): for i, env, rep in zip(*[batch.column(k).to_pylist() for k in ['id', 'envio_fecha', 'entrada_repetida']]): if rep or i not in tard: continue for e in str(env).split('|'): d_ = fecha(e) if d_ is not None: prev[i].append(d_) for i, v in P.items(): v['tarde_ultima_version'] = v['tarde'] v['previo'] = bool(prev.get(i)) and min(prev[i]) <= dt.date.fromisoformat(v['tope']) v['tarde'] = v['tarde_ultima_version'] and not v['previo'] print('versiones', len(prev), round(time.time() - T0), flush=True) import pyarrow.parquet as pq2 m1 = pq2.read_table(f'/o/priv/medida_{D}_{Hh}.parquet', columns=['id', 'anio', 'ff', 'form_env', 'tope', 'tarde', 'tarde_ultima_version', 'previo_en_plazo', 'sara', 'd']).to_pylist() M = {x['id']: x for x in m1} solo_1 = sorted(set(M) - set(P)); solo_2 = sorted(set(P) - set(M)) dif = [] for i in set(M) & set(P): a, b = M[i], P[i] if str(a['ff']) != b['ff'] or str(a['form_env']) != b['env'] or str(a['tope']) != b['tope'] or bool(a['tarde']) != b['tarde'] or bool(a['tarde_ultima_version']) != b['tarde_ultima_version'] or bool(a['previo_en_plazo']) != b['previo'] or bool(a['sara']) != b['sara'] or a['d'] != b['d'] or a['anio'] != b['anio']: dif.append((i, a, b)) res_out = dict(desde=D, hasta=Hh, impl1=len(M), impl2=len(P), solo_impl1=len(solo_1), solo_impl2=len(solo_2), filas_distintas=len(dif), impl2_tardios_sara=sum(1 for v in P.values() if v['sara'] and v['tarde']), impl2_n_sara=sum(1 for v in P.values() if v['sara']), impl1_tardios_sara=sum(1 for v in M.values() if v['sara'] and v['tarde']), impl1_n_sara=sum(1 for v in M.values() if v['sara']), segundos=round(time.time() - T0), muestra_dif=[(x[0]) for x in dif[:5]], muestra_solo1=solo_1[:5], muestra_solo2=solo_2[:5]) res_out['cero_diferencias'] = (not solo_1 and not solo_2 and not dif) json.dump(res_out, open(f'/o/salidas/20_impl2_{D}_{Hh}.json', 'w'), indent=1, ensure_ascii=False, default=str) print(json.dumps(res_out, indent=1, ensure_ascii=False, default=str))