"""X2 · segunda implementación independiente (DEFINICIONES_PREVIAS §6): pyarrow + Python puro, sin DuckDB. Técnicas: RF-SUMA, RF-LIMITE, RF-LIMITE-AGG, RF-DISTRIB, RF-DIARIO, RF-FAVORITO, RF-TEMP-ESTACIONAL-ANUAL (+@10), RF-CPV, y RF-OBJETO por fuerza bruta en 300 organismos-ejercicio al azar (tokenizador reescrito). Métricas: todas las celdas, con la AUC por pares. Escribe salidas/07_cotejo_impl2.md.""" import ast, csv, math, random, statistics, sys, time from collections import defaultdict import pyarrow.parquet as pq T0 = time.time() def log(*a): print(f"[{time.time()-T0:6.1f} s]", *a, flush=True) out = [] def w(s=""): out.append(s); print(s, flush=True) P = pq.read_table('/w/interno/p.parquet').to_pydict() N = len(P["rid"]); log("población", N) rid, org, emp, tc, base, fa, fp, cpv, on = (P[k] for k in ("rid", "organo_clave", "empresa_clave", "tipo_contrato", "base", "fecha_adj", "fecha_pub", "cpv", "objeto_norm")) U = [40000 if t == "Obras" else 15000 for t in tc] OB = [t == "Obras" for t in tc] Y = [d.year for d in fa] S1 = pq.read_table('/w/interno/senal.parquet').to_pydict() sen1 = defaultdict(set) for t, r in zip(S1["tec"], S1["rid"]): sen1[t].add(r) sen2 = {} # RF-SUMA g = defaultdict(list) for i in range(N): g[(org[i], emp[i], OB[i], Y[i])].append(i) sen2["RF-SUMA"] = {rid[i] for L in g.values() if len(L) >= 2 and round(sum(base[j] for j in L), 2) > U[L[0]] for i in L} # céntimos exactos # RF-LIMITE band = [0.90 * U[i] <= base[i] <= U[i] for i in range(N)] sen2["RF-LIMITE"] = {rid[i] for i in range(N) if band[i]} # RF-LIMITE-AGG g = defaultdict(list) for i in range(N): if band[i]: g[(org[i], emp[i], Y[i])].append(i) sen2["RF-LIMITE-AGG"] = {rid[i] for L in g.values() if len(L) >= 3 for i in L} # por organismo-ejercicio oy = defaultdict(list) for i in range(N): oy[(org[i], Y[i])].append(i) s = set() for L in oy.values(): nb = sum(1 for i in L if 0.85 * U[i] <= base[i] <= U[i]) if nb >= 5 and nb >= 0.30 * len(L): s.update(rid[i] for i in L) sen2["RF-DISTRIB"] = s for tec, mn in (("RF-TEMP-ESTACIONAL-ANUAL", 1), ("RF-TEMP-ESTACIONAL-ANUAL@10", 10)): s = set() for L in oy.values(): dic = sum(1 for i in L if fa[i].month == 12); q4 = sum(1 for i in L if fa[i].month >= 10) if len(L) >= mn and (dic > 0.25 * len(L) or q4 > 0.50 * len(L)): s.update(rid[i] for i in L) sen2[tec] = s s = set() for L in oy.values(): if len(L) < 10: continue pe_n, pe_i = defaultdict(int), defaultdict(float) for i in L: pe_n[emp[i]] += 1; pe_i[emp[i]] += base[i] if len(pe_n) < 3: continue rn = sorted(pe_n.items(), key=lambda kv: (-kv[1], kv[0])); ri = sorted(pe_i.items(), key=lambda kv: (-kv[1], kv[0])) fav = set() if rn[0][1] >= 0.40 * len(L) and rn[0][1] >= 2 * rn[1][1]: fav.add(rn[0][0]) tot = sum(pe_i.values()) if ri[0][1] >= 0.40 * tot and ri[0][1] >= 2 * ri[1][1]: fav.add(ri[0][0]) s.update(rid[i] for i in L if emp[i] in fav) sen2["RF-FAVORITO"] = s s = set() for L in oy.values(): if sum(1 for i in L if cpv[i] is not None) < 0.15 * len(L): continue cl = defaultdict(list) for i in L: if cpv[i] is not None: cl[cpv[i][:4]].append(i) for K in cl.values(): imp = sum(base[i] for i in K) if len(K) < 5 or imp < 10000: continue pe = defaultdict(float) for i in K: pe[emp[i]] += base[i] if sum(v * v for v in pe.values()) / imp ** 2 >= 0.5: s.update(rid[i] for i in K) sen2["RF-CPV"] = s # RF-DIARIO, con días-lote rehechos lote = defaultdict(int) for i in range(N): if fa[i] == fp[i]: lote[(org[i], fp[i])] += 1 fuera = {i for i in range(N) if fa[i] == fp[i] and lote[(org[i], fp[i])] >= 20} g = defaultdict(list) for i in range(N): if i not in fuera: g[(org[i], emp[i], fa[i])].append(i) sen2["RF-DIARIO"] = {rid[i] for L in g.values() if len(L) >= 2 for i in L} log("técnicas impl2 hechas") w("# X2 · Segunda implementación independiente (pyarrow + Python puro, sin DuckDB)\n") w(f"Generado {time.strftime('%F %T')}. Misma población exportada (`interno/p.parquet`, {N:,} menores); umbral, banda, grupos, " "días-lote y tokens rehechos en este código.\n") w("## 1. Contratos señalados: primera frente a segunda\n") w("| Técnica | primera | segunda | solo en la primera | solo en la segunda | ¿coincide? |\n|---|---:|---:|---:|---:|---|") todo_ok = True for t, s2 in sen2.items(): s1 = sen1[t]; a, b = len(s1 - s2), len(s2 - s1); ok = a == 0 and b == 0; todo_ok &= ok w(f"| {t} | {len(s1):,} | {len(s2):,} | {a} | {b} | {'sí' if ok else 'NO'} |") # RF-OBJETO por fuerza bruta en 300 organismos-ejercicio src = open('/app/etl/series_licitadas.py', encoding='utf-8').read() cons = {n.targets[0].id: ast.literal_eval(n.value) for n in ast.parse(src).body if isinstance(n, ast.Assign) and isinstance(n.targets[0], ast.Name) and n.targets[0].id in ("PALABRAS_VACIAS", "GENERICOS")} VAC, GEN = set(cons["PALABRAS_VACIAS"]), set(cons["GENERICOS"]) def singular(x): if len(x) >= 5 and x.endswith("es") and x[-3] in "rnldjy": return x[:-2] if len(x) >= 4 and x.endswith("s") and not x.endswith("ss"): return x[:-1] return x def toks(o): return frozenset(t for t in (singular(x) for x in (o or "").split(" ") if len(x) >= 3 and x not in VAC) if t not in GEN) rng = random.Random(20261021) claves = sorted(k for k, L in oy.items() if len(L) <= 1500) muestra = rng.sample(claves, 300) s2, ncomp, npares = set(), 0, 0 for k in muestra: L = [i for i in oy[k] if toks(on[i])] T = [toks(on[i]) for i in L] par = list(range(len(L))) def f(a): while par[a] != a: par[a] = par[par[a]]; a = par[a] return a for a in range(len(L)): for b in range(a + 1, len(L)): inter = len(T[a] & T[b]) if inter and inter / math.sqrt(len(T[a]) * len(T[b])) >= 0.70 - 1e-9: ra, rb = f(a), f(b) if ra != rb: par[max(ra, rb)] = min(ra, rb) npares += 1 comp = defaultdict(list) for a in range(len(L)): comp[f(a)].append(L[a]) for C in comp.values(): if len(C) >= 3: s2.update(rid[i] for i in C); ncomp += 1 rids_m = {rid[i] for k in muestra for i in oy[k]} s1 = sen1["RF-OBJETO"] & rids_m a, b = len(s1 - s2), len(s2 - s1); okob = a == 0 and b == 0; todo_ok &= okob w(f"\nRF-OBJETO por fuerza bruta (todos los pares de contratos, tokenizador reescrito) en 300 organismos-ejercicio al azar " f"(semilla 20261021, de los {len(claves):,} con ≤ 1.500 contratos; {len(rids_m):,} contratos, {npares:,} pares ≥ 0,70, {ncomp} componentes): " f"primera {len(s1):,}, segunda {len(s2):,}, solo en la primera {a}, solo en la segunda {b} → **{'coincide' if okob else 'NO coincide'}**.\n") log("objeto hecho") # ── métricas por otro código: AUC por pares ────────────────────────────────────────────────────────────────────── EU = pq.read_table('/w/interno/et_u.parquet').to_pydict(); ER = pq.read_table('/w/interno/et_r.parquet').to_pydict() cont = defaultdict(list) for e, u_, r in zip(ER["etiqueta"], ER["uid"], ER["rid"]): cont[(e, u_)].append(r) VAR = {("EM2B", "principal"): "v_principal", ("EM2B", "mas2021"): "v_mas2021", ("EM2B", "solo_series"): "v_series", ("L1", "principal"): "v_principal", ("L1", "C_y_discrepantes_como_0"): "v_mas2021", ("P6", "a"): "v_a", ("P6", "d"): "v_d"} ref = {(r["etiqueta"], r["variante"], r["tecnica"]): r for r in csv.DictReader(open('/w/salidas/40_tabla.csv', encoding='utf-8'), delimiter=';')} def auc_pares(pos, neg): """pos, neg: listas de (puntuación, peso). Pares explícitos; los negativos se agrupan por (puntuación, peso) idénticos.""" gn = defaultdict(float) for s_, w_ in neg: gn[(s_, w_)] += 1 num = 0.0 for sp, wp in pos: for (sn, wn), m in gn.items(): num += wp * wn * m * (1.0 if sp > sn else 0.5 if sp == sn else 0.0) return num / (sum(x[1] for x in pos) * sum(x[1] for x in neg)) maxd, nceldas, malas = 0.0, 0, [] for (et, var), col in VAR.items(): idx = [i for i in range(len(EU["uid"])) if EU["etiqueta"][i] == et and EU[col][i]] for tec in sorted(sen1): S = sen1[tec] filas = [(EU["y"][i], EU["w"][i], 1 if any(r in S for r in cont[(et, EU["uid"][i])]) else 0) for i in idx] sw = sum(x[1] for x in filas); swy = sum(x[1] for x in filas if x[0]); swf = sum(x[1] for x in filas if x[2]) swyf = sum(x[1] for x in filas if x[0] and x[2]) mine = {"base": swy / sw, "precision": swyf / swf if swf else None, "recall": swyf / swy, "lift": (swyf / swf) / (swy / sw) if swf else None, "auc": auc_pares([(x[2], x[1]) for x in filas if x[0]], [(x[2], x[1]) for x in filas if not x[0]])} r = ref[(et, var, tec)]; nceldas += 1 for k, v in mine.items(): o = None if r[k] == "" else float(r[k]) if (o is None) != (v is None): malas.append((et, var, tec, k, o, v)); continue if v is not None: d = abs(o - v); maxd = max(maxd, d) if d > 1e-5: malas.append((et, var, tec, k, o, v)) # 40_tabla.csv redondea a 6 decimales log("métricas hechas") # Sintético con resultado conocido para esta implementación (AUC 0,7; cero señaladas → AUC 0,5 y precisión vacía) sint_pos = [(1, 1.0), (1, 1.0), (0, 1.0), (0, 1.0)]; sint_neg = [(1, 1.0)] + [(0, 1.0)] * 4 + [(0, 5.0)] a_s = auc_pares(sint_pos, sint_neg); a_0 = auc_pares([(0, 1.0)] * 4, [(0, 1.0)] * 6) okm = not malas and abs(a_s - 0.7) < 1e-12 and abs(a_0 - 0.5) < 1e-12 todo_ok &= okm w("## 2. Métricas: otro código (AUC por pares explícitos)\n") w(f"- Celdas comparadas: {nceldas} (todas las técnicas y variantes, 5 métricas cada una) frente a `salidas/40_tabla.csv`.") w(f"- Diferencia máxima: {maxd:.2e} (la tabla está redondeada a 6 decimales); celdas que no coinciden: {len(malas)}.") for m in malas[:20]: w(f" - {m}") w(f"- Banco sintético de esta implementación: AUC {a_s:.6f} (esperada 0,7) y, con cero señaladas, {a_0:.6f} (esperada 0,5). " "El de la primera (`py/41_sintetico.py`, `salidas/41_sintetico.txt`) pasa.") w(f"\n**Veredicto del cotejo: {'COINCIDE' if todo_ok else 'NO COINCIDE (ver arriba)'}.**") open('/w/salidas/07_cotejo_impl2.md', 'w', encoding='utf-8').write("\n".join(out) + "\n") log("fin")