"""UTE1 · 40 · SEGUNDA IMPLEMENTACIÓN (Python puro + pyarrow para leer; sin DuckDB, sin importar nada de la primera): pares duales, n, k, p, τ² sin winsorizar, z_D y hecho por celda-año; AUC S1 estratificado frente a V3. Compara con la 1.ª.""" import math, json, csv, hashlib, itertools, collections, pyarrow.parquet as pq T = pq.read_table('/w/tmp/i2_lot.parquet').to_pydict() U = pq.read_table('/w/tmp/i2_ute.parquet').to_pydict() mem = collections.defaultdict(set) for u, e in zip(U['ute_clave'], U['emp']): mem[u].add(e) mem = {u: s for u, s in mem.items() if 2 <= len(s) <= 6} N = len(T['source']); print('filas', N) joint = collections.defaultdict(set) # (a,b,cpv,prov,anio) -> lics uteyear = collections.defaultdict(set) # (ute,cpv,prov,anio) presente nodos = set() for i in range(N): if T['es_ute'][i] and T['empresa_clave'][i] in mem and T['cpv2'][i] != '??' and T['prov'][i] is not None: for a, b in itertools.combinations(sorted(mem[T['empresa_clave'][i]]), 2): joint[(a, b, T['cpv2'][i], T['prov'][i], T['anio'][i])].add(T['lic'][i]); nodos.add(a); nodos.add(b) solo = collections.defaultdict(set) for i in range(N): e = T['empresa_clave'][i] if not T['es_ute'][i] and not T['pf'][i] and e in nodos and T['cpv2'][i] != '??' and T['prov'][i] is not None and not T['dup'][i]: solo[(e, T['cpv2'][i], T['prov'][i], T['anio'][i])].add(T['lic'][i]) pairs_cell = collections.defaultdict(set) for (a, b, c, p, y) in joint: pairs_cell[(a, b, c, p)].add(y) dual = set() for (a, b, c, p), ys in pairs_cell.items(): for t in range(2014, 2026): J = set() for y in (t - 2, t - 1, t): J |= joint.get((a, b, c, p, y), set()) if not J: continue sa = set(); sb = set() for y in (t - 2, t - 1, t): sa |= solo.get((a, c, p, y), set()); sb |= solo.get((b, c, p, y), set()) if (sa - J) and (sb - J): dual.add((a, b, c, p, t)) dmem = collections.defaultdict(set) for a, b, c, p, t in dual: dmem[(c, p, t)].update((a, b)) dpairs = collections.defaultdict(set) for a, b, c, p, t in dual: dpairs[(c, p, t)].add((a, b)) n = collections.Counter(); k = collections.Counter() for i in range(N): c, p, y = T['cpv2'][i], T['prov'][i], T['anio'][i] if c == '??' or p is None or not (2014 <= y <= 2025): continue key = (c, p, y); n[key] += 1; e = T['empresa_clave'][i] if T['es_ute'][i]: if e in mem and any((a, b) in dpairs[key] for a, b in itertools.combinations(sorted(mem[e]), 2)): k[key] += 1 elif e is not None and e in dmem[key]: k[key] += 1 Nm = collections.Counter(); Km = collections.Counter() for (c, p, y), v in n.items(): Nm[(c, y)] += v; Km[(c, y)] += k[(c, p, y)] pe = {key: ((Km[(key[0], key[2])] - k[key]) / (Nm[(key[0], key[2])] - n[key]) if Nm[(key[0], key[2])] - n[key] > 0 else None) for key in n} tau = {} for y in range(2014, 2026): rows = [(k[key] / n[key], pe[key], math.sqrt(pe[key] * (1 - pe[key]) / n[key])) for key in n if key[2] == y and n[key] >= 30 and pe[key] is not None and 0 < pe[key] < 1] I = len(rows); phi = sum(((a - b) / s) ** 2 for a, b, s in rows) / I w = [1 / s ** 2 for _, _, s in rows]; S1 = sum(w); S2 = sum(v * v for v in w) tau[y] = max(0.0, (I * phi - (I - 1)) / (S1 - S2 / S1)) if I * phi > I - 1 else 0.0 zd = {}; hecho = {} for key in n: p = pe[key] if n[key] < 30: zd[key] = None; hecho[key] = False; continue if p is not None and p == 0 and k[key] == 0: zd[key] = 0.0; hecho[key] = False; continue if p is None or p <= 0 or p >= 1: zd[key] = None; hecho[key] = False; continue zd[key] = (k[key] / n[key] - p) / math.sqrt(p * (1 - p) / n[key] + tau[key[2]]); hecho[key] = zd[key] > 3.090232 # comparación con la 1.ª C1 = pq.read_table('/w/tmp/i1_celf.parquet').to_pydict() D1 = pq.read_table('/w/tmp/i1_dual.parquet').to_pydict() d1 = set(zip(D1['a'], D1['b'], D1['cpv2'], D1['prov'], D1['t'])) dif = collections.Counter(); m = len(C1['cpv2']) for i in range(m): key = (C1['cpv2'][i], C1['prov'][i], C1['anio'][i]) if n.get(key) != C1['n'][i]: dif['n'] += 1 if k.get(key, 0) != C1['k'][i]: dif['k'] += 1 p1, p2 = C1['p'][i], pe.get(key) if (p1 is None) != (p2 is None) or (p1 is not None and abs(p1 - p2) > 1e-9): dif['p'] += 1 z1, z2 = C1['zd'][i], zd.get(key) if (z1 is None) != (z2 is None) or (z1 is not None and abs(z1 - z2) > 1e-6 * max(1, abs(z1))): dif['zd'] += 1 if bool(C1['hecho'][i]) != hecho.get(key, False): dif['hecho'] += 1 R = {'celdas_1': m, 'celdas_2': len(n), 'duales_1': len(d1), 'duales_2': len(dual), 'duales_diferentes': len(d1 ^ dual), 'diferencias': dict(dif), 'pct_celdas_con_diferencia': round(100 * max(dif.values() or [0]) / m, 4), 'hechos_2': sum(hecho.values()), 'tau2_2': {y: round(v, 6) for y, v in tau.items()}} # AUC S1 estratificado frente a V3 (marco reconstruido aquí) PER = {'S/DC/0627/18': (2012, 2018), 'S/DC/0614/17': (2012, 2017), 'S/DC/0598/16': (2012, 2016), 'S/DC/0565/15': (2012, 2017), 'S/0644/18': (2014, 2018), 'S/0626/18': (2014, 2018), 'S/0025/19': (2016, 2019), 'S/0011/19': (2012, 2019), 'S/0001/23': (2021, 2022), 'S/DC/0519/14': (2012, 2014), 'S/DC/0512/14': (2012, 2016), 'SAMUR/01/24': (2019, 2019), 'ACCO 101/2018': (2017, 2017), 'CGC S 8/2021': (2021, 2021), 'S/0013/19': (2014, 2018), 'S/0008/21': (2016, 2021), 'S/0016/21': (2015, 2022)} lab = collections.defaultdict(set) # md5 -> {(exp, var)} for r in csv.DictReader(open('/et/colusion/etiquetas/casado/salidas/lotes_cartel_v2.csv')): if r['var'] in ('V2', 'V3'): lab[(r['source'], r['id_md5'])].add((r['exp'], r['var'])) idx = {} for i in range(N): h = hashlib.md5((T['source'][i] + '|' + T['id_canonical'][i]).encode()).hexdigest() if (T['source'][i], h) in lab: idx[i] = lab[(T['source'][i], h)] cnt = collections.defaultdict(lambda: collections.defaultdict(set)) for i, s in idx.items(): for e, v in s: if T['cpv2'][i] != '??': cnt[e][T['cpv2'][i]].add((T['source'][i], T['id_canonical'][i])) merc = {e: {c for c, v in d.items() if len(v) >= 5 and len(v) >= 0.10 * sum(len(w) for w in d.values())} for e, d in cnt.items()} pos = collections.defaultdict(set); srcs = collections.defaultdict(set) for i, s in idx.items(): for e, v in s: if v == 'V3': pos[e].add(i); srcs[e].add(T['source'][i]) fr = collections.defaultdict(list) # (exp,cpv) -> [(imp, y, s1)] for e in merc: if e not in PER or not pos[e]: continue a0, a1 = PER[e] for i in range(N): if T['cpv2'][i] in merc[e] and a0 <= T['anio'][i] <= a1 and T['source'][i] in srcs[e]: y = i in pos[e] if y or i not in idx: fr[(e, T['cpv2'][i])].append((T['imp'][i], int(y), int(bool(T['es_ute'][i])))) num = den = 0.0 for key, v in fr.items(): nn_ = sorted([r for r in v if r[0] is not None], key=lambda r: r[0]); L = len(nn_) st = collections.defaultdict(list) for j, r in enumerate(nn_): st[min(4, j * 5 // L)].append(r) # ntile(5) st[-1] = [r for r in v if r[0] is None] for q, rs in st.items(): a = sum(1 for r in rs if r[1] and r[2]); b = sum(1 for r in rs if r[1] and not r[2]); c = sum(1 for r in rs if not r[1] and r[2]); d = sum(1 for r in rs if not r[1] and not r[2]) if a + b and c + d: num += a * d + 0.5 * (a * c + b * d); den += (a + b) * (c + d) R['auc_S1_V3_impl2'] = round(num / den, 4) if den else None json.dump(R, open('/w/salidas/40_impl2.json', 'w'), indent=1, default=str); print(json.dumps(R, default=str))