"""D5b · Paso 2e. Segunda implementación INDEPENDIENTE del cálculo principal (BAJA_COND) escrita desde DEFINICIONES_D1C0.md, sin reutilizar código de c0_screens.py ni de c0_medir.py: tramo, baja, mediana por mercado × tramo (celda ≥ 10), mercado ≥ 20, percentil orientado con rango medio, AUC estratificada por conteo con bisección, y bootstrap en dos etapas con OTRO generador (random.Random(4242 + r), random.choices) y AUC ponderada por sumas acumuladas. Comparte con impl1 solo la base (P0 y etiqueta). Entrada: interno/c0_p0_crudo.parquet, interno/c0_et.parquet. Salida: interno/c0_impl2.json.""" import duckdb, json, random, bisect, math, time, sys from collections import defaultdict, Counter from multiprocessing import Pool T0 = time.time() def log(*a): print(*a, round(time.time() - T0), flush=True) NB = int(sys.argv[1]) if len(sys.argv) > 1 else 2000 con = duckdb.connect() filas = con.execute("SELECT id_canonical, source, lic, org, cpv, tipo, anio, num_ofertas, adj, pbl_lote FROM read_parquet('/c/interno/c0_p0_crudo.parquet')").fetchall() log('p0', len(filas)) def tramo(n): return '3' if n == 3 else ('4-5' if n <= 5 else ('6-9' if n <= 9 else '10+')) merc = Counter(); celdas = defaultdict(list); base = {} for i, s, l, o, c, t, a, n, adj, pbl in filas: m = (s, c, t, a); merc[m] += 1 b = None if pbl is not None and pbl > 0 and adj / pbl <= 1.25: b = 1 - adj / pbl base[i] = (m, tramo(n), b, s + '|' + l, o, a) if b is not None: celdas[(m, tramo(n))].append(b) def mediana(xs): xs = sorted(xs); n = len(xs); h = n // 2 return xs[h] if n % 2 else xs[h - 1] + 0.5 * (xs[h] - xs[h - 1]) med = {k: mediana(v) for k, v in celdas.items() if len(v) >= 10} del celdas grupos = defaultdict(list) for i, (m, tr, b, l, o, a) in base.items(): if merc[m] >= 20 and b is not None and (m, tr) in med: grupos[m].append((round(-(b - med[(m, tr)]), 12), i)) pb = {} for m, xs in grupos.items(): vals = sorted(v for v, _ in xs); N = len(vals) for v, i in xs: lo = bisect.bisect_left(vals, v); hi = bisect.bisect_right(vals, v) pb[i] = ((lo + 1 + hi) / 2 - 0.5) / N del grupos log('percentiles', len(pb)) et = con.execute("SELECT rol, banco, cl, id_canonical FROM read_parquet('/c/interno/c0_et.parquet') WHERE rol = 'ctl' OR origen = 'unidad'").fetchall() BANCOS = {'previo': {'previo'}, 'ampliado': {'previo', 'd5', 'v2', 'acco'}} pos = {b: defaultdict(set) for b in BANCOS}; ctl = defaultdict(set) for rol, banco, cl, i in et: if rol == 'pos': for b, bs in BANCOS.items(): if banco in bs: pos[b][cl].add(i) else: ctl[cl].add(i) def unidades(cls, banco): """(estrato, órgano, licitación, valor, etiqueta) para BAJA_COND.""" U = [] for c in cls: P = [i for i in pos[banco][c] if i in pb]; anios = {base[i][5] for i in P} U += [((c, base[i][5]), base[i][4], base[i][3], pb[i], 1) for i in P] U += [((c, base[i][5]), base[i][4], base[i][3], pb[i], 0) for i in ctl[c] if i in pb and i not in pos[banco][c] and base[i][5] in anios] return U def auc(U): est = defaultdict(lambda: ([], [])) for e, o, l, v, y in U: est[e][1 - y].append(v) num = den = 0.0 for p, c in est.values(): if not p or not c: continue c = sorted(c) for v in p: num += bisect.bisect_left(c, v) + 0.5 * (bisect.bisect_right(c, v) - bisect.bisect_left(c, v)) den += len(p) * len(c) return num / den if den else None def prep(U): est = defaultdict(lambda: ([], [])) for e, o, l, v, y in U: est[e][1 - y].append((v, l)) S = [] for p, c in est.values(): if not p or not c: continue cv = sorted({v for v, _ in c}); pos_v = {v: k for k, v in enumerate(cv)} cl = [[] for _ in cv] for v, l in c: cl[pos_v[v]].append(l) pp = [(bisect.bisect_left(cv, v), (cv[bisect.bisect_left(cv, v)] == v) if bisect.bisect_left(cv, v) < len(cv) else False, l) for v, l in p] S.append((cl, pp)) return S def auc_w(S, w): num = den = 0.0 for cl, pp in S: cum = [0.0]; for ls in cl: cum.append(cum[-1] + sum(w.get(l, 0) for l in ls)) tc = cum[-1]; tp = 0.0 for k, igual, l in pp: wp = w.get(l, 0) if not wp: continue tp += wp; num += wp * (cum[k] + (0.5 * (cum[k + 1] - cum[k]) if igual else 0.0)) den += tp * tc return num / den if den else None DATA = {} def _b(args): key, r0, r1 = args orgs, lics, S = DATA[key]; out = [] for r in range(r0, r1): rng = random.Random(4242 + r); w = Counter() for o in rng.choices(orgs, k=len(orgs)): for l in rng.choices(lics[o], k=len(lics[o])): w[l] += 1 out.append(auc_w(S, w)) return out def boot(U, key): lics = defaultdict(set) for e, o, l, v, y in U: lics[o].add(l) orgs = sorted(lics); DATA[key] = (orgs, {o: sorted(lics[o]) for o in orgs}, prep(U)) with Pool(6) as pool: partes = pool.map(_b, [(key, k * NB // 6, (k + 1) * NB // 6) for k in range(6)]) del DATA[key] b = sorted(x for p in partes for x in p if x is not None) q = lambda f: b[math.floor((len(b) - 1) * f)] + (b[math.ceil((len(b) - 1) * f)] - b[math.floor((len(b) - 1) * f)]) * ((len(b) - 1) * f - math.floor((len(b) - 1) * f)) return (q(0.025), q(0.975)) if b else (None, None) OUT = dict(baja_cond_pct={i: pb[i] for i in {x[3] for x in et} if i in pb}) for banco in BANCOS: ev = [] for c in sorted(pos[banco]): U = unidades([c], banco); st = defaultdict(lambda: [0, 0]) for e, o, l, v, y in U: st[e][y] += 1 if len({l for e, o, l, v, y in U if y and st[e][0] > 0}) >= 10: ev.append(c) r = dict(evaluables=ev, por_cluster={}) for c in ev: U = unidades([c], banco); lo, hi = boot(U, (banco, c)); r['por_cluster'][c] = dict(auc=auc(U), lo=lo, hi=hi, n=len(U)) U = unidades(ev, banco); lo, hi = boot(U, (banco, 'conjunto')); r['conjunto'] = dict(auc=auc(U), lo=lo, hi=hi, n=len(U)) OUT[banco] = r; log(banco, r['conjunto']) json.dump(OUT, open('/c/interno/c0_impl2.json', 'w'), indent=0) log('fin')