"""D1b · segunda implementación de D-a y D-b (DEFINICIONES_PREVIAS §7), escrita desde el texto sellado. Python puro + pyarrow solo para leer las filas crudas. Recalcula mercado, cuota, K-CUOTA, tamaño, cortes, deciles, cuota relativa, estratos y AUC (contando pares con bisect). Bootstrap con otro generador. Salida: interno/30_impl2.json.""" import json, math, random, sys, time from bisect import bisect_left, bisect_right from collections import defaultdict import pyarrow.parquet as pq T0 = time.time(); B = int(sys.argv[1]) if len(sys.argv) > 1 else 2000 def log(*a): print(f"[{time.time()-T0:7.1f} s]", *a, flush=True) t = pq.read_table('/w/interno/29_crudo.parquet') C = {k: t.column(k).to_pylist() for k in t.column_names}; n = len(C['id_canonical']); del t log('filas', n) # mercado: tipo × cpv2 × año si ≥ 20 lotes (y cpv2 no nulo); si no, tipo × año cnt = defaultdict(int) for i in range(n): cnt[(C['tipo'][i], C['cpv2'][i], C['anio'][i])] += 1 merc = [(C['tipo'][i], C['cpv2'][i], C['anio'][i]) if (C['cpv2'][i] is not None and cnt[(C['tipo'][i], C['cpv2'][i], C['anio'][i])] >= 20) else (C['tipo'][i], '*', C['anio'][i]) for i in range(n)] # cuota en el órgano-año (suma SQL: ignora nulos, incluye ≤ 0) so = defaultdict(float); se = defaultdict(float) for i in range(n): a = C['adj'][i] if a is not None: so[(C['organo_clave'][i], C['anio'][i])] += a; se[(C['organo_clave'][i], C['anio'][i], C['empresa_clave'][i])] += a cuota = [se[(C['organo_clave'][i], C['anio'][i], C['empresa_clave'][i])] / so[(C['organo_clave'][i], C['anio'][i])] if (C['adj'][i] is not None and C['adj'][i] > 0) else None for i in range(n)] def cume_desc(val): """p = #(mismo mercado con valor ≥ x) / #(mismo mercado con valor).""" g = defaultdict(list) for i in range(n): if val[i] is not None: g[merc[i]].append(val[i]) for k in g: g[k].sort() out = [None] * n for i in range(n): if val[i] is not None: v = g[merc[i]]; out[i] = (len(v) - bisect_left(v, val[i])) / len(v) return out kp = cume_desc([round(c, 12) if c is not None else None for c in cuota]); log('K-CUOTA') # cambio posterior técnico: empates # tamaño: importe anual nacional (adj > 0) S = defaultdict(float) for i in range(n): a = C['adj'][i] if a is not None and a > 0: S[(C['empresa_clave'][i], C['anio'][i])] += a tam = [S.get((C['empresa_clave'][i], C['anio'][i]), 0.0) for i in range(n)] def qc(v, p): h = (len(v) - 1) * p; l = int(math.floor(h)); u = min(l + 1, len(v) - 1) return v[l] + (v[u] - v[l]) * (h - l) por_anio = defaultdict(list) for i in range(n): por_anio[C['anio'][i]].append(tam[i]) emp_anio = defaultdict(list) for (e, a), s in S.items(): emp_anio[a].append(s) # empresas-año con lotes pero sin importe positivo cuentan con 0 (como en impl1) vistos = defaultdict(set) for i in range(n): vistos[C['anio'][i]].add(C['empresa_clave'][i]) for a, es in vistos.items(): emp_anio[a] += [0.0] * sum(1 for e in es if (e, a) not in S) cl, ce, cc = {}, {}, {} for a, v in por_anio.items(): v.sort(); cl[a] = [qc(v, k / 10) for k in range(1, 10)]; cc[a] = [qc(v, k / 100) for k in range(1, 100)] for a, v in emp_anio.items(): v.sort(); ce[a] = [qc(v, k / 10) for k in range(1, 10)] def rango(x, cortes): return 1 + sum(1 for c in cortes if x > c * (1 + 1e-9) + 1e-6) # cambio posterior técnico: tolerancia dec_l = [rango(tam[i], cl[C['anio'][i]]) for i in range(n)] dec_e = [rango(tam[i], ce[C['anio'][i]]) for i in range(n)] cen_l = [rango(tam[i], cc[C['anio'][i]]) for i in range(n)] log('deciles') # cuota relativa: media simple de las cuotas de sus celdas órgano-año en otros órganos celda = {} for i in range(n): if cuota[i] is not None: celda[(C['empresa_clave'][i], C['organo_clave'][i], C['anio'][i])] = cuota[i] from fractions import Fraction tot = defaultdict(lambda: [Fraction(0), 0]); org = defaultdict(lambda: [Fraction(0), 0]) # suma exacta (cambio posterior técnico) for (e, o, a), c in celda.items(): c = Fraction(c); tot[e][0] += c; tot[e][1] += 1; org[(e, o)][0] += c; org[(e, o)][1] += 1 ratio = [None] * n for i in range(n): if cuota[i] is not None: e = C['empresa_clave'][i]; T = tot[e]; O = org[(e, C['organo_clave'][i])] if T[1] > O[1]: ratio[i] = float(f'{cuota[i] / (float(T[0] - O[0]) / (T[1] - O[1])):.8e}') rp = cume_desc(ratio); log('cuota relativa') # unidades pos_anios = {c: {C['anio'][i] for i in range(n) if C[f'pos_{s}'][i] == 1} for c, s in [('alimentos', 'alim'), ('militar', 'mil'), ('carreteras', 'carr')]} U = [] for i in range(n): for c, s in [('alimentos', 'alim'), ('militar', 'mil'), ('carreteras', 'carr')]: if C[f'pos_{s}'][i] == 1: U.append((c, i, 1)) elif C[f'm_{s}'][i] and not C['excluido'][i] and C['anio'][i] in pos_anios[c]: U.append((c, i, 0)) if 2019 <= C['anio'][i] <= 2025: if C['pos_cant'][i] == 1: U.append(('cantabria', i, 1)) elif C['m_obras'][i] and not C['en_cantabria'][i] and not C['excluido'][i]: U.append(('cantabria', i, 0)) log('unidades', len(U)) CART = ['alimentos', 'militar', 'carreteras'] def grupo(g): return CART if g == 'conjunto' else ([c for c in CART if c != g[4:]] if g.startswith('sin_') else [g]) # AUC por pares: P(p_pos < p_ctl) + ½ P(=), por estrato def auc_pares(items): est = defaultdict(lambda: ([], [])) for e, p, y in items: est[e][0 if y else 1].append(p) num = den = 0.0 for ps, cs in est.values(): if not ps or not cs: continue cs.sort() for p in ps: lo = bisect_left(cs, p); hi = bisect_right(cs, p) num += (len(cs) - hi) + 0.5 * (hi - lo) den += len(ps) * len(cs) return num / den if den else None def auc_peso(items, w): """con pesos por conglomerado: barrido por estrato ordenado de mayor a menor p (= de menor a mayor puntuación).""" est = defaultdict(list) for e, p, y, k in items: if w[k]: est[e].append((p, y, w[k])) num = den = 0.0 for v in est.values(): v.sort(key=lambda r: -r[0]); acc_c = 0.0; tp = tc = 0.0; j = 0 while j < len(v): h = j; gp = gc = 0.0 while h < len(v) and v[h][0] == v[j][0]: if v[h][1]: gp += v[h][2] else: gc += v[h][2] h += 1 num += gp * (acc_c + 0.5 * gc); acc_c += gc; tp += gp; tc += gc; j = h den += tp * tc return num / den if den else None def conf(g, k): cs = grupo(g); out = [] for c, i, y in U: if c not in cs: continue if k == 'REF0': out.append(((c, C['anio'][i]), kp[i] if kp[i] is not None else 1.0, y, i)) elif k in ('DA', 'DA_e', 'DA_c') and kp[i] is not None: d = {'DA': dec_l, 'DA_e': dec_e, 'DA_c': cen_l}[k][i]; out.append(((c, merc[i], d), kp[i], y, i)) elif k == 'DB' and rp[i] is not None: out.append(((c, C['anio'][i]), rp[i], y, i)) elif k == 'DB_a' and rp[i] is not None: out.append(((c, merc[i], dec_l[i]), rp[i], y, i)) return out res = {} rng = random.Random(4242) # otro generador y otra semilla que impl1 for g, k in [(g, k) for g in CART + ['conjunto'] for k in ['REF0', 'DA', 'DA_e', 'DA_c', 'DB', 'DB_a']] + [(f'sin_{c}', 'DA') for c in CART] + \ [('cantabria', k) for k in ['REF0', 'DA', 'DA_e', 'DB', 'DB_a']]: it = conf(g, k); r = {'auc': auc_pares([(e, p, y) for e, p, y, _ in it])} if k in ('DA', 'DB') and g != 'cantabria': for esq, col in [('org', 'organo_clave'), ('emp', 'empresa_clave')]: ks = sorted({C[col][i] for _, _, _, i in it}); kid = {v: j for j, v in enumerate(ks)} it2 = [(e, p, y, kid[C[col][i]]) for e, p, y, i in it]; bs = [] for b in range(B): w = [0] * len(ks) for j in rng.choices(range(len(ks)), k=len(ks)): w[j] += 1 bs.append(auc_peso(it2, w)) bs = sorted(v for v in bs if v is not None) r[f'ic_{esq}'] = [qc(bs, 0.025), qc(bs, 0.975)] res[f'{g}|{k}'] = r; log(g, k, r) # volcado para el cotejo fila a fila (por id_canonical) fil = {C['id_canonical'][i]: [cuota[i], kp[i], ratio[i], rp[i], dec_l[i], dec_e[i], cen_l[i], tam[i]] for _, i, _ in U} json.dump({'res': res, 'B': B}, open('/w/interno/30_impl2.json', 'w'), indent=1) json.dump(fil, open('/w/interno/30_impl2_filas.json', 'w')) log('fin')