"""MOD-REC · paso 31: SEGUNDA IMPLEMENTACIÓN (pre-registro 2 §4), escrita desde el texto del pre-registro y de la receta de MP1 (d) sin importar py/30_modelos.py: lectura del CSV con el módulo csv (sin pandas), variables con numpy, logística L2 con scipy L-BFGS sobre C·Σ w·logloss + ½‖β‖² (sin penalizar el término independiente), AUC por rangos medios ponderada, base órgano × CPV con su propia máxima verosimilitud beta-binomial. Imagen bquantagents-worker, sin red. Uso: python3 -I 31_impl2.py P|A → /w/salidas/31_impl2_.json (AUC del modelo y de B1 por año de prueba). """ import sys, csv, json, math, time import numpy as np from scipy.optimize import minimize from scipy.special import gammaln X_ = sys.argv[1] t0 = time.time() filas = list(csv.DictReader(open(f'/w/priv/ds_{X_}.csv', newline=''))) n = len(filas) def fl(v): return float(v) if v not in ('', None) else math.nan def bo(v): return {'True': 1.0, 'true': 1.0, 'False': 0.0, 'false': 0.0}.get(v, math.nan) col = lambda k: np.array([fl(r[k]) for r in filas]) colb = lambda k: np.array([bo(r[k]) for r in filas]) cols = lambda k: np.array([r[k] if r[k] != '' else '∅' for r in filas], dtype=object) T = col('T').astype(int); y = col('y'); w = col('w') org = np.array([r['organo_clave'] if r['organo_clave'] else '__' + r['licitacion_id'] for r in filas], dtype=object) def mes(k): return np.array([float(r[k][5:7]) if r[k] else math.nan for r in filas]) # variables numéricas del anuncio (receta de MP1 (d) v2) pbl = col('pbl_exp'); pbl = np.where(np.isnan(pbl), col('pbl_lote'), pbl); ve = col('ve_exp') dur = col('dur'); plazo = col('plazo'); nl = col('n_lotes_an'); sara = colb('sara'); ue = colb('ue'); cnd = col('cpv_n_div') ncrit = col('n_crit'); s100 = colb('suman100'); nst = col('n_sin_tipo'); psum = col('peso_suma'); ppre = col('peso_precio') pjv0 = col('peso_jv'); pcap = colb('pcap'); ppt = colb('ppt') nz = lambda a, v: np.where(np.isnan(a), v, a) ok = (nz(s100, 0) == 1) & (nz(nst, 1) == 0) ps = np.where(psum == 0, np.nan, psum) with np.errstate(invalid='ignore', divide='ignore'): pp = np.where(ok, nz(nz(ppre, 0) / ps, 0), -1.0) pj = np.where(ok, nz(nz(pjv0, 0) / ps, 0), -1.0) V = [np.log10(np.maximum(nz(pbl, 1), 1)), np.isnan(pbl) * 1.0, np.log10(np.maximum(nz(ve, 1), 1)), np.isnan(ve) * 1.0, np.where(np.isnan(dur), -1, np.minimum(nz(dur, 0), 120)), np.isnan(dur) * 1.0, np.minimum(nz(plazo, -1), 120), (nz(plazo, 99) < 15) * 1.0, np.log1p(nz(nl, 1)), nz(sara, 0), np.isnan(sara) * 1.0, nz(ue, 0), nz(cnd, -1), np.isnan(ncrit) * 1.0, np.minimum(nz(ncrit, 0), 15), (ncrit == 1) * 1.0, pp, pj, (pj > 0.5) * 1.0, nz(pcap, 0), nz(ppt, 0)] if X_ == 'A': om = col('of_max'); omin = col('of_min'); an = colb('anormales'); asum = col('adj_sum'); lsum = col('lic_sum') disp = col('dispersion'); la = col('lotes_adj'); ld = col('lotes_des'); nadj = col('n_adjudicatarios') with np.errstate(invalid='ignore', divide='ignore'): baja = 1 - asum / lsum V += [np.log1p(nz(om, 0)), np.isnan(om) * 1.0, (omin == 1) * 1.0, nz(an, 0), np.clip(nz(baja, 0), -0.5, 1), np.isnan(baja) * 1.0, np.clip(nz(disp, 0), 0, 1), np.isnan(disp) * 1.0, np.log1p(nz(la, 0)), np.log1p(nz(nadj, 0)), nz(colb('inf_jv'), 0), nz(colb('inf_anormal'), 0), np.log1p(nz(col('n_actas'), 0)), np.log1p(nz(col('n_aclaraciones'), 0)), np.maximum(nz(col('n_fechas_pliego'), 0) - 1, 0)] NUMS = np.column_stack(V) CATN = ['proc_nat', 'tram', 'tipo_contrato', 'cpv_div', 'tipo_organo', 'ccaa'] CATV = [cols(c) for c in CATN] + [np.array([str(int(m)) if not math.isnan(m) else '∅' for m in mes('fecha_pub' if X_ == 'P' else 'fecha_adj')], dtype=object)] def lml(k, nn, mu, m): a, b = m * mu, m * (1 - mu) return np.sum(gammaln(k + a) + gammaln(nn - k + b) - gammaln(nn + a + b) - (gammaln(a) + gammaln(b) - gammaln(a + b))) def mejor_m(k, nn, mu): mu = np.clip(mu, 1e-9, 1 - 1e-9) grid = [math.exp(0.05 * i) for i in range(241)] vals = [lml(k, nn, mu, m) for m in grid] return grid[int(np.argmax(vals))] def agrupa(keys, val): u, inv = np.unique(keys, return_inverse=True) return u, inv, np.bincount(inv, weights=val, minlength=len(u)), np.bincount(inv, minlength=len(u)).astype(float) def tasa_eb(hmask, fmask): """Base órgano × CPV para las filas fmask con los datos de hmask (cadena global → tipo×ccaa → órgano → órgano×CPV).""" yw = (y * w)[hmask]; g = yw.sum() / hmask.sum() mu_h = np.full(hmask.sum(), g); mu_f = np.full(fmask.sum(), g) tipo, ccaa, cpv = CATV[4], CATV[5], CATV[3] for claves in ((tipo, ccaa), (org,), (org, cpv)): kh = np.array(['|'.join(str(c[i]) for c in claves) for i in np.where(hmask)[0]], dtype=object) kf = np.array(['|'.join(str(c[i]) for c in claves) for i in np.where(fmask)[0]], dtype=object) u, inv, k, nn = agrupa(kh, yw) mu_g = np.bincount(inv, weights=mu_h, minlength=len(u)) / nn m = mejor_m(k, nn, mu_g) post = (k + m * mu_g) / (nn + m) d = dict(zip(u, post)) mu_h = post[inv] mu_f = np.array([d.get(kk, mf) for kk, mf in zip(kf, mu_f)]) return mu_f def hist(dd_mask_all): hn = np.zeros(n); hr = np.full(n, np.nan) for s in sorted(set(T.tolist())): cur = T == s; prev = T < s if prev.sum() == 0: continue yw = (y * w)[prev]; g = yw.sum() / prev.sum() u, inv, k, nn = agrupa(org[prev], yw) m = mejor_m(k, nn, np.full(len(u), g)) dn = dict(zip(u, nn)); dr = dict(zip(u, (k + m * g) / (nn + m))) hn[cur] = [math.log1p(dn.get(o, 0)) for o in org[cur]] hr[cur] = [dr.get(o, g) for o in org[cur]] return hn, hr def ajusta(Xa, ya, wa, C=0.1): p = Xa.shape[1] def f(b): z = Xa @ b[:p] + b[p] ll = np.sum(wa * (np.logaddexp(0, z) - ya * z)) r = 1 / (1 + np.exp(-z)) gr = Xa.T @ (wa * (r - ya)); gb = np.sum(wa * (r - ya)) return C * ll + 0.5 * b[:p] @ b[:p], np.concatenate([C * gr + b[:p], [C * gb]]) r = minimize(f, np.zeros(p + 1), jac=True, method='L-BFGS-B', options={'maxiter': 20000, 'gtol': 1e-10, 'ftol': 1e-15}) return r.x def auc(s, yy, ww): # rangos medios con empates; AUC ponderada = Σ_pos w·(Σ_neg w por debajo + ½ empatados) / (Wp·Wn) o = np.argsort(s, kind='mergesort'); s2, y2, w2 = s[o], yy[o], ww[o] tot = 0.0; neg_abajo = 0.0; i = 0; N = len(s2) while i < N: j = i while j + 1 < N and s2[j + 1] == s2[i]: j += 1 wp = np.sum(w2[i:j + 1] * y2[i:j + 1]); wn = np.sum(w2[i:j + 1] * (1 - y2[i:j + 1])) tot += wp * (neg_abajo + 0.5 * wn); neg_abajo += wn; i = j + 1 return float(tot / (np.sum(ww * yy) * np.sum(ww * (1 - yy)))) out = {'modelo': X_, 'n': n} for t in (2025, 2024): TR = T < t; TE = T == t M = NUMS.copy() if X_ == 'A': hn, hr = hist(None) fill = np.nanmean(hr[TR]) if np.isfinite(hr[TR]).any() else 0 M = np.column_stack([M, hn, np.isnan(hr) * 1.0, np.where(np.isnan(hr), fill, hr)]) mu = M[TR].mean(0); sd = M[TR].std(0); sd[sd == 0] = 1 Z = (M - mu) / sd D = [] for cv in CATV: vals, cnt = np.unique(cv[TR], return_counts=True) niv = sorted(v for v, c in zip(vals, cnt) if c >= 30) cv2 = np.where(np.isin(cv, niv), cv, 'otros') D += [(cv2 == lv) * 1.0 for lv in niv + ['otros']] Xm = np.column_stack([Z] + D) b = ajusta(Xm[TR], y[TR], w[TR]) s = Xm[TE] @ b[:-1] + b[-1] b1 = tasa_eb(TR, TE) out[f't{t}'] = {'auc': auc(s, y[TE], w[TE]), 'auc_b1': auc(b1, y[TE], w[TE]), 'n_test': int(TE.sum()), 'pos_test': int(y[TE].sum())} out['segundos'] = round(time.time() - t0, 1) json.dump(out, open(f'/w/salidas/31_impl2_{X_}.json', 'w'), indent=1) print(json.dumps(out))