"""MOD-EJEC · 31 · SEGUNDA implementación del PREREG_1 (md5 ddf7da57…), escrita desde el texto del pre-registro sin importar rpc_comun ni 30_modelo: numpy/pandas, beta-binomial propia con gammaln, logística L2 con scipy L-BFGS-B (Σ log-pérdida + ||w||²/(2C), intercepto sin penalizar), AUC por rangos. Compara con salidas/30_rpc.json. -> salidas/31_impl2.json""" import json, time import numpy as np, pandas as pd from scipy.special import gammaln, expit from scipy.optimize import minimize from scipy.stats import rankdata T0 = time.time() ds = pd.read_csv('/w/priv/ds_rpc.csv', keep_default_na=False, na_values=[''], dtype=str) num = lambda c: pd.to_numeric(ds[c], errors='coerce') fa = pd.to_datetime(ds['fecha_adj']); T = fa.dt.year.to_numpy(); Y = num('y').to_numpy().astype(int) fm = (fa + pd.to_timedelta(num('dias_mod'), unit='D')).to_numpy() amb = ds['ambit'].fillna('∅').to_numpy(object); org = ds['org'].fillna('∅').to_numpy(object) cpv = ds['cpv_div'].fillna('∅').to_numpy(object); tip = ds['tipo_contrato'].fillna('∅').to_numpy(object) def lml(k, n, a, b): # log verosimilitud marginal beta-binomial (sin la combinatoria, constante en m) return gammaln(k + a) + gammaln(n - k + b) - gammaln(n + a + b) - (gammaln(a) + gammaln(b) - gammaln(a + b)) def fuerza(k, n, mu): mu = np.clip(mu, 1e-9, 1 - 1e-9); mejor = (-np.inf, None) for i in range(241): m = np.exp(0.05 * i); v = lml(k, n, m * mu, m * (1 - mu)).sum() if v > mejor[0]: mejor = (v, m) return mejor[1] def grupos(cols, idx): s = np.array(['\x1f'.join(t) for t in zip(*[c[idx] for c in cols])], dtype=object) return s def tasas(hidx, yh, fidx, niveles): """cadena: global ← niveles sucesivos; devuelve por nivel (tasa en fidx, n en fidx)""" g = yh.mean(); mu_h = np.full(len(hidx), g); mu_f = np.full(len(fidx), g); res = [] for cols in niveles: kh = grupos(cols, hidx); kf = grupos(cols, fidx) u, inv = np.unique(kh, return_inverse=True) k = np.bincount(inv, yh.astype(float), len(u)); n = np.bincount(inv, None, len(u)); mu = np.bincount(inv, mu_h, len(u)) / n m = fuerza(k, n, mu); post = (k + m * mu) / (n + m) mu_h = post[inv] pos = np.searchsorted(u, kf); ok = (pos < len(u)) & (u[np.minimum(pos, len(u) - 1)] == kf) nf = np.where(ok, n[np.minimum(pos, len(u) - 1)], 0.0) mu_f = np.where(ok, post[np.minimum(pos, len(u) - 1)], mu_f) res.append((mu_f.copy(), nf)) return res # historial conocido al día 1 del mes de la adjudicación Hh = np.full((len(ds), 4), np.nan); Hn = np.zeros((len(ds), 2)) mes = fa.dt.year * 12 + fa.dt.month - 1 for mm in sorted(set(mes[T >= 2018])): corte = pd.Timestamp(year=mm // 12, month=mm % 12 + 1, day=1) fidx = np.where(mes.to_numpy() == mm)[0]; hidx = np.where(fa.to_numpy() < np.datetime64(corte))[0] if len(hidx) < 100: continue yh = ((Y[hidx] == 1) & (fm[hidx] < np.datetime64(corte))).astype(int) r1 = tasas(hidx, yh, fidx, [[amb], [amb, org], [amb, org, cpv]]); r2 = tasas(hidx, yh, fidx, [[amb], [amb, org], [amb, org, tip]]) Hh[fidx, 0], Hh[fidx, 1], Hh[fidx, 2], Hh[fidx, 3] = r1[0][0], r1[1][0], r1[2][0], r2[2][0] Hn[fidx, 0], Hn[fidx, 1] = np.log1p(r1[1][1]), np.log1p(r1[2][1]) print('historial', round(time.time() - T0), 's', flush=True) imp = num('imp_adj').to_numpy(); dur = num('dur').to_numpy(); nl = num('n_lotes').to_numpy(); na = num('n_adj_exp').to_numpy(); ln = num('lote_num').to_numpy() cols = [np.log10(np.maximum(imp, 1)), np.where(np.isnan(dur), -1, np.minimum(dur, 120)), np.isnan(dur).astype(float), np.log1p(np.where(np.isnan(dur), 0, np.maximum(dur, 0))), np.log1p(nl), np.log1p(na), (np.where(np.isnan(ln), 1, ln) > 1).astype(float)] for j in range(4): v = Hh[:, j]; rel = np.where(np.isnan(v), np.nanmean(v), v); rel = np.clip(rel, 1e-4, 1 - 1e-4) cols += [np.log(rel / (1 - rel)), np.isnan(v).astype(float)] cols += [Hn[:, 0], Hn[:, 1]] Z0 = np.column_stack(cols) CAT = [tip, ds['proc_nat'].fillna('∅').to_numpy(object), cpv, amb] def ajuste(X, y, C=0.1): n, p = X.shape def f(wb): w, b = wb[:p], wb[p]; z = X @ w + b ll = np.sum(np.logaddexp(0, z) - y * z); g = expit(z) - y return ll + w @ w / (2 * C), np.concatenate([X.T @ g + w / C, [g.sum()]]) r = minimize(f, np.zeros(p + 1), jac=True, method='L-BFGS-B', options={'maxiter': 20000, 'gtol': 1e-7}) return r.x[:p], r.x[p] def auc_rangos(s, y): r = rankdata(s); n1 = y.sum(); n0 = len(y) - n1 return float((r[y == 1].sum() - n1 * (n1 + 1) / 2) / (n1 * n0)) prim = json.load(open('/w/salidas/30_rpc.json')); OUT = {} for t in (2024, 2023): tr = np.where((T >= 2018) & (T <= t - 1))[0]; te = np.where(T == t)[0] mu = Z0[tr].mean(0); sd = Z0[tr].std(0); sd[sd == 0] = 1 D = [] for c in CAT: vals, cnt = np.unique(c[tr], return_counts=True); lv = sorted(vals[cnt >= 30]); cc = np.where(np.isin(c, lv), c, 'otros') D += [(cc == l).astype(float) for l in lv + ['otros']] X = np.column_stack([(Z0 - mu) / sd] + D) w, b = ajuste(X[tr], Y[tr].astype(float)) s = X[te] @ w + b a_mod = auc_rangos(s, Y[te]) b1 = tasas(tr, Y[tr], te, [[amb], [amb, org], [amb, org, cpv]])[2][0] a_b1 = auc_rangos(b1, Y[te]) OUT[str(t)] = {'auc_impl2': a_mod, 'auc_impl1': prim[str(t)]['auc']['p'], 'dif': a_mod - prim[str(t)]['auc']['p'], 'auc_b1_impl2': a_b1, 'auc_b1_impl1': prim[str(t)]['auc']['b1'], 'dif_b1': a_b1 - prim[str(t)]['auc']['b1'], 'n_test': int(len(te)), 'pos_test': int(Y[te].sum())} print(t, OUT[str(t)], round(time.time() - T0), 's', flush=True) OUT['pasa'] = all(abs(OUT[k]['dif']) <= 0.002 for k in ('2024', '2023')) json.dump(OUT, open('/w/salidas/31_impl2.json', 'w'), indent=1) print('pasa', OUT['pasa'])