"""MOD-REC · paso 30 (implementación 1): (P) recurso contra el pliego y (A) recurso contra la adjudicación o la exclusión, estimados por un tribunal (pre-registro 2). Imagen bquantagents-worker (sklearn 1.9.1, numpy 2.1.3, pandas 2.2.3, statsmodels 0.15.0), sin red. Uso: python3 -I 30_modelos.py P|A → /w/salidas/30_.json, /w/priv/pred__.csv y, con PRODUCCION=1, el modelo entrenado con todos los años ≤ 2025 en /w/priv/modelo_.json (para la web). La receta de (P) es la de MP1 (d) v2 (mp1/py/22_recurso.py) sin cambios en variables, niveles, C y Platt. """ import sys, json, os, time import numpy as np, pandas as pd import sklearn from scipy.special import logit, expit, betaln from sklearn.linear_model import LogisticRegression from sklearn.model_selection import GroupKFold import statsmodels.api as sm X_ = sys.argv[1] T0 = time.time(); SEM = 20261026; B = 2000 d = pd.read_csv(f'/w/priv/ds_{X_}.csv', dtype={'licitacion_id': 'string', 'organo_clave': 'string', 'tipo_organo': 'string', 'ccaa': 'string', 'cpv_div': 'string', 'proc_nat': 'string', 'tram': 'string', 'tipo_contrato': 'string', 'tribunal': 'string'}, keep_default_na=False, na_values=[''], low_memory=False) for c in ['fecha_pub', 'ffo'] + (['fecha_adj'] if X_ == 'A' else []): d[c] = pd.to_datetime(d[c], errors='coerce') BOOL = ['sara', 'ue', 'suman100', 'pcap', 'ppt'] + (['anormales', 'inf_jv', 'inf_anormal'] if X_ == 'A' else []) for c in BOOL: d[c] = d[c].map({True: True, False: False, 'True': True, 'False': False, 'true': True, 'false': False}) d['mes_pub'] = d['fecha_pub'].dt.month if X_ == 'A': d['mes_adj'] = d['fecha_adj'].dt.month d['org'] = d['organo_clave'].fillna('__' + d['licitacion_id']) def num_mp1(x): """mp1/py/22_recurso.py num(), sin cambios.""" f = pd.DataFrame(index=x.index) pbl = x['pbl_exp'].fillna(x['pbl_lote']); ve = x['ve_exp'] f['log_pbl'] = np.log10(np.maximum(pbl.fillna(1), 1)); f['pbl_na'] = pbl.isna().astype(float) f['log_ve'] = np.log10(np.maximum(ve.fillna(1), 1)); f['ve_na'] = ve.isna().astype(float) f['dur'] = np.where(x['dur'].isna(), -1, np.minimum(x['dur'].fillna(0), 120)); f['dur_na'] = x['dur'].isna().astype(float) f['plazo'] = np.minimum(x['plazo'].fillna(-1), 120); f['plazo_lt15'] = (x['plazo'] < 15).astype(float) f['log_nlotes'] = np.log1p(x['n_lotes_an'].fillna(1)) f['sara'] = x['sara'].fillna(False).astype(float); f['sara_na'] = x['sara'].isna().astype(float) f['ue'] = x['ue'].fillna(False).astype(float) f['cpv_n_div'] = x['cpv_n_div'].fillna(-1) ok_tipo = x['suman100'].fillna(False) & (x['n_sin_tipo'].fillna(1) == 0) ps = x['peso_suma'].replace(0, np.nan) f['sin_crit'] = x['n_crit'].isna().astype(float); f['n_crit'] = np.minimum(x['n_crit'].fillna(0), 15) f['un_solo_crit'] = (x['n_crit'] == 1).astype(float) f['peso_precio'] = np.where(ok_tipo, (x['peso_precio'].fillna(0) / ps).fillna(0), -1) pjv = np.where(ok_tipo, (x['peso_jv'].fillna(0) / ps).fillna(0), -1) f['peso_jv'] = pjv; f['peso_jv_gt50'] = (pjv > 0.5).astype(float) f['pcap'] = x['pcap'].fillna(False).astype(float); f['ppt'] = x['ppt'].fillna(False).astype(float) return f def num_adj(x): f = pd.DataFrame(index=x.index) f['log_of_max'] = np.log1p(x['of_max'].fillna(0)); f['of_na'] = x['of_max'].isna().astype(float) f['of_min_1'] = (x['of_min'] == 1).astype(float) f['anormales'] = x['anormales'].fillna(False).astype(float) baja = 1 - x['adj_sum'] / x['lic_sum'] f['baja'] = np.clip(baja.fillna(0), -0.5, 1); f['baja_na'] = baja.isna().astype(float) f['dispersion'] = x['dispersion'].fillna(0).clip(0, 1); f['disp_na'] = x['dispersion'].isna().astype(float) f['log_lotes_adj'] = np.log1p(x['lotes_adj'].fillna(0)) f['log_adjudicatarios'] = np.log1p(x['n_adjudicatarios'].fillna(0)) f['inf_jv'] = x['inf_jv'].fillna(False).astype(float); f['inf_anormal'] = x['inf_anormal'].fillna(False).astype(float) f['log_actas'] = np.log1p(x['n_actas'].fillna(0)); f['log_aclaraciones'] = np.log1p(x['n_aclaraciones'].fillna(0)) f['versiones_pliego'] = np.maximum(x['n_fechas_pliego'].fillna(0) - 1, 0) return f CATS = ['proc_nat', 'tram', 'tipo_contrato', 'cpv_div', 'tipo_organo', 'ccaa'] + (['mes_pub'] if X_ == 'P' else ['mes_adj']) F0 = num_mp1(d) if X_ == 'P' else pd.concat([num_mp1(d), num_adj(d)], axis=1) # ── base órgano × CPV por empírico bayesiano (m por verosimilitud marginal beta-binomial) ───────────────────────────── def m_ml(k, n, mu): k, n, mu = np.asarray(k, float), np.asarray(n, float), np.clip(np.asarray(mu, float), 1e-9, 1 - 1e-9) mejor, mm = -np.inf, None for lm in np.arange(0, 12.0001, 0.05): m = np.exp(lm); a, b = m * mu, m * (1 - mu) ll = np.sum(betaln(k + a, n - k + b) - betaln(a, b)) if ll > mejor: mejor, mm = ll, m return float(mm) def eb_cadena(hist, y, w, filas): """hist: DataFrame de años < t (y, w). filas: a las que se da la tasa. Cadena global ← tipo×ccaa ← órgano ← órgano×CPV.""" h = hist.assign(yw=y * w) g = h['yw'].sum() / len(h) out, ms = {}, {} mu_prev_h = pd.Series(g, index=h.index); mu_prev_f = pd.Series(g, index=filas.index) for nivel, claves in (('tc', ['tipo_organo', 'ccaa']), ('o', ['org']), ('oc', ['org', 'cpv_div'])): kk = [h[c].astype('string').fillna('∅') for c in claves]; kf = [filas[c].astype('string').fillna('∅') for c in claves] hk = pd.Series(list(zip(*kk)), index=h.index); fk = pd.Series(list(zip(*kf)), index=filas.index) agg = pd.DataFrame({'k': h['yw'], 'n': 1.0, 'mu': mu_prev_h, 'key': hk}).groupby('key').agg(k=('k', 'sum'), n=('n', 'sum'), mu=('mu', 'mean')) m = m_ml(agg['k'], agg['n'], agg['mu']); ms[nivel] = m post = (agg['k'] + m * agg['mu']) / (agg['n'] + m) mu_prev_h = hk.map(post).astype(float) mu_prev_f = fk.map(post).astype(float).fillna(mu_prev_f) return mu_prev_f.to_numpy(), ms def historial_organo(dd): """Para cada fila de año s: log(1 + filas del órgano en años < s) y su tasa EB de órgano hacia la global de años < s (m del órgano estimado con todos los años < s).""" hn = np.zeros(len(dd)); hr = np.full(len(dd), np.nan) for s in sorted(dd['T'].unique()): cur = (dd['T'] == s).to_numpy(); prev = (dd['T'] < s).to_numpy() if prev.sum() == 0: continue h = dd.loc[prev]; yw = h['y'] * h['w']; g = yw.sum() / len(h) agg = pd.DataFrame({'k': yw, 'n': 1.0, 'org': h['org']}).groupby('org').agg(k=('k', 'sum'), n=('n', 'sum')) m = m_ml(agg['k'], agg['n'], np.full(len(agg), g)) rate = (agg['k'] + m * g) / (agg['n'] + m) o = dd.loc[cur, 'org'] hn[cur] = np.log1p(o.map(agg['n']).fillna(0).to_numpy()) hr[cur] = o.map(rate).fillna(g).to_numpy() return hn, hr def auc_w(s, yy, ww): u, g = np.unique(s, return_inverse=True); Gn = len(u) wp = np.bincount(g, ww * yy, Gn); wn = np.bincount(g, ww * (1 - yy), Gn); below = np.cumsum(wn) - wn return float(np.sum(wp * (below + 0.5 * wn)) / (wp.sum() * wn.sum())) def evaluar(t, produccion=False): TR = (d['T'] < t).to_numpy() if not produccion else (d['T'] <= 2025).to_numpy() TE = (d['T'] == t).to_numpy() if not produccion else np.zeros(len(d), bool) F = F0.copy() if X_ == 'A': hn, hr = historial_organo(d) F['hist_n'] = hn; F['hist_na'] = np.isnan(hr).astype(float) F['hist_rate'] = np.where(np.isnan(hr), np.nanmean(hr[TR]) if np.isfinite(hr[TR]).any() else 0, hr) NUM = list(F.columns) CAT_NIV = {}; Cd = pd.DataFrame(index=d.index) for c in CATS: v = d[c].astype('string').fillna('∅') cnt = v[TR].value_counts(); niv = sorted([x for x, n in cnt.items() if n >= 30]); CAT_NIV[c] = niv Cd[c] = v.where(v.isin(niv), 'otros') MU = F.loc[TR].mean().to_numpy(); SD = F.loc[TR].std(ddof=0).to_numpy(); SD[SD == 0] = 1 def X(mask): Z = (F.loc[mask, NUM].to_numpy(float) - MU) / SD Dm = [np.column_stack([(Cd.loc[mask, c].to_numpy() == lv).astype(float) for lv in CAT_NIV[c] + ['otros']]) for c in CATS] return np.column_stack([Z] + Dm) y = d['y'].to_numpy(float); w = d['w'].to_numpy(float); grp = d['org'].to_numpy() Xtr = X(TR) aj = lambda Xa, ya, wa: LogisticRegression(C=0.1, solver='lbfgs', max_iter=5000).fit(Xa, ya, sample_weight=wa) m = aj(Xtr, y[TR], w[TR]) oof = np.zeros(TR.sum()) for a, b in GroupKFold(n_splits=5).split(Xtr, y[TR], grp[TR]): oof[b] = aj(Xtr[a], y[TR][a], w[TR][a]).predict_proba(Xtr[b])[:, 1] platt = LogisticRegression(C=1e6).fit(logit(np.clip(oof, 1e-6, 1 - 1e-6))[:, None], y[TR], sample_weight=w[TR]) cal = lambda p: platt.predict_proba(logit(np.clip(p, 1e-6, 1 - 1e-6))[:, None])[:, 1] if produccion: return {'NUM': NUM, 'CATS': CATS, 'CAT_NIV': CAT_NIV, 'MU': MU.tolist(), 'SD': SD.tolist(), 'coef': m.coef_[0].tolist(), 'intercept': float(m.intercept_[0]), 'platt': {'a': float(platt.coef_[0][0]), 'b': float(platt.intercept_[0])}, 'n_train': int(TR.sum()), 'positivas_train': float((y * w)[TR].sum()), 'versiones': {'sklearn': sklearn.__version__, 'numpy': np.__version__, 'pandas': pd.__version__}} Xte = X(TE) p_raw = m.predict_proba(Xte)[:, 1]; p = cal(p_raw) # comparadores b1, ms = eb_cadena(d.loc[TR], d.loc[TR, 'y'], d.loc[TR, 'w'], d.loc[TE]) xi = lambda mask: np.log10(np.maximum(d.loc[mask, 've_exp'].fillna(d.loc[mask, 'pbl_exp']).fillna(1), 1)).to_numpy()[:, None] b3 = LogisticRegression(C=1e6).fit(xi(TR), y[TR], sample_weight=w[TR]).predict_proba(xi(TE))[:, 1] yt, wt = y[TE], w[TE] cl = pd.factorize(d.loc[TE, 'org'])[0]; Cn = cl.max() + 1 pt = {'p': auc_w(p, yt, wt), 'p_raw': auc_w(p_raw, yt, wt), 'b1': auc_w(b1, yt, wt), 'b3': auc_w(b3, yt, wt)} rep = {k: [] for k in pt}; rng = np.random.default_rng(SEM) for _ in range(B): mult = np.bincount(rng.integers(0, Cn, Cn), minlength=Cn)[cl].astype(float) if (yt * mult).sum() == 0 or ((1 - yt) * mult).sum() == 0: continue for k, s in (('p', p), ('p_raw', p_raw), ('b1', b1), ('b3', b3)): rep[k].append(auc_w(s, yt, wt * mult)) rep = {k: np.array(v) for k, v in rep.items()} ic = {k: [float(np.percentile(v, 2.5)), float(np.percentile(v, 97.5))] for k, v in rep.items()} dl = {k: {'delta': pt['p'] - pt[k], 'ic': [float(np.percentile(rep['p'] - rep[k], 2.5)), float(np.percentile(rep['p'] - rep[k], 97.5))]} for k in ('b1', 'b3')} o_e = float((wt * yt).sum() / p.sum()) sl = sm.GLM(yt, sm.add_constant(logit(np.clip(p, 1e-6, 1 - 1e-6))), family=sm.families.Binomial(), freq_weights=wt).fit() q = pd.qcut(p, 10, labels=False, duplicates='drop') dec = pd.DataFrame({'p': p, 'yw': yt * wt, 'q': q}).groupby('q').agg(n=('p', 'size'), predicha=('p', 'mean'), obs_sum=('yw', 'sum')).reset_index() dec['observada'] = dec['obs_sum'] / dec['n']; dec['dif_puntos'] = 100 * (dec['predicha'] - dec['observada']) nd = len(dec) cal_ok = bool((dec['dif_puntos'].abs().iloc[:nd - 1] <= 5).all() and abs(dec['dif_puntos'].iloc[-1]) <= 8) pos = int(yt.sum()) por_ccaa = {c: {'n': int((d.loc[TE, 'ccaa'] == c).sum()), 'pos': int(yt[(d.loc[TE, 'ccaa'] == c).to_numpy()].sum()), 'auc': (auc_w(p[(d.loc[TE, 'ccaa'] == c).to_numpy()], yt[(d.loc[TE, 'ccaa'] == c).to_numpy()], wt[(d.loc[TE, 'ccaa'] == c).to_numpy()]) if 0 < yt[(d.loc[TE, 'ccaa'] == c).to_numpy()].sum() < (d.loc[TE, 'ccaa'] == c).sum() else None)} for c in sorted(d.loc[TE, 'ccaa'].unique())} te = d.loc[TE, ['licitacion_id', 'org', 'ccaa', 'T', 'y', 'w']].copy(); te['p'] = p; te['p_raw'] = p_raw; te['b1'] = b1; te['b3'] = b3 te.to_csv(f'/w/priv/pred_{X_}_{t}.csv', index=False) return {'t': t, 'n_train': int(TR.sum()), 'pos_train': int(y[TR].sum()), 'n_test': int(TE.sum()), 'pos_test': pos, 'pos_test_ponderadas': float((yt * wt).sum()), 'auc': pt, 'auc_ic': ic, 'delta': dl, 'o_e': o_e, 'pendiente': float(sl.params[1]), 'brier': float(np.mean(wt * (p - yt) ** 2) / np.mean(wt)), 'deciles': dec.to_dict(orient='records'), 'calibracion_ok': cal_ok, 'm_b1': ms, 'por_ccaa': por_ccaa, 'n_variables': len(NUM), 'niveles': {c: len(v) for c, v in CAT_NIV.items()}, 'coeficientes': dict(zip(NUM + [f'{c}={lv}' for c in CATS for lv in CAT_NIV[c] + ['otros']], map(float, m.coef_[0])))} inf = {'modelo': X_, 'versiones': {'sklearn': sklearn.__version__, 'numpy': np.__version__, 'pandas': pd.__version__}, 'n': int(len(d)), 'por_T': d.groupby('T')['y'].agg(['size', 'sum']).reset_index().values.tolist()} inf['t2025'] = evaluar(2025); inf['t2024'] = evaluar(2024) # prueba ex ante (pre-registro 2 §5): estática (ninguna columna de la lista negra entre las variables) y dinámica (con las # etiquetas de la prueba invertidas, las variables de la prueba, el historial incluido, no cambian) NEGRA = ('estado', 'formalizacion', 'adjudicatario_nif', 'adjudicatario_nombre', 'tabla_origen', 'recurso', 'suspension', 'nulidad', 'desistimiento', 'modificacion', 'cambio') usadas = list(F0.columns) + CATS + (['hist_n', 'hist_na', 'hist_rate'] if X_ == 'A' else []) ex = {'estatica_columnas_negras': [c for c in usadas if any(n in c for n in NEGRA)]} for t in (2025, 2024): TE = (d['T'] == t).to_numpy() F1 = (num_mp1(d) if X_ == 'P' else pd.concat([num_mp1(d), num_adj(d)], axis=1)).loc[TE] d2 = d.copy(); d2.loc[TE, 'y'] = 1 - d2.loc[TE, 'y'] F2 = (num_mp1(d2) if X_ == 'P' else pd.concat([num_mp1(d2), num_adj(d2)], axis=1)).loc[TE] dif = int((~np.isclose(F1.to_numpy(float), F2.to_numpy(float), equal_nan=True)).sum()) if X_ == 'A': h1 = historial_organo(d); h2 = historial_organo(d2) dif += int((~np.isclose(h1[0][TE], h2[0][TE], equal_nan=True)).sum() + (~np.isclose(h1[1][TE], h2[1][TE], equal_nan=True)).sum()) ex[f'dinamica_{t}_diferencias'] = dif inf['prueba_ex_ante'] = ex a5, a4 = inf['t2025'], inf['t2024'] pot = a5['pos_test'] >= (100 if X_ == 'A' else 30) and a4['pos_test'] >= 30 crit = {'auc_2025>=0.68': a5['auc']['p'] >= 0.68, 'auc_2024>=0.66': a4['auc']['p'] >= 0.66, 'calibracion_2025': a5['calibracion_ok'], 'calibracion_2024': a4['calibracion_ok'], 'delta_b1_2025>=0.015': a5['delta']['b1']['delta'] >= 0.015, 'potencia': pot} inf['criterios_impl1'] = crit inf['veredicto_provisional'] = ('NO CONFIRMADA (potencia)' if not pot else ('PASA a falta de la segunda implementación' if all(crit.values()) else 'NO PASA')) if os.environ.get('PRODUCCION') == '1': json.dump(evaluar(None, produccion=True), open(f'/w/priv/modelo_{X_}.json', 'w'), indent=1) inf['segundos'] = round(time.time() - T0, 1) json.dump(inf, open(f'/w/salidas/30_{X_}.json', 'w'), indent=1, ensure_ascii=False, default=float) print(json.dumps({k: inf[k] for k in ('n', 'por_T', 'criterios_impl1', 'veredicto_provisional')}, ensure_ascii=False, default=float)) for t in ('t2025', 't2024'): r = inf[t] print(t, 'n', r['n_test'], 'pos', r['pos_test'], 'AUC', round(r['auc']['p'], 4), r['auc_ic']['p'], 'B1', round(r['auc']['b1'], 4), 'B3', round(r['auc']['b3'], 4), 'dB1', round(r['delta']['b1']['delta'], 4), 'O/E', round(r['o_e'], 3), 'pend', round(r['pendiente'], 3))