"""MOD-EJEC · 25 · DESARROLLO (antes del sello): solo cohortes <= 2022 (entrena 2017-2021, valida 2022). No lee ni calcula nada de 2023-2025: la primera línea tras leer descarta esas filas. Sirve para elegir receta (variables, modelo) antes de sellar el pre-registro 1. -> salidas/25_dev.json""" import json, time import numpy as np, pandas as pd from scipy.special import logit, betaln from sklearn.linear_model import LogisticRegression from sklearn.ensemble import HistGradientBoostingClassifier from sklearn.model_selection import GroupKFold from sklearn.metrics import roc_auc_score T0 = time.time() d = pd.read_csv('/w/priv/ds_rpc.csv', dtype={'org': 'string', 'agrup': 'string', 'ambit': 'string', 'cpv_div': 'string', 'proc_nat': 'string', 'tipo_contrato': 'string', 'tram': 'string', 'link_src': 'string', 'licitacion_id': 'string'}, keep_default_na=False, na_values=['']) d = d[d['T'] <= 2022].reset_index(drop=True) # NADA de 2023 en adelante assert d['T'].max() == 2022 d['fecha_adj'] = pd.to_datetime(d['fecha_adj']); d['mes'] = d['fecha_adj'].dt.month.astype(str) def m_ml(k, n, mu): k, n, mu = np.asarray(k, float), np.asarray(n, float), np.clip(np.asarray(mu, float), 1e-9, 1 - 1e-9) best, mm = -np.inf, None for lm in np.arange(0, 12.0001, 0.05): m = np.exp(lm); a, b = m * mu, m * (1 - mu); ll = np.sum(betaln(k + a, n - k + b) - betaln(a, b)) if ll > best: best, mm = ll, m return float(mm) def eb(hist_y, hist_keys, f_keys, mu_h, mu_f): agg = pd.DataFrame({'k': hist_y, 'n': 1.0, 'mu': mu_h, 'key': hist_keys}).groupby('key').agg(k=('k', 'sum'), n=('n', 'sum'), mu=('mu', 'mean')) m = m_ml(agg['k'], agg['n'], agg['mu']); post = (agg['k'] + m * agg['mu']) / (agg['n'] + m) return hist_keys.map(post).astype(float), f_keys.map(post).astype(float).fillna(mu_f), m, f_keys.map(agg['n']).fillna(0).to_numpy() def key(df, cols): return pd.Series(list(zip(*[df[c].astype('string').fillna('∅') for c in cols])), index=df.index) def cadena(h, f, niveles): g = h['y'].mean(); mh = pd.Series(g, index=h.index); mf = pd.Series(g, index=f.index); out = {} for nm, cols in niveles: mh, mf, m, nf = eb(h['y'], key(h, cols), key(f, cols), mh, mf); out[nm] = (mf.to_numpy(), nf) return out def historial(dd): """por cohorte s: tasas EB (ámbito ← órgano ← órgano×CPV ← órgano×tipo) con las cohortes < s""" cols = {c: np.full(len(dd), np.nan) for c in ('h_amb', 'h_org', 'h_oc', 'h_ot')}; hn = np.zeros(len(dd)); hnoc = np.zeros(len(dd)) for s in sorted(dd['T'].unique()): cur = (dd['T'] == s).to_numpy(); prev = (dd['T'] < s).to_numpy() if prev.sum() < 100: continue h, f = dd.loc[prev], dd.loc[cur] o = cadena(h, f, (('h_amb', ['ambit']), ('h_org', ['ambit', 'org']), ('h_oc', ['ambit', 'org', 'cpv_div']))) o2 = cadena(h, f, (('a', ['ambit']), ('b', ['ambit', 'org']), ('h_ot', ['ambit', 'org', 'tipo_contrato']))) for c in ('h_amb', 'h_org', 'h_oc'): cols[c][cur] = o[c][0] cols['h_ot'][cur] = o2['h_ot'][0]; hn[cur] = np.log1p(o['h_org'][1]); hnoc[cur] = np.log1p(o['h_oc'][1]) return cols, hn, hnoc cols, hn, hnoc = historial(d) # variante «conocido a 1-ene de s»: la etiqueta de cada contrato anterior solo con modificaciones aprobadas antes del 1-ene de s d['f_mod'] = pd.to_datetime(d['fecha_adj']) + pd.to_timedelta(d['dias_mod'], unit='D') def historial_conocido(dd): cols = {c: np.full(len(dd), np.nan) for c in ('h_amb', 'h_org', 'h_oc', 'h_ot')}; hn = np.zeros(len(dd)); hnoc = np.zeros(len(dd)) for s_ in sorted(dd['T'].unique()): cur = (dd['T'] == s_).to_numpy(); prev = (dd['T'] < s_).to_numpy() if prev.sum() < 100: continue h = dd.loc[prev].copy(); f = dd.loc[cur] h['y'] = ((h['y'] == 1) & (h['f_mod'] < pd.Timestamp(f'{s_}-01-01'))).astype(int) o = cadena(h, f, (('h_amb', ['ambit']), ('h_org', ['ambit', 'org']), ('h_oc', ['ambit', 'org', 'cpv_div']))) o2 = cadena(h, f, (('a', ['ambit']), ('b', ['ambit', 'org']), ('h_ot', ['ambit', 'org', 'tipo_contrato']))) for c in ('h_amb', 'h_org', 'h_oc'): cols[c][cur] = o[c][0] cols['h_ot'][cur] = o2['h_ot'][0]; hn[cur] = np.log1p(o['h_org'][1]); hnoc[cur] = np.log1p(o['h_oc'][1]) return cols, hn, hnoc colsK, hnK, hnocK = historial_conocido(d) def historial_mes(dd): """conocido al día 1 del mes de la adjudicación: contratos adjudicados antes de ese día; su etiqueta, con las modificaciones aprobadas antes""" cols = {c: np.full(len(dd), np.nan) for c in ('h_amb', 'h_org', 'h_oc', 'h_ot')}; hn = np.zeros(len(dd)); hnoc = np.zeros(len(dd)) mes = dd['fecha_adj'].dt.to_period('M') for pm in sorted(mes[dd['T'] >= 2018].unique()): corte = pm.to_timestamp() cur = (mes == pm).to_numpy(); prev = (dd['fecha_adj'] < corte).to_numpy() if prev.sum() < 100: continue h = dd.loc[prev].copy(); f = dd.loc[cur] h['y'] = ((h['y'] == 1) & (h['f_mod'] < corte)).astype(int) o = cadena(h, f, (('h_amb', ['ambit']), ('h_org', ['ambit', 'org']), ('h_oc', ['ambit', 'org', 'cpv_div']))) o2 = cadena(h, f, (('a', ['ambit']), ('b', ['ambit', 'org']), ('h_ot', ['ambit', 'org', 'tipo_contrato']))) for c in ('h_amb', 'h_org', 'h_oc'): cols[c][cur] = o[c][0] cols['h_ot'][cur] = o2['h_ot'][0]; hn[cur] = np.log1p(o['h_org'][1]); hnoc[cur] = np.log1p(o['h_oc'][1]) return cols, hn, hnoc colsM, hnM, hnocM = historial_mes(d) def feats(x): f = pd.DataFrame(index=x.index) f['log_imp'] = np.log10(np.maximum(x['imp_adj'], 1)) f['dur'] = np.minimum(x['dur'].fillna(-1), 120); f['dur_na'] = x['dur'].isna().astype(float); f['log_dur'] = np.log1p(np.maximum(x['dur'].fillna(0), 0)) f['log_nlotes'] = np.log1p(x['n_lotes']); f['log_nadj'] = np.log1p(x['n_adj_exp']); f['lote_gt1'] = (x['lote_num'].fillna(1) > 1).astype(float) f['log_of'] = np.log1p(x['nof'].fillna(0)); f['of_na'] = x['nof'].isna().astype(float); f['of1'] = (x['nof'] == 1).astype(float) baja = 1 - x['imp_adj'] / x['imp_lic'] f['baja'] = np.clip(baja.fillna(0), -0.5, 1); f['baja_na'] = baja.isna().astype(float) f['dpa'] = np.clip(x['dias_pub_adj'].fillna(-1), -1, 730); f['dpa_na'] = x['dias_pub_adj'].isna().astype(float) f['link_na'] = x['link_src'].isna().astype(float) return f F0 = feats(d) for c in cols: v = cols[c]; F0[c] = logit(np.clip(np.where(np.isnan(v), np.nanmean(v), v), 1e-4, 1 - 1e-4)); F0[c + '_na'] = np.isnan(v).astype(float) F0['hn'] = hn; F0['hnoc'] = hnoc CATS = ['tipo_contrato', 'proc_nat', 'cpv_div', 'ambit', 'mes', 'tram'] def design(F, NUM, TR, TE): Cd = {}; niv = {} for c in CATS: v = d[c].astype('string').fillna('∅'); cnt = v[TR].value_counts(); niv[c] = sorted([a for a, n in cnt.items() if n >= 30]); Cd[c] = v.where(v.isin(niv[c]), 'otros') MU = F.loc[TR, NUM].mean().to_numpy(); SD = F.loc[TR, NUM].std(ddof=0).to_numpy(); SD[SD == 0] = 1 def X(mask): Z = (F.loc[mask, NUM].to_numpy(float) - MU) / SD return np.column_stack([Z] + [np.column_stack([(Cd[c][mask].to_numpy() == lv).astype(float) for lv in niv[c] + ['otros']]) for c in CATS]) return X(TR), X(TE) y = d['y'].to_numpy(); grp = d['org'].fillna('∅').to_numpy() TR = (d['T'] <= 2021).to_numpy() & (d['T'] >= 2018).to_numpy(); TE = (d['T'] == 2022).to_numpy() BASE = ['log_imp', 'dur', 'dur_na', 'log_dur', 'log_nlotes', 'log_nadj', 'lote_gt1'] ANUN = ['log_of', 'of_na', 'of1', 'baja', 'baja_na', 'dpa', 'dpa_na', 'link_na'] HIST = ['h_amb', 'h_amb_na', 'h_org', 'h_org_na', 'h_oc', 'h_oc_na', 'h_ot', 'h_ot_na', 'hn', 'hnoc'] R = {'n_tr': int(TR.sum()), 'pos_tr': int(y[TR].sum()), 'n_te': int(TE.sum()), 'pos_te': int(y[TE].sum())} # base órgano × CPV con las cohortes de entrenamiento b = cadena(d.loc[TR], d.loc[TE], (('amb', ['ambit']), ('o', ['ambit', 'org']), ('oc', ['ambit', 'org', 'cpv_div']))) R['b1_oc'] = roc_auc_score(y[TE], b['oc'][0]); R['b1_o'] = roc_auc_score(y[TE], b['o'][0]) R['solo_importe'] = roc_auc_score(y[TE], F0.loc[TE, 'log_imp']) for nm, NUM in (('rpc', BASE), ('rpc+anuncio', BASE + ANUN), ('rpc+hist', BASE + HIST), ('todo', BASE + ANUN + HIST)): for C in (0.1, 1.0): Xtr, Xte = design(F0, NUM, TR, TE) m = LogisticRegression(C=C, max_iter=5000).fit(Xtr, y[TR]); p = m.predict_proba(Xte)[:, 1] R[f'log_{nm}_C{C}'] = roc_auc_score(y[TE], p) print(nm, {k: round(v, 4) for k, v in R.items() if nm in k}, flush=True) CATS0 = list(CATS) for nm, cats, NUM in (('sin_tram', [c for c in CATS0 if c != 'tram'], BASE + HIST), ('sin_tram_sin_dur', [c for c in CATS0 if c != 'tram'], [v for v in BASE if 'dur' not in v] + HIST), ('sin_tram_sin_ambit', [c for c in CATS0 if c not in ('tram', 'ambit')], BASE + HIST), ('sin_tram_sin_mes', [c for c in CATS0 if c not in ('tram', 'mes')], BASE + HIST), ('sin_tram_sin_hot', [c for c in CATS0 if c != 'tram'], BASE + [h for h in HIST if not h.startswith('h_ot')])): CATS[:] = cats for C in (0.03, 0.1): Xtr, Xte = design(F0, NUM, TR, TE); m = LogisticRegression(C=C, max_iter=5000).fit(Xtr, y[TR]); p = m.predict_proba(Xte)[:, 1] R[f'v_{nm}_C{C}'] = roc_auc_score(y[TE], p) print(nm, {k: round(v, 4) for k, v in R.items() if nm in k}, flush=True) # calibración con Platt por validación cruzada por órgano (receta de MOD-REC) en la variante sin_tram CATS[:] = [c for c in CATS0 if c != 'tram'] Xtr, Xte = design(F0, BASE + HIST, TR, TE) aj = lambda Xa, ya: LogisticRegression(C=0.1, max_iter=5000).fit(Xa, ya) m = aj(Xtr, y[TR]); oof = np.zeros(TR.sum()) for a_, b_ in GroupKFold(n_splits=5).split(Xtr, y[TR], grp[TR]): oof[b_] = aj(Xtr[a_], y[TR][a_]).predict_proba(Xtr[b_])[:, 1] platt = LogisticRegression(C=1e6).fit(logit(np.clip(oof, 1e-6, 1 - 1e-6))[:, None], y[TR]) p = platt.predict_proba(logit(np.clip(m.predict_proba(Xte)[:, 1], 1e-6, 1 - 1e-6))[:, None])[:, 1] q = pd.qcut(p, 10, labels=False, duplicates='drop'); dec = pd.DataFrame({'p': p, 'y': y[TE], 'q': q}).groupby('q').agg(pred=('p', 'mean'), obs=('y', 'mean'), n=('p', 'size')) dec['dif'] = 100 * (dec['pred'] - dec['obs']); R['dev_platt_auc'] = roc_auc_score(y[TE], p); R['dev_oe'] = float(y[TE].sum() / p.sum()); R['dev_deciles'] = dec.round(4).to_dict(orient='records') print(dec.round(4)); print('O/E', R['dev_oe']) FK = feats(d) for c in colsK: v = colsK[c]; FK[c] = logit(np.clip(np.where(np.isnan(v), np.nanmean(v), v), 1e-4, 1 - 1e-4)); FK[c + '_na'] = np.isnan(v).astype(float) FK['hn'] = hnK; FK['hnoc'] = hnocK CATS[:] = ['tipo_contrato', 'proc_nat', 'cpv_div', 'ambit'] FM = feats(d) for c in colsM: v = colsM[c]; FM[c] = logit(np.clip(np.where(np.isnan(v), np.nanmean(v), v), 1e-4, 1 - 1e-4)); FM[c + '_na'] = np.isnan(v).astype(float) FM['hn'] = hnM; FM['hnoc'] = hnocM for nm, FF in (('etiqueta_completa', F0), ('conocido_1ene', FK), ('conocido_mes', FM)): Xtr, Xte = design(FF, BASE + HIST, TR, TE); m = LogisticRegression(C=0.1, max_iter=5000).fit(Xtr, y[TR]); R[f'hist_{nm}'] = roc_auc_score(y[TE], m.predict_proba(Xte)[:, 1]) print('historial', R['hist_etiqueta_completa'], R['hist_conocido_1ene'], R['hist_conocido_mes'], round(time.time()-T0), flush=True) CATS[:] = CATS0 Xtr, Xte = design(F0, BASE + ANUN + HIST, TR, TE) for lr, it in ((0.05, 300), (0.1, 200)): g = HistGradientBoostingClassifier(learning_rate=lr, max_iter=it, max_leaf_nodes=15, min_samples_leaf=100, l2_regularization=1.0, random_state=0).fit(Xtr, y[TR]) R[f'hgb_todo_lr{lr}'] = roc_auc_score(y[TE], g.predict_proba(Xte)[:, 1]) # sensibilidades de la etiqueta (solo 2022) R['tasa_te'] = float(y[TE].mean()) print({k: (round(v, 4) if isinstance(v, float) else v) for k, v in R.items()}, round(time.time() - T0), 's') json.dump(R, open('/w/salidas/25_dev.json', 'w'), indent=1)