"""ML1 · 10 · ajusta los modelos de DEFINICIONES_PREVIAS §4-§6 por esquema y pliegue; guarda predicciones fuera de muestra. Uso: 10_modelos.py [esquemas] modelo ∈ P I2 LR D1 D3 D4 S1""" import sys, time, json; sys.path.insert(0, '/w/py') import numpy as np, pandas as pd from comun import carga, matriz, CONJ, CAT, LOG1P, SEM from sklearn.ensemble import HistGradientBoostingClassifier from sklearn.linear_model import LogisticRegression T0 = time.time() def log(*a): print(f"[{time.time()-T0:6.0f}s]", *a, flush=True) MOD = sys.argv[1]; ESQ = sys.argv[2].split(',') if len(sys.argv) > 2 else ['VT', 'VE', 'VM'] L, S = carga() conj = CONJ['P'] if MOD in ('P', 'I2', 'LR', 'S1') else CONJ[MOD] X, catm = matriz(L, conj) if MOD != 'S1': S = S[S.gris == 0] E = sorted(S.exp.unique()) def pliegues(): if 'VT' in ESQ: yield 'VT', 'VT', S.anio < 2019, S.anio >= 2019 if 'VE' in ESQ: for e in E: yield 'VE', e, S.exp != e, S.exp == e if 'VM' in ESQ: pos = S[S.y == 1].groupby('cpv_division').size() for d in sorted(pos[pos >= 20].index): yield 'VM', d, S.cpv_division != d, S.cpv_division == d def ajusta(Xtr, ytr): if MOD == 'I2': import lightgbm as lgb m = lgb.LGBMClassifier(n_estimators=300, learning_rate=0.05, num_leaves=31, min_child_samples=50, reg_lambda=1.0, random_state=SEM, deterministic=True, force_row_wise=True, verbose=-1, n_jobs=4) m.fit(Xtr, ytr, categorical_feature=[c for c, k in zip(conj, catm) if k]); return m, lambda Z: m.predict_proba(Z)[:, 1] if MOD == 'LR': med = {}; mu = {}; sd = {} def prep(Z, fit): cols = [] for c, k in zip(conj, catm): v = Z[c].values if k: cats = np.arange(len(L[c].cat.categories)) cols.append((v[:, None] == cats[None, :]).astype(float)); continue v = v.copy() if c in LOG1P: v = np.log1p(np.clip(v, 0, None)) ind = np.isnan(v).astype(float) if fit: med[c] = np.nanmedian(v) if (~np.isnan(v)).any() else 0.0 v = np.where(np.isnan(v), med[c], v) if fit: mu[c] = v.mean(); sd[c] = v.std() or 1.0 cols.append(((v - mu[c]) / sd[c])[:, None]); cols.append(ind[:, None]) return np.hstack(cols) A = prep(Xtr, True); m = LogisticRegression(C=1.0, max_iter=2000); m.fit(A, ytr) return m, lambda Z: m.predict_proba(prep(Z, False))[:, 1] m = HistGradientBoostingClassifier(learning_rate=0.05, max_iter=300, max_leaf_nodes=31, min_samples_leaf=50, l2_regularization=1.0, early_stopping=False, categorical_features=catm, random_state=SEM) m.fit(Xtr, ytr); return m, lambda Z: m.predict_proba(Z)[:, 1] out = {}; meta = [] for esq, fold, mtr, mte in pliegues(): te = S[mte]; tr = S[mtr & ~S.idx.isin(te.idx.unique())] ytr = tr.groupby('idx').y.max(); itr = ytr.index.values ite = np.unique(te.idx.values) m, pred = ajusta(X.iloc[itr], ytr.values) sc = pred(X.iloc[ite]) out[f'{esq}|{fold}'] = np.c_[ite, sc] meta.append({'esquema': esq, 'pliegue': fold, 'n_entreno': int(len(itr)), 'pos_entreno': int(ytr.sum()), 'n_prueba_lotes': int(len(ite)), 'pos_prueba': int(te.y.sum())}) log(MOD, esq, fold, meta[-1]) if MOD == 'P' and esq == 'VT': # importancia por permutación dentro de división × año (§5), 5 repeticiones from comun import Estrat tel = te.copy(); smap = dict(zip(ite, sc)); tel['s'] = tel.idx.map(smap) est0 = Estrat(tel.exp + '|' + tel.cpv_division + '|' + tel.anio.astype(str), tel.s.values, tel.y.values); a0 = est0.auc() Xte = X.iloc[ite].reset_index(drop=True); grp = L.loc[ite, 'cpv_division'].astype(str).values + '|' + L.loc[ite, 'anio'].astype(str).values rng = np.random.default_rng(SEM); imp = [] gidx = pd.Series(np.arange(len(ite))).groupby(grp).indices for c in conj: caidas = [] for r in range(5): Xp = Xte.copy(); v = Xp[c].values.copy() for g, ii in gidx.items(): v[ii] = v[rng.permutation(ii)] Xp[c] = v; sp = dict(zip(ite, pred(Xp))) e2 = Estrat(tel.exp + '|' + tel.cpv_division + '|' + tel.anio.astype(str), tel.idx.map(sp).values, tel.y.values) caidas.append(a0 - e2.auc()) imp.append({'variable': c, 'caida_auc_media': float(np.mean(caidas)), 'caida_auc_sd': float(np.std(caidas))}) log('perm', c, imp[-1]['caida_auc_media']) json.dump({'auc_base': a0, 'importancias': imp}, open('/w/tmp/perm_P_VT.json', 'w'), indent=1) # calibración isotónica con predicciones fuera de pliegue del entrenamiento (5 pliegues por órgano) from sklearn.model_selection import GroupKFold from sklearn.isotonic import IsotonicRegression oof = np.zeros(len(itr)); orgs = L.loc[itr, 'org'].values for a, b in GroupKFold(5).split(itr, ytr.values, orgs): _, pr = ajusta(X.iloc[itr[a]], ytr.values[a]); oof[b] = pr(X.iloc[itr[b]]) iso = IsotonicRegression(out_of_bounds='clip', y_min=0, y_max=1).fit(oof, ytr.values) cal = iso.predict(sc); yte = te.groupby('idx').y.max().reindex(ite).values json.dump({'idx': ite.tolist(), 'cal': cal.tolist(), 'raw': sc.tolist(), 'y': yte.tolist(), 'prev_entreno': float(ytr.mean())}, open('/w/tmp/cal_P_VT.json', 'w')) log('calibración hecha') np.savez_compressed(f'/w/tmp/pred_{MOD}.npz', **{k.replace('/', '_'): v for k, v in out.items()}) json.dump({'modelo': MOD, 'variables': conj, 'pliegues': meta, 'claves': list(out)}, open(f'/w/tmp/pred_{MOD}.json', 'w'), indent=1, ensure_ascii=False) log('fin')