# impl2 · §6 test de Carril: MCO grado 3 + ficticias de redondeo, ventana excluida, bootstrap por órganos (1.000, semilla 20261020) import sys, json, numpy as np, pandas as pd print('python', sys.version.split()[0], 'numpy', np.__version__, 'pandas', pd.__version__) A = pd.read_csv('/w/tmp/carril_agg.csv', keep_default_na=False, na_values=[]) O = pd.read_csv('/w/tmp/carril_org.csv', keep_default_na=False, na_values=[]) A = A.merge(O[['oid','tipo_org','agente_secti']], on='oid', how='left') assert A.tipo_org.notna().all() SEED = 20261020; NREP = 1000 def cfg_ss(grado=3, fit_lo=5000, win_lo=14500, dummies=True): return dict(clase='SS', lo=5000, hi=15000, w=100, thr=15000, grado=grado, fit_lo=fit_lo, win_lo=win_lo, dums=[500,1000] if dummies else [], tramo0=50) def cfg_ob(grado=3): return dict(clase='OB', lo=12000, hi=40000, w=250, thr=40000, grado=grado, fit_lo=12000, win_lo=39000, dums=[1000,5000], tramo0=48) def design(c): nb = (c['hi']-c['lo'])//c['w'] los = c['lo'] + c['w']*np.arange(nb) z = (los + c['w']/2.0 - c['thr'])/1000.0 cols = [z**k for k in range(c['grado']+1)] + [(los % m == 0).astype(float) for m in c['dums']] X = np.column_stack(cols) fit = (los >= c['fit_lo']) & (los < c['win_lo']) win = (los >= c['win_lo']) & (los < c['hi']) return los, X, fit, win def estima(y, c, D=None): """y: (..., nb) recuentos. Devuelve dict de arrays con B, b, e, r.""" los, X, fit, win = D if D is not None else design(c) Y = np.atleast_2d(y).astype(float) beta = np.linalg.lstsq(X[fit], Y[:, fit].T, rcond=None)[0] # (k, R) cf = (X[win] @ beta).T # (R, nwin): polinomio + redondeo que toque B = (Y[:, win] - cf).sum(1) S = cf.sum(1) b = B / (S / win.sum()) e = B / S out = dict(B=B, b=b, e=e, cf_sum=S, obs_win=Y[:, win].sum(1)) if c['clase']=='SS': nw = Y[:, (los>=14500)&(los<15000)].sum(1); nr = Y[:, (los>=10000)&(los<14500)].sum(1) out['r'] = (nw/500.0)/(nr/4500.0) return out def matrices(sel, c): s = A[sel & (A.clase==c['clase'])] nb = (c['hi']-c['lo'])//c['w'] idx = (s.tramo.values - c['tramo0']) assert idx.min() >= 0 and idx.max() < nb, (idx.min(), idx.max()) orgs, oi = np.unique(s.oid.values, return_inverse=True) H = {} for k in ('a','b','c'): H[k] = np.zeros((len(orgs), nb)) np.add.at(H[k], (oi, idx), s[f'n_{k}'].values) keep = H['a'].sum(1) > 0 return orgs[keep], {k: v[keep] for k, v in H.items()} def zona(x): return 'recorte' if x >= 0.70 else ('troceo' if x <= 0.40 else 'mixto') def corre(nombre, sel, c, boot=True): D = design(c); los, X, fit, win = D orgs, H = matrices(sel, c) res = dict(test=nombre, clase=c['clase'], grado=c['grado'], fit_lo=c['fit_lo'], win_lo=c['win_lo'], ficticias='+'.join(map(str, c['dums'])) or 'ninguna', n_organos=len(orgs)) pt = {} for k in ('a','b','c'): y = H[k].sum(0) res[f'n_{k}'] = int(y.sum()); res[f'n_{k}_ventana'] = int(y[win].sum()) if c['clase']=='SS': res[f'n_{k}_10000_14500'] = int(y[(los>=10000)&(los<14500)].sum()) o = estima(y, c, D); pt[k] = o for m in o: res[f'{m}_{k}'] = float(o[m][0]) res['rho_b'] = res['b_b']/res['b_a']; res['rho_c'] = res['b_c']/res['b_a'] insuf = (res['n_a'] < 5000) or (res['n_b'] < 1000) or (res['n_c'] < 1000) res['muestra_insuficiente'] = bool(insuf) if boot: rng = np.random.default_rng(SEED); N = len(orgs) Wt = np.empty((NREP, N)) for i in range(NREP): Wt[i] = np.bincount(rng.integers(0, N, N), minlength=N) bs = {k: estima(Wt @ H[k], c, D) for k in ('a','b','c')} rb = bs['b']['b']/bs['a']['b']; rc = bs['c']['b']/bs['a']['b'] def ic(v): return np.percentile(v, [2.5, 97.5]) for nm, v in (('b_a',bs['a']['b']),('b_b',bs['b']['b']),('b_c',bs['c']['b']),('rho_b',rb),('rho_c',rc), ('e_a',bs['a']['e']),('e_b',bs['b']['e']),('e_c',bs['c']['e'])): lo_, hi_ = ic(v); res[f'{nm}_ic_lo'] = float(lo_); res[f'{nm}_ic_hi'] = float(hi_) res['_boot_rb'] = rb; res['_orgs'] = orgs; res['_Wt'] = None # resultado if not (res['b_a_ic_lo'] > 0 or res['b_a_ic_hi'] < 0): res['resultado'] = 'sin amontonamiento que explicar'; res['firmeza'] = '' else: zb, zc = zona(res['rho_b']), zona(res['rho_c']) if zb == zc == 'recorte': r_ = 'recorte' elif zb == zc == 'troceo': r_ = 'troceo' else: r_ = 'mixto' + (' (discordante)' if {zb, zc} == {'recorte','troceo'} else '') zr = r_.split()[0] def dentro(lo_, hi_, z): if z == 'recorte': return lo_ >= 0.70 if z == 'troceo': return hi_ <= 0.40 return lo_ > 0.40 and hi_ < 0.70 firme = dentro(res['rho_b_ic_lo'], res['rho_b_ic_hi'], zr) and dentro(res['rho_c_ic_lo'], res['rho_c_ic_hi'], zr) res['resultado'] = r_; res['firmeza'] = 'firme' if firme else 'en el limite' res['zona_rho_b'] = zb; res['zona_rho_c'] = zc return res SS = A.clase=='SS' noinv = A.tipo_org != 'investigacion' anio = A.anio_ref tests = [] tests.append(('principal_SS_2019_2025', noinv & anio.between(2019,2025), cfg_ss())) tests.append(('obras_2019_2025', anio.between(2019,2025), cfg_ob())) tests.append(('SS_2019', noinv & (anio==2019), cfg_ss())) tests.append(('SS_2020_2025', noinv & anio.between(2020,2025), cfg_ss())) # sensibilidad del principal tests.append(('sens_grado2', noinv & anio.between(2019,2025), cfg_ss(grado=2))) tests.append(('sens_grado4', noinv & anio.between(2019,2025), cfg_ss(grado=4))) tests.append(('sens_ajuste_7000', noinv & anio.between(2019,2025), cfg_ss(fit_lo=7000))) tests.append(('sens_ajuste_10000', noinv & anio.between(2019,2025), cfg_ss(fit_lo=10000))) tests.append(('sens_ventana_14000', noinv & anio.between(2019,2025), cfg_ss(win_lo=14000))) tests.append(('sens_sin_ficticias', noinv & anio.between(2019,2025), cfg_ss(dummies=False))) tests.append(('sens_sin_agente_secti', noinv & anio.between(2019,2025) & (A.agente_secti==0), cfg_ss())) tests.append(('sens_sin_ccaa_nula', noinv & anio.between(2019,2025) & (A.ccaa_nula==0), cfg_ss())) # descripción for y in range(2019, 2026): tests.append((f'SS_{y}', noinv & (anio==y), cfg_ss())) tests.append(('SS_PLACSP_sola', noinv & anio.between(2019,2025) & (A.placsp==1), cfg_ss())) for t in sorted(O.tipo_org.unique()): tests.append((f'SS_tipo_{t}', (A.tipo_org==t) & anio.between(2019,2025), cfg_ss())) out = [] for nombre, sel, c in tests: r = corre(nombre, sel, c); out.append(r) print(f"{nombre:28s} org={r['n_organos']:6d} n_a={r['n_a']:8d} n_b={r['n_b']:7d} n_c={r['n_c']:7d} | b_a={r['b_a']:8.4f} b_b={r['b_b']:8.4f} b_c={r['b_c']:8.4f} rho_b={r['rho_b']:.4f} [{r.get('rho_b_ic_lo',np.nan):.4f},{r.get('rho_b_ic_hi',np.nan):.4f}] rho_c={r['rho_c']:.4f} [{r.get('rho_c_ic_lo',np.nan):.4f},{r.get('rho_c_ic_hi',np.nan):.4f}] -> {r.get('resultado')} {r.get('firmeza')}{' INSUF' if r['muestra_insuficiente'] else ''}") # hipótesis: rho_b(2020-2025) - rho_b(2019), remuestreo conjunto de órganos del principal c = cfg_ss(); D = design(c) orgs_all, _ = matrices(noinv & anio.between(2019,2025), c) def Hfor(sel): s = A[sel & SS]; nb = 100 pos = np.searchsorted(orgs_all, s.oid.values); assert (orgs_all[pos] == s.oid.values).all() H = {k: np.zeros((len(orgs_all), nb)) for k in 'abc'} for k in 'abc': np.add.at(H[k], (pos, s.tramo.values - 50), s[f'n_{k}'].values) return H H19 = Hfor(noinv & (anio==2019)); H25 = Hfor(noinv & anio.between(2020,2025)) rng = np.random.default_rng(SEED); N = len(orgs_all) Wt = np.stack([np.bincount(rng.integers(0, N, N), minlength=N) for _ in range(NREP)]) def rho(H, k): return estima(Wt @ H[k], c, D)['b'] / estima(Wt @ H['a'], c, D)['b'] def rho_pt(H, k): return estima(H[k].sum(0), c, D)['b'][0] / estima(H['a'].sum(0), c, D)['b'][0] hip = {} for k in 'bc': d_pt = rho_pt(H25, k) - rho_pt(H19, k); d_bs = rho(H25, k) - rho(H19, k) lo_, hi_ = np.percentile(d_bs, [2.5, 97.5]) hip[f'dif_rho_{k}_2020_25_menos_2019'] = dict(punto=float(d_pt), ic_lo=float(lo_), ic_hi=float(hi_), organos=int(N)) print(f'dif rho_{k} (2020-25 - 2019) = {d_pt:.4f} [{lo_:.4f}, {hi_:.4f}] (N órganos conjunto={N})') json.dump(hip, open('/w/out/carril_hipotesis.json','w'), indent=1) df = pd.DataFrame([{k: v for k, v in r.items() if not k.startswith('_')} for r in out]) df.to_csv('/w/out/carril.csv', index=False) print('escrito carril.csv', df.shape)