"""SELLO-SEM · segunda implementación, fase B (numpy, sin app/): con la tabla de la fase A y el JSON del modelo, recalcula P(recurso) y la población, y compara con el sello. uso: python3 21_impl2_b_numpy.py """ import json, sys, math import numpy as np, pandas as pd POB, MOD, PMP1, SD, SR, SAL, PREV = sys.argv[1:8] p = pd.read_csv(POB.replace(".parquet",".csv"), dtype=str, keep_default_na=False, na_values=[""]) for col in ("sara","ue","suman100","pcap","ppt"): p[col] = p[col].map({"true":True,"false":False,"True":True,"False":False}) m = json.load(open(MOD)) tipos_validos = None d = {} pmp1 = pd.read_csv(POB.replace("pob.parquet","pmp1.csv"), dtype={"id_canonical":str,"licitacion_id":str}) # ---- desierta ---- prev = {l.strip() for l in open(PREV) if l.strip()} s_d = json.load(open(SD)); s_r = json.load(open(SR)) fd = pd.DataFrame(s_d["filas"]) cand = pmp1.merge(p[["id_canonical"]], on="id_canonical", how="inner") cand = cand[~cand.licitacion_id.isin(prev)] cand = cand.drop_duplicates("licitacion_id") ids_c = set(cand.id_canonical); ids_f = set(fd.id_canonical) dif = cand.merge(fd, on="id_canonical", how="inner", suffixes=("_i", "_s")) d["desierta"] = {"impl2_filas": len(cand), "sello_filas": len(fd), "en_impl2_no_sello": len(ids_c - ids_f), "en_sello_no_impl2": len(ids_f - ids_c), "dif_p_max": float((dif.p_i - dif.p_s).abs().max()), "dif_lo_max": float((dif.p_lo_i - dif.p_lo_s).abs().max()), "dif_hi_max": float((dif.p_hi_i - dif.p_hi_s).abs().max())} # ---- recurso ---- OBR = {"Obras", "Concesion_obras", "Concesion_servicios", "Gestion_servicios_publicos"} PROC = {"Abierto", "Restringido", "Negociado_publicidad", "Dialogo_competitivo", "Asociacion_innovacion"} CCAA = {"AN", "CT", "AR", "CN", "CL"} df = p.copy() for c in ("pbl_exp", "imp_lic_eur", "ve_exp", "dur", "peso_jv", "peso_suma", "peso_precio", "n_crit", "n_sin_tipo", "cpv_n_div", "n_lotes_an"): df[c] = pd.to_numeric(df[c], errors="coerce") fp = pd.to_datetime(df.fecha_pub); ff = pd.to_datetime(df.fecha_fin_ofertas) df["plazo_d"] = (ff - fp).dt.days df["plazo"] = df.plazo_d.where((df.plazo_d > 0) & (df.plazo_d <= 180)) ve = df.ve_exp.fillna(df.pbl_exp).fillna(df.imp_lic_eur) recur = ((df.tipo_contrato.isin(["Servicios", "Suministros"])) & (ve >= 100_000)) | ((df.tipo_contrato.isin(OBR)) & (ve >= 3_000_000)) ok = df.ccaa.isin(CCAA) & (df.tipo_organo != "age") & df.procedimiento.isin(PROC) & (df.tramitacion.fillna("") != "emergencia") & recur & df.plazo.notna() df = df[ok].copy() print("recurso, población impl2", len(df)) def nz(s, v): return s.fillna(v) pbl = df.pbl_exp.combine_first(df.imp_lic_eur) F = pd.DataFrame(index=df.index) F["log_pbl"] = np.log10(np.maximum(nz(pbl, 1), 1)); F["pbl_na"] = pbl.isna().astype(float) F["log_ve"] = np.log10(np.maximum(nz(df.ve_exp, 1), 1)); F["ve_na"] = df.ve_exp.isna().astype(float) F["dur"] = df.dur.where(df.dur.notna(), -1.0).clip(upper=120); F["dur_na"] = df.dur.isna().astype(float) F["plazo"] = nz(df.plazo, -1).clip(upper=120); F["plazo_lt15"] = (df.plazo.notna() & (df.plazo < 15)).astype(float) F["log_nlotes"] = np.log1p(nz(df.n_lotes_an, 1)) bsara = df.sara.map(lambda v: None if pd.isna(v) else bool(v)); F["sara"] = bsara.map(lambda v: float(bool(v))); F["sara_na"] = bsara.isna().astype(float) F["ue"] = df.ue.fillna(False).astype(bool).astype(float) F["cpv_n_div"] = nz(df.cpv_n_div, -1) s100 = df.suman100.fillna(False).astype(bool) ok_tipo = s100 & (nz(df.n_sin_tipo, 1) == 0) ps = df.peso_suma.where((df.peso_suma != 0) & df.peso_suma.notna()) F["sin_crit"] = df.n_crit.isna().astype(float); F["n_crit"] = nz(df.n_crit, 0).clip(upper=15); F["un_solo_crit"] = (df.n_crit == 1).astype(float) ppr = (nz(df.peso_precio, 0) / ps.where(ps.notna(), 1)).where(ps.notna(), 0.0) F["peso_precio"] = ppr.where(ok_tipo, -1.0) pjv = (nz(df.peso_jv, 0) / ps.where(ps.notna(), 1)).where(ps.notna(), 0.0) pjv = pjv.where(ok_tipo, -1.0) F["peso_jv"] = pjv; F["peso_jv_gt50"] = (pjv > 0.5).astype(float) F["pcap"] = df.pcap.fillna(False).astype(bool).astype(float); F["ppt"] = df.ppt.fillna(False).astype(bool).astype(float) Xn = (F[m["NUM"]].to_numpy(float) - np.array(m["MU"])) / np.array(m["SD"]) C = pd.DataFrame(index=df.index) C["proc_nat"] = df.proc_nat; C["tram"] = df.tramitacion; C["tipo_contrato"] = df.tipo_contrato; C["cpv_div"] = df.cpv_div; C["tipo_organo"] = df.tipo_organo; C["ccaa"] = df.ccaa; C["mes_pub"] = fp[df.index].dt.month.astype(str) Xc = [] for cat in m["CATS"]: s = C[cat].map(lambda v: "∅" if v is None or (isinstance(v, float) and math.isnan(v)) else str(v)) niv = list(m["CAT_NIV"][cat]); val = s.where(s.isin(niv), "otros") for lv in niv + ["otros"]: Xc.append((val == lv).astype(float).to_numpy()[:, None]) X = np.hstack([Xn] + Xc) sig = lambda t: np.where(t >= 0, 1 / (1 + np.exp(-t)), np.exp(t) / (1 + np.exp(t))) def platt(p): p = np.clip(p, 1e-6, 1 - 1e-6); return sig(m["platt"]["a"] * np.log(p / (1 - p)) + m["platt"]["b"]) p0 = platt(sig(m["intercept"] + X @ np.array(m["coef"]))) B = np.array([platt(sig(b["intercept"] + X @ np.array(b["coef"]))) for b in m["bootstrap"]]).T B.sort(axis=1) nb = B.shape[1]; j5 = int(round(0.05 * (nb - 1))); j95 = int(round(0.95 * (nb - 1))) lo = B[:, j5]; hi = B[:, j95] imp = pd.DataFrame({"licitacion_id": df.licitacion_id.values, "id_canonical": df.id_canonical.values, "p": p0, "p_lo": lo, "p_hi": hi}).drop_duplicates("licitacion_id") fr = pd.DataFrame(s_r["filas"]) dif = imp.merge(fr, on="licitacion_id", how="inner", suffixes=("_i", "_s")) d["recurso"] = {"impl2_filas": len(imp), "sello_filas": len(fr), "en_impl2_no_sello": len(set(imp.licitacion_id) - set(fr.licitacion_id)), "en_sello_no_impl2": len(set(fr.licitacion_id) - set(imp.licitacion_id)), "dif_p_max": float((dif.p_i - dif.p_s).abs().max()), "dif_lo_max": float((dif.p_lo_i - dif.p_lo_s).abs().max()), "dif_hi_max": float((dif.p_hi_i - dif.p_hi_s).abs().max())} json.dump(d, open(SAL, "w"), ensure_ascii=False, indent=1) print(json.dumps(d, ensure_ascii=False))