"""21 · Segunda implementación INDEPENDIENTE (PRE_B3_MULT1.md §9): pyarrow + Python puro, sin DuckDB, sin numpy y sin importar reglas.py. Población reescrita fila a fila; p por barrido de intervalos exacto (claves enteras = valor·L, L = mcm(1..150)); j y k_fit con otras fórmulas. Uso: 21_impl2.py . Compara con /o/priv/firma_impl1_desc.json. Salida: /o/salidas/21_impl2_desc.json""" import pyarrow.dataset as ds, pyarrow.compute as pc, pyarrow.parquet as pq import json, re, sys, hashlib, unicodedata, math, time, datetime as dt sys.path.insert(0, '/o/py'); from agente import R_AGENTE T0 = time.time(); CAS = None PT = json.load(open('/o/py/patrones_t5118.json')); U_ = '/d/unified_20261108' RX = {k: re.compile(v) for k, v in { 'C2': r"acuerdo marco|contratacion centralizada|catalogo|central de compras|derivad|basado en", 'C3': r"arancel|notari|registrador|procurador|franqueo|correos|peaje", 'C3b': r"\btasas?\b|precio publico|\bcuotas?\b", 'C5': r"suscripci|revista|base(s)? de datos|publicaciones periodicas|hemeroteca|prensa digital", 'C6i': r"(arrendamiento|alquiler)( de| del)? (local|inmueble|nave|oficina|edificio|terreno|vivienda|finca|solar|parcela|garaje|plaza)|ocupacion del dominio publico|autorizacion demanial|concesion demanial", 'C8': PT['T4'], 'C8m': r"\b(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|setiembre|octubre|noviembre|diciembre|xaneiro|febreiro|maio|xuno|xullo|setembro|outubro|novembro|decembro|gener|febrer|maig|juny|juliol|agost|setembre|desembre)\b|\bmensual|\bmes (de|d')", 'C9': r"emergencia|covid|coronavirus|sars-cov|\bdana\b", 'C21': r"asistencia sanitaria|traslado de pacientes|transporte (sanitario|de pacientes)", 'C22': r"\bencargo|medio propio|convenio", 'C23': r"accion concertada|concierto", 'OBRAS': PT['OBRAS'], 'GAL_OBRA': r"edificios e outras construc|construc|\bobra", 'AG': R_AGENTE}.items()} NIFOK = re.compile(r'^[ABCDFGNRUVW][0-9]{7}[0-9A-J]$') def sa(s): return ''.join(ch for ch in unicodedata.normalize('NFD', s or '') if unicodedata.category(ch) != 'Mn') def low(s): return sa(s).lower() L = 1 for i in range(1, 151): L = L * i // math.gcd(L, i) KMAX, PMIN = 150, 10000 PMAX = {'resto': 750000, 'agente': 2500000, 'obras': 2000000} TIPOS_FUERA = {'Patrimonial', 'Concesion_servicios', 'Concesion_obras', 'Gestion_servicios_publicos', 'Administrativo_especial'} def poblacion(cas, d0, d1): vfal = set(pc.filter(t := pq.read_table(f'{U_}.vinculos/filas.parquet', columns=['source', 'id_canonical', 'cuenta'], filters=[('source', '=', cas), ('cuenta', '=', False)]), pc.equal(t['cuenta'], False))['id_canonical'].to_pylist()) a = pq.read_table(f'{U_}.identidad/adjudicatarios_filas.parquet', columns=['adjudicatario_nif_raw', 'adjudicatario_nif', 'adjudicatario_nombre', 'persona_fisica', 'entidad_de_personas']) marca = pc.or_kleene(pc.fill_null(a['persona_fisica'], False), pc.fill_null(a['entidad_de_personas'], False)); a = a.filter(marca) adjx = set(zip(a['adjudicatario_nif_raw'].to_pylist(), a['adjudicatario_nif'].to_pylist(), a['adjudicatario_nombre'].to_pylist())) cols = ['id_canonical', 'objeto', 'tipo_contrato', 'cpv_code', 'organo_nombre', 'organo_nif', 'expediente', 'lote_id', 'fecha_adj', 'fecha_formalizacion', 'fecha_pub', 'imp_adj_eur', 'imp_adj_eur_sin_iva', 'imp_adj_eur_con_iva', 'adjudicatario_nif', 'adjudicatario_nif_raw', 'adjudicatario_nombre', 'copia_de', 'anulado_en_origen', 'correccion_propuesta', 'rol_am', 'menor_dudoso_motivo', 'organo_ccaa', 'ccaa', 'nuts'] dset = ds.dataset(f'{U_}/source={cas}', format='parquet', partitioning='hive') have = set(dset.schema.names); f0, f1 = dt.date.fromisoformat(d0), dt.date.fromisoformat(d1); filas = [] for b in dset.to_batches(columns=[c for c in cols if c in have], filter=ds.field('es_menor') == True, batch_size=200000): d = b.to_pydict(); n = b.num_rows for i in range(n): g = lambda c: d[c][i] if c in d else None fe = g('fecha_pub') if cas == 'GALICIA' else (g('fecha_adj') or g('fecha_formalizacion') or g('fecha_pub')) if fe is None: continue if isinstance(fe, dt.datetime): fe = fe.date() if not (f0 <= fe <= f1): continue n0 = re.sub('[^A-Z0-9]', '', (g('adjudicatario_nif') or '').upper()); nn = n0[2:] if len(n0) == 11 and n0[:2] == 'ES' else n0 if not NIFOK.match(nn): continue if (g('adjudicatario_nif_raw'), g('adjudicatario_nif'), g('adjudicatario_nombre')) in adjx: continue ob = low(g('objeto')); on = low(g('organo_nombre')); t = lambda k: RX[k].search(ob) is not None if cas == 'GALICIA': imp, con, dud = g('imp_adj_eur'), None, True else: imp = g('imp_adj_eur_sin_iva') if g('imp_adj_eur_sin_iva') is not None else g('imp_adj_eur'); con = g('imp_adj_eur_con_iva') dud = cas == 'PLACSP' and con is not None and imp is not None and abs(con - imp) < 0.01 obra = RX['GAL_OBRA'].search(ob) is not None if cas == 'GALICIA' else (g('tipo_contrato') == 'Obras' or (g('cpv_code') or '')[:2] == '45' or ((g('tipo_contrato') or '') not in ('Servicios', 'Suministros') and t('OBRAS'))) agente = RX['AG'].search(on) is not None est = 'obras' if obra else ('agente' if agente else 'resto'); u = {'obras': 4000000, 'agente': 5000000, 'resto': 1500000}[est] okey = (g('organo_nif') or '') + '|' + re.sub('[\t\n\f\r ]+', ' ', on).strip(' ') letra = re.sub('[^A-Za-z0-9]', '', g('organo_nif') or '')[:1].upper(); ccaa = low(g('organo_ccaa') if g('organo_ccaa') is not None else (g('ccaa') if g('ccaa') is not None else '')) if g('id_canonical') in vfal or g('copia_de') is not None or g('anulado_en_origen') is True or g('correccion_propuesta') in ('varias_empresas', 'repetido'): ex = 'estructural' elif imp is None or imp <= 0 or imp * 100 > 10 * u: ex = 'importe' elif fe < dt.date(2018, 3, 9): ex = 'antes_lcsp' elif 'navarra' in ccaa or (g('nuts') or '').startswith('ES22'): ex = 'navarra' elif letra in ('A', 'B'): ex = 'C25_organo_321' elif g('tipo_contrato') in TIPOS_FUERA: ex = 'tipo' elif g('rol_am') is not None or t('C2'): ex = 'C2' elif t('C5'): ex = 'P3a_C5' elif t('C6i'): ex = 'P3a_C6i' elif t('C3b'): ex = 'P3a_C3b' elif t('C9') or 'emergencia' in (g('menor_dudoso_motivo') or ''): ex = 'P3a_C9' elif t('C3'): ex = 'P3b_C3' elif t('C8'): ex = 'C8' elif t('C8m'): ex = 'C8_mes' elif t('C21') and imp < 30000: ex = 'C21' elif t('C22'): ex = 'C22' elif t('C23'): ex = 'C23' else: ex = None lote = g('lote_id') is not None and str(g('lote_id')) not in ('', '0') yc = 0 if imp is not None and imp > 0: v = imp * 100; yc = math.floor(v); yc += 1 if v - yc >= 0.5 else 0 filas.append((g('id_canonical'), fe.year, fe.year * 10 + (1 if fe.month <= 6 else 2), okey, nn, est, u, est == 'agente', lote, dud, yc, ex, g('expediente') or '')) # lotes: (okey, nn, expediente, importe) repetido con lote -> el de menor id vis = {} for r in sorted(filas, key=lambda r: r[0]): if r[8]: vis.setdefault((r[3], r[4], r[12], r[10]), r[0]) return [r for r in filas if not r[8] or vis[(r[3], r[4], r[12], r[10])] == r[0]] def mejor_p(ys, pmax, kmax=KMAX, pmin=PMIN, estricta=False, umbral=0.8): cnt = {} for y in ys: cnt[y] = cnt.get(y, 0) + 1 ev = [] for y, w in cnt.items(): jlo = max(1, (y - 1) // pmax); jhi = min(kmax, (y + 1) // pmin) for j in range(jlo, jhi + 1): if estricta: if y % j: continue ev.append(((y // j) * L, 0, w)); ev.append(((y // j) * L, 2, -w)) else: ev.append(((y - 1) * (L // j), 0, w)); ev.append(((y + 1) * (L // j), 2, -w)) for k0 in range(max(1, -(-y // pmax)), min(kmax, y // pmin) + 1): if estricta and y % k0: continue ev.append((y * (L // k0), 1, (y, k0))) ev.sort(key=lambda e: (e[0], e[1])); s = 0; best = None; cands = [] for v, tp, w in ev: if tp == 1: c = (s, v, -w[0]); cands.append((v, -w[0], s, w)) if best is None or c > best[0]: best = (c, w) else: s += w if best is None: return None # B4: la mayor cantidad (después menor x) que casa >= umbral*N filas con >= 2 valores de j; si no hay, la de máximo X N = len(ys) for v, mx, X, (x0, k0) in sorted(cands, reverse=True): if X * 100 < round(umbral * 100) * N: continue if len({j for j in jotas(ys, x0, k0, estricta, kmax) if j}) >= 2: return (X, x0, k0) return (best[0][0], best[1][0], best[1][1]) def jotas(ys, x0, k0, estricta=False, kmax=KMAX): out = [] for y in ys: if estricta: pcent = x0 // k0; j = y // pcent if y % pcent == 0 else 0 else: jl = -(-(y - 1) * k0 // x0); jh = (y + 1) * k0 // x0; j = jl if jl <= jh else 0 out.append(j if 1 <= j <= kmax else 0) return out from fractions import Fraction as Fr def es_redondo(x0, k0): p = Fr(x0, k0) for t in (Fr(1), Fr(104, 100), Fr(110, 100), Fr(121, 100)): for v in (p * t, p / t): if abs(v - 500 * round(v / 500)) <= 1: return True return False def hecho(fs, est, umbral=0.8, estricta=False): ys = [f[10] for f in fs]; r = mejor_p(ys, PMAX[est], estricta=estricta); N = len(fs) if r is None: return dict(N=N, X=0, Y=0, patron=False, x0=None, k0=None, js=[0] * N) X, x0, k0 = r; js = jotas(ys, x0, k0, estricta) kf = lambda u, le: (u * k0 // x0) if le else (-(-u * k0 // x0) - 1) redondo = es_redondo(x0, k0); nk = len({j for j in js if j}) patron = N >= 10 and X * 10 >= umbral * 10 * N - 1e-9 and nk >= 2 and not redondo Y = 0; det = [] for f, j in zip(fs, js): d = (not f[9] or kf(f[6], f[7]) == kf(f[6] * 121 // 100, f[7])) and not (CAS == 'GALICIA' and f[5] == 'obras'); det.append(d) if patron and j and j == kf(f[6], f[7]) and not f[8] and d: Y += 1 return dict(N=N, X=X, Y=Y, patron=patron, x0=x0, k0=k0, js=js, det=det, kf=kf) def est_par(fs): return max(['resto', 'agente', 'obras'], key=lambda e: (sum(f[5] == e for f in fs), {'resto': 0, 'obras': 1, 'agente': 2}[e])) def grupos(rows, key): g = {} for r in rows: g.setdefault(key(r), []).append(r) return {k: sorted(v, key=lambda r: r[0]) for k, v in g.items() if len(v) >= 10} def h(*a): return hashlib.md5('|'.join(map(str, a)).encode()).hexdigest() def main(): global CAS SELLO = sys.argv[1] assert hashlib.md5(open('/o/prereg/PRE_B3_MULT1.original.md', 'rb').read()).hexdigest() == SELLO out = {'sello': SELLO, 'casillas': {}}; firma2 = [] FICT = {'resto': (1050000, 2100000), 'obras': (2800000, 5600000), 'agente': (3500000, 7000000)} for CAS in ['PLACSP', 'ANDALUCIA', 'GALICIA']: t1 = time.time(); rows = [r for r in poblacion(CAS, '2019-01-01', '2024-12-31') if r[11] is None] pares = grupos(rows, lambda r: (r[1], r[3], r[4])); ev = {} for k, fs in pares.items(): est = est_par(fs); e = hecho(fs, est); e['est'] = est; ev[k] = e firma2.append([CAS, k[0], h(k[1], k[2]), e['N'], e['x0'], e['k0'], e['X'], e['Y'], e['patron']]) C = dict(N=len(ev), K=sum(e['patron'] for e in ev.values()), Y=sum(e['Y'] for e in ev.values()), X=sum(e['X'] for e in ev.values() if e['patron'])) C['K_V4'] = sum(e['patron'] and e['X'] * 10 >= 9 * e['N'] for e in ev.values()) C['K_V3'] = sum(hecho(fs, est_par(fs))['patron'] for fs in grupos(rows, lambda r: (r[2], r[3], r[4])).values()) C['K_V5'] = sum(hecho(fs, ev[k]['est'], estricta=True)['patron'] for k, fs in pares.items()) p1 = {} for s in ('resto', 'obras', 'agente'): n = [0, 0, 0, 0] for k, e in ev.items(): if not e['patron']: continue for f, j, d in zip(pares[k], e['js'], e['det']): if f[5] != s or not j or f[8] or not d: continue u1, u2 = FICT[s]; kf, k1, k2 = e['kf'](f[6], f[7]), e['kf'](u1, f[7]), e['kf'](u2, f[7]) if kf == k1 or kf == k2: continue if f[9] and (k1 != e['kf'](u1 * 121 // 100, f[7]) or k2 != e['kf'](u2 * 121 // 100, f[7])): continue n[0] += 1; n[1] += j == kf; n[2] += j == k1; n[3] += j == k2 p1[s] = n C['P1'] = p1; C['segundos'] = round(time.time() - t1); out['casillas'][CAS] = C print(CAS, json.dumps(C), flush=True) f1 = {tuple(r[:3]): r for r in json.load(open('/o/priv/firma_impl1_desc.json'))}; f2 = {tuple(r[:3]): r for r in firma2} dif = [k for k in set(f1) | set(f2) if f1.get(k) != f2.get(k)] m1 = json.load(open('/o/salidas/11_medida_desc.json'))['casillas']; cmpk = {} for c, C in out['casillas'].items(): M = m1[c]; cmpk[c] = dict(K=(M['V1']['K'], C['K']), Y=(M['V1']['Y'], C['Y']), X=(M['V1']['X'], C['X']), K_V3=(M['V3']['K'], C['K_V3']), K_V4=(M['V4']['K'], C['K_V4']), K_V5=(M['V5']['K'], C['K_V5']), P1={s: ([M['P1'][s][x] for x in ('S', 'en_U', 'en_07U', 'en_14U')], C['P1'][s]) for s in C['P1']}) iguales = all(a == b for c in cmpk.values() for k, v in c.items() if k != 'P1' for a, b in [v]) and all(a == b for c in cmpk.values() for a, b in c['P1'].values()) out.update(pares_impl1=len(f1), pares_impl2=len(f2), diferencias_firma=len(dif), ejemplo_dif=[[f1.get(k), f2.get(k)] for k in sorted(dif)[:10]], comparacion=cmpk, cero_diferencias=len(dif) == 0 and iguales, segundos=round(time.time() - T0)) json.dump(out, open('/o/salidas/21_impl2_desc.json', 'w'), ensure_ascii=False, indent=1, default=str) print('diferencias', len(dif), 'iguales_agregados', iguales, out['segundos']) if __name__ == '__main__': main()