"""ETIQ-3 · impl. 2 · puntuaciones S1, S2, V por entidad-año sin DuckDB ni SQL (pyarrow + Python puro; sumas en Decimal, percentiles con bisect). Definiciones de V-INT §3-§4. Objetivo: los bancos de la impl. 2 (V-INT2 y E9). Uso (etiq3-sc2): python3 -I scores_i2.py """ import csv, sys, math from bisect import bisect_left, bisect_right from collections import defaultdict from decimal import Decimal import pyarrow.parquet as pq IND, BV, BE, SAL = sys.argv[1:5] objetivo = set() for r in csv.DictReader(open(BV), delimiter='\t'): if r['nif']: objetivo.add((r['nif'], int(r['ejercicio']))) for r in csv.DictReader(open(BE), delimiter='\t'): if int(r['ejercicio']) >= 2019: objetivo.add((r['nif'], int(r['ejercicio']))) ent_de = {r['organo_clave']: r['entidad_clave'] for r in pq.read_table(f'{IND}/organos.parquet', columns=['organo_clave', 'entidad_clave']).to_pylist()} def leer(n, cols): return pq.read_table(f'{IND}/{n}', columns=cols).to_pylist() def D(x): return Decimal(0) if x is None else (x if isinstance(x, Decimal) else Decimal(str(x))) oa = [] for r in leer('descriptores_organo_anio.parquet', ['organo_clave', 'anio', 'tipo', 'menores', 'importe', 'm1a_importe', 'm1r_importe', 'm1e_importe', 'm2_ss_n', 'm2_ss_borde100', 'm6_n', 'm6_n3mas', 'm4_elegible', 'm4_cuota_top1_pct']): o, a, t, imp, me = r['organo_clave'], r['anio'], r['tipo'], r['importe'], r['menores'] for ind, col in (('M1r', 'm1r_importe'), ('M1e', 'm1e_importe'), ('M1a', 'm1a_importe')): if imp is not None and r[col] is not None: oa.append((ind, 'PLACSP', t, a, o, D(r[col]), D(imp), me)) if r['m2_ss_n'] is not None and r['m2_ss_borde100'] is not None: oa.append(('M2', 'PLACSP', t, a, o, D(r['m2_ss_borde100']), D(r['m2_ss_n']), r['m2_ss_n'])) if r['m6_n'] is not None and r['m6_n3mas'] is not None: oa.append(('M6', 'PLACSP', t, a, o, D(r['m6_n'] - r['m6_n3mas']), D(r['m6_n']), r['m6_n'])) if r['m4_elegible'] and r['m4_cuota_top1_pct'] is not None and imp is not None: oa.append(('M4', 'PLACSP', t, a, o, Decimal(repr(float(r['m4_cuota_top1_pct']) * float(imp))), D(imp), me)) for r in leer('m3_organo_anio.parquet', ['organo_clave', 'anio', 'tipo', 'menores', 'm3_n']): if r['menores'] is not None and r['m3_n'] is not None: oa.append(('M3', 'PLACSP', r['tipo'], r['anio'], r['organo_clave'], D(r['m3_n']), D(r['menores']), r['menores'])) for r in leer('c1_organo_anio.parquet', ['organo_clave', 'anio', 'tipo', 'fuente', 'n', 'n_fuera_trim_sig', 'no_comparable']): if r['no_comparable'] is None and r['n'] is not None: oa.append(('C1', r['fuente'], r['tipo'], r['anio'], r['organo_clave'], D(r['n_fuera_trim_sig']), D(r['n']), r['n'])) for r in leer('c2_organo_anio.parquet', ['organo_clave', 'anio', 'tipo', 'fuente', 'poblacion', 'n', 'n_incompleta']): if r['n'] is not None and r['poblacion'] == 'menores': oa.append(('C2', r['fuente'], r['tipo'], r['anio'], r['organo_clave'], D(r['n_incompleta']), D(r['n']), r['n'])) for r in leer('documentos_organo_anio.parquet', ['organo_clave', 'anio', 'tipo', 'licitaciones', 'pcap_n', 'memoria_n', 'aprobacion_n', 'actas_n']): L = r['licitaciones'] if L is not None: oa.append(('DOC4', 'PLACSP', r['tipo'], r['anio'], r['organo_clave'], D(r['pcap_n']) + D(r['memoria_n']) + D(r['aprobacion_n']) + D(r['actas_n']), D(4 * L), L)) def coc(n, d): return round(float(n / d), 9) if d and d > 0 else None ref = defaultdict(list) for ind, f, t, a, o, n, d, ne in oa: if o in ent_de and d and d > 0 and ne is not None and ne >= 20: ref[(ind, f, t, a)].append(coc(n, d)) for k in ref: ref[k].sort() def rm(lista, v): lo, hi = bisect_left(lista, v), bisect_right(lista, v) return 100.0 * (lo + 0.5 * (hi - lo)) / len(lista) partes = defaultdict(lambda: [Decimal(0), Decimal(0), 0, defaultdict(int)]) for ind, f, t, a, o, n, d, ne in oa: e = ent_de.get(o) if e is None or d is None or d == 0: continue p = partes[(e, a, ind, f)] p[0] += n; p[1] += d; p[2] += ne or 0 p[3][t] = max(p[3][t], ne or 0) PRIO = {'ayuntamiento': 9, 'local_supramunicipal': 8} def tipo_de(cnt): return max(cnt.items(), key=lambda kv: (kv[1], PRIO.get(kv[0], 1)))[0] pares = defaultdict(list) for (e, a, ind, f), (n, d, ne, cnt) in partes.items(): v = coc(n, d) if ne >= 20 and v is not None: pares[(ind, f, tipo_de(cnt), a)].append((math.log(ne), e, v)) for k in pares: pares[k].sort() por_ea = defaultdict(list) for (e, a, ind, f), val in partes.items(): if (e, a) in objetivo: por_ea[(e, a, ind)].append((f, val)) def s2_de(key, e, ne, v): L = pares.get(key, []) if not L: return None x = math.log(ne) i = bisect_left(L, (x,)) cand = [p for p in L[max(0, i - 400): i + 400] if p[1] != e] # ventana amplia; luego orden exacto (distancia, clave) if len(cand) < len([p for p in L if p[1] != e]) and len(cand) < 120: cand = [p for p in L if p[1] != e] cand.sort(key=lambda p: (abs(p[0] - x), p[1])) cand = cand[:50] if len(cand) < 20: return None return rm(sorted(p[2] for p in cand), v) with open(SAL, 'w', newline='') as fh: w = csv.writer(fh) w.writerow(['nif', 'anio', 'ind', 'n_el', 'v_dir', 's1_dir', 's2_dir']) for (e, a, ind), lst in sorted(por_ea.items()): N = sum(val[0] for _, val in lst); Dd = sum(val[1] for _, val in lst); NE = sum(val[2] for _, val in lst) s1n = s1d = s2n = s2d = 0.0 for f, (n, d, ne, cnt) in lst: v = coc(n, d) if v is None or ne < 1: continue tp = tipo_de(cnt) rl = ref.get((ind, f, tp, a)) if rl and len(rl) >= 20: s1n += rm(rl, v) * ne; s1d += ne s2 = s2_de((ind, f, tp, a), e, ne, v) if s2 is not None: s2n += s2 * ne; s2d += ne dr = -1 if ind == 'DOC4' else 1 v = coc(N, Dd) s1 = s1n / s1d if s1d else None s2 = s2n / s2d if s2d else None w.writerow([e, a, ind, NE, '' if v is None else dr * v, '' if s1 is None else (s1 if dr == 1 else 100 - s1), '' if s2 is None else (s2 if dr == 1 else 100 - s2)]) print('ok', len(por_ea))