#!/bin/bash # uso: q.sh sql/fichero.sql [nombre] [salida.md] -> DuckDB (imagen etl:v2), datos :ro, -m 8g, sin red, timeout. # Vistas: u = unified_20261018; idn/* identidad; p8 y actas montados :ro. Escribe solo en colusion/ (montado en /c). # Sentencias separadas por ';\n--;'. Si se da salida, se escribe además en ese fichero. set -u C=/colusion SQL="/c/$1"; N="colusion-${2:-sql}"; OUT="${3:-}" AV=$(free -g | awk '/Mem:/{print $7}'); while [ "$AV" -lt 12 ]; do echo "available=${AV}G < 12G: espero" >&2; sleep 30; AV=$(free -g | awk '/Mem:/{print $7}'); done timeout ${TMO:-3000} docker run -i --rm -m 8g --memory-swap 8g --cpus 4 --network none --user 1000:1000 --name "$N" \ -v /:/d:ro -v /p8:/p8:ro \ -v /actas:/actas:ro -v $C:/c \ buscalicitaciones-etl:v2 python3 - "$SQL" <<'PY' | if [ -n "$OUT" ]; then tee "$C/$OUT"; else cat; fi import sys, duckdb, time sql = open(sys.argv[1]).read() con = duckdb.connect('/c/interno/col.duckdb') con.execute("SET memory_limit='6GB'; SET threads=4; SET temp_directory='/c/tmp/duck'; SET max_temp_directory_size='30GB'; SET preserve_insertion_order=false;") con.execute("CREATE OR REPLACE TEMP VIEW u AS SELECT * FROM read_parquet('/d/unified_20261018/*/*/*.parquet', hive_partitioning=1, union_by_name=1)") for t in ['adjudicatarios_filas','organos_filas','organos_tipo','ute_socios','empresas','organos']: con.execute(f"CREATE OR REPLACE TEMP VIEW idn_{t} AS SELECT * FROM read_parquet('/d/unified_20261018.identidad/{t}.parquet')") con.execute("CREATE OR REPLACE TEMP VIEW vf AS SELECT * FROM read_parquet('/d/unified_20261018.vinculos/filas.parquet')") print(f"DuckDB {duckdb.__version__} · {time.strftime('%Y-%m-%d %H:%M:%S')}\n") for stmt in [s for s in sql.split(';\n--;') if s.strip()]: t=time.time() try: r = con.execute(stmt) if r.description: cols=[d[0] for d in r.description]; rows=r.fetchall() tit=[l for l in stmt.strip().splitlines() if l.startswith('-- @')] print('\n## '+(tit[0][4:].strip() if tit else stmt.strip().splitlines()[0][:110])) print('\n| '+' | '.join(cols)+' |\n|'+'|'.join('---' for _ in cols)+'|') for row in rows[:500]: print('| '+' | '.join('' if v is None else (f'{v:.4g}' if isinstance(v,float) else str(v)) for v in row)+' |') if len(rows)>500: print(f'... {len(rows)} filas') except Exception as e: print('## ERROR en:', stmt.strip().splitlines()[0][:110]); print(e) print(f'\n_{time.time()-t:.1f} s_', flush=True) PY