"""PG1 · descarga por lotes. Uso: get.py . Cada línea: urldestino (relativo a /out). Registra url, destino, fecha, licencia, bytes, md5, status, tipo en /out/descargas.jsonl. 1 petición/s. Tope global 3.000.""" import urllib.request, sys, datetime, hashlib, json, time, os TOPE = 3000 UA = "buscalicitaciones-PG1/1.0 (investigacion estadistica de transparencia, sin animo de lucro; datos abiertos; 1 peticion/s)" lista, lic = sys.argv[1], sys.argv[2] LOG = '/out/descargas.jsonl' def n_hechas(): try: return sum(1 for _ in open(LOG)) except FileNotFoundError: return 0 for ln in open(lista): ln = ln.rstrip('\n') if not ln or ln.startswith('#'): continue u, dst = ln.split('\t')[:2] if os.path.exists('/out/' + dst) and os.path.getsize('/out/' + dst) > 0: continue if n_hechas() >= TOPE: print('TOPE ALCANZADO'); break os.makedirs(os.path.dirname('/out/' + dst) or '/out', exist_ok=True) t = time.time() try: req = urllib.request.Request(u, headers={"User-Agent": UA, "Accept": "application/json, application/xml;q=0.9, */*;q=0.8"}) r = urllib.request.urlopen(req, timeout=120); b = r.read() open('/out/' + dst, 'wb').write(b) d = dict(url=u, fichero=dst, fecha=datetime.datetime.now().isoformat(timespec='seconds'), licencia=lic, bytes=len(b), md5=hashlib.md5(b).hexdigest(), status=r.status, ctype=r.headers.get('Content-Type')) except Exception as e: d = dict(url=u, fichero=dst, fecha=datetime.datetime.now().isoformat(timespec='seconds'), licencia=lic, error=repr(e)[:300]) open(LOG, 'a').write(json.dumps(d, ensure_ascii=False) + '\n') print(json.dumps(d, ensure_ascii=False)[:250], flush=True) time.sleep(max(0, 1.0 - (time.time() - t)) + 0.05)