"""E3 · Cotejo con TED (API de búsqueda v3 y XML de avisos). Uso: python3 -I pedir.py lote.json Tope 30 peticiones en todo el encargo (contador persistente), >= 6 s entre peticiones, parada con 403/429, UA del proyecto sin correo. Manifiesto: hora UTC, nº, id, método y URL, consulta, campos, HTTP, bytes, sha256.""" import hashlib import json import os import sys import time import urllib.error import urllib.request BASE = '/e3/cotejo' UA = 'buscalicitaciones.com-investigacion/1.0 (+https://buscalicitaciones.com)' URL = 'https://api.ted.europa.eu/v3/notices/search' TOPE = 30 cnt_f = f'{BASE}/contador.txt' n = int(open(cnt_f).read()) if os.path.exists(cnt_f) else 0 log = open(f'{BASE}/peticiones.log', 'a') for q in json.load(open(sys.argv[1])): if n >= TOPE: print('TOPE alcanzado') break if q.get('get'): req = urllib.request.Request(q['get'], method='GET', headers={'User-Agent': UA}) url = q['get'] else: url = URL body = {"query": q["query"], "fields": q["fields"], "page": 1, "limit": q.get("limit", 50), "scope": "ALL", "checkQuerySyntax": False, "paginationMode": "PAGE_NUMBER"} req = urllib.request.Request(URL, data=json.dumps(body).encode(), method='POST', headers={'User-Agent': UA, 'Accept': 'application/json', 'Content-Type': 'application/json'}) t0 = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()) try: with urllib.request.urlopen(req, timeout=60) as r: st, raw = r.status, r.read() except urllib.error.HTTPError as e: st, raw = e.code, e.read() except Exception as e: # noqa: BLE001 st, raw = -1, repr(e).encode() n += 1 open(cnt_f, 'w').write(str(n)) ext = '.xml' if q.get('get') else '.json' open(f'{BASE}/resp/{q["id"]}{ext}', 'wb').write(raw) sha = hashlib.sha256(raw).hexdigest() log.write(f'{t0} UTC\t#{n}\t{q["id"]}\t{"GET" if q.get("get") else "POST"} {url}\t{q.get("query", "")}\t' f'{",".join(q.get("fields", []))}\tHTTP {st}\t{len(raw)} B\tsha256 {sha}\n') log.flush() print(n, q['id'], st, len(raw), sha[:16], raw[:600].decode('utf-8', 'replace').replace('\n', ' '), flush=True) if st in (403, 429): print('PARADA por', st) break time.sleep(6)