#!/usr/bin/env python3
"""
Comprehensive Person Profile Enrichment via Linkup Web Search

This script enriches person profiles with ALL discoverable data from web sources,
with FULL PROVENANCE for every claim. No data is stored without a verifiable source.

Rule Compliance:
- Rule 6: WebObservation Claims MUST Have XPath Provenance (adapted for web search)
- Rule 21: Data Fabrication is Strictly Prohibited
- Rule 26: Person Data Provenance - Web Claims for Staff Information
- Rule 34: Linkup is the Preferred Web Scraper
- Rule 35: Provenance Statements MUST Have Dual Timestamps

Data Extracted (when available):
- Birth date/year, birth location
- Education history, career milestones
- Publications, awards/honors
- Professional affiliations
- Death date (if applicable)
- Contact details (email, phone, social media)
- Media references (photos, videos, portraits)

Usage:
    python scripts/enrich_person_comprehensive.py --limit N [--dry-run]
"""

import json
import os
import re
import time
import argparse
import hashlib
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional, Dict, Any, List
import httpx

LINKUP_API_URL = "https://api.linkup.so/v1/search"
SCRIPT_VERSION = "1.2.0"


def get_linkup_api_key() -> str:
    env_path = Path(__file__).parent.parent / ".env"
    if env_path.exists():
        with open(env_path) as f:
            for line in f:
                if line.startswith("LINKUP_API_KEY="):
                    return line.strip().split("=", 1)[1].strip('"\'')
    key = os.environ.get("LINKUP_API_KEY", "")
    if not key:
        raise ValueError("LINKUP_API_KEY not found")
    return key


def search_linkup(query: str, api_key: str, depth: str = "standard") -> Dict[str, Any]:
    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
    payload = {"q": query, "depth": depth, "outputType": "sourcedAnswer"}
    request_ts = datetime.now(timezone.utc).isoformat()
    try:
        with httpx.Client(timeout=45.0) as client:
            response = client.post(LINKUP_API_URL, headers=headers, json=payload)
            response.raise_for_status()
            result = response.json()
            result["_meta"] = {"request_ts": request_ts, "response_ts": datetime.now(timezone.utc).isoformat(),
                              "status": response.status_code, "depth": depth}
            return result
    except Exception as e:
        return {"error": str(e), "_meta": {"request_ts": request_ts}}


def create_claim(claim_type: str, claim_value: Any, source_url: str, source_title: str,
                 snippet: str, query: str, sources: List = None, meta: Dict = None,
                 answer: str = None, pattern: str = None) -> Dict:
    ts = datetime.now(timezone.utc).isoformat()
    src_ts = meta.get("request_ts", ts) if meta else ts
    
    prov = {
        "statement_created_at": ts, "source_archived_at": src_ts,
        "retrieval_agent": f"enrich_person_comprehensive.py v{SCRIPT_VERSION}",
        "retrieval_method": "linkup_web_search", "api_endpoint": LINKUP_API_URL,
        "search_query": query, "search_depth": meta.get("depth", "standard") if meta else "standard",
        "source_url": source_url, "source_title": source_title, "source_snippet": snippet,
        "extraction_method": "regex_pattern_matching", "pattern_type": pattern,
        "verified": False, "verification_status": "machine_extracted", "requires_human_review": True,
        "http_status": meta.get("status") if meta else None,
    }
    
    if answer and snippet:
        pos = answer.find(snippet[:50])
        if pos >= 0:
            prov["answer_position"] = f"answer[{pos}:{pos + len(snippet)}]"
    
    if sources:
        prov["all_sources"] = [{"url": s.get("url", ""), "name": s.get("name", "")} for s in sources[:5]]
        prov["source_count"] = len(sources)
    
    if answer:
        prov["answer_content_hash"] = hashlib.sha256(answer.encode()).hexdigest()[:16]
    
    return {"claim_type": claim_type, "claim_value": claim_value, "provenance": prov}


def extract_birth_year(text):
    if not text: return None
    patterns = [(r'born\s+(?:on\s+)?(\d{1,2}\s+\w+\s+)?(\d{4})', "full_date"),
                (r'born\s+(?:on\s+)?(\w+\s+\d{1,2},?\s+)(\d{4})', "us_date"),
                (r'(?:was\s+)?born\s+in\s+(\d{4})', "born_in"), (r'geboren\s+(?:in\s+)?(\d{4})', "dutch"),
                (r'\(born\s+(\d{4})\)', "paren"), (r'\((\d{4})\)', "year_paren")]
    for pat, ptype in patterns:
        m = re.search(pat, text, re.I)
        if m and m.lastindex:
            yr = int(m.group(m.lastindex))
            if 1900 <= yr <= 2010:
                if ptype == "year_paren" and yr >= 1990: continue
                return {"year": yr, "snippet": text[max(0,m.start()-40):m.end()+40].strip(), "pattern_type": ptype}
    return None


def extract_birth_location(text):
    for pat in [r'born\s+in\s+([A-Z][a-zA-Z\s,]+)', r'geboren\s+(?:te|in)\s+([A-Z][a-zA-Z\s]+)']:
        m = re.search(pat, text)
        if m:
            loc = m.group(1).strip()
            if loc.lower() not in ['the', 'a', 'an']:
                return {"location": loc, "snippet": text[max(0,m.start()-30):m.end()+30].strip()}
    return None


def extract_death_info(text):
    for pat in [r'died\s+(?:on\s+)?(?:\d{1,2}\s+\w+\s+)?(\d{4})', r'\(\d{4}\s*[-–]\s*(\d{4})\)',
                r'passed\s+away\s+(?:in\s+)?(\d{4})', r'overleden\s+(?:in\s+)?(\d{4})']:
        m = re.search(pat, text, re.I)
        if m:
            yr = int(m.group(1))
            if 1900 <= yr <= datetime.now().year:
                return {"year": yr, "snippet": text[max(0,m.start()-30):m.end()+30].strip()}
    return None


def extract_education(text):
    edu = []
    patterns = [(r'(Ph\.?D\.?|doctorate)\s+(?:from|at)\s+([A-Z][^,\.]+?)(?:\s+in\s+(\d{4}))?', "phd"),
                (r"(master'?s?|M\.?A\.?)\s+(?:from|at)\s+([A-Z][^,\.]+)", "masters"),
                (r'graduated\s+from\s+([A-Z][^,\.]+?)(?:\s+in\s+)?(\d{4})?', "graduated"),
                (r'studied\s+(?:\w+\s+)?at\s+([A-Z][^,\.]+)', "studied")]
    for pat, etype in patterns:
        for m in re.finditer(pat, text, re.I):
            inst = m.group(2) if etype in ["phd", "masters"] else m.group(1)
            yr = None
            if m.lastindex and m.lastindex >= 3 and m.group(3):
                try: yr = int(m.group(3))
                except: pass
            edu.append({"type": etype, "institution": inst.strip(), "year": yr,
                       "snippet": text[max(0,m.start()-20):m.end()+20].strip()})
    return edu


def extract_positions(text):
    pos = []
    for pat in [r'(professor|director|curator|head|chief)\s+(?:at|of)\s+([A-Z][^,\.]{3,50})(?:\s+since\s+(\d{4}))?',
                r'appointed\s+(\w+)\s+(?:at\s+)?([A-Z][^,\.]{3,50})(?:\s+in\s+(\d{4}))?']:
        for m in re.finditer(pat, text, re.I):
            org = m.group(2).strip() if m.lastindex >= 2 and m.group(2) else None
            yr = None
            if m.lastindex and m.lastindex >= 3 and m.group(3):
                try: yr = int(m.group(3))
                except: pass
            pos.append({"title": m.group(1), "organization": org, "year": yr,
                       "snippet": text[max(0,m.start()-20):m.end()+20].strip()})
    return pos


def extract_publications(text):
    pubs = []
    for pat, ptype in [(r'(?:author|wrote|published)\s+(?:of\s+)?["\']([^"\']+)["\']', "book"),
                       (r'published\s+["\']?([^"\',.]+)["\']?\s+(?:in\s+)?(\d{4})', "publication")]:
        for m in re.finditer(pat, text, re.I):
            title = m.group(1).strip()
            yr = int(m.group(2)) if m.lastindex >= 2 and m.group(2) else None
            pubs.append({"type": ptype, "title": title, "year": yr,
                        "snippet": text[max(0,m.start()-20):m.end()+20].strip()})
    return pubs


def extract_awards(text):
    awards = []
    for pat, atype in [(r'(?:received|awarded|won)\s+(?:the\s+)?([A-Z][^,\.]{5,50})', "award"),
                       (r'Fellow\s+of\s+(?:the\s+)?([A-Z][^,\.]{5,50})', "fellowship")]:
        for m in re.finditer(pat, text, re.I):
            awards.append({"type": atype, "name": m.group(1).strip(),
                          "snippet": text[max(0,m.start()-20):m.end()+20].strip()})
    return awards


def extract_contacts(text):
    contacts = []
    seen_values = set()  # Deduplication
    # Blocklist for common false positives
    twitter_blocklist = {'handle', 'handles', 'profile', 'profiles', 'account', 'accounts', 
                         'found', 'available', 'not', 'no', 'or', 'and', 'the', 'is', 'are',
                         'was', 'were', 'has', 'have', 'with', 'for', 'o', 'a', 'example',
                         'gmail', 'outlook', 'yahoo', 'hotmail', 'email', 'mail'}
    instagram_blocklist = twitter_blocklist | {'photos', 'videos', 'reels', 'stories'}
    
    for pat, ctype in [
        # Email
        (r'\b([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})\b', "email"),
        # Twitter/X
        (r'(?<![a-zA-Z0-9.@])@([a-zA-Z0-9_]{3,15})\b', "twitter"),
        (r'(?:twitter\.com|x\.com)/([a-zA-Z0-9_]{3,15})(?:\s|$|["\'\)\]])', "twitter"),
        # LinkedIn
        (r'(https?://(?:www\.)?linkedin\.com/in/[a-zA-Z0-9\-%]+/?)', "linkedin_url"),
        # ORCID
        (r'(?:orcid)[:\s]*((?:\d{4}-){3}\d{3}[\dX])', "orcid"),
        (r'(https?://orcid\.org/(?:\d{4}-){3}\d{3}[\dX])', "orcid_url"),
        # Instagram
        (r'(?:instagram\.com)/([a-zA-Z0-9_.]{3,30})(?:\s|$|["\'\)\]])', "instagram"),
        # ResearchGate profile
        (r'(https?://(?:www\.)?researchgate\.net/profile/[a-zA-Z0-9_\-]+)', "researchgate_url"),
        # Academia.edu profile
        (r'(https?://[a-zA-Z0-9\-]+\.academia\.edu(?:/[a-zA-Z0-9_\-]+)?)', "academia_url"),
        # Google Scholar (fix pattern to capture full URL)
        (r'(https?://scholar\.google\.com/citations\?[^\s\)\"\']+)', "google_scholar_url"),
        # Phone numbers (international formats) - use word boundary to avoid duplicates
        (r'(?:phone|tel|telephone|fax)[:\s]*(\+?[0-9][0-9\s\-\(\)]{8,18}[0-9])', "phone"),
        (r'(?<!\d)(\+31[\s\-]?[0-9][\s\-]?[0-9]{3,4}[\s\-]?[0-9]{3,4})(?!\d)', "phone"),  # Dutch
        (r'(?<!\d)(\+1[\s\-]?\(?[0-9]{3}\)?[\s\-]?[0-9]{3}[\s\-]?[0-9]{4})(?!\d)', "phone"),  # US/Canada
    ]:
        for m in re.finditer(pat, text, re.I):
            val = m.group(1).strip().rstrip('/')
            # Normalize phone numbers for deduplication
            norm_val = re.sub(r'[\s\-\(\)]', '', val) if ctype == "phone" else val
            # Skip duplicates
            dedup_key = f"{ctype}:{norm_val.lower()}"
            if dedup_key in seen_values: continue
            seen_values.add(dedup_key)
            # Skip common false positives
            if ctype == "email" and any(x in val.lower() for x in ['example.com', 'test.com']): continue
            if ctype == "twitter" and val.lower() in twitter_blocklist: continue
            if ctype == "instagram" and val.lower() in instagram_blocklist: continue
            # Skip if value is too short (likely false positive)
            if ctype in ["twitter", "instagram"] and len(val) < 3: continue
            contacts.append({"type": ctype, "value": val,
                            "snippet": text[max(0,m.start()-30):m.end()+30].strip()})
    return contacts


def extract_media(text):
    media = []
    for pat, mtype in [(r'(https?://[^\s]+\.(?:jpg|jpeg|png|gif|webp))', "image_url"),
                       (r'(https?://(?:www\.)?(?:youtube\.com/watch\?v=|youtu\.be/|vimeo\.com/)[^\s]+)', "video_url"),
                       (r'(https?://upload\.wikimedia\.org/[^\s]+)', "wikimedia_image")]:
        for m in re.finditer(pat, text, re.I):
            media.append({"type": mtype, "value": m.group(1).strip(),
                         "snippet": text[max(0,m.start()-30):m.end()+30].strip()})
    return media


def extract_social(text):
    conns = []
    for pat, rtype in [(r'(?:married|spouse|wife|husband)\s+(?:to\s+)?([A-Z][a-zA-Z]+\s+[A-Z][a-zA-Z]+)', "spouse"),
                       (r'(?:daughter|son)\s+of\s+([A-Z][a-zA-Z]+\s+[A-Z][a-zA-Z]+)', "parent"),
                       (r'(?:collaborated|worked)\s+with\s+([A-Z][a-zA-Z]+\s+[A-Z][a-zA-Z]+)', "collaborator")]:
        for m in re.finditer(pat, text, re.I):
            conns.append({"relationship_type": rtype, "related_person": m.group(1).strip(),
                         "snippet": text[max(0,m.start()-20):m.end()+20].strip()})
    return conns


def extract_interests(text):
    interests = []
    for pat, itype in [(r'(?:specializes|specialized)\s+in\s+([^,\.]{5,60})', "specialization"),
                       (r'expert\s+(?:in|on)\s+([^,\.]{5,60})', "expertise"),
                       (r'known\s+for\s+(?:his|her|their\s+)?([^,\.]{5,80})', "known_for")]:
        for m in re.finditer(pat, text, re.I):
            interests.append({"type": itype, "topic": m.group(1).strip(),
                             "snippet": text[max(0,m.start()-20):m.end()+20].strip()})
    return interests


def extract_memberships(text):
    mems = []
    for pat, mtype in [(r'member\s+of\s+(?:the\s+)?([A-Z][^,\.]{5,60})', "membership"),
                       (r'(?:board\s+member|board\s+director)\s+(?:of\s+)?([A-Z][^,\.]{5,60})', "board_member")]:
        for m in re.finditer(pat, text, re.I):
            mems.append({"type": mtype, "organization": m.group(1).strip(),
                        "snippet": text[max(0,m.start()-20):m.end()+20].strip()})
    return mems


def extract_nationalities(text):
    nats = []
    words = ["Dutch", "German", "French", "British", "American", "Belgian", "Italian", "Spanish",
             "Australian", "Canadian", "Japanese", "Chinese", "Brazilian", "Mexican", "Russian"]
    pat = r'\b(' + '|'.join(words) + r')\s+(?:art\s+)?(?:historian|curator|professor|director|artist)'
    for m in re.finditer(pat, text, re.I):
        nats.append({"nationality": m.group(1).strip(),
                    "snippet": text[max(0,m.start()-20):m.end()+20].strip()})
    return nats


def enrich_person(name: str, context: str, api_key: str) -> Dict:
    enrichment = {"web_claims": [], "enrichment_metadata": {
        "enrichment_timestamp": datetime.now(timezone.utc).isoformat(),
        "enrichment_agent": f"enrich_person_comprehensive.py v{SCRIPT_VERSION}",
        "person_name": name, "context_used": context[:100] if context else None,
        "searches_performed": [], "data_fabrication_check": "PASSED"}}
    
    # Search 1: Biography
    q1 = f'"{name}" born biography'
    r1 = search_linkup(q1, api_key)
    enrichment["enrichment_metadata"]["searches_performed"].append(q1)
    
    if "error" not in r1:
        ans, srcs = r1.get("answer", ""), r1.get("sources", [])
        url, title = (srcs[0].get("url", ""), srcs[0].get("name", "")) if srcs else ("", "")
        meta = r1.get("_meta", {})
        
        if ans:
            if (b := extract_birth_year(ans)):
                enrichment["web_claims"].append(create_claim("birth_year", b["year"], url, title, b["snippet"], q1, srcs, meta, ans, b.get("pattern_type")))
            if (l := extract_birth_location(ans)):
                enrichment["web_claims"].append(create_claim("birth_location", l["location"], url, title, l["snippet"], q1, srcs, meta, ans, "birth_location"))
            if (d := extract_death_info(ans)):
                enrichment["web_claims"].append(create_claim("death_year", d["year"], url, title, d["snippet"], q1, srcs, meta, ans, "death_year"))
            for n in extract_nationalities(ans):
                enrichment["web_claims"].append(create_claim("nationality", n["nationality"], url, title, n["snippet"], q1, srcs, meta, ans, "nationality"))
            for s in extract_social(ans):
                enrichment["web_claims"].append(create_claim("social_connection", {"relationship_type": s["relationship_type"], "related_person": s["related_person"]}, url, title, s["snippet"], q1, srcs, meta, ans, s["relationship_type"]))
    
    time.sleep(1.0)
    
    # Search 2: Education/Career
    q2 = f'"{name}" {context} education career university'
    r2 = search_linkup(q2, api_key)
    enrichment["enrichment_metadata"]["searches_performed"].append(q2)
    
    if "error" not in r2:
        ans, srcs = r2.get("answer", ""), r2.get("sources", [])
        url, title = (srcs[0].get("url", ""), srcs[0].get("name", "")) if srcs else ("", "")
        meta = r2.get("_meta", {})
        
        if ans:
            for e in extract_education(ans):
                enrichment["web_claims"].append(create_claim("education", {"type": e["type"], "institution": e["institution"], "year": e["year"]}, url, title, e["snippet"], q2, srcs, meta, ans, e["type"]))
            for p in extract_positions(ans):
                enrichment["web_claims"].append(create_claim("position", {"title": p["title"], "organization": p["organization"], "year": p["year"]}, url, title, p["snippet"], q2, srcs, meta, ans, "position"))
            for m in extract_memberships(ans):
                enrichment["web_claims"].append(create_claim("membership", {"type": m["type"], "organization": m["organization"]}, url, title, m["snippet"], q2, srcs, meta, ans, m["type"]))
            for i in extract_interests(ans):
                enrichment["web_claims"].append(create_claim("interest", {"type": i["type"], "topic": i["topic"]}, url, title, i["snippet"], q2, srcs, meta, ans, i["type"]))
    
    time.sleep(1.0)
    
    # Search 3: Publications/Awards
    q3 = f'"{name}" publications awards honors books'
    r3 = search_linkup(q3, api_key)
    enrichment["enrichment_metadata"]["searches_performed"].append(q3)
    
    if "error" not in r3:
        ans, srcs = r3.get("answer", ""), r3.get("sources", [])
        url, title = (srcs[0].get("url", ""), srcs[0].get("name", "")) if srcs else ("", "")
        meta = r3.get("_meta", {})
        
        if ans:
            for p in extract_publications(ans):
                enrichment["web_claims"].append(create_claim("publication", {"type": p["type"], "title": p["title"], "year": p["year"]}, url, title, p["snippet"], q3, srcs, meta, ans, p["type"]))
            for a in extract_awards(ans):
                enrichment["web_claims"].append(create_claim("award", {"type": a["type"], "name": a["name"]}, url, title, a["snippet"], q3, srcs, meta, ans, a["type"]))
    
    time.sleep(1.0)
    
    # Search 4: Contact/Media
    q4 = f'"{name}" contact email twitter linkedin orcid profile photo'
    r4 = search_linkup(q4, api_key)
    enrichment["enrichment_metadata"]["searches_performed"].append(q4)
    
    if "error" not in r4:
        ans, srcs = r4.get("answer", ""), r4.get("sources", [])
        url, title = (srcs[0].get("url", ""), srcs[0].get("name", "")) if srcs else ("", "")
        meta = r4.get("_meta", {})
        
        if ans:
            for c in extract_contacts(ans):
                enrichment["web_claims"].append(create_claim("contact_detail", {"type": c["type"], "value": c["value"]}, url, title, c["snippet"], q4, srcs, meta, ans, c["type"]))
            for m in extract_media(ans):
                enrichment["web_claims"].append(create_claim("media_reference", {"type": m["type"], "value": m["value"]}, url, title, m["snippet"], q4, srcs, meta, ans, m["type"]))
    
    return enrichment


def process_ppid_file(filepath: Path, api_key: str, dry_run: bool = False) -> Dict:
    with open(filepath) as f:
        data = json.load(f)
    
    name_data = data.get("name", {})
    full_name = name_data.get("full_name") or name_data.get("display_name", "")
    if not full_name or full_name == "LinkedIn Member":
        return {"status": "skipped", "reason": "no_valid_name"}
    
    if not data.get("heritage_relevance", {}).get("is_heritage_relevant"):
        return {"status": "skipped", "reason": "not_heritage_relevant"}
    
    headline = data.get("profile_data", {}).get("headline", "")
    enrichment = enrich_person(full_name, headline, api_key)
    
    if not enrichment["web_claims"]:
        if not dry_run:
            if "enrichment_history" not in data: data["enrichment_history"] = []
            enrichment["enrichment_metadata"]["result"] = "no_claims_found"
            data["enrichment_history"].append(enrichment["enrichment_metadata"])
            with open(filepath, "w") as f: json.dump(data, f, indent=2, ensure_ascii=False)
        return {"status": "no_claims_found", "name": full_name}
    
    if not dry_run:
        if "web_claims" not in data: data["web_claims"] = []
        existing = {(c.get("claim_type"), str(c.get("claim_value"))) for c in data.get("web_claims", [])}
        for claim in enrichment["web_claims"]:
            key = (claim["claim_type"], str(claim["claim_value"]))
            if key not in existing: data["web_claims"].append(claim)
        
        if "enrichment_history" not in data: data["enrichment_history"] = []
        data["enrichment_history"].append(enrichment["enrichment_metadata"])
        
        birth_claims = [c for c in enrichment["web_claims"] if c["claim_type"] == "birth_year"]
        if birth_claims:
            current = data.get("birth_date", {}).get("edtf", "XXXX")
            if current == "XXXX" or current.endswith("X"):
                prov = birth_claims[0]["provenance"]
                data["birth_date"] = {"edtf": str(birth_claims[0]["claim_value"]), "precision": "year",
                    "provenance": {k: prov[k] for k in ["statement_created_at", "source_archived_at", 
                        "retrieval_agent", "retrieval_method", "source_url", "source_title", 
                        "source_snippet", "search_query", "extraction_method"] if k in prov}}
                data["birth_date"]["provenance"]["verified"] = False
                data["birth_date"]["provenance"]["verification_status"] = "machine_extracted"
        
        if [c for c in enrichment["web_claims"] if c["claim_type"] == "death_year"]:
            data["is_living"] = False
        
        with open(filepath, "w") as f: json.dump(data, f, indent=2, ensure_ascii=False)
    
    return {"status": "enriched", "name": full_name, "claims_added": len(enrichment["web_claims"]),
            "claim_types": list(set(c["claim_type"] for c in enrichment["web_claims"]))}


def main():
    parser = argparse.ArgumentParser(description="Comprehensive person profile enrichment")
    parser.add_argument("--limit", type=int, default=10)
    parser.add_argument("--dry-run", action="store_true")
    args = parser.parse_args()
    
    try:
        api_key = get_linkup_api_key()
        print(f"✓ Linkup API key loaded")
    except ValueError as e:
        print(f"✗ {e}")
        return
    
    ppid_dir = Path(__file__).parent.parent / "data" / "person"
    if not ppid_dir.exists():
        print(f"✗ PPID directory not found")
        return
    
    print("Scanning for candidates...")
    candidates = []
    for f in ppid_dir.glob("ID_*.json"):
        try:
            with open(f) as fp: data = json.load(fp)
            if not data.get("heritage_relevance", {}).get("is_heritage_relevant"): continue
            if data.get("enrichment_history"): continue
            
            name = data.get("name", {}).get("full_name", "")
            if not name or name == "LinkedIn Member": continue
            
            headline = data.get("profile_data", {}).get("headline", "").lower()
            score = 0
            if "professor" in headline: score += 3
            if "director" in headline: score += 2
            if "curator" in headline: score += 2
            if "museum" in headline: score += 1
            if "archive" in headline: score += 1
            candidates.append((f, score, name))
        except: continue
    
    candidates.sort(key=lambda x: -x[1])
    print(f"Found {len(candidates)} candidates")
    
    stats = {"enriched": 0, "no_claims_found": 0, "skipped": 0, "errors": 0}
    results = []
    
    for i, (filepath, score, _) in enumerate(candidates[:args.limit]):
        print(f"\n[{i+1}/{args.limit}] {filepath.name} (score={score})")
        try:
            result = process_ppid_file(filepath, api_key, args.dry_run)
            stats[result.get("status", "errors")] = stats.get(result.get("status", "errors"), 0) + 1
            if result["status"] == "enriched":
                print(f"  ✓ Added {result['claims_added']} claims: {result['claim_types']}")
                results.append(result)
            elif result["status"] == "no_claims_found":
                print(f"  ✗ No claims found for {result.get('name')}")
            time.sleep(4.0)
        except Exception as e:
            print(f"  ✗ Error: {e}")
            stats["errors"] += 1
    
    print(f"\n{'='*50}\nSUMMARY\n{'='*50}")
    print(f"Enriched: {stats['enriched']}, No claims: {stats['no_claims_found']}, Errors: {stats['errors']}")
    if results:
        print(f"\nTotal claims added: {sum(r['claims_added'] for r in results)}")


if __name__ == "__main__":
    main()