472 lines
15 KiB
Python
472 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Build entity resolution candidates between WCMS and LinkedIn profiles.
|
|
|
|
This script:
|
|
1. Indexes all profiles by normalized name
|
|
2. Finds potential matches based on multiple signals
|
|
3. Scores matches but NEVER auto-merges
|
|
4. Outputs candidates for manual review
|
|
|
|
CRITICAL: No auto-merging! Entity resolution requires human verification.
|
|
|
|
Matching signals:
|
|
- Name similarity (primary)
|
|
- Email domain matches employer
|
|
- Overlapping affiliations
|
|
- Location overlap
|
|
|
|
Usage:
|
|
python scripts/build_entity_resolution.py --limit 10000
|
|
python scripts/build_entity_resolution.py --output candidates.json
|
|
"""
|
|
|
|
import json
|
|
import argparse
|
|
import re
|
|
from pathlib import Path
|
|
from datetime import datetime, timezone
|
|
import unicodedata
|
|
from typing import Dict, List, Optional, Set, Tuple
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass, asdict
|
|
|
|
PERSON_DIR = Path('/Users/kempersc/apps/glam/data/person')
|
|
OUTPUT_DIR = Path('/Users/kempersc/apps/glam/data/entity_resolution')
|
|
|
|
|
|
def normalize_name(name) -> str:
|
|
"""Normalize name for comparison."""
|
|
if not name:
|
|
return ""
|
|
|
|
# Handle dict or other types
|
|
if isinstance(name, dict):
|
|
name = name.get('full_name', name.get('name', str(name)))
|
|
if not isinstance(name, str):
|
|
name = str(name)
|
|
|
|
# Remove titles
|
|
name = re.sub(r'\b(Dr|Prof|Mr|Mrs|Ms|Drs|Ir|Ing|PhD|MA|MSc|MBA|BSc|Jr|Sr)\b\.?', '', name, flags=re.IGNORECASE)
|
|
|
|
# Normalize unicode
|
|
nfkd = unicodedata.normalize('NFKD', name)
|
|
ascii_name = ''.join(c for c in nfkd if not unicodedata.combining(c))
|
|
|
|
# Lowercase, remove punctuation
|
|
clean = re.sub(r'[^a-z\s]', '', ascii_name.lower())
|
|
|
|
# Normalize whitespace
|
|
return ' '.join(clean.split())
|
|
|
|
|
|
def extract_name_parts(name: str) -> Tuple[str, str]:
|
|
"""Extract first and last name parts."""
|
|
parts = name.split()
|
|
if len(parts) >= 2:
|
|
return parts[0], parts[-1]
|
|
elif len(parts) == 1:
|
|
return parts[0], ""
|
|
return "", ""
|
|
|
|
|
|
def extract_employer_domains(profile: dict) -> Set[str]:
|
|
"""Extract domains from employer information in profile."""
|
|
domains = set()
|
|
|
|
# From affiliations
|
|
for aff in profile.get('affiliations', []) or []:
|
|
if not isinstance(aff, dict):
|
|
continue
|
|
org = aff.get('organization', {})
|
|
if isinstance(org, str):
|
|
continue # Skip if org is just a string
|
|
if not isinstance(org, dict):
|
|
continue
|
|
website = org.get('website', '') or ''
|
|
if website and isinstance(website, str):
|
|
# Extract domain from URL
|
|
domain = re.sub(r'^https?://(www\.)?', '', website.lower())
|
|
domain = domain.split('/')[0]
|
|
if domain:
|
|
domains.add(domain)
|
|
|
|
# From profile_data
|
|
profile_data = profile.get('profile_data', {}) or {}
|
|
if not isinstance(profile_data, dict):
|
|
return domains
|
|
|
|
for exp in profile_data.get('experience', []) or []:
|
|
if not isinstance(exp, dict):
|
|
continue
|
|
company_url = exp.get('company_url', '') or ''
|
|
if company_url and isinstance(company_url, str):
|
|
domain = re.sub(r'^https?://(www\.)?', '', company_url.lower())
|
|
domain = domain.split('/')[0]
|
|
if domain:
|
|
domains.add(domain)
|
|
|
|
return domains
|
|
|
|
|
|
def extract_employer_names(profile: dict) -> Set[str]:
|
|
"""Extract employer names from profile."""
|
|
names = set()
|
|
|
|
# From affiliations
|
|
for aff in profile.get('affiliations', []) or []:
|
|
if not isinstance(aff, dict):
|
|
continue
|
|
org = aff.get('organization', {})
|
|
if isinstance(org, str):
|
|
# org is just a string name
|
|
normalized = normalize_name(org)
|
|
if normalized:
|
|
names.add(normalized)
|
|
continue
|
|
if isinstance(org, dict):
|
|
name = org.get('name', '')
|
|
if name and isinstance(name, str):
|
|
normalized = normalize_name(name)
|
|
if normalized:
|
|
names.add(normalized)
|
|
|
|
# From profile_data experience
|
|
profile_data = profile.get('profile_data', {}) or {}
|
|
if not isinstance(profile_data, dict):
|
|
return names
|
|
|
|
for exp in profile_data.get('experience', []) or []:
|
|
if not isinstance(exp, dict):
|
|
continue
|
|
company = exp.get('company', '')
|
|
if company and isinstance(company, str):
|
|
normalized = normalize_name(company)
|
|
if normalized:
|
|
names.add(normalized)
|
|
|
|
return names
|
|
|
|
|
|
@dataclass
|
|
class MatchCandidate:
|
|
"""A potential entity resolution match."""
|
|
wcms_ppid: str
|
|
wcms_name: str
|
|
wcms_email: Optional[str]
|
|
wcms_email_domain: Optional[str]
|
|
linkedin_ppid: str
|
|
linkedin_name: str
|
|
linkedin_slug: Optional[str]
|
|
|
|
# Scoring signals
|
|
name_match_score: float # 0-1, how similar are names
|
|
email_domain_matches_employer: bool
|
|
employer_name_overlap: List[str]
|
|
|
|
# Overall score
|
|
confidence_score: float
|
|
match_signals: List[str]
|
|
|
|
# Review status
|
|
requires_review: bool = True
|
|
reviewed: bool = False
|
|
review_decision: Optional[str] = None # "match", "not_match", "uncertain"
|
|
|
|
|
|
def calculate_name_similarity(name1: str, name2: str) -> float:
|
|
"""Calculate name similarity score (0-1)."""
|
|
n1 = normalize_name(name1)
|
|
n2 = normalize_name(name2)
|
|
|
|
if not n1 or not n2:
|
|
return 0.0
|
|
|
|
# Exact match
|
|
if n1 == n2:
|
|
return 1.0
|
|
|
|
# Check first/last name match
|
|
first1, last1 = extract_name_parts(n1)
|
|
first2, last2 = extract_name_parts(n2)
|
|
|
|
# Both first and last match
|
|
if first1 == first2 and last1 == last2:
|
|
return 0.95
|
|
|
|
# Last name matches, first name is initial or similar
|
|
if last1 == last2:
|
|
if first1 and first2 and first1[0] == first2[0]:
|
|
return 0.85
|
|
if first1 == first2:
|
|
return 0.9
|
|
|
|
# First name matches, last name similar
|
|
if first1 == first2 and last1 and last2:
|
|
# Check if one is substring of other
|
|
if last1 in last2 or last2 in last1:
|
|
return 0.8
|
|
|
|
# Token overlap
|
|
tokens1 = set(n1.split())
|
|
tokens2 = set(n2.split())
|
|
overlap = tokens1 & tokens2
|
|
if overlap:
|
|
jaccard = len(overlap) / len(tokens1 | tokens2)
|
|
return 0.5 + (jaccard * 0.4)
|
|
|
|
return 0.0
|
|
|
|
|
|
def build_name_index(profiles: List[dict]) -> Dict[str, List[dict]]:
|
|
"""Build index of profiles by normalized name components."""
|
|
index = defaultdict(list)
|
|
|
|
for profile in profiles:
|
|
# Skip profiles without ppid
|
|
if not profile.get('ppid'):
|
|
continue
|
|
|
|
name = profile.get('name', '')
|
|
normalized = normalize_name(name)
|
|
|
|
if normalized:
|
|
# Index by full normalized name
|
|
index[normalized].append(profile)
|
|
|
|
# Also index by last name
|
|
_, last = extract_name_parts(normalized)
|
|
if last:
|
|
index[f"_last_{last}"].append(profile)
|
|
|
|
return index
|
|
|
|
|
|
def find_candidates(wcms_profile: dict, linkedin_index: Dict[str, List[dict]]) -> List[MatchCandidate]:
|
|
"""Find potential LinkedIn matches for a WCMS profile."""
|
|
candidates = []
|
|
|
|
wcms_name = wcms_profile.get('name', '')
|
|
wcms_normalized = normalize_name(wcms_name)
|
|
wcms_email = wcms_profile.get('contact_details', {}).get('email')
|
|
wcms_domain = wcms_profile.get('contact_details', {}).get('email_domain')
|
|
|
|
if not wcms_normalized:
|
|
return []
|
|
|
|
# Get candidate LinkedIn profiles
|
|
potential_matches = set()
|
|
|
|
# Exact name match
|
|
for p in linkedin_index.get(wcms_normalized, []):
|
|
ppid = p.get('ppid')
|
|
if ppid:
|
|
potential_matches.add(ppid)
|
|
|
|
# Last name match
|
|
_, last = extract_name_parts(wcms_normalized)
|
|
if last:
|
|
for p in linkedin_index.get(f"_last_{last}", []):
|
|
ppid = p.get('ppid')
|
|
if ppid:
|
|
potential_matches.add(ppid)
|
|
|
|
# Score each potential match
|
|
for linkedin_ppid in potential_matches:
|
|
# Get full profile
|
|
profile_path = PERSON_DIR / f"{linkedin_ppid}.json"
|
|
if not profile_path.exists():
|
|
continue
|
|
|
|
try:
|
|
with open(profile_path) as f:
|
|
linkedin_profile = json.load(f)
|
|
except:
|
|
continue
|
|
|
|
# Skip if same profile (WCMS profile, not LinkedIn)
|
|
if 'wcms' in linkedin_profile.get('data_sources', []):
|
|
continue
|
|
|
|
linkedin_name = linkedin_profile.get('name', '')
|
|
|
|
# Calculate signals
|
|
name_score = calculate_name_similarity(wcms_name, linkedin_name)
|
|
|
|
if name_score < 0.5:
|
|
continue # Too low to consider
|
|
|
|
# Check email domain vs employer
|
|
employer_domains = extract_employer_domains(linkedin_profile)
|
|
domain_matches = wcms_domain and wcms_domain in employer_domains
|
|
|
|
# Check employer name overlap
|
|
employer_names = extract_employer_names(linkedin_profile)
|
|
# Could enhance with WCMS organization data if available
|
|
|
|
# Build match signals
|
|
signals = []
|
|
if name_score >= 0.95:
|
|
signals.append("exact_name_match")
|
|
elif name_score >= 0.85:
|
|
signals.append("strong_name_match")
|
|
elif name_score >= 0.7:
|
|
signals.append("partial_name_match")
|
|
|
|
if domain_matches:
|
|
signals.append("email_domain_matches_employer")
|
|
|
|
# Calculate overall confidence
|
|
confidence = name_score * 0.6 # Name is 60% of score
|
|
if domain_matches:
|
|
confidence += 0.3 # Domain match is strong signal
|
|
|
|
# Create candidate
|
|
candidate = MatchCandidate(
|
|
wcms_ppid=wcms_profile['ppid'],
|
|
wcms_name=wcms_name,
|
|
wcms_email=wcms_email,
|
|
wcms_email_domain=wcms_domain,
|
|
linkedin_ppid=linkedin_ppid,
|
|
linkedin_name=linkedin_name,
|
|
linkedin_slug=linkedin_profile.get('linkedin_slug'),
|
|
name_match_score=name_score,
|
|
email_domain_matches_employer=domain_matches,
|
|
employer_name_overlap=list(employer_names)[:5],
|
|
confidence_score=confidence,
|
|
match_signals=signals,
|
|
requires_review=True
|
|
)
|
|
|
|
candidates.append(candidate)
|
|
|
|
# Sort by confidence
|
|
candidates.sort(key=lambda c: c.confidence_score, reverse=True)
|
|
|
|
return candidates[:5] # Return top 5 candidates per WCMS profile
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Build entity resolution candidates')
|
|
parser.add_argument('--limit', type=int, default=None, help='Limit profiles to process')
|
|
parser.add_argument('--output', type=str, default='entity_resolution_candidates.json', help='Output file name')
|
|
parser.add_argument('--min-confidence', type=float, default=0.5, help='Minimum confidence threshold')
|
|
args = parser.parse_args()
|
|
|
|
print("=" * 70)
|
|
print("ENTITY RESOLUTION CANDIDATE BUILDER")
|
|
print("=" * 70)
|
|
print(" CRITICAL: No auto-merging! All candidates require manual review.")
|
|
|
|
OUTPUT_DIR.mkdir(exist_ok=True)
|
|
|
|
# Phase 1: Load all profiles
|
|
print("\nPhase 1: Loading profiles...")
|
|
|
|
wcms_profiles = []
|
|
linkedin_profiles = []
|
|
|
|
count = 0
|
|
for f in PERSON_DIR.glob('ID_*.json'):
|
|
count += 1
|
|
if count % 20000 == 0:
|
|
print(f" Loaded {count:,} profiles...")
|
|
|
|
try:
|
|
with open(f) as fp:
|
|
data = json.load(fp)
|
|
|
|
sources = data.get('data_sources', [])
|
|
|
|
if 'wcms' in sources:
|
|
wcms_profiles.append(data)
|
|
elif 'linkedin' in sources or data.get('linkedin_slug'):
|
|
linkedin_profiles.append(data)
|
|
except:
|
|
pass
|
|
|
|
print(f" Loaded {len(wcms_profiles):,} WCMS profiles")
|
|
print(f" Loaded {len(linkedin_profiles):,} LinkedIn profiles")
|
|
|
|
if args.limit:
|
|
wcms_profiles = wcms_profiles[:args.limit]
|
|
print(f" Limited WCMS profiles to {args.limit}")
|
|
|
|
# Phase 2: Build LinkedIn index
|
|
print("\nPhase 2: Building LinkedIn name index...")
|
|
linkedin_index = build_name_index(linkedin_profiles)
|
|
print(f" Index size: {len(linkedin_index):,} name keys")
|
|
|
|
# Phase 3: Find candidates
|
|
print("\nPhase 3: Finding match candidates...")
|
|
|
|
all_candidates = []
|
|
profiles_with_matches = 0
|
|
|
|
for i, wcms in enumerate(wcms_profiles):
|
|
candidates = find_candidates(wcms, linkedin_index)
|
|
|
|
# Filter by confidence
|
|
candidates = [c for c in candidates if c.confidence_score >= args.min_confidence]
|
|
|
|
if candidates:
|
|
profiles_with_matches += 1
|
|
all_candidates.extend(candidates)
|
|
|
|
if (i + 1) % 10000 == 0:
|
|
print(f" Processed {i+1:,}/{len(wcms_profiles):,} - "
|
|
f"Found {len(all_candidates):,} candidates from {profiles_with_matches:,} profiles")
|
|
|
|
# Phase 4: Output results
|
|
print("\nPhase 4: Saving results...")
|
|
|
|
output_path = OUTPUT_DIR / args.output
|
|
|
|
# Convert to dict for JSON serialization
|
|
results = {
|
|
"metadata": {
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"wcms_profiles_processed": len(wcms_profiles),
|
|
"linkedin_profiles_indexed": len(linkedin_profiles),
|
|
"profiles_with_matches": profiles_with_matches,
|
|
"total_candidates": len(all_candidates),
|
|
"min_confidence_threshold": args.min_confidence,
|
|
"requires_manual_review": True
|
|
},
|
|
"candidates": [asdict(c) for c in all_candidates]
|
|
}
|
|
|
|
with open(output_path, 'w') as f:
|
|
json.dump(results, f, indent=2, ensure_ascii=False)
|
|
|
|
# Summary
|
|
print("\n" + "=" * 70)
|
|
print("ENTITY RESOLUTION SUMMARY")
|
|
print("=" * 70)
|
|
print(f" WCMS profiles processed: {len(wcms_profiles):,}")
|
|
print(f" LinkedIn profiles indexed: {len(linkedin_profiles):,}")
|
|
print(f" Profiles with potential matches: {profiles_with_matches:,}")
|
|
print(f" Total match candidates: {len(all_candidates):,}")
|
|
print(f" Output saved to: {output_path}")
|
|
|
|
# Show confidence distribution
|
|
if all_candidates:
|
|
high = sum(1 for c in all_candidates if c.confidence_score >= 0.8)
|
|
medium = sum(1 for c in all_candidates if 0.6 <= c.confidence_score < 0.8)
|
|
low = sum(1 for c in all_candidates if c.confidence_score < 0.6)
|
|
|
|
print(f"\n Confidence distribution:")
|
|
print(f" High (>=0.8): {high:,}")
|
|
print(f" Medium (0.6-0.8): {medium:,}")
|
|
print(f" Low (<0.6): {low:,}")
|
|
|
|
# Show sample candidates
|
|
if all_candidates:
|
|
print(f"\n Sample high-confidence candidates:")
|
|
for c in sorted(all_candidates, key=lambda x: x.confidence_score, reverse=True)[:5]:
|
|
print(f" {c.wcms_name} <-> {c.linkedin_name}")
|
|
print(f" Score: {c.confidence_score:.2f}, Signals: {', '.join(c.match_signals)}")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|