mirror of
https://github.com/cfinke/EpsteOut.git
synced 2026-08-24 10:04:08 -05:00
523 lines
16 KiB
Python
Executable File
523 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Search Epstein files for mentions of LinkedIn connections.
|
|
|
|
Usage:
|
|
python EpsteOut.py --connections <linkedin_csv> [--output <report.html>]
|
|
|
|
Prerequisites:
|
|
pip install requests
|
|
"""
|
|
|
|
import argparse
|
|
import base64
|
|
import csv
|
|
from datetime import datetime
|
|
import html
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
|
|
try:
|
|
import requests
|
|
HAS_REQUESTS = True
|
|
except ImportError:
|
|
HAS_REQUESTS = False
|
|
|
|
API_BASE_URL = "https://analytics.dugganusa.com/api/v1/search"
|
|
PDF_BASE_URL = "https://www.justice.gov/epstein/files/"
|
|
API_KEY_PATH = os.path.join(os.getcwd(), ".epstein_api_key")
|
|
CACHE_PATH = os.path.join(os.getcwd(), ".epstein_cache.json")
|
|
|
|
|
|
def load_cache():
|
|
"""Load cached search results from disk."""
|
|
if os.path.exists(CACHE_PATH):
|
|
with open(CACHE_PATH, 'r', encoding='utf-8') as f:
|
|
return json.load(f)
|
|
return {}
|
|
|
|
|
|
def save_cache(cache):
|
|
"""Write cached search results to disk."""
|
|
with open(CACHE_PATH, 'w', encoding='utf-8') as f:
|
|
json.dump(cache, f, indent=2, ensure_ascii=False)
|
|
|
|
|
|
def get_api_key():
|
|
"""Load API key from disk, or prompt the user for one."""
|
|
if os.path.exists(API_KEY_PATH):
|
|
with open(API_KEY_PATH, 'r') as f:
|
|
key = f.read().strip()
|
|
if key:
|
|
return key
|
|
|
|
print("An API key is required to search the Epstein files.")
|
|
print("To obtain one, visit: https://epstein.dugganusa.com/register.html")
|
|
print()
|
|
api_key = input("Enter your API key: ").strip()
|
|
|
|
if not api_key:
|
|
print("Error: No API key provided.", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
with open(API_KEY_PATH, 'w') as f:
|
|
f.write(api_key)
|
|
|
|
print(f"API key saved to {API_KEY_PATH}\n")
|
|
return api_key
|
|
|
|
|
|
def parse_linkedin_contacts(csv_path):
|
|
"""
|
|
Parse LinkedIn connections CSV export.
|
|
LinkedIn exports have columns: First Name, Last Name, Email Address, Company, Position, Connected On
|
|
"""
|
|
contacts = []
|
|
|
|
with open(csv_path, 'r', encoding='utf-8-sig') as f:
|
|
# Skip lines until we find the header row
|
|
# LinkedIn includes a "Notes" section at the top that must be skipped.
|
|
header_line = None
|
|
for line in f:
|
|
if 'First Name' in line and 'Last Name' in line:
|
|
header_line = line
|
|
break
|
|
|
|
if not header_line:
|
|
return contacts
|
|
|
|
# Create a reader from the header line onwards
|
|
remaining_content = header_line + f.read()
|
|
reader = csv.DictReader(remaining_content.splitlines())
|
|
|
|
for row in reader:
|
|
first_name = row.get('First Name', '').strip()
|
|
last_name = row.get('Last Name', '').strip()
|
|
|
|
# Remove credentials/certifications (everything after the first comma)
|
|
if ',' in last_name:
|
|
last_name = last_name.split(',')[0].strip()
|
|
|
|
if first_name and last_name:
|
|
full_name = f"{first_name} {last_name}"
|
|
contacts.append({
|
|
'first_name': first_name,
|
|
'last_name': last_name,
|
|
'full_name': full_name,
|
|
'company': row.get('Company', ''),
|
|
'position': row.get('Position', '')
|
|
})
|
|
|
|
return contacts
|
|
|
|
|
|
def search_epstein_files(name, delay, api_key):
|
|
"""
|
|
Search the Epstein files API for a name.
|
|
Returns (result_dict, delay) where delay may be increased on 429 responses.
|
|
"""
|
|
# Wrap name in quotes for exact phrase matching
|
|
quoted_name = f'"{name}"'
|
|
encoded_name = urllib.parse.quote(quoted_name)
|
|
url = f"{API_BASE_URL}?q={encoded_name}&indexes=epstein_files"
|
|
headers = {"Authorization": f"Bearer {api_key}"}
|
|
|
|
while True:
|
|
try:
|
|
response = requests.get(url, headers=headers, timeout=30)
|
|
|
|
if response.status_code == 429:
|
|
retry_after = response.headers.get('Retry-After')
|
|
|
|
if retry_after:
|
|
delay = int(retry_after)
|
|
else:
|
|
delay *= 2
|
|
|
|
print(f" [429 rate limited, retrying in {delay}s]", end='', flush=True)
|
|
time.sleep(delay)
|
|
continue
|
|
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
if data.get('success'):
|
|
return {
|
|
'total_hits': data.get('data', {}).get('totalHits', 0),
|
|
'hits': data.get('data', {}).get('hits', [])
|
|
}, delay
|
|
except requests.exceptions.ConnectTimeout:
|
|
delay *= 2
|
|
print(f" [connect timeout, retrying in {delay}s]", end='', flush=True)
|
|
time.sleep(delay)
|
|
continue
|
|
except requests.exceptions.RequestException as e:
|
|
print(f"Warning: API request failed for '{name}': {e}", file=sys.stderr)
|
|
return {'total_hits': 0, 'hits': [], 'error': str(e)}, delay
|
|
|
|
return {'total_hits': 0, 'hits': []}, delay
|
|
|
|
|
|
def generate_html_report(results, output_path):
|
|
contacts_with_mentions = len([r for r in results if r['total_mentions'] > 0])
|
|
|
|
# Read and encode logo as base64 data URI, or fall back to text header
|
|
script_dir = os.path.dirname(os.path.abspath(__file__))
|
|
logo_path = os.path.join(script_dir, 'assets', 'logo.png')
|
|
if os.path.exists(logo_path):
|
|
with open(logo_path, 'rb') as f:
|
|
logo_base64 = base64.b64encode(f.read()).decode('utf-8')
|
|
logo_html = f'<img src="data:image/png;base64,{logo_base64}" alt="EpsteOut" class="logo">'
|
|
else:
|
|
logo_html = '<h1 class="logo" style="text-align: center;">EpsteOut</h1>'
|
|
|
|
html_content = f"""<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>EpsteOut: Which LinkedIn Connections Appear in the Epstein Files?</title>
|
|
<style>
|
|
* {{
|
|
box-sizing: border-box;
|
|
}}
|
|
body {{
|
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif;
|
|
line-height: 1.6;
|
|
max-width: 1200px;
|
|
margin: 0 auto;
|
|
padding: 20px;
|
|
background-color: #f5f5f5;
|
|
}}
|
|
.logo {{
|
|
display: block;
|
|
max-width: 300px;
|
|
margin: 0 auto 20px auto;
|
|
}}
|
|
.summary {{
|
|
background: #fff;
|
|
padding: 20px;
|
|
border-radius: 8px;
|
|
margin-bottom: 30px;
|
|
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
|
}}
|
|
.contact {{
|
|
background: #fff;
|
|
padding: 20px;
|
|
margin-bottom: 20px;
|
|
border-radius: 8px;
|
|
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
|
}}
|
|
.contact-header {{
|
|
display: flex;
|
|
justify-content: space-between;
|
|
align-items: center;
|
|
border-bottom: 1px solid #eee;
|
|
padding-bottom: 10px;
|
|
margin-bottom: 15px;
|
|
}}
|
|
.contact-name {{
|
|
font-size: 1.4em;
|
|
font-weight: bold;
|
|
color: #333;
|
|
}}
|
|
.contact-info {{
|
|
color: #666;
|
|
font-size: 0.9em;
|
|
}}
|
|
.hit-count {{
|
|
background: #e74c3c;
|
|
color: white;
|
|
padding: 5px 15px;
|
|
border-radius: 20px;
|
|
font-weight: bold;
|
|
}}
|
|
.hit {{
|
|
background: #f9f9f9;
|
|
padding: 15px;
|
|
margin-bottom: 10px;
|
|
border-radius: 4px;
|
|
border-left: 3px solid #3498db;
|
|
}}
|
|
.hit-preview {{
|
|
color: #444;
|
|
margin-bottom: 10px;
|
|
font-size: 0.95em;
|
|
}}
|
|
.hit-link {{
|
|
display: inline-block;
|
|
color: #3498db;
|
|
text-decoration: none;
|
|
font-size: 0.85em;
|
|
}}
|
|
.hit-link:hover {{
|
|
text-decoration: underline;
|
|
}}
|
|
.no-results {{
|
|
color: #999;
|
|
font-style: italic;
|
|
}}
|
|
.footer {{
|
|
margin-top: 40px;
|
|
padding-top: 20px;
|
|
border-top: 1px solid #ddd;
|
|
text-align: center;
|
|
color: #666;
|
|
font-size: 0.9em;
|
|
}}
|
|
.footer a {{
|
|
color: #3498db;
|
|
text-decoration: none;
|
|
}}
|
|
.footer a:hover {{
|
|
text-decoration: underline;
|
|
}}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
{logo_html}
|
|
|
|
<div class="summary">
|
|
<strong>Total connections searched:</strong> {len(results)}<br>
|
|
<strong>Connections with mentions:</strong> {contacts_with_mentions}
|
|
</div>
|
|
"""
|
|
|
|
for result in results:
|
|
if result['total_mentions'] == 0:
|
|
continue
|
|
|
|
contact_info = []
|
|
if result['position']:
|
|
contact_info.append(html.escape(result['position']))
|
|
if result['company']:
|
|
contact_info.append(html.escape(result['company']))
|
|
|
|
html_content += f"""
|
|
<div class="contact">
|
|
<div class="contact-header">
|
|
<div>
|
|
<div class="contact-name">{html.escape(result['name'])}</div>
|
|
<div class="contact-info">{' at '.join(contact_info) if contact_info else ''}</div>
|
|
</div>
|
|
<div class="hit-count">{result['total_mentions']:,} mentions</div>
|
|
</div>
|
|
"""
|
|
|
|
if result['hits']:
|
|
for hit in result['hits']:
|
|
preview = hit.get('content_preview') or (hit.get('content') or '')[:500]
|
|
|
|
pdf_url = hit.get('doj_url', '')
|
|
|
|
if not pdf_url:
|
|
file_path = hit.get('file_path', '')
|
|
if file_path:
|
|
file_path = file_path.replace('dataset', 'DataSet')
|
|
base_url = PDF_BASE_URL.rstrip('/') if file_path.startswith('/') else PDF_BASE_URL
|
|
pdf_url = base_url + urllib.parse.quote(file_path, safe='/')
|
|
else:
|
|
pdf_url = ''
|
|
|
|
html_content += f"""
|
|
<div class="hit">
|
|
<div class="hit-preview">{html.escape(preview)}</div>
|
|
{f'<a class="hit-link" href="{html.escape(pdf_url)}" target="_blank">View PDF: {html.escape(pdf_url)}</a>' if pdf_url else ''}
|
|
</div>
|
|
"""
|
|
else:
|
|
html_content += """
|
|
<div class="no-results">Hit details not available</div>
|
|
"""
|
|
|
|
html_content += """
|
|
</div>
|
|
"""
|
|
|
|
html_content += """
|
|
<div class="footer">
|
|
Epstein files indexed by <a href="https://dugganusa.com" target="_blank">DugganUSA.com</a>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
with open(output_path, 'w', encoding='utf-8') as f:
|
|
f.write(html_content)
|
|
|
|
|
|
def main():
|
|
if not HAS_REQUESTS:
|
|
print("Error: 'requests' library is required. Install with: pip install requests", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description='Search Epstein files for mentions of LinkedIn connections'
|
|
)
|
|
parser.add_argument(
|
|
'--connections', '-c',
|
|
required=False,
|
|
help='Path to LinkedIn connections CSV export'
|
|
)
|
|
parser.add_argument(
|
|
'--output', '-o',
|
|
default='EpsteOut.html',
|
|
help='Output HTML file for the report (default: EpsteOut.html)'
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
# Validate inputs
|
|
if not args.connections:
|
|
print("""
|
|
No connections file specified.
|
|
|
|
To export your LinkedIn connections:
|
|
1. Go to linkedin.com and log in
|
|
2. Click your profile icon in the top right
|
|
3. Select "Settings & Privacy"
|
|
4. Click "Data privacy" in the left sidebar
|
|
5. Under "How LinkedIn uses your data", click "Get a copy of your data"
|
|
6. Select "Connections" (or "Want something in particular?" and check Connections)
|
|
7. Click "Request archive"
|
|
8. Wait for LinkedIn's email (may take up to 24 hours)
|
|
9. Download and extract the ZIP file
|
|
10. Use the Connections.csv file with this script:
|
|
|
|
python EpsteOut.py --connections /path/to/Connections.csv
|
|
""")
|
|
sys.exit(1)
|
|
|
|
if not os.path.exists(args.connections):
|
|
print(f"Error: Connections file not found: {args.connections}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
# Parse LinkedIn connections
|
|
print(f"Reading LinkedIn connections from: {args.connections}")
|
|
contacts = parse_linkedin_contacts(args.connections)
|
|
print(f"Found {len(contacts)} connections")
|
|
|
|
if not contacts:
|
|
print("No connections found in CSV. Check the file format.", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
# Get API key (prompts user if not stored)
|
|
api_key = get_api_key()
|
|
|
|
# Load cached results from previous runs
|
|
cache = load_cache()
|
|
|
|
# Sort contacts: never-searched first, then oldest-searched first
|
|
def sort_key(contact):
|
|
cached = cache.get(contact['full_name'])
|
|
if cached is None:
|
|
return (0, '') # Never searched — highest priority
|
|
return (1, cached.get('last_searched', ''))
|
|
|
|
contacts.sort(key=sort_key)
|
|
|
|
# Search for each contact
|
|
print("Searching Epstein files API...")
|
|
print("(Press Ctrl+C to stop and generate a partial report)\n")
|
|
searched_this_run = set()
|
|
|
|
delay = 0.25
|
|
|
|
try:
|
|
for i, contact in enumerate(contacts):
|
|
print(f" [{i+1}/{len(contacts)}] {contact['full_name']}", end='', flush=True)
|
|
|
|
# Skip contacts searched in the last 23 hours
|
|
cached_entry = cache.get(contact['full_name'])
|
|
if cached_entry and 'last_searched' in cached_entry:
|
|
age = datetime.now() - datetime.fromisoformat(cached_entry['last_searched'])
|
|
if age.total_seconds() < 23 * 3600:
|
|
print(f" -> skipped (cached {age.total_seconds() / 3600:.1f}h ago)")
|
|
continue
|
|
|
|
search_result, delay = search_epstein_files(contact['full_name'], delay, api_key)
|
|
total_mentions = search_result['total_hits']
|
|
|
|
print(f" -> {total_mentions} hits")
|
|
|
|
# Update cache immediately so interrupted runs keep progress
|
|
cache[contact['full_name']] = {
|
|
'last_searched': datetime.now().isoformat(),
|
|
'total_hits': total_mentions,
|
|
'hits': search_result['hits'],
|
|
'first_name': contact['first_name'],
|
|
'last_name': contact['last_name'],
|
|
'company': contact['company'],
|
|
'position': contact['position'],
|
|
}
|
|
save_cache(cache)
|
|
searched_this_run.add(contact['full_name'])
|
|
|
|
# Rate limiting
|
|
if i < len(contacts) - 1:
|
|
time.sleep(delay)
|
|
|
|
except KeyboardInterrupt:
|
|
print("\n\nSearch interrupted by user (Ctrl+C).")
|
|
|
|
# Build results: fresh searches + cached entries for remaining contacts
|
|
fresh_count = len(searched_this_run)
|
|
cached_count = 0
|
|
results = []
|
|
|
|
for contact in contacts:
|
|
name = contact['full_name']
|
|
if name in searched_this_run:
|
|
entry = cache[name]
|
|
elif name in cache:
|
|
entry = cache[name]
|
|
cached_count += 1
|
|
else:
|
|
continue
|
|
|
|
results.append({
|
|
'name': name,
|
|
'first_name': entry['first_name'],
|
|
'last_name': entry['last_name'],
|
|
'company': entry['company'],
|
|
'position': entry['position'],
|
|
'total_mentions': entry['total_hits'],
|
|
'hits': entry['hits'],
|
|
})
|
|
|
|
print(f"\n{fresh_count} contacts searched fresh, {cached_count} loaded from cache.")
|
|
|
|
if not results:
|
|
print("No results collected yet. Exiting without generating report.")
|
|
sys.exit(0)
|
|
|
|
# Sort by mentions (descending)
|
|
results.sort(key=lambda x: x['total_mentions'], reverse=True)
|
|
|
|
# Write HTML report
|
|
print(f"\nWriting report to: {args.output}")
|
|
generate_html_report(results, args.output)
|
|
|
|
# Print summary
|
|
contacts_with_mentions = [r for r in results if r['total_mentions'] > 0]
|
|
print(f"\n{'='*60}")
|
|
print("SUMMARY")
|
|
print(f"{'='*60}")
|
|
print(f"Total connections searched: {len(results)}")
|
|
print(f"Connections with mentions: {len(contacts_with_mentions)}")
|
|
|
|
if contacts_with_mentions:
|
|
print(f"\nTop mentions:")
|
|
for r in contacts_with_mentions[:20]:
|
|
print(f" {r['total_mentions']:6,} - {r['name']}")
|
|
else:
|
|
print("\nNo connections found in the Epstein files.")
|
|
|
|
print(f"\nFull report saved to: {args.output}")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|