#!/bin/bash
# Monitor Swiss ISIL scraper progress

SWISS_DIR="/Users/kempersc/apps/glam/data/isil/switzerland"
LOG_FILE="$SWISS_DIR/scraper_background.log"

echo "==================== Swiss ISIL Scraper Progress ===================="
echo ""

# Check if process is running
if ps -p 96896 > /dev/null 2>&1; then
    echo "✓ Scraper process (PID 96896) is RUNNING"
else
    echo "✗ Scraper process (PID 96896) is NOT running"
fi
echo ""

# Count batch files
batch_count=$(ls -1 "$SWISS_DIR"/swiss_isil_complete_batch_*.json 2>/dev/null | wc -l | tr -d ' ')
echo "Batch files created: $batch_count"

# Show latest batch
if [ $batch_count -gt 0 ]; then
    latest_batch=$(ls -1t "$SWISS_DIR"/swiss_isil_complete_batch_*.json | head -1 | xargs basename)
    latest_num=$(echo "$latest_batch" | grep -oE '[0-9]+' | tail -1)
    echo "Latest batch: $latest_batch (institution $latest_num of 2379)"
    echo "Progress: $(echo "scale=1; $latest_num * 100 / 2379" | bc)%"
    echo "Remaining: $((2379 - latest_num)) institutions"
fi
echo ""

# Show last 10 log lines
echo "Last 10 log entries:"
echo "--------------------"
tail -10 "$LOG_FILE" 2>/dev/null || echo "Log file not yet created"
echo ""

echo "======================================================================"
echo "To stop the scraper: kill 96896"
echo "To view live log: tail -f $LOG_FILE"
echo "======================================================================"