41 lines
1.4 KiB
Bash
Executable file
41 lines
1.4 KiB
Bash
Executable file
#!/bin/bash
|
|
# Monitor Swiss ISIL scraper progress
|
|
|
|
SWISS_DIR="/Users/kempersc/apps/glam/data/isil/switzerland"
|
|
LOG_FILE="$SWISS_DIR/scraper_background.log"
|
|
|
|
echo "==================== Swiss ISIL Scraper Progress ===================="
|
|
echo ""
|
|
|
|
# Check if process is running
|
|
if ps -p 96896 > /dev/null 2>&1; then
|
|
echo "✓ Scraper process (PID 96896) is RUNNING"
|
|
else
|
|
echo "✗ Scraper process (PID 96896) is NOT running"
|
|
fi
|
|
echo ""
|
|
|
|
# Count batch files
|
|
batch_count=$(ls -1 "$SWISS_DIR"/swiss_isil_complete_batch_*.json 2>/dev/null | wc -l | tr -d ' ')
|
|
echo "Batch files created: $batch_count"
|
|
|
|
# Show latest batch
|
|
if [ $batch_count -gt 0 ]; then
|
|
latest_batch=$(ls -1t "$SWISS_DIR"/swiss_isil_complete_batch_*.json | head -1 | xargs basename)
|
|
latest_num=$(echo "$latest_batch" | grep -oE '[0-9]+' | tail -1)
|
|
echo "Latest batch: $latest_batch (institution $latest_num of 2379)"
|
|
echo "Progress: $(echo "scale=1; $latest_num * 100 / 2379" | bc)%"
|
|
echo "Remaining: $((2379 - latest_num)) institutions"
|
|
fi
|
|
echo ""
|
|
|
|
# Show last 10 log lines
|
|
echo "Last 10 log entries:"
|
|
echo "--------------------"
|
|
tail -10 "$LOG_FILE" 2>/dev/null || echo "Log file not yet created"
|
|
echo ""
|
|
|
|
echo "======================================================================"
|
|
echo "To stop the scraper: kill 96896"
|
|
echo "To view live log: tail -f $LOG_FILE"
|
|
echo "======================================================================"
|