chore(ci): remove GitHub dspy-eval workflow (replaced by Forgejo workflow)
This commit is contained in:
kempersc
parent
8470bf5860
commit
7e9df1d600
1 changed files with 0 additions and 355 deletions
355
.github/workflows/dspy-eval.yml
vendored
355
.github/workflows/dspy-eval.yml
vendored
|
|
@ -1,355 +0,0 @@
|
||||||
# DSPy RAG Evaluation Workflow
|
|
||||||
# Automated testing and evaluation for Heritage RAG system
|
|
||||||
#
|
|
||||||
# Layers:
|
|
||||||
# - Layer 1: Fast unit tests (no LLM)
|
|
||||||
# - Layer 2: DSPy module tests with LLM
|
|
||||||
# - Layer 3: Integration tests (requires SSH tunnel to Oxigraph)
|
|
||||||
# - Layer 4: Comprehensive evaluation (nightly)
|
|
||||||
|
|
||||||
name: DSPy RAG Evaluation
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
paths:
|
|
||||||
- 'backend/rag/**'
|
|
||||||
- 'tests/dspy_gitops/**'
|
|
||||||
- 'src/glam_extractor/api/**'
|
|
||||||
pull_request:
|
|
||||||
branches: [main]
|
|
||||||
paths:
|
|
||||||
- 'backend/rag/**'
|
|
||||||
- 'tests/dspy_gitops/**'
|
|
||||||
- 'src/glam_extractor/api/**'
|
|
||||||
workflow_dispatch:
|
|
||||||
inputs:
|
|
||||||
evaluation_level:
|
|
||||||
description: 'Evaluation depth'
|
|
||||||
required: true
|
|
||||||
default: 'standard'
|
|
||||||
type: choice
|
|
||||||
options:
|
|
||||||
- smoke
|
|
||||||
- standard
|
|
||||||
- comprehensive
|
|
||||||
schedule:
|
|
||||||
# Nightly comprehensive evaluation at 2 AM UTC
|
|
||||||
- cron: '0 2 * * *'
|
|
||||||
|
|
||||||
env:
|
|
||||||
PYTHON_VERSION: '3.11'
|
|
||||||
SERVER_IP: '91.98.224.44'
|
|
||||||
SERVER_USER: 'root'
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
# ==========================================================================
|
|
||||||
# Layer 1: Fast Unit Tests (no LLM calls)
|
|
||||||
# ==========================================================================
|
|
||||||
unit-tests:
|
|
||||||
name: Layer 1 - Unit Tests
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 5
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ env.PYTHON_VERSION }}
|
|
||||||
cache: 'pip'
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
run: |
|
|
||||||
pip install -e ".[dev]"
|
|
||||||
pip install rapidfuzz
|
|
||||||
|
|
||||||
- name: Run Layer 1 unit tests
|
|
||||||
run: |
|
|
||||||
pytest tests/dspy_gitops/test_layer1_unit.py \
|
|
||||||
-v --tb=short \
|
|
||||||
-m "layer1 or not (layer2 or layer3 or layer4)" \
|
|
||||||
--junit-xml=layer1-results.xml
|
|
||||||
|
|
||||||
- name: Upload test results
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
if: always()
|
|
||||||
with:
|
|
||||||
name: layer1-test-results
|
|
||||||
path: layer1-results.xml
|
|
||||||
|
|
||||||
# ==========================================================================
|
|
||||||
# Layer 2: DSPy Module Tests (with LLM)
|
|
||||||
# ==========================================================================
|
|
||||||
dspy-module-tests:
|
|
||||||
name: Layer 2 - DSPy Module Tests
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 20
|
|
||||||
needs: unit-tests
|
|
||||||
|
|
||||||
# Run on PRs, scheduled runs, or manual triggers
|
|
||||||
if: github.event_name == 'pull_request' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ env.PYTHON_VERSION }}
|
|
||||||
cache: 'pip'
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
run: |
|
|
||||||
pip install -e ".[dev]"
|
|
||||||
pip install dspy-ai httpx rapidfuzz litellm
|
|
||||||
|
|
||||||
- name: Run Layer 2 DSPy tests
|
|
||||||
env:
|
|
||||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
||||||
run: |
|
|
||||||
pytest tests/dspy_gitops/test_layer2_dspy.py \
|
|
||||||
-v --tb=short \
|
|
||||||
-m "layer2 or not (layer1 or layer3 or layer4)" \
|
|
||||||
--junit-xml=layer2-results.xml
|
|
||||||
|
|
||||||
- name: Upload test results
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
if: always()
|
|
||||||
with:
|
|
||||||
name: layer2-test-results
|
|
||||||
path: layer2-results.xml
|
|
||||||
|
|
||||||
- name: Comment PR with Layer 2 results
|
|
||||||
if: github.event_name == 'pull_request'
|
|
||||||
uses: actions/github-script@v7
|
|
||||||
with:
|
|
||||||
script: |
|
|
||||||
const fs = require('fs');
|
|
||||||
try {
|
|
||||||
const results = fs.readFileSync('layer2-results.xml', 'utf8');
|
|
||||||
const testsMatch = results.match(/tests="(\d+)"/);
|
|
||||||
const failuresMatch = results.match(/failures="(\d+)"/);
|
|
||||||
const errorsMatch = results.match(/errors="(\d+)"/);
|
|
||||||
|
|
||||||
const tests = testsMatch ? testsMatch[1] : '0';
|
|
||||||
const failures = failuresMatch ? failuresMatch[1] : '0';
|
|
||||||
const errors = errorsMatch ? errorsMatch[1] : '0';
|
|
||||||
const passed = parseInt(tests) - parseInt(failures) - parseInt(errors);
|
|
||||||
|
|
||||||
const body = '## DSPy Layer 2 Evaluation Results\n\n' +
|
|
||||||
'| Metric | Value |\n' +
|
|
||||||
'|--------|-------|\n' +
|
|
||||||
'| Tests Passed | ' + passed + '/' + tests + ' |\n' +
|
|
||||||
'| Failures | ' + failures + ' |\n' +
|
|
||||||
'| Errors | ' + errors + ' |\n' +
|
|
||||||
'| Status | ' + ((parseInt(failures) + parseInt(errors)) > 0 ? '❌ FAILED' : '✅ PASSED') + ' |\n';
|
|
||||||
|
|
||||||
github.rest.issues.createComment({
|
|
||||||
issue_number: context.issue.number,
|
|
||||||
owner: context.repo.owner,
|
|
||||||
repo: context.repo.repo,
|
|
||||||
body: body
|
|
||||||
});
|
|
||||||
} catch (e) {
|
|
||||||
console.log('Could not parse results:', e);
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==========================================================================
|
|
||||||
# Layer 3: Integration Tests (requires SSH tunnel to Oxigraph)
|
|
||||||
# ==========================================================================
|
|
||||||
integration-tests:
|
|
||||||
name: Layer 3 - Integration Tests
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 15
|
|
||||||
needs: unit-tests
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ env.PYTHON_VERSION }}
|
|
||||||
cache: 'pip'
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
run: |
|
|
||||||
pip install -e ".[dev]"
|
|
||||||
pip install httpx pytest-asyncio
|
|
||||||
|
|
||||||
- name: Setup SSH for tunnel
|
|
||||||
run: |
|
|
||||||
mkdir -p ~/.ssh
|
|
||||||
echo "${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}" > ~/.ssh/deploy_key
|
|
||||||
chmod 600 ~/.ssh/deploy_key
|
|
||||||
ssh-keyscan -H ${{ env.SERVER_IP }} >> ~/.ssh/known_hosts 2>/dev/null || true
|
|
||||||
|
|
||||||
- name: Create SSH tunnel to Oxigraph
|
|
||||||
run: |
|
|
||||||
# Create SSH tunnel: local port 7878 -> server localhost:7878
|
|
||||||
ssh -f -N -L 7878:127.0.0.1:7878 \
|
|
||||||
-i ~/.ssh/deploy_key \
|
|
||||||
-o StrictHostKeyChecking=no \
|
|
||||||
${{ env.SERVER_USER }}@${{ env.SERVER_IP }}
|
|
||||||
|
|
||||||
# Wait for tunnel to establish
|
|
||||||
sleep 3
|
|
||||||
|
|
||||||
# Verify tunnel is working
|
|
||||||
curl -sf "http://127.0.0.1:7878/query" \
|
|
||||||
-H "Accept: application/sparql-results+json" \
|
|
||||||
--data-urlencode "query=SELECT (1 AS ?test) WHERE {}" \
|
|
||||||
|| (echo "SSH tunnel failed" && exit 1)
|
|
||||||
|
|
||||||
echo "SSH tunnel established successfully"
|
|
||||||
|
|
||||||
- name: Run Layer 3 integration tests
|
|
||||||
env:
|
|
||||||
OXIGRAPH_ENDPOINT: "http://127.0.0.1:7878"
|
|
||||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
||||||
run: |
|
|
||||||
pytest tests/dspy_gitops/test_layer3_integration.py \
|
|
||||||
-v --tb=short \
|
|
||||||
-m "layer3 or not (layer1 or layer2 or layer4)" \
|
|
||||||
--junit-xml=layer3-results.xml
|
|
||||||
|
|
||||||
- name: Upload test results
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
if: always()
|
|
||||||
with:
|
|
||||||
name: layer3-test-results
|
|
||||||
path: layer3-results.xml
|
|
||||||
|
|
||||||
# ==========================================================================
|
|
||||||
# Layer 4: Comprehensive Evaluation (nightly only)
|
|
||||||
# ==========================================================================
|
|
||||||
comprehensive-eval:
|
|
||||||
name: Layer 4 - Comprehensive Evaluation
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 60
|
|
||||||
needs: [unit-tests, dspy-module-tests, integration-tests]
|
|
||||||
|
|
||||||
# Only run on schedule or manual trigger with 'comprehensive'
|
|
||||||
if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.evaluation_level == 'comprehensive')
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ env.PYTHON_VERSION }}
|
|
||||||
cache: 'pip'
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
run: |
|
|
||||||
pip install -e ".[dev]"
|
|
||||||
pip install dspy-ai httpx rapidfuzz pandas pytest-json-report litellm
|
|
||||||
|
|
||||||
- name: Setup SSH for tunnel
|
|
||||||
run: |
|
|
||||||
mkdir -p ~/.ssh
|
|
||||||
echo "${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}" > ~/.ssh/deploy_key
|
|
||||||
chmod 600 ~/.ssh/deploy_key
|
|
||||||
ssh-keyscan -H ${{ env.SERVER_IP }} >> ~/.ssh/known_hosts 2>/dev/null || true
|
|
||||||
|
|
||||||
- name: Create SSH tunnel to Oxigraph
|
|
||||||
run: |
|
|
||||||
ssh -f -N -L 7878:127.0.0.1:7878 \
|
|
||||||
-i ~/.ssh/deploy_key \
|
|
||||||
-o StrictHostKeyChecking=no \
|
|
||||||
${{ env.SERVER_USER }}@${{ env.SERVER_IP }}
|
|
||||||
sleep 3
|
|
||||||
|
|
||||||
- name: Run comprehensive evaluation
|
|
||||||
env:
|
|
||||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
||||||
OXIGRAPH_ENDPOINT: "http://127.0.0.1:7878"
|
|
||||||
run: |
|
|
||||||
pytest tests/dspy_gitops/test_layer4_comprehensive.py \
|
|
||||||
-v --tb=short \
|
|
||||||
-m "layer4 or not (layer1 or layer2 or layer3)" \
|
|
||||||
--junit-xml=layer4-results.xml \
|
|
||||||
--json-report \
|
|
||||||
--json-report-file=eval-report.json
|
|
||||||
|
|
||||||
- name: Generate metrics summary
|
|
||||||
run: |
|
|
||||||
python -c "
|
|
||||||
import json
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
try:
|
|
||||||
with open('eval-report.json') as f:
|
|
||||||
report = json.load(f)
|
|
||||||
|
|
||||||
metrics = {
|
|
||||||
'timestamp': datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
|
|
||||||
'commit': '${{ github.sha }}',
|
|
||||||
'total_tests': report.get('summary', {}).get('total', 0),
|
|
||||||
'passed': report.get('summary', {}).get('passed', 0),
|
|
||||||
'failed': report.get('summary', {}).get('failed', 0),
|
|
||||||
'duration': report.get('duration', 0),
|
|
||||||
}
|
|
||||||
|
|
||||||
with open('metrics.json', 'w') as f:
|
|
||||||
json.dump(metrics, f, indent=2)
|
|
||||||
|
|
||||||
print('Metrics saved to metrics.json')
|
|
||||||
print(json.dumps(metrics, indent=2))
|
|
||||||
except Exception as e:
|
|
||||||
print(f'Error generating metrics: {e}')
|
|
||||||
"
|
|
||||||
|
|
||||||
- name: Upload evaluation artifacts
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: comprehensive-eval-results
|
|
||||||
path: |
|
|
||||||
layer4-results.xml
|
|
||||||
eval-report.json
|
|
||||||
metrics.json
|
|
||||||
|
|
||||||
# ==========================================================================
|
|
||||||
# Quality Gate Check
|
|
||||||
# ==========================================================================
|
|
||||||
quality-gate:
|
|
||||||
name: Quality Gate
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: [unit-tests, dspy-module-tests, integration-tests]
|
|
||||||
if: always()
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Check all required tests passed
|
|
||||||
run: |
|
|
||||||
echo "Checking quality gates..."
|
|
||||||
|
|
||||||
# Layer 1 (unit tests) is always required
|
|
||||||
if [[ "${{ needs.unit-tests.result }}" != "success" ]]; then
|
|
||||||
echo "❌ Layer 1 (Unit Tests) failed"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "✅ Layer 1 (Unit Tests) passed"
|
|
||||||
|
|
||||||
# Layer 2 (DSPy module tests) required for PRs
|
|
||||||
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
|
|
||||||
if [[ "${{ needs.dspy-module-tests.result }}" != "success" ]]; then
|
|
||||||
echo "❌ Layer 2 (DSPy Module Tests) failed - required for PRs"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "✅ Layer 2 (DSPy Module Tests) passed"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Layer 3 (integration tests) is warning-only for now
|
|
||||||
if [[ "${{ needs.integration-tests.result }}" != "success" ]]; then
|
|
||||||
echo "⚠️ Layer 3 (Integration Tests) failed - non-blocking"
|
|
||||||
else
|
|
||||||
echo "✅ Layer 3 (Integration Tests) passed"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "============================================"
|
|
||||||
echo " All required quality gates passed!"
|
|
||||||
echo "============================================"
|
|
||||||
Loading…
Reference in a new issue