finrag-eval/Makefile at main · Ruthwik-Data/finrag-eval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
.PHONY: help install dev test setup db-up db-down db-init download ingest ask eval eval-fast eval-dedup eval-routing ablation ui ui-build clean

# Defaults match this project's own eval dataset (Apple's two most recent
# 10-Ks). Override on the command line, e.g. `make download TICKER=MSFT COUNT=1`.
TICKER ?= AAPL
COUNT ?= 2

help:
	@echo "finrag-eval — common tasks"
	@echo ""
	@echo "  make install                        Install runtime dependencies only"
	@echo "  make dev                            Install dev/test dependencies (includes runtime deps)"
	@echo "  make test                            Run the offline test suite (no Docker/Ollama/DB/network)"
	@echo ""
	@echo "  make setup                           Start the local DB stack + print required Ollama pulls"
	@echo "  make download TICKER=AAPL COUNT=2    Download recent 10-K filing(s) from SEC EDGAR"
	@echo "  make ingest FILE=... TICKER=AAPL     Parse, chunk, embed, and store a downloaded filing"
	@echo "  make ask Q=\"...\"                     Ask the live RAG pipeline + see the grounding verdict"
	@echo "  make eval                            Run the evaluation suite (writes eval_runs/*.jsonl + .summary.json)"
	@echo "  make eval-fast                        Same, skipping DeepEval's judge-model metrics"
	@echo "  make eval-dedup                       Same, with chunk deduplication enabled"
	@echo "  make eval-routing                     Same, with metadata-aware section routing enabled"
	@echo "  make ablation                         Run raw / dedup / routing variants for comparison"
	@echo ""
	@echo "  make ui                              Start the frontend dashboard (fixture data works without Ollama/Docker)"
	@echo "  make ui-build                         Production build + type-check the frontend"
	@echo ""
	@echo "  make clean                           Remove __pycache__ / pytest cache"

install:
	pip install -r requirements.txt

dev:
	pip install -r requirements-dev.txt

test:
	pytest -q

# Starts the local Postgres/pgvector container and initializes it, then
# prints the exact Ollama models this repo's config expects — read from
# configs/pipeline.yaml, never hardcoded here.
setup: db-up db-init
	@echo ""
	@echo "Local DB is up. Pull the models this config actually uses:"
	@python3 -c "import yaml; c = yaml.safe_load(open('configs/pipeline.yaml')); print('  ollama pull ' + c['generation']['model']); print('  ollama pull ' + c['embedding']['model'])"
	@echo ""
	@echo "Then:"
	@echo "  make download TICKER=$(TICKER) COUNT=$(COUNT)"
	@echo "  make ingest FILE=data/raw/<downloaded-file> TICKER=$(TICKER)"
	@echo "  make ask Q=\"What was Apple's FY2025 total net revenue?\""

db-up:
	docker-compose up -d

db-down:
	docker-compose down

db-init:
	python scripts/init_db.py

download:
	python -m src.ingestion.edgar_download --ticker $(TICKER) --count $(COUNT)

# Usage: make ingest FILE=data/raw/AAPL_10K_2024-11-01.htm TICKER=AAPL
ingest:
	@if [ -z "$(FILE)" ]; then \
		echo "Usage: make ingest FILE=data/raw/<downloaded-filing> TICKER=AAPL"; \
		echo "  e.g.: make ingest FILE=data/raw/AAPL_10K_2024-11-01.htm TICKER=AAPL"; \
		echo "  (filename pattern comes from 'make download' — see docs/USE_CASES.md)"; \
		exit 1; \
	fi
	python -m src.ingestion.ingest --input $(FILE) --ticker $(TICKER)

# Usage: make ask Q="What was Apple's FY2025 total net revenue?"
# Requires the live stack (make setup + make download + make ingest first).
ask:
	python -m src.retrieval.ask "$(Q)"

eval:
	python -m src.eval.run_eval

# Same, without DeepEval's 4 judge-model metrics (each costs its own extra
# LLM call on top of generation) — useful when the local judge model can't
# practically absorb that load. numeric_grounding/verdict never depend on
# DeepEval either way.
eval-fast:
	python -m src.eval.run_eval --skip-deepeval

eval-dedup:
	python -m src.eval.run_eval --deduplicate

eval-routing:
	python -m src.eval.run_eval --metadata-routing

# Ablation: the project's core thesis is "overlap helps retrieval, hurts naive
# eval." These three runs isolate the effect of dedup and metadata routing.
ablation: eval eval-dedup eval-routing
	@echo "Ablation complete — compare the three stored runs in eval_runs/ (and the eval_runs DB table)."

# Static dashboard over eval_runs/ JSON + bundled fixtures — works without
# Ollama/Docker. See frontend/README.md and docs/USE_CASES.md.
ui:
	cd frontend && npm install && npm run dev

ui-build:
	cd frontend && npm install && npm run build

clean:
	find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
	rm -rf .pytest_cache