Feature/dual classifier yossio

.gitignore

@@ -21,6 +21,9 @@ __pycache__/
 *.test
 *.out
 extproc-server
+main
+semantic_router/main
+cmd/main
 
 # IDE
 .idea/
@@ -61,19 +64,77 @@ bin/
 */models/*.h5
 */models/*.json
 */models/*.txt
+
 # Allow README files in model directories
 !*/trained_model/README.md
 !*/models/README.md
 
-# Added by Claude Task Master
-# Logs
-logs
+# Dual classifier training outputs
+dual_classifier/training_outputs/**/checkpoints/
+dual_classifier/training_outputs/**/*.pt
+dual_classifier/training_outputs/**/*.bin
+dual_classifier/training_outputs/**/*.pth
+dual_classifier/training_outputs/**/vocab.txt
+
+# Large dataset files
+dual_classifier/datasets/real_train_dataset.json
+dual_classifier/datasets/real_val_dataset.json
+dual_classifier/datasets/generators/extended_train_dataset.json
+dual_classifier/datasets/generators/extended_val_dataset.json
+**/real_train_dataset.json
+**/real_val_dataset.json
+
+# Dual classifier training output directories
+dual_classifier/enhanced_training_maximum/final_model/vocab.txt
+dual_classifier/enhanced_training_maximum/training_history.json
+dual_classifier/enhanced_training_maximum/final_model/config.json
+dual_classifier/enhanced_training_maximum/final_model/special_tokens_map.json
+dual_classifier/enhanced_training_maximum/final_model/tokenizer_config.json
+dual_classifier/enhanced_training_maximum/final_model/training_config.json
+dual_classifier/training_output/normal/training_history.json
+dual_classifier/training_output/normal/final_model/config.json
+dual_classifier/training_output/normal/final_model/special_tokens_map.json
+dual_classifier/training_output/normal/final_model/tokenizer_config.json
+dual_classifier/training_output/normal/final_model/training_config.json
+dual_classifier/training_output/normal/final_model/vocab.txt
+dual_classifier/training_output/maximum/training_history.json
+dual_classifier/training_output/maximum/final_model/config.json
+dual_classifier/training_output/maximum/final_model/special_tokens_map.json
+dual_classifier/training_output/maximum/final_model/tokenizer_config.json
+dual_classifier/training_output/maximum/final_model/training_config.json
+
+# Fine-tuning models
+finetune-model/
+
+# Task Master related files
+.taskmaster/
+tasks.json
+tasks/
+.taskmasterconfig
+.env.taskmaster
+package.json
+package-lock.json
+
+# Task Master logs
 *.log
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 dev-debug.log
+logs/
 node_modules/
+
+# Task Master examples and templates
+example_prd.txt
+scripts/prd.txt
+
+# Other development files
+.cursor/
+.roo/
+.env.example
+.roomodes
+.windsurfrules
+
 # Editor directories and files
 .idea
 .vscode
@@ -82,17 +143,3 @@ node_modules/
 *.njsproj
 *.sln
 *.sw?
-# Task files
-tasks.json
-tasks/ 
-.cursor/
-.roo/
-.env.example
-.taskmasterconfig
-example_prd.txt
-.roomodes
-.windsurfrules
-scripts/prd.txt
-.env.taskmaster
-package-lock.json
-package.json

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all build clean test docker-build podman-build docker-run podman-run
+.PHONY: all build clean test docker-build podman-build docker-run podman-run test-pii test-pii-unit test-pii-integration test-existing-functionality
 
 # Default target
 all: build
@@ -66,6 +66,8 @@ else
 		./bin/router -config=config/config.yaml
 endif
 
+# Removed run-router-pii target - PII detection is now enabled by default in config.yaml
+
 # Run Envoy proxy
 run-envoy:
 	@echo "Starting Envoy..."
@@ -87,6 +89,34 @@ else
 		cd candle-binding && CGO_ENABLED=1 go test -v
 endif
 
+# Test PII detection unit tests only
+test-pii-unit: rust
+	@echo "Running PII detection unit tests..."
+ifeq ($(USE_CONTAINER),true)
+	$(RUN_PREFIX) -d $(IMAGE_NAME) sleep infinity
+	$(EXEC_PREFIX) bash -c "cd candle-binding && CGO_ENABLED=1 go test -v -run TestPII"
+	$(EXEC_PREFIX) bash -c "cd semantic_router/pkg/extproc && go test -v -run TestPII"
+	$(CONTAINER_CMD) stop $(CONTAINER_NAME)
+else
+	@export LD_LIBRARY_PATH=${PWD}/candle-binding/target/release && \
+		cd candle-binding && CGO_ENABLED=1 go test -v -run TestPII
+	@cd semantic_router/pkg/extproc && go test -v -run TestPII
+endif
+
+# Test PII detection integration tests (requires running services)
+test-pii-integration:
+	@echo "Running PII integration tests..."
+	@cd tests && python3 03-pii-detection-test.py
+
+# Test that existing functionality still works (regression test)
+test-existing-functionality:
+	@echo "Running regression tests to ensure existing functionality works..."
+	@cd tests && python3 run_all_tests.py --pattern "*test.py" --skip-check || echo "Some tests failed - check if this is due to PII changes"
+
+# Comprehensive PII testing
+test-pii: test-pii-unit test-pii-integration test-existing-functionality
+	@echo "All PII tests completed!"
+
 # Test with the candle-binding library
 test-classifier: rust
 	@echo "Testing domain classifier with candle-binding..."
@@ -131,11 +161,19 @@ test-prompt:
 	curl -X POST http://localhost:8801/v1/chat/completions \
 		-H "Content-Type: application/json" \
 		-d '{"model": "auto", "messages": [{"role": "assistant", "content": "You are a professional math teacher. Explain math concepts clearly and show step-by-step solutions to problems."}, {"role": "user", "content": "What is the derivative of f(x) = x^3 + 2x^2 - 5x + 7?"}], "temperature": 0.7}'
-	@echo "Testing Envoy extproc with curl (Creative Writing)..."
+	@echo "Testing Envoy extproc with curl (History)..."
+	curl -X POST http://localhost:8801/v1/chat/completions \
+		-H "Content-Type: application/json" \
+		-d '{"model": "auto", "messages": [{"role": "assistant", "content": "You are a history teacher. Provide accurate historical information and context."}, {"role": "user", "content": "Tell me about the causes of World War I."}], "temperature": 0.7}'
+	@echo "Testing Envoy extproc with curl (Health)..."
+	curl -X POST http://localhost:8801/v1/chat/completions \
+		-H "Content-Type: application/json" \
+		-d '{"model": "auto", "messages": [{"role": "assistant", "content": "You are a health advisor. Provide helpful health and wellness information."}, {"role": "user", "content": "What are the benefits of regular exercise?"}], "temperature": 0.7}'
+	@echo "Testing Envoy extproc with curl (Programming)..."
 	curl -X POST http://localhost:8801/v1/chat/completions \
 		-H "Content-Type: application/json" \
-		-d '{"model": "auto", "messages": [{"role": "assistant", "content": "You are a story writer. Create interesting stories with good characters and settings."}, {"role": "user", "content": "Write a short story about a space cat."}], "temperature": 0.7}'
-	@echo "Testing Envoy extproc with curl (Default/General)..."
+		-d '{"model": "auto", "messages": [{"role": "assistant", "content": "You are a programming expert. Help with code and software development."}, {"role": "user", "content": "How do I implement a binary search in Python?"}], "temperature": 0.7}'
+	@echo "Testing Envoy extproc with curl (General)..."
 	curl -X POST http://localhost:8801/v1/chat/completions \
 		-H "Content-Type: application/json" \
 		-d '{"model": "auto", "messages": [{"role": "assistant", "content": "You are a helpful assistant."}, {"role": "user", "content": "What is the capital of France?"}], "temperature": 0.7}'
@@ -166,6 +204,29 @@ test-pii:
 		-H "Content-Type: application/json" \
 		-d '{"model": "auto", "messages": [{"role": "assistant", "content": "You are a helpful assistant."}, {"role": "user", "content": "What is the weather today?"}], "temperature": 0.7}'
 
+# Test PII detection specifically with sample prompts
+test-pii-prompt:
+	@echo "Testing PII detection with sample prompts..."
+	@echo "Testing with email..."
+	curl -X POST http://localhost:8801/v1/chat/completions \
+		-H "Content-Type: application/json" \
+		-d '{"model": "auto", "messages": [{"role": "user", "content": "Please contact me at [email protected] for further assistance"}], "temperature": 0.1, "max_tokens": 50}'
+	@echo ""
+	@echo "Testing with phone number..."
+	curl -X POST http://localhost:8801/v1/chat/completions \
+		-H "Content-Type: application/json" \
+		-d '{"model": "auto", "messages": [{"role": "user", "content": "Call me at 555-123-4567 if you need anything"}], "temperature": 0.1, "max_tokens": 50}'
+	@echo ""
+	@echo "Testing with multiple PII types..."
+	curl -X POST http://localhost:8801/v1/chat/completions \
+		-H "Content-Type: application/json" \
+		-d '{"model": "auto", "messages": [{"role": "user", "content": "John Smith can be reached at [email protected] or 555-0123"}], "temperature": 0.1, "max_tokens": 50}'
+	@echo ""
+	@echo "Testing with clean text (no PII)..."
+	curl -X POST http://localhost:8801/v1/chat/completions \
+		-H "Content-Type: application/json" \
+		-d '{"model": "auto", "messages": [{"role": "user", "content": "What is the weather like today?"}], "temperature": 0.1, "max_tokens": 50}'
+
 test-vllm:
 	curl -X POST $(VLLM_ENDPOINT)/v1/chat/completions \
 		-H "Content-Type: application/json" \

README.md

@@ -16,20 +16,40 @@ The router is implemented in two ways: Golang (with Rust FFI based on Candle) an
 
 ## Usage
 
-### Run the Envoy Proxy
+### Build the Project
 
+```bash
+make build
+```
+
+### Create Dataset (if needed)
+
+```bash
+python dual_classifier/create_enhanced_dataset.py
+```
+
+### Train Model (if needed)
+
+```bash
+python dual_classifier/train_enhanced_model.py --create-dataset --training-strength quick --max-length 256
+```
+
+### Start Services (2 terminals)
+
+#### Terminal 1: Run the Envoy Proxy
 This listens for incoming requests and uses the ExtProc filter.
 ```bash
 make run-envoy
 ```
 
-### Run the Semantic Router (Go Implementation)
-
-This builds the Rust binding and the Go router, then starts the ExtProc gRPC server that Envoy communicates with.
+#### Terminal 2: Run the Semantic Router
+This builds the Rust binding and the Go router, then starts the ExtProc gRPC server that Envoy communicates with. Includes PII detection via the trained dual classifier.
 ```bash
 make run-router
 ```
 
+### Test the System
+
 Once both Envoy and the router are running, you can test the routing logic using predefined prompts:
 
 ```bash
@@ -38,6 +58,49 @@ make test-prompt
 
 This will send curl requests simulating different types of user prompts (Math, Creative Writing, General) to the Envoy endpoint (`http://localhost:8801`). The router should direct these to the appropriate backend model configured in `config/config.yaml`.
 
+### Test PII Detection
+
+To test Personally Identifiable Information (PII) detection capabilities:
+
+**Note:** PII detection is enabled by default in `config/config.yaml`.
+
+#### Unit Tests (No Envoy Required)
+
+Test PII detection logic directly without external services:
+```bash
+make test-pii-unit
+```
+
+#### Integration Tests (Requires Envoy + Router)
+
+Make sure both services are running:
+```bash
+make run-envoy  # In one terminal
+make run-router # In another terminal
+```
+
+Then test PII detection with sample prompts containing various types of PII:
+```bash
+make test-pii-prompt
+```
+
+This will test detection of:
+- Email addresses (`[email protected]`)
+- Phone numbers (`555-123-4567`) 
+- Multiple PII types together
+- Clean text (no PII) as a control
+
+#### Comprehensive PII Testing
+
+Run all PII tests (unit tests, integration tests, and regression tests):
+```bash
+make test-pii
+```
+
+**Note:** The integration and comprehensive tests require both Envoy and the router to be running.
+
+The PII detection system uses BERT-based classification to identify and optionally sanitize sensitive information before routing requests to backend models.
+
 ## Testing
 
 A comprehensive test suite is available to validate the functionality of the Semantic Router. The tests follow the data flow through the system, from client request to routing decision.