Spaces:
Paused
Paused
Architectural simplifications - merge ValueHarmonizer into ResultHarmonizer
Browse filesHigh priority simplifications:
1. Merged ValueHarmonizer into ResultHarmonizer as private methods
- ValueHarmonizer had only one consumer (ResultHarmonizer)
- All methods now prefixed with _ to indicate internal use
- Reduces code complexity and import dependencies
- Deleted schema_translator/value_harmonizer.py (288 lines)
2. Removed unused query_history_path from Config
- QueryHistory stores data in memory only
- query_history.json file never created or used
- Removed from config.py and .env.example
Impact:
- ~300 lines of code eliminated
- 1 fewer file to maintain
- Simpler architecture (1 fewer class)
- More cohesive design
- All 156 tests passing
- data/feedback.jsonl +2 -0
- schema_translator/config.py +1 -8
- schema_translator/result_harmonizer.py +275 -4
- schema_translator/value_harmonizer.py +0 -288
- tests/test_result_harmonization.py +30 -37
data/feedback.jsonl
CHANGED
|
@@ -4,3 +4,5 @@
|
|
| 4 |
{"query_text": "test query", "semantic_plan": {"intent": "find_contracts", "filters": [], "projections": ["contract_identifier"], "aggregations": [], "group_by": null, "order_by": null, "limit": 10}, "feedback_type": "good", "feedback_text": "Works great!", "correct_result": null, "timestamp": "2025-11-08T01:18:35.490463Z"}
|
| 5 |
{"query_text": "test query", "semantic_plan": {"intent": "find_contracts", "filters": [], "projections": ["contract_identifier"], "aggregations": [], "group_by": null, "order_by": null, "limit": 10}, "feedback_type": "good", "feedback_text": "Works great!", "correct_result": null, "timestamp": "2025-11-08T01:19:25.274999Z"}
|
| 6 |
{"query_text": "test query", "semantic_plan": {"intent": "find_contracts", "filters": [], "projections": ["contract_identifier"], "aggregations": [], "group_by": null, "order_by": null, "limit": 10}, "feedback_type": "good", "feedback_text": "Works great!", "correct_result": null, "timestamp": "2025-11-08T01:29:04.264865Z"}
|
|
|
|
|
|
|
|
|
| 4 |
{"query_text": "test query", "semantic_plan": {"intent": "find_contracts", "filters": [], "projections": ["contract_identifier"], "aggregations": [], "group_by": null, "order_by": null, "limit": 10}, "feedback_type": "good", "feedback_text": "Works great!", "correct_result": null, "timestamp": "2025-11-08T01:18:35.490463Z"}
|
| 5 |
{"query_text": "test query", "semantic_plan": {"intent": "find_contracts", "filters": [], "projections": ["contract_identifier"], "aggregations": [], "group_by": null, "order_by": null, "limit": 10}, "feedback_type": "good", "feedback_text": "Works great!", "correct_result": null, "timestamp": "2025-11-08T01:19:25.274999Z"}
|
| 6 |
{"query_text": "test query", "semantic_plan": {"intent": "find_contracts", "filters": [], "projections": ["contract_identifier"], "aggregations": [], "group_by": null, "order_by": null, "limit": 10}, "feedback_type": "good", "feedback_text": "Works great!", "correct_result": null, "timestamp": "2025-11-08T01:29:04.264865Z"}
|
| 7 |
+
{"query_text": "test query", "semantic_plan": {"intent": "find_contracts", "filters": [], "projections": ["contract_identifier"], "aggregations": [], "group_by": null, "order_by": null, "limit": 10}, "feedback_type": "good", "feedback_text": "Works great!", "correct_result": null, "timestamp": "2025-11-08T01:38:31.687884Z"}
|
| 8 |
+
{"query_text": "test query", "semantic_plan": {"intent": "find_contracts", "filters": [], "projections": ["contract_identifier"], "aggregations": [], "group_by": null, "order_by": null, "limit": 10}, "feedback_type": "good", "feedback_text": "Works great!", "correct_result": null, "timestamp": "2025-11-08T01:39:40.175343Z"}
|
schema_translator/config.py
CHANGED
|
@@ -54,19 +54,13 @@ class Config(BaseSettings):
|
|
| 54 |
description="Path to knowledge graph JSON file"
|
| 55 |
)
|
| 56 |
|
| 57 |
-
# Query History Configuration
|
| 58 |
-
query_history_path: Path = Field(
|
| 59 |
-
default=Path("./query_history.json"),
|
| 60 |
-
description="Path to query history JSON file"
|
| 61 |
-
)
|
| 62 |
-
|
| 63 |
# Logging Configuration
|
| 64 |
log_level: str = Field(
|
| 65 |
default="INFO",
|
| 66 |
description="Logging level"
|
| 67 |
)
|
| 68 |
|
| 69 |
-
@field_validator("database_dir", "knowledge_graph_path",
|
| 70 |
@classmethod
|
| 71 |
def convert_to_path(cls, v) -> Path:
|
| 72 |
"""Convert string paths to Path objects."""
|
|
@@ -97,7 +91,6 @@ class Config(BaseSettings):
|
|
| 97 |
|
| 98 |
# Ensure parent directories exist for other paths
|
| 99 |
self.knowledge_graph_path.parent.mkdir(parents=True, exist_ok=True)
|
| 100 |
-
self.query_history_path.parent.mkdir(parents=True, exist_ok=True)
|
| 101 |
|
| 102 |
def get_database_path(self, customer_id: str) -> Path:
|
| 103 |
"""Get path to a specific customer database.
|
|
|
|
| 54 |
description="Path to knowledge graph JSON file"
|
| 55 |
)
|
| 56 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
# Logging Configuration
|
| 58 |
log_level: str = Field(
|
| 59 |
default="INFO",
|
| 60 |
description="Logging level"
|
| 61 |
)
|
| 62 |
|
| 63 |
+
@field_validator("database_dir", "knowledge_graph_path", mode="before")
|
| 64 |
@classmethod
|
| 65 |
def convert_to_path(cls, v) -> Path:
|
| 66 |
"""Convert string paths to Path objects."""
|
|
|
|
| 91 |
|
| 92 |
# Ensure parent directories exist for other paths
|
| 93 |
self.knowledge_graph_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
| 94 |
|
| 95 |
def get_database_path(self, customer_id: str) -> Path:
|
| 96 |
"""Get path to a specific customer database.
|
schema_translator/result_harmonizer.py
CHANGED
|
@@ -2,18 +2,20 @@
|
|
| 2 |
|
| 3 |
import time
|
| 4 |
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 5 |
-
from
|
|
|
|
| 6 |
|
| 7 |
from schema_translator.database_executor import DatabaseExecutor
|
| 8 |
from schema_translator.knowledge_graph import SchemaKnowledgeGraph
|
| 9 |
from schema_translator.models import (
|
| 10 |
HarmonizedResult,
|
| 11 |
HarmonizedRow,
|
|
|
|
| 12 |
QueryResult,
|
| 13 |
SemanticQueryPlan,
|
|
|
|
| 14 |
)
|
| 15 |
from schema_translator.query_compiler import QueryCompiler
|
| 16 |
-
from schema_translator.value_harmonizer import ValueHarmonizer
|
| 17 |
|
| 18 |
|
| 19 |
class ResultHarmonizer:
|
|
@@ -33,7 +35,6 @@ class ResultHarmonizer:
|
|
| 33 |
self.knowledge_graph = knowledge_graph
|
| 34 |
self.executor = executor or DatabaseExecutor()
|
| 35 |
self.compiler = QueryCompiler(knowledge_graph)
|
| 36 |
-
self.value_harmonizer = ValueHarmonizer(knowledge_graph)
|
| 37 |
|
| 38 |
def execute_across_customers(
|
| 39 |
self,
|
|
@@ -205,7 +206,7 @@ class ResultHarmonizer:
|
|
| 205 |
|
| 206 |
# Harmonize each row
|
| 207 |
for row in result.data:
|
| 208 |
-
harmonized_data = self.
|
| 209 |
row, customer_id, field_mappings
|
| 210 |
)
|
| 211 |
|
|
@@ -427,3 +428,273 @@ class ResultHarmonizer:
|
|
| 427 |
errors=harmonized_result.errors,
|
| 428 |
execution_time_ms=harmonized_result.execution_time_ms
|
| 429 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
import time
|
| 4 |
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 5 |
+
from datetime import datetime, timedelta
|
| 6 |
+
from typing import Any, Dict, List, Optional
|
| 7 |
|
| 8 |
from schema_translator.database_executor import DatabaseExecutor
|
| 9 |
from schema_translator.knowledge_graph import SchemaKnowledgeGraph
|
| 10 |
from schema_translator.models import (
|
| 11 |
HarmonizedResult,
|
| 12 |
HarmonizedRow,
|
| 13 |
+
NormalizedValue,
|
| 14 |
QueryResult,
|
| 15 |
SemanticQueryPlan,
|
| 16 |
+
SemanticType,
|
| 17 |
)
|
| 18 |
from schema_translator.query_compiler import QueryCompiler
|
|
|
|
| 19 |
|
| 20 |
|
| 21 |
class ResultHarmonizer:
|
|
|
|
| 35 |
self.knowledge_graph = knowledge_graph
|
| 36 |
self.executor = executor or DatabaseExecutor()
|
| 37 |
self.compiler = QueryCompiler(knowledge_graph)
|
|
|
|
| 38 |
|
| 39 |
def execute_across_customers(
|
| 40 |
self,
|
|
|
|
| 206 |
|
| 207 |
# Harmonize each row
|
| 208 |
for row in result.data:
|
| 209 |
+
harmonized_data = self._harmonize_row(
|
| 210 |
row, customer_id, field_mappings
|
| 211 |
)
|
| 212 |
|
|
|
|
| 428 |
errors=harmonized_result.errors,
|
| 429 |
execution_time_ms=harmonized_result.execution_time_ms
|
| 430 |
)
|
| 431 |
+
|
| 432 |
+
# Value normalization methods (formerly ValueHarmonizer)
|
| 433 |
+
|
| 434 |
+
def _normalize_value(
|
| 435 |
+
self,
|
| 436 |
+
value: Any,
|
| 437 |
+
customer_id: str,
|
| 438 |
+
concept_id: str,
|
| 439 |
+
target_type: Optional[SemanticType] = None
|
| 440 |
+
) -> NormalizedValue:
|
| 441 |
+
"""Normalize a value from a customer schema to a common format.
|
| 442 |
+
|
| 443 |
+
Args:
|
| 444 |
+
value: The value to normalize
|
| 445 |
+
customer_id: Customer ID for context
|
| 446 |
+
concept_id: The semantic concept this value represents
|
| 447 |
+
target_type: Optional target semantic type to convert to
|
| 448 |
+
|
| 449 |
+
Returns:
|
| 450 |
+
NormalizedValue with original and normalized forms
|
| 451 |
+
"""
|
| 452 |
+
# Get concept mapping for this customer
|
| 453 |
+
mapping = self.knowledge_graph.get_mapping(concept_id, customer_id)
|
| 454 |
+
if not mapping:
|
| 455 |
+
# No mapping found, return as-is
|
| 456 |
+
return NormalizedValue(
|
| 457 |
+
original_value=value,
|
| 458 |
+
normalized_value=value,
|
| 459 |
+
original_type="unknown",
|
| 460 |
+
normalized_type="unknown",
|
| 461 |
+
transformation_applied=None
|
| 462 |
+
)
|
| 463 |
+
|
| 464 |
+
# Handle semantic_type as either SemanticType enum or string
|
| 465 |
+
if isinstance(mapping.semantic_type, SemanticType):
|
| 466 |
+
original_type = mapping.semantic_type.value
|
| 467 |
+
semantic_type_enum = mapping.semantic_type
|
| 468 |
+
else:
|
| 469 |
+
original_type = str(mapping.semantic_type)
|
| 470 |
+
semantic_type_enum = SemanticType(mapping.semantic_type)
|
| 471 |
+
|
| 472 |
+
transformation = mapping.transformation
|
| 473 |
+
|
| 474 |
+
# Apply transformation if specified
|
| 475 |
+
if transformation:
|
| 476 |
+
normalized = self._apply_transformation(
|
| 477 |
+
value, transformation, customer_id, concept_id
|
| 478 |
+
)
|
| 479 |
+
transformation_applied = transformation
|
| 480 |
+
else:
|
| 481 |
+
normalized = value
|
| 482 |
+
transformation_applied = None
|
| 483 |
+
|
| 484 |
+
# Convert type if target specified
|
| 485 |
+
if target_type and target_type != semantic_type_enum:
|
| 486 |
+
normalized = self._convert_type(normalized, semantic_type_enum, target_type)
|
| 487 |
+
if transformation_applied:
|
| 488 |
+
transformation_applied += f" + type_conversion_to_{target_type.value}"
|
| 489 |
+
else:
|
| 490 |
+
transformation_applied = f"type_conversion_to_{target_type.value}"
|
| 491 |
+
|
| 492 |
+
return NormalizedValue(
|
| 493 |
+
original_value=value,
|
| 494 |
+
normalized_value=normalized,
|
| 495 |
+
original_type=original_type,
|
| 496 |
+
normalized_type=target_type.value if target_type else original_type,
|
| 497 |
+
transformation_applied=transformation_applied
|
| 498 |
+
)
|
| 499 |
+
|
| 500 |
+
def _apply_transformation(
|
| 501 |
+
self,
|
| 502 |
+
value: Any,
|
| 503 |
+
transformation: str,
|
| 504 |
+
customer_id: str,
|
| 505 |
+
concept_id: str
|
| 506 |
+
) -> Any:
|
| 507 |
+
"""Apply a transformation to a value.
|
| 508 |
+
|
| 509 |
+
Args:
|
| 510 |
+
value: Value to transform
|
| 511 |
+
transformation: Transformation SQL or expression
|
| 512 |
+
customer_id: Customer ID for context
|
| 513 |
+
concept_id: Concept ID for context
|
| 514 |
+
|
| 515 |
+
Returns:
|
| 516 |
+
Transformed value
|
| 517 |
+
"""
|
| 518 |
+
# Handle days_remaining -> end_date conversion
|
| 519 |
+
if "CURRENT_DATE" in transformation or "julianday" in transformation:
|
| 520 |
+
return self._days_to_date(value)
|
| 521 |
+
|
| 522 |
+
# Handle annual_value -> lifetime_value conversion
|
| 523 |
+
if "contract_length" in transformation or "*" in transformation:
|
| 524 |
+
# Need to get contract_length for this specific row
|
| 525 |
+
# For now, apply a default multiplier (this should be done at query time)
|
| 526 |
+
# In real implementation, this would need row-level context
|
| 527 |
+
return value # Return as-is; transformation happens at SQL level
|
| 528 |
+
|
| 529 |
+
# Other transformations
|
| 530 |
+
return value
|
| 531 |
+
|
| 532 |
+
def _days_to_date(self, days_remaining: Any) -> Optional[str]:
|
| 533 |
+
"""Convert days remaining to an end date.
|
| 534 |
+
|
| 535 |
+
Args:
|
| 536 |
+
days_remaining: Number of days remaining
|
| 537 |
+
|
| 538 |
+
Returns:
|
| 539 |
+
ISO format date string or None if invalid
|
| 540 |
+
"""
|
| 541 |
+
if days_remaining is None:
|
| 542 |
+
return None
|
| 543 |
+
|
| 544 |
+
try:
|
| 545 |
+
days = int(days_remaining)
|
| 546 |
+
end_date = datetime.now() + timedelta(days=days)
|
| 547 |
+
return end_date.strftime("%Y-%m-%d")
|
| 548 |
+
except (ValueError, TypeError):
|
| 549 |
+
return None
|
| 550 |
+
|
| 551 |
+
def _convert_type(
|
| 552 |
+
self,
|
| 553 |
+
value: Any,
|
| 554 |
+
from_type: SemanticType,
|
| 555 |
+
to_type: SemanticType
|
| 556 |
+
) -> Any:
|
| 557 |
+
"""Convert a value from one semantic type to another.
|
| 558 |
+
|
| 559 |
+
Args:
|
| 560 |
+
value: Value to convert
|
| 561 |
+
from_type: Current semantic type
|
| 562 |
+
to_type: Target semantic type
|
| 563 |
+
|
| 564 |
+
Returns:
|
| 565 |
+
Converted value
|
| 566 |
+
"""
|
| 567 |
+
if value is None:
|
| 568 |
+
return None
|
| 569 |
+
|
| 570 |
+
# Date conversions
|
| 571 |
+
if to_type == SemanticType.DATE:
|
| 572 |
+
if from_type == SemanticType.INTEGER:
|
| 573 |
+
# Assume integer is days remaining
|
| 574 |
+
return self._days_to_date(value)
|
| 575 |
+
elif from_type == SemanticType.TEXT:
|
| 576 |
+
# Parse text date
|
| 577 |
+
try:
|
| 578 |
+
dt = datetime.fromisoformat(str(value))
|
| 579 |
+
return dt.strftime("%Y-%m-%d")
|
| 580 |
+
except (ValueError, TypeError):
|
| 581 |
+
return str(value)
|
| 582 |
+
|
| 583 |
+
# Numeric conversions
|
| 584 |
+
if to_type == SemanticType.FLOAT:
|
| 585 |
+
if from_type in (SemanticType.INTEGER, SemanticType.TEXT):
|
| 586 |
+
try:
|
| 587 |
+
return float(value)
|
| 588 |
+
except (ValueError, TypeError):
|
| 589 |
+
return value
|
| 590 |
+
|
| 591 |
+
if to_type == SemanticType.INTEGER:
|
| 592 |
+
if from_type in (SemanticType.FLOAT, SemanticType.TEXT):
|
| 593 |
+
try:
|
| 594 |
+
return int(float(value))
|
| 595 |
+
except (ValueError, TypeError):
|
| 596 |
+
return value
|
| 597 |
+
|
| 598 |
+
# Text conversion (always works)
|
| 599 |
+
if to_type == SemanticType.TEXT:
|
| 600 |
+
return str(value)
|
| 601 |
+
|
| 602 |
+
# No conversion available
|
| 603 |
+
return value
|
| 604 |
+
|
| 605 |
+
def _normalize_field_name(
|
| 606 |
+
self,
|
| 607 |
+
customer_field_name: str,
|
| 608 |
+
customer_id: str
|
| 609 |
+
) -> Optional[str]:
|
| 610 |
+
"""Map a customer-specific field name to its semantic concept.
|
| 611 |
+
|
| 612 |
+
Args:
|
| 613 |
+
customer_field_name: Field name in customer schema
|
| 614 |
+
customer_id: Customer ID
|
| 615 |
+
|
| 616 |
+
Returns:
|
| 617 |
+
Semantic concept ID or None if not mapped
|
| 618 |
+
"""
|
| 619 |
+
# Check all concepts for this customer
|
| 620 |
+
for concept_id in self.knowledge_graph.concepts.keys():
|
| 621 |
+
mapping = self.knowledge_graph.get_mapping(concept_id, customer_id)
|
| 622 |
+
if mapping and mapping.column_name == customer_field_name:
|
| 623 |
+
return concept_id
|
| 624 |
+
|
| 625 |
+
return None
|
| 626 |
+
|
| 627 |
+
def _normalize_industry_name(self, industry: Optional[str]) -> Optional[str]:
|
| 628 |
+
"""Normalize industry names to common format.
|
| 629 |
+
|
| 630 |
+
Args:
|
| 631 |
+
industry: Industry name from customer data
|
| 632 |
+
|
| 633 |
+
Returns:
|
| 634 |
+
Normalized industry name
|
| 635 |
+
"""
|
| 636 |
+
if not industry:
|
| 637 |
+
return None
|
| 638 |
+
|
| 639 |
+
# Convert to lowercase for comparison
|
| 640 |
+
industry_lower = industry.lower().strip()
|
| 641 |
+
|
| 642 |
+
# Map common variations
|
| 643 |
+
industry_mapping = {
|
| 644 |
+
"tech": "Technology",
|
| 645 |
+
"technology": "Technology",
|
| 646 |
+
"it": "Technology",
|
| 647 |
+
"information technology": "Technology",
|
| 648 |
+
"healthcare": "Healthcare",
|
| 649 |
+
"health": "Healthcare",
|
| 650 |
+
"medical": "Healthcare",
|
| 651 |
+
"finance": "Financial Services",
|
| 652 |
+
"financial": "Financial Services",
|
| 653 |
+
"financial services": "Financial Services",
|
| 654 |
+
"banking": "Financial Services",
|
| 655 |
+
"retail": "Retail",
|
| 656 |
+
"manufacturing": "Manufacturing",
|
| 657 |
+
"mfg": "Manufacturing",
|
| 658 |
+
"education": "Education",
|
| 659 |
+
"edu": "Education",
|
| 660 |
+
"government": "Government",
|
| 661 |
+
"gov": "Government",
|
| 662 |
+
"public sector": "Government",
|
| 663 |
+
}
|
| 664 |
+
|
| 665 |
+
return industry_mapping.get(industry_lower, industry.title())
|
| 666 |
+
|
| 667 |
+
def _harmonize_row(
|
| 668 |
+
self,
|
| 669 |
+
row: Dict[str, Any],
|
| 670 |
+
customer_id: str,
|
| 671 |
+
field_mappings: Dict[str, str]
|
| 672 |
+
) -> Dict[str, Any]:
|
| 673 |
+
"""Harmonize a single row of data.
|
| 674 |
+
|
| 675 |
+
Args:
|
| 676 |
+
row: Raw row data from customer database
|
| 677 |
+
customer_id: Customer ID
|
| 678 |
+
field_mappings: Map of customer field names to concept IDs
|
| 679 |
+
|
| 680 |
+
Returns:
|
| 681 |
+
Harmonized row with normalized field names and values
|
| 682 |
+
"""
|
| 683 |
+
harmonized = {}
|
| 684 |
+
|
| 685 |
+
for customer_field, concept_id in field_mappings.items():
|
| 686 |
+
if customer_field in row:
|
| 687 |
+
value = row[customer_field]
|
| 688 |
+
|
| 689 |
+
# Special handling for industry
|
| 690 |
+
if concept_id == "industry_sector":
|
| 691 |
+
harmonized[concept_id] = self._normalize_industry_name(value)
|
| 692 |
+
else:
|
| 693 |
+
# Normalize the value
|
| 694 |
+
normalized = self._normalize_value(value, customer_id, concept_id)
|
| 695 |
+
harmonized[concept_id] = normalized.normalized_value
|
| 696 |
+
else:
|
| 697 |
+
# Field not present in row
|
| 698 |
+
harmonized[concept_id] = None
|
| 699 |
+
|
| 700 |
+
return harmonized
|
schema_translator/value_harmonizer.py
DELETED
|
@@ -1,288 +0,0 @@
|
|
| 1 |
-
"""Value harmonization for normalizing data across customer schemas."""
|
| 2 |
-
|
| 3 |
-
from datetime import datetime, timedelta
|
| 4 |
-
from decimal import Decimal
|
| 5 |
-
from typing import Any, Dict, Optional
|
| 6 |
-
|
| 7 |
-
from schema_translator.knowledge_graph import SchemaKnowledgeGraph
|
| 8 |
-
from schema_translator.models import NormalizedValue, SemanticType
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
class ValueHarmonizer:
|
| 12 |
-
"""Harmonizes values across different customer schemas."""
|
| 13 |
-
|
| 14 |
-
def __init__(self, knowledge_graph: SchemaKnowledgeGraph):
|
| 15 |
-
"""Initialize the value harmonizer.
|
| 16 |
-
|
| 17 |
-
Args:
|
| 18 |
-
knowledge_graph: Knowledge graph with concept mappings and transformations
|
| 19 |
-
"""
|
| 20 |
-
self.knowledge_graph = knowledge_graph
|
| 21 |
-
|
| 22 |
-
def normalize_value(
|
| 23 |
-
self,
|
| 24 |
-
value: Any,
|
| 25 |
-
customer_id: str,
|
| 26 |
-
concept_id: str,
|
| 27 |
-
target_type: Optional[SemanticType] = None
|
| 28 |
-
) -> NormalizedValue:
|
| 29 |
-
"""Normalize a value from a customer schema to a common format.
|
| 30 |
-
|
| 31 |
-
Args:
|
| 32 |
-
value: The value to normalize
|
| 33 |
-
customer_id: Customer ID for context
|
| 34 |
-
concept_id: The semantic concept this value represents
|
| 35 |
-
target_type: Optional target semantic type to convert to
|
| 36 |
-
|
| 37 |
-
Returns:
|
| 38 |
-
NormalizedValue with original and normalized forms
|
| 39 |
-
"""
|
| 40 |
-
# Get concept mapping for this customer
|
| 41 |
-
mapping = self.knowledge_graph.get_mapping(concept_id, customer_id)
|
| 42 |
-
if not mapping:
|
| 43 |
-
# No mapping found, return as-is
|
| 44 |
-
return NormalizedValue(
|
| 45 |
-
original_value=value,
|
| 46 |
-
normalized_value=value,
|
| 47 |
-
original_type="unknown",
|
| 48 |
-
normalized_type="unknown",
|
| 49 |
-
transformation_applied=None
|
| 50 |
-
)
|
| 51 |
-
|
| 52 |
-
# Handle semantic_type as either SemanticType enum or string
|
| 53 |
-
if isinstance(mapping.semantic_type, SemanticType):
|
| 54 |
-
original_type = mapping.semantic_type.value
|
| 55 |
-
semantic_type_enum = mapping.semantic_type
|
| 56 |
-
else:
|
| 57 |
-
original_type = str(mapping.semantic_type)
|
| 58 |
-
semantic_type_enum = SemanticType(mapping.semantic_type)
|
| 59 |
-
|
| 60 |
-
transformation = mapping.transformation
|
| 61 |
-
|
| 62 |
-
# Apply transformation if specified
|
| 63 |
-
if transformation:
|
| 64 |
-
normalized = self._apply_transformation(
|
| 65 |
-
value, transformation, customer_id, concept_id
|
| 66 |
-
)
|
| 67 |
-
transformation_applied = transformation
|
| 68 |
-
else:
|
| 69 |
-
normalized = value
|
| 70 |
-
transformation_applied = None
|
| 71 |
-
|
| 72 |
-
# Convert type if target specified
|
| 73 |
-
if target_type and target_type != semantic_type_enum:
|
| 74 |
-
normalized = self._convert_type(normalized, semantic_type_enum, target_type)
|
| 75 |
-
if transformation_applied:
|
| 76 |
-
transformation_applied += f" + type_conversion_to_{target_type.value}"
|
| 77 |
-
else:
|
| 78 |
-
transformation_applied = f"type_conversion_to_{target_type.value}"
|
| 79 |
-
|
| 80 |
-
return NormalizedValue(
|
| 81 |
-
original_value=value,
|
| 82 |
-
normalized_value=normalized,
|
| 83 |
-
original_type=original_type,
|
| 84 |
-
normalized_type=target_type.value if target_type else original_type,
|
| 85 |
-
transformation_applied=transformation_applied
|
| 86 |
-
)
|
| 87 |
-
|
| 88 |
-
def _apply_transformation(
|
| 89 |
-
self,
|
| 90 |
-
value: Any,
|
| 91 |
-
transformation: str,
|
| 92 |
-
customer_id: str,
|
| 93 |
-
concept_id: str
|
| 94 |
-
) -> Any:
|
| 95 |
-
"""Apply a transformation to a value.
|
| 96 |
-
|
| 97 |
-
Args:
|
| 98 |
-
value: Value to transform
|
| 99 |
-
transformation: Transformation SQL or expression
|
| 100 |
-
customer_id: Customer ID for context
|
| 101 |
-
concept_id: Concept ID for context
|
| 102 |
-
|
| 103 |
-
Returns:
|
| 104 |
-
Transformed value
|
| 105 |
-
"""
|
| 106 |
-
# Handle days_remaining -> end_date conversion
|
| 107 |
-
if "CURRENT_DATE" in transformation or "julianday" in transformation:
|
| 108 |
-
return self._days_to_date(value)
|
| 109 |
-
|
| 110 |
-
# Handle annual_value -> lifetime_value conversion
|
| 111 |
-
if "contract_length" in transformation or "*" in transformation:
|
| 112 |
-
# Need to get contract_length for this specific row
|
| 113 |
-
# For now, apply a default multiplier (this should be done at query time)
|
| 114 |
-
# In real implementation, this would need row-level context
|
| 115 |
-
return value # Return as-is; transformation happens at SQL level
|
| 116 |
-
|
| 117 |
-
# Other transformations
|
| 118 |
-
return value
|
| 119 |
-
|
| 120 |
-
def _days_to_date(self, days_remaining: Any) -> Optional[str]:
|
| 121 |
-
"""Convert days remaining to an end date.
|
| 122 |
-
|
| 123 |
-
Args:
|
| 124 |
-
days_remaining: Number of days remaining
|
| 125 |
-
|
| 126 |
-
Returns:
|
| 127 |
-
ISO format date string or None if invalid
|
| 128 |
-
"""
|
| 129 |
-
if days_remaining is None:
|
| 130 |
-
return None
|
| 131 |
-
|
| 132 |
-
try:
|
| 133 |
-
days = int(days_remaining)
|
| 134 |
-
end_date = datetime.now() + timedelta(days=days)
|
| 135 |
-
return end_date.strftime("%Y-%m-%d")
|
| 136 |
-
except (ValueError, TypeError):
|
| 137 |
-
return None
|
| 138 |
-
|
| 139 |
-
def _convert_type(
|
| 140 |
-
self,
|
| 141 |
-
value: Any,
|
| 142 |
-
from_type: SemanticType,
|
| 143 |
-
to_type: SemanticType
|
| 144 |
-
) -> Any:
|
| 145 |
-
"""Convert a value from one semantic type to another.
|
| 146 |
-
|
| 147 |
-
Args:
|
| 148 |
-
value: Value to convert
|
| 149 |
-
from_type: Current semantic type
|
| 150 |
-
to_type: Target semantic type
|
| 151 |
-
|
| 152 |
-
Returns:
|
| 153 |
-
Converted value
|
| 154 |
-
"""
|
| 155 |
-
if value is None:
|
| 156 |
-
return None
|
| 157 |
-
|
| 158 |
-
# Date conversions
|
| 159 |
-
if to_type == SemanticType.DATE:
|
| 160 |
-
if from_type == SemanticType.INTEGER:
|
| 161 |
-
# Assume integer is days remaining
|
| 162 |
-
return self._days_to_date(value)
|
| 163 |
-
elif from_type == SemanticType.TEXT:
|
| 164 |
-
# Parse text date
|
| 165 |
-
try:
|
| 166 |
-
dt = datetime.fromisoformat(str(value))
|
| 167 |
-
return dt.strftime("%Y-%m-%d")
|
| 168 |
-
except (ValueError, TypeError):
|
| 169 |
-
return str(value)
|
| 170 |
-
|
| 171 |
-
# Numeric conversions
|
| 172 |
-
if to_type == SemanticType.FLOAT:
|
| 173 |
-
if from_type in (SemanticType.INTEGER, SemanticType.TEXT):
|
| 174 |
-
try:
|
| 175 |
-
return float(value)
|
| 176 |
-
except (ValueError, TypeError):
|
| 177 |
-
return value
|
| 178 |
-
|
| 179 |
-
if to_type == SemanticType.INTEGER:
|
| 180 |
-
if from_type in (SemanticType.FLOAT, SemanticType.TEXT):
|
| 181 |
-
try:
|
| 182 |
-
return int(float(value))
|
| 183 |
-
except (ValueError, TypeError):
|
| 184 |
-
return value
|
| 185 |
-
|
| 186 |
-
# Text conversion (always works)
|
| 187 |
-
if to_type == SemanticType.TEXT:
|
| 188 |
-
return str(value)
|
| 189 |
-
|
| 190 |
-
# No conversion available
|
| 191 |
-
return value
|
| 192 |
-
|
| 193 |
-
def normalize_field_name(
|
| 194 |
-
self,
|
| 195 |
-
customer_field_name: str,
|
| 196 |
-
customer_id: str
|
| 197 |
-
) -> Optional[str]:
|
| 198 |
-
"""Map a customer-specific field name to its semantic concept.
|
| 199 |
-
|
| 200 |
-
Args:
|
| 201 |
-
customer_field_name: Field name in customer schema
|
| 202 |
-
customer_id: Customer ID
|
| 203 |
-
|
| 204 |
-
Returns:
|
| 205 |
-
Semantic concept ID or None if not mapped
|
| 206 |
-
"""
|
| 207 |
-
# Check all concepts for this customer
|
| 208 |
-
for concept_id in self.knowledge_graph.concepts.keys():
|
| 209 |
-
mapping = self.knowledge_graph.get_mapping(concept_id, customer_id)
|
| 210 |
-
if mapping and mapping.column_name == customer_field_name:
|
| 211 |
-
return concept_id
|
| 212 |
-
|
| 213 |
-
return None
|
| 214 |
-
|
| 215 |
-
def normalize_industry_name(self, industry: Optional[str]) -> Optional[str]:
|
| 216 |
-
"""Normalize industry names to common format.
|
| 217 |
-
|
| 218 |
-
Args:
|
| 219 |
-
industry: Industry name from customer data
|
| 220 |
-
|
| 221 |
-
Returns:
|
| 222 |
-
Normalized industry name
|
| 223 |
-
"""
|
| 224 |
-
if not industry:
|
| 225 |
-
return None
|
| 226 |
-
|
| 227 |
-
# Convert to lowercase for comparison
|
| 228 |
-
industry_lower = industry.lower().strip()
|
| 229 |
-
|
| 230 |
-
# Map common variations
|
| 231 |
-
industry_mapping = {
|
| 232 |
-
"tech": "Technology",
|
| 233 |
-
"technology": "Technology",
|
| 234 |
-
"it": "Technology",
|
| 235 |
-
"information technology": "Technology",
|
| 236 |
-
"healthcare": "Healthcare",
|
| 237 |
-
"health": "Healthcare",
|
| 238 |
-
"medical": "Healthcare",
|
| 239 |
-
"finance": "Financial Services",
|
| 240 |
-
"financial": "Financial Services",
|
| 241 |
-
"financial services": "Financial Services",
|
| 242 |
-
"banking": "Financial Services",
|
| 243 |
-
"retail": "Retail",
|
| 244 |
-
"manufacturing": "Manufacturing",
|
| 245 |
-
"mfg": "Manufacturing",
|
| 246 |
-
"education": "Education",
|
| 247 |
-
"edu": "Education",
|
| 248 |
-
"government": "Government",
|
| 249 |
-
"gov": "Government",
|
| 250 |
-
"public sector": "Government",
|
| 251 |
-
}
|
| 252 |
-
|
| 253 |
-
return industry_mapping.get(industry_lower, industry.title())
|
| 254 |
-
|
| 255 |
-
def harmonize_row(
|
| 256 |
-
self,
|
| 257 |
-
row: Dict[str, Any],
|
| 258 |
-
customer_id: str,
|
| 259 |
-
field_mappings: Dict[str, str]
|
| 260 |
-
) -> Dict[str, Any]:
|
| 261 |
-
"""Harmonize a single row of data.
|
| 262 |
-
|
| 263 |
-
Args:
|
| 264 |
-
row: Raw row data from customer database
|
| 265 |
-
customer_id: Customer ID
|
| 266 |
-
field_mappings: Map of customer field names to concept IDs
|
| 267 |
-
|
| 268 |
-
Returns:
|
| 269 |
-
Harmonized row with normalized field names and values
|
| 270 |
-
"""
|
| 271 |
-
harmonized = {}
|
| 272 |
-
|
| 273 |
-
for customer_field, concept_id in field_mappings.items():
|
| 274 |
-
if customer_field in row:
|
| 275 |
-
value = row[customer_field]
|
| 276 |
-
|
| 277 |
-
# Special handling for industry
|
| 278 |
-
if concept_id == "industry_sector":
|
| 279 |
-
harmonized[concept_id] = self.normalize_industry_name(value)
|
| 280 |
-
else:
|
| 281 |
-
# Normalize the value
|
| 282 |
-
normalized = self.normalize_value(value, customer_id, concept_id)
|
| 283 |
-
harmonized[concept_id] = normalized.normalized_value
|
| 284 |
-
else:
|
| 285 |
-
# Field not present in row
|
| 286 |
-
harmonized[concept_id] = None
|
| 287 |
-
|
| 288 |
-
return harmonized
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_result_harmonization.py
CHANGED
|
@@ -17,7 +17,6 @@ from schema_translator.models import (
|
|
| 17 |
SemanticType,
|
| 18 |
)
|
| 19 |
from schema_translator.result_harmonizer import ResultHarmonizer
|
| 20 |
-
from schema_translator.value_harmonizer import ValueHarmonizer
|
| 21 |
|
| 22 |
|
| 23 |
@pytest.fixture
|
|
@@ -34,12 +33,6 @@ def knowledge_graph(config):
|
|
| 34 |
return kg
|
| 35 |
|
| 36 |
|
| 37 |
-
@pytest.fixture
|
| 38 |
-
def value_harmonizer(knowledge_graph):
|
| 39 |
-
"""Create a value harmonizer."""
|
| 40 |
-
return ValueHarmonizer(knowledge_graph)
|
| 41 |
-
|
| 42 |
-
|
| 43 |
@pytest.fixture
|
| 44 |
def result_harmonizer(knowledge_graph):
|
| 45 |
"""Create a result harmonizer."""
|
|
@@ -47,11 +40,11 @@ def result_harmonizer(knowledge_graph):
|
|
| 47 |
|
| 48 |
|
| 49 |
class TestValueHarmonizer:
|
| 50 |
-
"""Tests for
|
| 51 |
|
| 52 |
-
def test_normalize_value_no_transformation(self,
|
| 53 |
"""Test normalizing a value with no transformation."""
|
| 54 |
-
normalized =
|
| 55 |
value="Active",
|
| 56 |
customer_id="customer_a",
|
| 57 |
concept_id="contract_status"
|
|
@@ -62,10 +55,10 @@ class TestValueHarmonizer:
|
|
| 62 |
assert normalized.original_type == "text"
|
| 63 |
assert normalized.transformation_applied is None
|
| 64 |
|
| 65 |
-
def test_normalize_value_with_transformation(self,
|
| 66 |
"""Test normalizing a value that requires transformation."""
|
| 67 |
# Customer D uses days_remaining instead of end_date
|
| 68 |
-
normalized =
|
| 69 |
value=365,
|
| 70 |
customer_id="customer_d",
|
| 71 |
concept_id="contract_expiration"
|
|
@@ -78,23 +71,23 @@ class TestValueHarmonizer:
|
|
| 78 |
assert normalized.normalized_value is not None
|
| 79 |
assert "-" in str(normalized.normalized_value) # Date format
|
| 80 |
|
| 81 |
-
def test_days_to_date_conversion(self,
|
| 82 |
"""Test converting days remaining to a date."""
|
| 83 |
# 30 days from now
|
| 84 |
-
date_str =
|
| 85 |
|
| 86 |
assert date_str is not None
|
| 87 |
assert len(date_str) == 10 # YYYY-MM-DD format
|
| 88 |
assert date_str.count("-") == 2
|
| 89 |
|
| 90 |
-
def test_days_to_date_invalid(self,
|
| 91 |
"""Test days to date with invalid input."""
|
| 92 |
-
assert
|
| 93 |
-
assert
|
| 94 |
|
| 95 |
-
def test_convert_type_int_to_float(self,
|
| 96 |
"""Test type conversion from integer to float."""
|
| 97 |
-
result =
|
| 98 |
100,
|
| 99 |
SemanticType.INTEGER,
|
| 100 |
SemanticType.FLOAT
|
|
@@ -103,9 +96,9 @@ class TestValueHarmonizer:
|
|
| 103 |
assert isinstance(result, float)
|
| 104 |
assert result == 100.0
|
| 105 |
|
| 106 |
-
def test_convert_type_float_to_int(self,
|
| 107 |
"""Test type conversion from float to integer."""
|
| 108 |
-
result =
|
| 109 |
99.9,
|
| 110 |
SemanticType.FLOAT,
|
| 111 |
SemanticType.INTEGER
|
|
@@ -114,9 +107,9 @@ class TestValueHarmonizer:
|
|
| 114 |
assert isinstance(result, int)
|
| 115 |
assert result == 99
|
| 116 |
|
| 117 |
-
def test_convert_type_to_text(self,
|
| 118 |
"""Test type conversion to text."""
|
| 119 |
-
result =
|
| 120 |
123,
|
| 121 |
SemanticType.INTEGER,
|
| 122 |
SemanticType.TEXT
|
|
@@ -125,26 +118,26 @@ class TestValueHarmonizer:
|
|
| 125 |
assert isinstance(result, str)
|
| 126 |
assert result == "123"
|
| 127 |
|
| 128 |
-
def test_normalize_industry_name(self,
|
| 129 |
"""Test normalizing industry names."""
|
| 130 |
-
assert
|
| 131 |
-
assert
|
| 132 |
-
assert
|
| 133 |
-
assert
|
| 134 |
-
assert
|
| 135 |
-
assert
|
| 136 |
|
| 137 |
-
def test_normalize_field_name(self,
|
| 138 |
"""Test mapping customer field names to concepts."""
|
| 139 |
# Customer A uses 'contract_id'
|
| 140 |
-
concept =
|
| 141 |
assert concept == "contract_identifier"
|
| 142 |
|
| 143 |
# Customer B uses 'id' for contract_identifier
|
| 144 |
-
concept =
|
| 145 |
assert concept == "contract_identifier"
|
| 146 |
|
| 147 |
-
def test_harmonize_row(self,
|
| 148 |
"""Test harmonizing a complete row."""
|
| 149 |
row = {
|
| 150 |
"contract_id": "A001",
|
|
@@ -158,13 +151,13 @@ class TestValueHarmonizer:
|
|
| 158 |
"contract_value": "contract_value"
|
| 159 |
}
|
| 160 |
|
| 161 |
-
harmonized =
|
| 162 |
|
| 163 |
assert harmonized["contract_identifier"] == "A001"
|
| 164 |
assert harmonized["contract_status"] == "Active"
|
| 165 |
assert harmonized["contract_value"] == 100000.0
|
| 166 |
|
| 167 |
-
def test_harmonize_row_with_industry(self,
|
| 168 |
"""Test harmonizing a row with industry normalization."""
|
| 169 |
row = {
|
| 170 |
"contract_id": "A001",
|
|
@@ -176,7 +169,7 @@ class TestValueHarmonizer:
|
|
| 176 |
"industry": "industry_sector"
|
| 177 |
}
|
| 178 |
|
| 179 |
-
harmonized =
|
| 180 |
|
| 181 |
assert harmonized["industry_sector"] == "Technology"
|
| 182 |
|
|
|
|
| 17 |
SemanticType,
|
| 18 |
)
|
| 19 |
from schema_translator.result_harmonizer import ResultHarmonizer
|
|
|
|
| 20 |
|
| 21 |
|
| 22 |
@pytest.fixture
|
|
|
|
| 33 |
return kg
|
| 34 |
|
| 35 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
@pytest.fixture
|
| 37 |
def result_harmonizer(knowledge_graph):
|
| 38 |
"""Create a result harmonizer."""
|
|
|
|
| 40 |
|
| 41 |
|
| 42 |
class TestValueHarmonizer:
|
| 43 |
+
"""Tests for value normalization (now part of ResultHarmonizer)."""
|
| 44 |
|
| 45 |
+
def test_normalize_value_no_transformation(self, result_harmonizer):
|
| 46 |
"""Test normalizing a value with no transformation."""
|
| 47 |
+
normalized = result_harmonizer._normalize_value(
|
| 48 |
value="Active",
|
| 49 |
customer_id="customer_a",
|
| 50 |
concept_id="contract_status"
|
|
|
|
| 55 |
assert normalized.original_type == "text"
|
| 56 |
assert normalized.transformation_applied is None
|
| 57 |
|
| 58 |
+
def test_normalize_value_with_transformation(self, result_harmonizer):
|
| 59 |
"""Test normalizing a value that requires transformation."""
|
| 60 |
# Customer D uses days_remaining instead of end_date
|
| 61 |
+
normalized = result_harmonizer._normalize_value(
|
| 62 |
value=365,
|
| 63 |
customer_id="customer_d",
|
| 64 |
concept_id="contract_expiration"
|
|
|
|
| 71 |
assert normalized.normalized_value is not None
|
| 72 |
assert "-" in str(normalized.normalized_value) # Date format
|
| 73 |
|
| 74 |
+
def test_days_to_date_conversion(self, result_harmonizer):
|
| 75 |
"""Test converting days remaining to a date."""
|
| 76 |
# 30 days from now
|
| 77 |
+
date_str = result_harmonizer._days_to_date(30)
|
| 78 |
|
| 79 |
assert date_str is not None
|
| 80 |
assert len(date_str) == 10 # YYYY-MM-DD format
|
| 81 |
assert date_str.count("-") == 2
|
| 82 |
|
| 83 |
+
def test_days_to_date_invalid(self, result_harmonizer):
|
| 84 |
"""Test days to date with invalid input."""
|
| 85 |
+
assert result_harmonizer._days_to_date(None) is None
|
| 86 |
+
assert result_harmonizer._days_to_date("invalid") is None
|
| 87 |
|
| 88 |
+
def test_convert_type_int_to_float(self, result_harmonizer):
|
| 89 |
"""Test type conversion from integer to float."""
|
| 90 |
+
result = result_harmonizer._convert_type(
|
| 91 |
100,
|
| 92 |
SemanticType.INTEGER,
|
| 93 |
SemanticType.FLOAT
|
|
|
|
| 96 |
assert isinstance(result, float)
|
| 97 |
assert result == 100.0
|
| 98 |
|
| 99 |
+
def test_convert_type_float_to_int(self, result_harmonizer):
|
| 100 |
"""Test type conversion from float to integer."""
|
| 101 |
+
result = result_harmonizer._convert_type(
|
| 102 |
99.9,
|
| 103 |
SemanticType.FLOAT,
|
| 104 |
SemanticType.INTEGER
|
|
|
|
| 107 |
assert isinstance(result, int)
|
| 108 |
assert result == 99
|
| 109 |
|
| 110 |
+
def test_convert_type_to_text(self, result_harmonizer):
|
| 111 |
"""Test type conversion to text."""
|
| 112 |
+
result = result_harmonizer._convert_type(
|
| 113 |
123,
|
| 114 |
SemanticType.INTEGER,
|
| 115 |
SemanticType.TEXT
|
|
|
|
| 118 |
assert isinstance(result, str)
|
| 119 |
assert result == "123"
|
| 120 |
|
| 121 |
+
def test_normalize_industry_name(self, result_harmonizer):
|
| 122 |
"""Test normalizing industry names."""
|
| 123 |
+
assert result_harmonizer._normalize_industry_name("tech") == "Technology"
|
| 124 |
+
assert result_harmonizer._normalize_industry_name("TECHNOLOGY") == "Technology"
|
| 125 |
+
assert result_harmonizer._normalize_industry_name("healthcare") == "Healthcare"
|
| 126 |
+
assert result_harmonizer._normalize_industry_name("finance") == "Financial Services"
|
| 127 |
+
assert result_harmonizer._normalize_industry_name("Unknown Industry") == "Unknown Industry"
|
| 128 |
+
assert result_harmonizer._normalize_industry_name(None) is None
|
| 129 |
|
| 130 |
+
def test_normalize_field_name(self, result_harmonizer):
|
| 131 |
"""Test mapping customer field names to concepts."""
|
| 132 |
# Customer A uses 'contract_id'
|
| 133 |
+
concept = result_harmonizer._normalize_field_name("contract_id", "customer_a")
|
| 134 |
assert concept == "contract_identifier"
|
| 135 |
|
| 136 |
# Customer B uses 'id' for contract_identifier
|
| 137 |
+
concept = result_harmonizer._normalize_field_name("id", "customer_b")
|
| 138 |
assert concept == "contract_identifier"
|
| 139 |
|
| 140 |
+
def test_harmonize_row(self, result_harmonizer):
|
| 141 |
"""Test harmonizing a complete row."""
|
| 142 |
row = {
|
| 143 |
"contract_id": "A001",
|
|
|
|
| 151 |
"contract_value": "contract_value"
|
| 152 |
}
|
| 153 |
|
| 154 |
+
harmonized = result_harmonizer._harmonize_row(row, "customer_a", field_mappings)
|
| 155 |
|
| 156 |
assert harmonized["contract_identifier"] == "A001"
|
| 157 |
assert harmonized["contract_status"] == "Active"
|
| 158 |
assert harmonized["contract_value"] == 100000.0
|
| 159 |
|
| 160 |
+
def test_harmonize_row_with_industry(self, result_harmonizer):
|
| 161 |
"""Test harmonizing a row with industry normalization."""
|
| 162 |
row = {
|
| 163 |
"contract_id": "A001",
|
|
|
|
| 169 |
"industry": "industry_sector"
|
| 170 |
}
|
| 171 |
|
| 172 |
+
harmonized = result_harmonizer._harmonize_row(row, "customer_a", field_mappings)
|
| 173 |
|
| 174 |
assert harmonized["industry_sector"] == "Technology"
|
| 175 |
|