Coverage for src/hieraconf/lazy_factory.py: 37%
454 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-02 21:44 +0000
1"""Generic lazy dataclass factory using flexible resolution."""
3# Standard library imports
4import dataclasses
5import logging
6import re
7import sys
8from abc import ABCMeta
9from dataclasses import dataclass, fields, is_dataclass, make_dataclass, MISSING, field
10from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union
12# OpenHCS imports
13from hieraconf.placeholder import LazyDefaultPlaceholderService
14# Optional: metaclass_registry for context provider registration
15try:
16 from metaclass_registry import AutoRegisterMeta, RegistryConfig
17except ImportError:
18 # Provide minimal fallback implementations
19 class AutoRegisterMeta(type):
20 """Fallback metaclass when metaclass_registry is not available."""
21 def __new__(mcs, name, bases, attrs, registry_config=None):
22 return super().__new__(mcs, name, bases, attrs)
24 class RegistryConfig:
25 """Fallback registry config when metaclass_registry is not available."""
26 def __init__(self, **kwargs):
27 pass
29# Note: dual_axis_resolver_recursive and lazy_placeholder imports kept inline to avoid circular imports
32# Type registry for lazy dataclass to base class mapping
33_lazy_type_registry: Dict[Type, Type] = {}
35# Cache for lazy classes to prevent duplicate creation
36_lazy_class_cache: Dict[str, Type] = {}
39# ContextEventCoordinator removed - replaced with contextvars-based context system
44def register_lazy_type_mapping(lazy_type: Type, base_type: Type) -> None:
45 """Register mapping between lazy dataclass type and its base type."""
46 _lazy_type_registry[lazy_type] = base_type
49def get_base_type_for_lazy(lazy_type: Type) -> Optional[Type]:
50 """Get the base type for a lazy dataclass type."""
51 return _lazy_type_registry.get(lazy_type)
53# Optional imports (handled gracefully)
54try:
55 from PyQt6.QtWidgets import QApplication
56 HAS_PYQT = True
57except ImportError:
58 QApplication = None
59 HAS_PYQT = False
61logger = logging.getLogger(__name__)
64# Constants for lazy configuration system - simplified from class to module-level
65MATERIALIZATION_DEFAULTS_PATH = "materialization_defaults"
66RESOLVE_FIELD_VALUE_METHOD = "_resolve_field_value"
67GET_ATTRIBUTE_METHOD = "__getattribute__"
68TO_BASE_CONFIG_METHOD = "to_base_config"
69WITH_DEFAULTS_METHOD = "with_defaults"
70WITH_OVERRIDES_METHOD = "with_overrides"
71LAZY_FIELD_DEBUG_TEMPLATE = "LAZY FIELD CREATION: {field_name} - original={original_type}, has_default={has_default}, final={final_type}"
73LAZY_CLASS_NAME_PREFIX = "Lazy"
75# Legacy helper functions removed - new context system handles all resolution
78# Functional fallback strategies
79def _get_raw_field_value(obj: Any, field_name: str) -> Any:
80 """
81 Get raw field value bypassing lazy property getters to prevent infinite recursion.
83 Uses object.__getattribute__() to access stored values directly without triggering
84 lazy resolution, which would create circular dependencies in the resolution chain.
86 Args:
87 obj: Object to get field from
88 field_name: Name of field to access
90 Returns:
91 Raw field value or None if field doesn't exist
93 Raises:
94 AttributeError: If field doesn't exist (fail-loud behavior)
95 """
96 try:
97 return object.__getattribute__(obj, field_name)
98 except AttributeError:
99 return None
102@dataclass(frozen=True)
103class LazyMethodBindings:
104 """Declarative method bindings for lazy dataclasses."""
106 @staticmethod
107 def create_resolver() -> Callable[[Any, str], Any]:
108 """Create field resolver method using new pure function interface."""
109 from hieraconf.dual_axis_resolver import resolve_field_inheritance
110 from hieraconf.context_manager import current_temp_global, extract_all_configs
112 def _resolve_field_value(self, field_name: str) -> Any:
113 # Get current context from contextvars
114 try:
115 current_context = current_temp_global.get()
116 # Extract available configs from current context
117 available_configs = extract_all_configs(current_context)
119 # Use pure function for resolution
120 return resolve_field_inheritance(self, field_name, available_configs)
121 except LookupError:
122 # No context available - return None (fail-loud approach)
123 logger.debug(f"No context available for resolving {type(self).__name__}.{field_name}")
124 return None
126 return _resolve_field_value
128 @staticmethod
129 def create_getattribute() -> Callable[[Any, str], Any]:
130 """Create lazy __getattribute__ method using new context system."""
131 from hieraconf.dual_axis_resolver import resolve_field_inheritance, _has_concrete_field_override
132 from hieraconf.context_manager import current_temp_global, extract_all_configs
134 def _find_mro_concrete_value(base_class, name):
135 """Extract common MRO traversal pattern."""
136 return next((getattr(cls, name) for cls in base_class.__mro__
137 if _has_concrete_field_override(cls, name)), None)
139 def _try_global_context_value(self, base_class, name):
140 """Extract global context resolution logic using new pure function interface."""
141 if not hasattr(self, '_global_config_type'):
142 return None
144 # Get current context from contextvars
145 try:
146 current_context = current_temp_global.get()
147 # Extract available configs from current context
148 available_configs = extract_all_configs(current_context)
150 # Use pure function for resolution
151 resolved_value = resolve_field_inheritance(self, name, available_configs)
152 if resolved_value is not None:
153 return resolved_value
154 except LookupError:
155 # No context available - fall back to MRO
156 pass
158 # Fallback to MRO concrete value
159 return _find_mro_concrete_value(base_class, name)
161 def __getattribute__(self: Any, name: str) -> Any:
162 """
163 Three-stage resolution using new context system.
165 Stage 1: Check instance value
166 Stage 2: Simple field path lookup in current scope's merged config
167 Stage 3: Inheritance resolution using same merged context
168 """
169 # Stage 1: Get instance value
170 value = object.__getattribute__(self, name)
171 if value is not None or name not in {f.name for f in fields(self.__class__)}:
172 return value
174 # Stage 2: Simple field path lookup in current scope's merged global
175 try:
176 current_context = current_temp_global.get()
177 if current_context is not None:
178 # Get the config type name for this lazy class
179 config_field_name = getattr(self, '_config_field_name', None)
180 if config_field_name:
181 try:
182 config_instance = getattr(current_context, config_field_name)
183 if config_instance is not None:
184 resolved_value = getattr(config_instance, name)
185 if resolved_value is not None:
186 return resolved_value
187 except AttributeError:
188 # Field doesn't exist in merged config, continue to inheritance
189 pass
190 except LookupError:
191 # No context available, continue to inheritance
192 pass
194 # Stage 3: Inheritance resolution using same merged context
195 try:
196 current_context = current_temp_global.get()
197 available_configs = extract_all_configs(current_context)
198 resolved_value = resolve_field_inheritance(self, name, available_configs)
200 if resolved_value is not None:
201 return resolved_value
203 # For nested dataclass fields, return lazy instance
204 field_obj = next((f for f in fields(self.__class__) if f.name == name), None)
205 if field_obj and is_dataclass(field_obj.type):
206 return field_obj.type()
208 return None
210 except LookupError:
211 # No context available - fallback to MRO concrete values
212 return _find_mro_concrete_value(get_base_type_for_lazy(self.__class__), name)
213 return __getattribute__
215 @staticmethod
216 def create_to_base_config(base_class: Type) -> Callable[[Any], Any]:
217 """Create base config converter method."""
218 def to_base_config(self):
219 # CRITICAL FIX: Use object.__getattribute__ to preserve raw None values
220 # getattr() triggers lazy resolution, converting None to static defaults
221 # None values must be preserved for dual-axis inheritance to work correctly
222 #
223 # Context: to_base_config() is called DURING config_context() setup (line 124 in context_manager.py)
224 # If we use getattr() here, it triggers resolution BEFORE the context is fully set up,
225 # causing resolution to use the wrong/stale context and losing the GlobalPipelineConfig base.
226 # We must extract raw None values here, let config_context() merge them into the hierarchy,
227 # and THEN resolution happens later with the properly built context.
228 field_values = {f.name: object.__getattribute__(self, f.name) for f in fields(self)}
229 return base_class(**field_values)
230 return to_base_config
232 @staticmethod
233 def create_class_methods() -> Dict[str, Any]:
234 """Create class-level utility methods."""
235 return {
236 WITH_DEFAULTS_METHOD: classmethod(lambda cls: cls()),
237 WITH_OVERRIDES_METHOD: classmethod(lambda cls, **kwargs: cls(**kwargs))
238 }
241class LazyDataclassFactory:
242 """Generic factory for creating lazy dataclasses with flexible resolution."""
248 @staticmethod
249 def _introspect_dataclass_fields(base_class: Type, debug_template: str, global_config_type: Type = None, parent_field_path: str = None, parent_instance_provider: Optional[Callable[[], Any]] = None) -> List[Tuple[str, Type, None]]:
250 """
251 Introspect dataclass fields for lazy loading.
253 Converts nested dataclass fields to lazy equivalents and makes fields Optional
254 if they lack defaults. Complex logic handles type unwrapping and lazy nesting.
255 """
256 base_fields = fields(base_class)
257 lazy_field_definitions = []
259 for field in base_fields:
260 # Check if field already has Optional type
261 origin = getattr(field.type, '__origin__', None)
262 is_already_optional = (origin is Union and
263 type(None) in getattr(field.type, '__args__', ()))
265 # Check if field has default value or factory
266 has_default = (field.default is not MISSING or
267 field.default_factory is not MISSING)
269 # Check if field type is a dataclass that should be made lazy
270 field_type = field.type
271 if is_dataclass(field.type):
272 # SIMPLIFIED: Create lazy version using simple factory
273 lazy_nested_type = LazyDataclassFactory.make_lazy_simple(
274 base_class=field.type,
275 lazy_class_name=f"Lazy{field.type.__name__}"
276 )
277 field_type = lazy_nested_type
278 logger.debug(f"Created lazy class for {field.name}: {field.type} -> {lazy_nested_type}")
280 # Complex type logic: make Optional if no default, preserve existing Optional types
281 if is_already_optional or not has_default:
282 final_field_type = Union[field_type, type(None)] if not is_already_optional else field_type
283 else:
284 final_field_type = field_type
286 # CRITICAL FIX: Create default factory for Optional dataclass fields
287 # This eliminates the need for field introspection and ensures UI always has instances to render
288 # CRITICAL: Always preserve metadata from original field (e.g., ui_hidden flag)
289 if (is_already_optional or not has_default) and is_dataclass(field.type):
290 # For Optional dataclass fields, create default factory that creates lazy instances
291 # This ensures the UI always has nested lazy instances to render recursively
292 # CRITICAL: field_type is already the lazy type, so use it directly
293 field_def = (field.name, final_field_type, dataclasses.field(default_factory=field_type, metadata=field.metadata))
294 elif field.metadata:
295 # For fields with metadata but no dataclass default factory, create a Field object to preserve metadata
296 # We need to replicate the original field's default behavior
297 if field.default is not MISSING:
298 field_def = (field.name, final_field_type, dataclasses.field(default=field.default, metadata=field.metadata))
299 elif field.default_factory is not MISSING:
300 field_def = (field.name, final_field_type, dataclasses.field(default_factory=field.default_factory, metadata=field.metadata))
301 else:
302 # Field has metadata but no default - use MISSING to indicate required field
303 field_def = (field.name, final_field_type, dataclasses.field(default=MISSING, metadata=field.metadata))
304 else:
305 # No metadata, no special handling needed
306 field_def = (field.name, final_field_type, None)
308 lazy_field_definitions.append(field_def)
310 # Debug logging with provided template (reduced to DEBUG level to reduce log pollution)
311 logger.debug(debug_template.format(
312 field_name=field.name,
313 original_type=field.type,
314 has_default=has_default,
315 final_type=final_field_type
316 ))
318 return lazy_field_definitions
320 @staticmethod
321 def _create_lazy_dataclass_unified(
322 base_class: Type,
323 instance_provider: Callable[[], Any],
324 lazy_class_name: str,
325 debug_template: str,
326 use_recursive_resolution: bool = False,
327 fallback_chain: Optional[List[Callable[[str], Any]]] = None,
328 global_config_type: Type = None,
329 parent_field_path: str = None,
330 parent_instance_provider: Optional[Callable[[], Any]] = None
331 ) -> Type:
332 """
333 Create lazy dataclass with declarative configuration.
335 Core factory method that creates lazy dataclass with introspected fields,
336 binds resolution methods, and registers type mappings. Complex orchestration
337 of field analysis, method binding, and class creation.
338 """
339 if not is_dataclass(base_class):
340 raise ValueError(f"{base_class} must be a dataclass")
342 # Check cache first to prevent duplicate creation
343 cache_key = f"{base_class.__name__}_{lazy_class_name}_{id(instance_provider)}"
344 if cache_key in _lazy_class_cache:
345 return _lazy_class_cache[cache_key]
347 # ResolutionConfig system removed - dual-axis resolver handles all resolution
349 # Create lazy dataclass with introspected fields
350 # CRITICAL FIX: Avoid inheriting from classes with custom metaclasses to prevent descriptor conflicts
351 # Exception: InheritAsNoneMeta is safe to inherit from as it only modifies field defaults
352 # Exception: Classes with _inherit_as_none marker are safe even with ABCMeta (processed by @global_pipeline_config)
353 base_metaclass = type(base_class)
354 has_inherit_as_none_marker = hasattr(base_class, '_inherit_as_none') and base_class._inherit_as_none
355 has_unsafe_metaclass = (
356 (hasattr(base_class, '__metaclass__') or base_metaclass != type) and
357 base_metaclass != InheritAsNoneMeta and
358 not has_inherit_as_none_marker
359 )
361 # Determine if base class is frozen to avoid frozen/non-frozen conflicts
362 base_is_frozen = base_class.__dataclass_params__.frozen if hasattr(base_class, '__dataclass_params__') else False
364 if has_unsafe_metaclass:
365 # Base class has unsafe custom metaclass - don't inherit, just copy interface
366 print(f"🔧 LAZY FACTORY: {base_class.__name__} has custom metaclass {base_metaclass.__name__}, avoiding inheritance")
367 lazy_class = make_dataclass(
368 lazy_class_name,
369 LazyDataclassFactory._introspect_dataclass_fields(
370 base_class, debug_template, global_config_type, parent_field_path, parent_instance_provider
371 ),
372 bases=(), # No inheritance to avoid metaclass conflicts
373 frozen=base_is_frozen # Match base class frozen state
374 )
375 else:
376 # Safe to inherit from regular dataclass
377 lazy_class = make_dataclass(
378 lazy_class_name,
379 LazyDataclassFactory._introspect_dataclass_fields(
380 base_class, debug_template, global_config_type, parent_field_path, parent_instance_provider
381 ),
382 bases=(base_class,),
383 frozen=base_is_frozen # Match base class frozen state
384 )
386 # Add constructor parameter tracking to detect user-set fields
387 original_init = lazy_class.__init__
388 def __init_with_tracking__(self, **kwargs):
389 # Track which fields were explicitly passed to constructor
390 object.__setattr__(self, '_explicitly_set_fields', set(kwargs.keys()))
391 # Store the global config type for inheritance resolution
392 object.__setattr__(self, '_global_config_type', global_config_type)
393 # Store the config field name for simple field path lookup
394 import re
395 def _camel_to_snake_local(name: str) -> str:
396 s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
397 return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
398 config_field_name = _camel_to_snake_local(base_class.__name__)
399 object.__setattr__(self, '_config_field_name', config_field_name)
400 original_init(self, **kwargs)
402 lazy_class.__init__ = __init_with_tracking__
404 # Bind methods declaratively - inline single-use method
405 method_bindings = {
406 RESOLVE_FIELD_VALUE_METHOD: LazyMethodBindings.create_resolver(),
407 GET_ATTRIBUTE_METHOD: LazyMethodBindings.create_getattribute(),
408 TO_BASE_CONFIG_METHOD: LazyMethodBindings.create_to_base_config(base_class),
409 **LazyMethodBindings.create_class_methods()
410 }
411 for method_name, method_impl in method_bindings.items():
412 setattr(lazy_class, method_name, method_impl)
414 # CRITICAL: Preserve original module for proper imports in generated code
415 # make_dataclass() sets __module__ to the caller's module (lazy_factory.py)
416 # We need to set it to the base class's original module for correct import paths
417 lazy_class.__module__ = base_class.__module__
419 # Automatically register the lazy dataclass with the type registry
420 register_lazy_type_mapping(lazy_class, base_class)
422 # Cache the created class to prevent duplicates
423 _lazy_class_cache[cache_key] = lazy_class
425 return lazy_class
431 @staticmethod
432 def make_lazy_simple(
433 base_class: Type,
434 lazy_class_name: str = None
435 ) -> Type:
436 """
437 Create lazy dataclass using new contextvars system.
439 SIMPLIFIED: No complex hierarchy providers or field path detection needed.
440 Uses new contextvars system for all resolution.
442 Args:
443 base_class: Base dataclass to make lazy
444 lazy_class_name: Optional name for the lazy class
446 Returns:
447 Generated lazy dataclass with contextvars-based resolution
448 """
449 # Generate class name if not provided
450 lazy_class_name = lazy_class_name or f"Lazy{base_class.__name__}"
452 # Simple provider that uses new contextvars system
453 def simple_provider():
454 """Simple provider using new contextvars system."""
455 return base_class() # Lazy __getattribute__ handles resolution
457 return LazyDataclassFactory._create_lazy_dataclass_unified(
458 base_class=base_class,
459 instance_provider=simple_provider,
460 lazy_class_name=lazy_class_name,
461 debug_template=f"Simple contextvars resolution for {base_class.__name__}",
462 use_recursive_resolution=False,
463 fallback_chain=[],
464 global_config_type=None,
465 parent_field_path=None,
466 parent_instance_provider=None
467 )
469 # All legacy methods removed - use make_lazy_simple() for all use cases
472# Generic utility functions for clean thread-local storage management
473def ensure_global_config_context(global_config_type: Type, global_config_instance: Any) -> None:
474 """Ensure proper thread-local storage setup for any global config type."""
475 from hieraconf.global_config import set_global_config_for_editing
476 set_global_config_for_editing(global_config_type, global_config_instance)
479# Context provider registry and metaclass for automatic registration
480CONTEXT_PROVIDERS = {}
483# Configuration for context provider registration
484_CONTEXT_PROVIDER_REGISTRY_CONFIG = RegistryConfig(
485 registry_dict=CONTEXT_PROVIDERS,
486 key_attribute='_context_type',
487 key_extractor=None, # Requires explicit _context_type
488 skip_if_no_key=True, # Skip if no _context_type set
489 secondary_registries=None,
490 log_registration=True,
491 registry_name='context provider'
492)
495class ContextProviderMeta(AutoRegisterMeta):
496 """Metaclass for automatic registration of context provider classes."""
498 def __new__(mcs, name, bases, attrs):
499 return super().__new__(mcs, name, bases, attrs,
500 registry_config=_CONTEXT_PROVIDER_REGISTRY_CONFIG)
503class ContextProvider(metaclass=ContextProviderMeta):
504 """Base class for objects that can provide context for lazy resolution."""
505 _context_type: Optional[str] = None # Override in subclasses
508def _detect_context_type(obj: Any) -> Optional[str]:
509 """
510 Detect what type of context object this is using registered providers.
512 Returns the context type name or None if not a recognized context type.
513 """
514 # Check for functions first (simple callable check)
515 if callable(obj) and hasattr(obj, '__name__'):
516 return "function"
518 # Check if object is an instance of any registered context provider
519 for context_type, provider_class in CONTEXT_PROVIDERS.items():
520 if isinstance(obj, provider_class):
521 return context_type
523 return None
526# ContextInjector removed - replaced with contextvars-based context system
531def resolve_hieraconfurations_for_serialization(data: Any) -> Any:
532 """
533 Recursively resolve lazy dataclass instances to concrete values for serialization.
535 CRITICAL: This function must be called WITHIN a config_context() block!
536 The context provides the hierarchy for lazy resolution.
538 How it works:
539 1. For lazy dataclasses: Access fields with getattr() to trigger resolution
540 2. The lazy __getattribute__ uses the active config_context() to resolve None values
541 3. Convert resolved values to base config for pickling
543 Example (from README.md):
544 with config_context(orchestrator.pipeline_config):
545 # Lazy resolution happens here via context
546 resolved_steps = resolve_hieraconfurations_for_serialization(steps)
547 """
548 # Check if this is a lazy dataclass
549 base_type = get_base_type_for_lazy(type(data))
550 if base_type is not None:
551 # This is a lazy dataclass - resolve fields using getattr() within the active context
552 # getattr() triggers lazy __getattribute__ which uses config_context() for resolution
553 resolved_fields = {}
554 for f in fields(data):
555 # CRITICAL: Use getattr() to trigger lazy resolution via context
556 # The active config_context() provides the hierarchy for resolution
557 resolved_value = getattr(data, f.name)
558 resolved_fields[f.name] = resolved_value
560 # Create base config instance with resolved values
561 resolved_data = base_type(**resolved_fields)
562 else:
563 # Not a lazy dataclass
564 resolved_data = data
566 # CRITICAL FIX: Handle step objects (non-dataclass objects with dataclass attributes)
567 step_context_type = _detect_context_type(resolved_data)
568 if step_context_type:
569 # This is a context object - inject it for its dataclass attributes
570 import inspect
571 frame = inspect.currentframe()
572 context_var_name = f"__{step_context_type}_context__"
573 frame.f_locals[context_var_name] = resolved_data
574 logger.debug(f"Injected {context_var_name} = {type(resolved_data).__name__}")
576 try:
577 # Process step attributes recursively
578 resolved_attrs = {}
579 for attr_name in dir(resolved_data):
580 if attr_name.startswith('_'):
581 continue
582 try:
583 attr_value = getattr(resolved_data, attr_name)
584 if not callable(attr_value): # Skip methods
585 logger.debug(f"Resolving {type(resolved_data).__name__}.{attr_name} = {type(attr_value).__name__}")
586 resolved_attrs[attr_name] = resolve_hieraconfurations_for_serialization(attr_value)
587 except (AttributeError, Exception):
588 continue
590 # Handle function objects specially - they can't be recreated with __new__
591 if step_context_type == "function":
592 # For functions, just process attributes for resolution but return original function
593 # The resolved config values will be stored in func plan by compiler
594 return resolved_data
596 # Create new step object with resolved attributes
597 # CRITICAL FIX: Copy all original attributes using __dict__ to preserve everything
598 new_step = type(resolved_data).__new__(type(resolved_data))
600 # Copy all attributes from the original object's __dict__
601 if hasattr(resolved_data, '__dict__'):
602 new_step.__dict__.update(resolved_data.__dict__)
604 # Update with resolved config attributes (these override the originals)
605 for attr_name, attr_value in resolved_attrs.items():
606 setattr(new_step, attr_name, attr_value)
607 return new_step
608 finally:
609 if context_var_name in frame.f_locals:
610 del frame.f_locals[context_var_name]
611 del frame
613 # Recursively process nested structures based on type
614 elif is_dataclass(resolved_data) and not isinstance(resolved_data, type):
615 # Process dataclass fields recursively - inline field processing pattern
616 # CRITICAL FIX: Inject parent object as context for sibling config inheritance
617 context_type = _detect_context_type(resolved_data) or "dataclass" # Default to "dataclass" for generic dataclasses
618 import inspect
619 frame = inspect.currentframe()
620 context_var_name = f"__{context_type}_context__"
621 frame.f_locals[context_var_name] = resolved_data
622 logger.debug(f"Injected {context_var_name} = {type(resolved_data).__name__}")
624 # Add debug to see which fields are being resolved
625 logger.debug(f"Resolving fields for {type(resolved_data).__name__}: {[f.name for f in fields(resolved_data)]}")
627 try:
628 resolved_fields = {}
629 for f in fields(resolved_data):
630 field_value = getattr(resolved_data, f.name)
631 logger.debug(f"Resolving {type(resolved_data).__name__}.{f.name} = {type(field_value).__name__}")
632 resolved_fields[f.name] = resolve_hieraconfurations_for_serialization(field_value)
633 return type(resolved_data)(**resolved_fields)
634 finally:
635 if context_var_name in frame.f_locals:
636 del frame.f_locals[context_var_name]
637 del frame
639 elif isinstance(resolved_data, dict):
640 # Process dictionary values recursively
641 return {
642 key: resolve_hieraconfurations_for_serialization(value)
643 for key, value in resolved_data.items()
644 }
646 elif isinstance(resolved_data, (list, tuple)):
647 # Process sequence elements recursively
648 resolved_items = [resolve_hieraconfurations_for_serialization(item) for item in resolved_data]
649 return type(resolved_data)(resolved_items)
651 else:
652 # Primitive type or unknown structure - return as-is
653 return resolved_data
656# Generic dataclass editing with configurable value preservation
657T = TypeVar('T')
660def create_dataclass_for_editing(dataclass_type: Type[T], source_config: Any, preserve_values: bool = False, context_provider: Optional[Callable[[Any], None]] = None) -> T:
661 """Create dataclass for editing with configurable value preservation."""
662 if not is_dataclass(dataclass_type):
663 raise ValueError(f"{dataclass_type} must be a dataclass")
665 # Set up context if provider is given (e.g., thread-local storage)
666 if context_provider:
667 context_provider(source_config)
669 # Mathematical simplification: Convert verbose loop to unified comprehension
670 from hieraconf.placeholder import LazyDefaultPlaceholderService
671 field_values = {
672 f.name: (getattr(source_config, f.name) if preserve_values
673 else f.type() if is_dataclass(f.type) and LazyDefaultPlaceholderService.has_lazy_resolution(f.type)
674 else None)
675 for f in fields(dataclass_type)
676 }
678 return dataclass_type(**field_values)
684def rebuild_hieraconf_with_new_global_reference(
685 existing_hieraconf: Any,
686 new_global_config: Any,
687 global_config_type: Optional[Type] = None
688) -> Any:
689 """
690 Rebuild lazy config to reference new global config while preserving field states.
692 This function preserves the exact field state of the existing lazy config:
693 - Fields that are None (using lazy resolution) remain None
694 - Fields that have been explicitly set retain their concrete values
695 - Nested dataclass fields are recursively rebuilt to reference new global config
696 - The underlying global config reference is updated for None field resolution
698 Args:
699 existing_hieraconf: Current lazy config instance
700 new_global_config: New global config to reference for lazy resolution
701 global_config_type: Type of the global config (defaults to type of new_global_config)
703 Returns:
704 New lazy config instance with preserved field states and updated global reference
705 """
706 if existing_hieraconf is None:
707 return None
709 # Determine global config type
710 if global_config_type is None:
711 global_config_type = type(new_global_config)
713 # Set new global config in thread-local storage
714 ensure_global_config_context(global_config_type, new_global_config)
716 # Extract current field values without triggering lazy resolution - inline field processing pattern
717 def process_field_value(field_obj):
718 raw_value = object.__getattribute__(existing_hieraconf, field_obj.name)
720 if raw_value is not None and hasattr(raw_value, '__dataclass_fields__'):
721 try:
722 # Check if this is a concrete dataclass that should be converted to lazy
723 is_lazy = LazyDefaultPlaceholderService.has_lazy_resolution(type(raw_value))
725 if not is_lazy:
726 lazy_type = LazyDefaultPlaceholderService._get_lazy_type_for_base(type(raw_value))
728 if lazy_type:
729 # Convert concrete dataclass to lazy version while preserving ONLY non-default field values
730 # This allows fields that match class defaults to inherit from context
731 concrete_field_values = {}
732 for f in fields(raw_value):
733 field_value = object.__getattribute__(raw_value, f.name)
735 # Get the class default for this field
736 class_default = getattr(type(raw_value), f.name, None)
738 # Only preserve values that differ from class defaults
739 # This allows default values to be inherited from context
740 if field_value != class_default:
741 concrete_field_values[f.name] = field_value
743 logger.debug(f"Converting concrete {type(raw_value).__name__} to lazy version {lazy_type.__name__} for placeholder resolution")
744 return lazy_type(**concrete_field_values)
746 # If already lazy or no lazy version available, rebuild recursively
747 nested_result = rebuild_hieraconf_with_new_global_reference(raw_value, new_global_config, global_config_type)
748 return nested_result
749 except Exception as e:
750 logger.debug(f"Failed to rebuild nested config {field_obj.name}: {e}")
751 return raw_value
752 return raw_value
754 current_field_values = {f.name: process_field_value(f) for f in fields(existing_hieraconf)}
756 return type(existing_hieraconf)(**current_field_values)
759# Declarative Global Config Field Injection System
760# Moved inline imports to top-level
762# Naming configuration
763GLOBAL_CONFIG_PREFIX = "Global"
764LAZY_CONFIG_PREFIX = "Lazy"
766# Registry to accumulate all decorations before injection
767_pending_injections = {}
771class InheritAsNoneMeta(ABCMeta):
772 """
773 Metaclass that applies inherit_as_none modifications during class creation.
775 This runs BEFORE @dataclass and modifies the class definition to add
776 field overrides with None defaults for inheritance.
777 """
779 def __new__(mcs, name, bases, namespace, **kwargs):
780 # Create the class first
781 cls = super().__new__(mcs, name, bases, namespace)
783 # Check if this class should have inherit_as_none applied
784 if hasattr(cls, '_inherit_as_none') and cls._inherit_as_none:
785 # Add multiprocessing safety marker
786 cls._multiprocessing_safe = True
787 # Get explicitly defined fields (in this class's namespace)
788 explicitly_defined_fields = set()
789 if '__annotations__' in namespace:
790 for field_name in namespace['__annotations__']:
791 if field_name in namespace:
792 explicitly_defined_fields.add(field_name)
794 # Process parent classes to find fields that need None overrides
795 processed_fields = set()
796 for base in bases:
797 if hasattr(base, '__annotations__'):
798 for field_name, field_type in base.__annotations__.items():
799 if field_name in processed_fields:
800 continue
802 # Check if parent has concrete default
803 parent_has_concrete_default = False
804 if hasattr(base, field_name):
805 parent_value = getattr(base, field_name)
806 parent_has_concrete_default = parent_value is not None
808 # Add None override if needed
809 if (field_name not in explicitly_defined_fields and parent_has_concrete_default):
810 # Set the class attribute to None
811 setattr(cls, field_name, None)
813 # Ensure annotation exists
814 if not hasattr(cls, '__annotations__'):
815 cls.__annotations__ = {}
816 cls.__annotations__[field_name] = field_type
818 processed_fields.add(field_name)
819 else:
820 processed_fields.add(field_name)
822 return cls
824 def __reduce__(cls):
825 """Make classes with this metaclass pickle-safe for multiprocessing."""
826 # Filter out problematic descriptors that cause conflicts during pickle/unpickle
827 safe_dict = {}
828 for key, value in cls.__dict__.items():
829 # Skip descriptors that cause conflicts
830 if hasattr(value, '__get__') and hasattr(value, '__set__'):
831 continue # Skip data descriptors
832 if hasattr(value, '__dict__') and hasattr(value, '__class__'):
833 # Skip complex objects that might have descriptor conflicts
834 if 'descriptor' in str(type(value)).lower():
835 continue
836 # Include safe attributes
837 safe_dict[key] = value
839 # Return reconstruction using the base type (not the metaclass)
840 return (type, (cls.__name__, cls.__bases__, safe_dict))
843def create_global_default_decorator(target_config_class: Type):
844 """
845 Create a decorator factory for a specific global config class.
847 The decorator accumulates all decorations, then injects all fields at once
848 when the module finishes loading. Also creates lazy versions of all decorated configs.
849 """
850 target_class_name = target_config_class.__name__
851 if target_class_name not in _pending_injections:
852 _pending_injections[target_class_name] = {
853 'target_class': target_config_class,
854 'configs_to_inject': []
855 }
857 def global_default_decorator(cls=None, *, optional: bool = False, inherit_as_none: bool = True, ui_hidden: bool = False):
858 """
859 Decorator that can be used with or without parameters.
861 Args:
862 cls: The class being decorated (when used without parentheses)
863 optional: Whether to wrap the field type with Optional (default: False)
864 inherit_as_none: Whether to set all inherited fields to None by default (default: True)
865 ui_hidden: Whether to hide from UI (apply decorator but don't inject into global config) (default: False)
866 """
867 def decorator(actual_cls):
868 # Apply inherit_as_none by modifying class BEFORE @dataclass (multiprocessing-safe)
869 if inherit_as_none:
870 # Mark the class for inherit_as_none processing
871 actual_cls._inherit_as_none = True
873 # Apply inherit_as_none logic by directly modifying the class definition
874 # This must happen BEFORE @dataclass processes the class
875 explicitly_defined_fields = set()
876 if hasattr(actual_cls, '__annotations__'):
877 for field_name in actual_cls.__annotations__:
878 # Check if field has a concrete default value in THIS class definition (not inherited)
879 if field_name in actual_cls.__dict__: # Only fields defined in THIS class
880 field_value = actual_cls.__dict__[field_name]
881 # Only consider it explicitly defined if it has a concrete value (not None)
882 if field_value is not None:
883 explicitly_defined_fields.add(field_name)
885 # Process parent classes to find fields that need None overrides
886 processed_fields = set()
887 fields_set_to_none = set() # Track which fields were actually set to None
888 for base in actual_cls.__bases__:
889 if hasattr(base, '__annotations__'):
890 for field_name, field_type in base.__annotations__.items():
891 if field_name in processed_fields:
892 continue
894 # Set inherited fields to None (except explicitly defined ones)
895 if field_name not in explicitly_defined_fields:
896 # CRITICAL: Force the field to be seen as locally defined by @dataclass
897 # We need to ensure @dataclass processes this as a local field, not inherited
899 # 1. Set the class attribute to None
900 setattr(actual_cls, field_name, None)
901 fields_set_to_none.add(field_name)
903 # 2. Ensure annotation exists in THIS class
904 if not hasattr(actual_cls, '__annotations__'):
905 actual_cls.__annotations__ = {}
906 actual_cls.__annotations__[field_name] = field_type
908 processed_fields.add(field_name)
910 # Note: We modify class attributes here, but we also need to fix the dataclass
911 # field definitions after @dataclass runs, since @dataclass processes the MRO
912 # and may use parent class field definitions instead of our modified attributes.
914 # Generate field and class names
915 field_name = _camel_to_snake(actual_cls.__name__)
916 lazy_class_name = f"{LAZY_CONFIG_PREFIX}{actual_cls.__name__}"
918 # Mark class with ui_hidden metadata for UI layer to check
919 # This allows the config to remain in the context (for lazy resolution)
920 # while being hidden from UI rendering
921 if ui_hidden:
922 actual_cls._ui_hidden = True
924 # Check if class is abstract (has unimplemented abstract methods)
925 # Abstract classes should NEVER be injected into GlobalPipelineConfig
926 # because they can't be instantiated
927 # NOTE: We need to check if the class ITSELF is abstract, not just if it inherits from ABC
928 # Concrete subclasses of abstract classes should still be injected
929 # We check for __abstractmethods__ attribute which exists even before @dataclass runs
930 # (it's set by ABCMeta when the class is created)
931 is_abstract = hasattr(actual_cls, '__abstractmethods__') and len(actual_cls.__abstractmethods__) > 0
933 # Add to pending injections for field injection
934 # Skip injection for abstract classes (they can't be instantiated)
935 # For concrete classes: inject even if ui_hidden (needed for lazy resolution context)
936 if not is_abstract:
937 _pending_injections[target_class_name]['configs_to_inject'].append({
938 'config_class': actual_cls,
939 'field_name': field_name,
940 'lazy_class_name': lazy_class_name,
941 'optional': optional, # Store the optional flag
942 'inherit_as_none': inherit_as_none, # Store the inherit_as_none flag
943 'ui_hidden': ui_hidden # Store the ui_hidden flag for field metadata
944 })
946 # Immediately create lazy version of this config (not dependent on injection)
949 lazy_class = LazyDataclassFactory.make_lazy_simple(
950 base_class=actual_cls,
951 lazy_class_name=lazy_class_name
952 )
954 # Export lazy class to config module immediately
955 config_module = sys.modules[actual_cls.__module__]
956 setattr(config_module, lazy_class_name, lazy_class)
958 # Also mark lazy class with ui_hidden metadata
959 if ui_hidden:
960 lazy_class._ui_hidden = True
962 # CRITICAL: Post-process dataclass fields after @dataclass has run
963 # This fixes the constructor behavior for inherited fields that should be None
964 if inherit_as_none and hasattr(actual_cls, '__dataclass_fields__'):
965 _fix_dataclass_field_defaults_post_processing(actual_cls, fields_set_to_none)
967 return actual_cls
969 # Handle both @decorator and @decorator() usage
970 if cls is None:
971 # Called with parentheses: @decorator(optional=True)
972 return decorator
973 else:
974 # Called without parentheses: @decorator
975 return decorator(cls)
977 return global_default_decorator
980def _fix_dataclass_field_defaults_post_processing(cls: Type, fields_set_to_none: set) -> None:
981 """
982 Fix dataclass field defaults after @dataclass has processed the class.
984 This is necessary because @dataclass processes the MRO and may use parent class
985 field definitions instead of our modified class attributes. We need to ensure
986 that fields we set to None actually use None as the default in the constructor.
987 """
988 import dataclasses
990 # Store the original __init__ method
991 original_init = cls.__init__
993 def custom_init(self, **kwargs):
994 """Custom __init__ that ensures inherited fields use None defaults."""
995 # For fields that should be None, set them to None if not explicitly provided
996 for field_name in fields_set_to_none:
997 if field_name not in kwargs:
998 kwargs[field_name] = None
1000 # Call the original __init__ with modified kwargs
1001 original_init(self, **kwargs)
1003 # Replace the __init__ method
1004 cls.__init__ = custom_init
1006 # Also update the field defaults for consistency
1007 for field_name in fields_set_to_none:
1008 if field_name in cls.__dataclass_fields__:
1009 # Get the field object
1010 field_obj = cls.__dataclass_fields__[field_name]
1012 # Update the field default to None (overriding any parent class default)
1013 field_obj.default = None
1014 field_obj.default_factory = dataclasses.MISSING
1016 # Also ensure the class attribute is None (should already be set, but double-check)
1017 setattr(cls, field_name, None)
1021def _inject_all_pending_fields():
1022 """Inject all accumulated fields at once."""
1023 for target_name, injection_data in _pending_injections.items():
1024 target_class = injection_data['target_class']
1025 configs = injection_data['configs_to_inject']
1027 if configs: # Only inject if there are configs to inject
1028 _inject_multiple_fields_into_dataclass(target_class, configs)
1030def _camel_to_snake(name: str) -> str:
1031 """Convert CamelCase to snake_case for field names."""
1032 s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
1033 return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
1035def _inject_multiple_fields_into_dataclass(target_class: Type, configs: List[Dict]) -> None:
1036 """Mathematical simplification: Batch field injection with direct dataclass recreation."""
1037 # Imports moved to top-level
1039 # Direct field reconstruction - guaranteed by dataclass contract
1040 existing_fields = [
1041 (f.name, f.type, field(default_factory=f.default_factory) if f.default_factory != MISSING
1042 else f.default if f.default != MISSING else f.type)
1043 for f in fields(target_class)
1044 ]
1046 # Mathematical simplification: Unified field construction with algebraic common factors
1047 def create_field_definition(config):
1048 """Create field definition with optional and inherit_as_none support."""
1049 field_type = config['config_class']
1050 is_optional = config.get('optional', False)
1051 is_ui_hidden = config.get('ui_hidden', False)
1053 # Algebraic simplification: factor out common default_value logic
1054 if is_optional:
1055 field_type = Union[field_type, type(None)]
1056 default_value = None
1057 else:
1058 # Both inherit_as_none and regular cases use same default factory
1059 # Add ui_hidden metadata to the field so UI layer can check it
1060 default_value = field(default_factory=field_type, metadata={'ui_hidden': is_ui_hidden})
1062 return (config['field_name'], field_type, default_value)
1064 all_fields = existing_fields + [create_field_definition(config) for config in configs]
1066 # Direct dataclass recreation - fail-loud
1067 new_class = make_dataclass(
1068 target_class.__name__,
1069 all_fields,
1070 bases=target_class.__bases__,
1071 frozen=target_class.__dataclass_params__.frozen
1072 )
1074 # CRITICAL: Preserve original module for proper imports in generated code
1075 # make_dataclass() sets __module__ to the caller's module (lazy_factory.py)
1076 # We need to set it to the target class's original module for correct import paths
1077 new_class.__module__ = target_class.__module__
1079 # Sibling inheritance is now handled by the dual-axis resolver system
1081 # Direct module replacement
1082 module = sys.modules[target_class.__module__]
1083 setattr(module, target_class.__name__, new_class)
1084 globals()[target_class.__name__] = new_class
1086 # Mathematical simplification: Extract common module assignment pattern
1087 def _register_lazy_class(lazy_class, class_name, module_name):
1088 """Register lazy class in both module and global namespace."""
1089 setattr(sys.modules[module_name], class_name, lazy_class)
1090 globals()[class_name] = lazy_class
1092 # Create lazy classes and recreate PipelineConfig inline
1093 for config in configs:
1094 lazy_class = LazyDataclassFactory.make_lazy_simple(
1095 base_class=config['config_class'],
1096 lazy_class_name=config['lazy_class_name']
1097 )
1098 _register_lazy_class(lazy_class, config['lazy_class_name'], config['config_class'].__module__)
1100 # Create lazy version of the updated global config itself with proper naming
1101 # Global configs must start with GLOBAL_CONFIG_PREFIX - fail-loud if not
1102 if not target_class.__name__.startswith(GLOBAL_CONFIG_PREFIX):
1103 raise ValueError(f"Target class '{target_class.__name__}' must start with '{GLOBAL_CONFIG_PREFIX}' prefix")
1105 # Remove global prefix (GlobalPipelineConfig → PipelineConfig)
1106 lazy_global_class_name = target_class.__name__[len(GLOBAL_CONFIG_PREFIX):]
1108 lazy_global_class = LazyDataclassFactory.make_lazy_simple(
1109 base_class=new_class,
1110 lazy_class_name=lazy_global_class_name
1111 )
1113 # Use extracted helper for consistent registration
1114 _register_lazy_class(lazy_global_class, lazy_global_class_name, target_class.__module__)
1120def auto_create_decorator(global_config_class):
1121 """
1122 Decorator that automatically creates:
1123 1. A field injection decorator for other configs to use
1124 2. A lazy version of the global config itself
1126 Global config classes must start with "Global" prefix.
1127 """
1128 # Validate naming convention
1129 if not global_config_class.__name__.startswith(GLOBAL_CONFIG_PREFIX):
1130 raise ValueError(f"Global config class '{global_config_class.__name__}' must start with '{GLOBAL_CONFIG_PREFIX}' prefix")
1132 decorator_name = _camel_to_snake(global_config_class.__name__)
1133 decorator = create_global_default_decorator(global_config_class)
1135 # Export decorator to module globals
1136 module = sys.modules[global_config_class.__module__]
1137 setattr(module, decorator_name, decorator)
1139 # Lazy global config will be created after field injection
1141 return global_config_class