fix: resolve Decimal serialisation and UUID generation errors

- Add orjson for high-performance JSON serialisation with Decimal support
- Convert Decimal to string (not float) to preserve financial precision
- Implement client-side UUID v5 generation to remove uuid-ossp dependency
- Configure Supabase service role client with proper authentication
- Improve error handling in portfolio save flow with return value checks

This fixes three critical issues:
1. Redis cache Decimal serialisation errors
2. PostgreSQL uuid_generate_v5 function missing errors
3. Supabase RLS failures due to improper authentication configuration

Files modified:
- backend/utils/serialisation.py (new): Decimal-safe JSON utilities
- backend/utils/uuid_generator.py (new): Client-side UUID generation
- backend/caching/redis_cache.py: Use orjson for cache serialisation
- backend/database.py: Python UUID generation + Decimal handling
- app.py: Improved error handling and validation
- pyproject.toml: Add orjson dependency

app.py

@@ -812,15 +812,22 @@ async def run_analysis_with_ui_update(
                     logger.error("No valid user_id available for portfolio creation")
                     raise ValueError("User ID not available")
 
-                await db.save_portfolio(
+                portfolio_saved = await db.save_portfolio(
                     portfolio_id=portfolio_id,
                     user_id=user_id,
                     name=f"Analysis {datetime.now().strftime('%Y-%m-%d %H:%M')}",
                     risk_tolerance='moderate'
                 )
-                logger.info(f"Portfolio {portfolio_id} created for user {user_id}")
+
+                if portfolio_saved:
+                    logger.info(f"✓ Portfolio {portfolio_id} created for user {user_id}")
+                else:
+                    logger.error(f"✗ Failed to create portfolio {portfolio_id} in database")
+                    raise ValueError(f"Portfolio creation failed for {portfolio_id}")
+
             except Exception as e:
-                logger.warning(f"Failed to save portfolio: {e}")
+                logger.error(f"✗ Failed to save portfolio: {e}")
+                raise ValueError(f"Database error: {e}") from e
         else:
             logger.info(f"Demo mode: Portfolio {portfolio_id} created (ephemeral, not saved to database)")
 
@@ -876,9 +883,12 @@ async def run_analysis_with_ui_update(
                 if save_result:
                     logger.info(f"✓ Successfully saved analysis for portfolio {portfolio_id}")
                 else:
-                    logger.warning(f"✗ Failed to save analysis for portfolio {portfolio_id} (returned False)")
+                    logger.error(f"✗ Failed to save analysis for portfolio {portfolio_id} (returned False)")
+                    # Note: We don't raise here because the analysis itself succeeded
+                    # The user still gets their results even if database save fails
             except Exception as e:
                 logger.error(f"✗ Exception saving analysis for portfolio {portfolio_id}: {e}", exc_info=True)
+                # Note: We don't raise here because the analysis itself succeeded
         else:
             logger.info(f"Demo mode: Analysis for portfolio {portfolio_id} not saved (ephemeral session)")
 

backend/caching/redis_cache.py

@@ -420,7 +420,14 @@ class HybridCache:
 
         try:
             if self.redis_client:
-                serialised = serialiser(value) if serialiser else json.dumps(value)
+                # Use custom serialiser if provided, otherwise use orjson with Decimal support
+                if serialiser:
+                    serialised = serialiser(value)
+                else:
+                    # Import here to avoid circular dependencies
+                    from backend.utils.serialisation import dumps
+                    serialised = dumps(value)
+
                 if ttl:
                     self.redis_client.setex(key, ttl, serialised)
                 else:

backend/database.py

@@ -4,42 +4,17 @@ Handles Supabase PostgreSQL connections and operations.
 """
 
 from typing import Optional, Dict, Any, List
-from decimal import Decimal
-from datetime import datetime, date
 from supabase import create_client, Client
 from backend.config import settings
+from backend.utils.serialisation import decimal_to_json_safe
+from backend.utils.uuid_generator import string_to_uuid
 import logging
 
 logger = logging.getLogger(__name__)
 
 
-def decimal_to_float(obj: Any) -> Any:
-    """Recursively convert non-JSON-serializable objects in nested structures.
-
-    Converts:
-    - Decimal objects to float (financial calculations)
-    - datetime/date objects to ISO format strings (timestamps)
-
-    This is needed because JSON serialization doesn't support these types,
-    but financial data often contains them.
-
-    Args:
-        obj: Any Python object (dict, list, Decimal, datetime, etc.)
-
-    Returns:
-        Same structure with non-serializable objects converted
-    """
-    if isinstance(obj, Decimal):
-        return float(obj)
-    elif isinstance(obj, (datetime, date)):
-        return obj.isoformat()
-    elif isinstance(obj, dict):
-        return {k: decimal_to_float(v) for k, v in obj.items()}
-    elif isinstance(obj, list):
-        return [decimal_to_float(item) for item in obj]
-    elif isinstance(obj, tuple):
-        return tuple(decimal_to_float(item) for item in obj)
-    return obj
+# Re-export for backwards compatibility
+__all__ = ['Database', 'decimal_to_json_safe']
 
 
 class Database:
@@ -59,19 +34,32 @@ class Database:
                 if settings.supabase_service_role_key:
                     api_key = settings.supabase_service_role_key
                     logger.info("Using Supabase service role key (bypasses RLS)")
+
+                    # Simple client creation for backend service operations
+                    # Service role automatically bypasses RLS, no special config needed
+                    self.client = create_client(
+                        settings.supabase_url,
+                        api_key
+                    )
+                    logger.info("Supabase service role client initialised successfully")
+
                 elif settings.supabase_key:
                     api_key = settings.supabase_key
-                    logger.warning("Using anon key - history features may not work due to RLS. Add SUPABASE_SERVICE_ROLE_KEY for full functionality.")
+                    logger.warning(
+                        "Using anon key - database operations may fail due to RLS policies. "
+                        "Add SUPABASE_SERVICE_ROLE_KEY environment variable for full functionality."
+                    )
+
+                    # Anon key client (RLS enforced, requires user context)
+                    self.client = create_client(
+                        settings.supabase_url,
+                        api_key
+                    )
+                    logger.info("Supabase anon client initialised (RLS enforced)")
+
                 else:
                     logger.error("No Supabase API key found (need either SUPABASE_SERVICE_ROLE_KEY or SUPABASE_KEY)")
                     return
-
-                # Simple client creation for backend service (no session management needed)
-                self.client = create_client(
-                    settings.supabase_url,
-                    api_key
-                )
-                logger.info("Supabase client initialised successfully")
             except Exception as e:
                 logger.error(f"Failed to initialise Supabase client: {e}")
                 logger.info("Running in demo mode without database")
@@ -87,28 +75,24 @@ class Database:
         return self.client is not None
 
     def _string_to_uuid(self, string_id: str) -> str:
-        """Convert string ID to deterministic UUID using uuid_generate_v5.
+        """Convert string ID to deterministic UUID.
+
+        Uses Python UUID v5 generation for deterministic conversion.
+        This is faster than database-side generation and doesn't
+        require the uuid-ossp extension.
 
         Args:
-            string_id: String identifier (e.g., 'demo_20251117_143022')
+            string_id: Human-readable ID (e.g., 'demo_20241119_105323')
 
         Returns:
-            UUID string
-        """
-        if not self.is_connected():
-            return string_id
-
-        try:
-            result = self.client.rpc('string_to_portfolio_uuid', {
-                'input_string': string_id
-            }).execute()
+            UUID string (deterministic, always same for same input)
 
-            if result.data:
-                return str(result.data)
-        except Exception as e:
-            logger.warning(f"Failed to convert string to UUID: {e}")
-
-        return string_id
+        Example:
+            >>> db = Database()
+            >>> db._string_to_uuid('demo_20241119_105323')
+            '...'  # Same UUID every time for this input
+        """
+        return string_to_uuid(string_id)
 
     async def ensure_demo_user_exists(self) -> str:
         """Ensure demo user exists in database.
@@ -200,9 +184,9 @@ class Database:
         try:
             uuid_portfolio_id = self._string_to_uuid(portfolio_id)
 
-            # Convert all Decimal objects to float for JSON serialization
-            # Financial calculations often return Decimals which aren't JSON-serializable
-            analysis_clean = decimal_to_float(analysis_results)
+            # Convert all Decimal objects to string for JSON serialisation
+            # This preserves precision - float conversion loses pennies in large numbers
+            analysis_clean = decimal_to_json_safe(analysis_results)
 
             data = {
                 'portfolio_id': uuid_portfolio_id,

backend/utils/__init__.py

@@ -0,0 +1 @@
+"""Utility modules for the Portfolio Intelligence Platform."""

backend/utils/serialisation.py

@@ -0,0 +1,120 @@
+"""JSON serialisation utilities with Decimal precision preservation.
+
+This module provides safe JSON serialisation for financial data,
+ensuring Decimal types are converted to strings (not floats) to
+preserve full precision. Converting Decimal to float can cause
+monetary errors in large values (e.g., £1,000,000,000.47 → £1,000,000,000.50).
+"""
+
+from typing import Any
+from decimal import Decimal
+from datetime import datetime, date
+import orjson
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def default_handler(obj: Any) -> Any:
+    """Custom default handler for orjson serialisation.
+
+    Handles types that aren't natively JSON-serialisable:
+    - Decimal → string (preserves precision for financial data)
+    - datetime/date → ISO format string
+
+    Args:
+        obj: Object to serialise
+
+    Returns:
+        JSON-serialisable representation
+
+    Raises:
+        TypeError: If object type is not handled
+
+    Example:
+        >>> import orjson
+        >>> from decimal import Decimal
+        >>> orjson.dumps({"price": Decimal("19.99")}, default=default_handler)
+        b'{"price":"19.99"}'
+    """
+    if isinstance(obj, Decimal):
+        # CRITICAL: Convert to string, NOT float
+        # Float loses precision in large numbers
+        return str(obj)
+    elif isinstance(obj, (datetime, date)):
+        return obj.isoformat()
+    raise TypeError(f"Type {type(obj)} not serialisable")
+
+
+def dumps(obj: Any) -> bytes:
+    """Serialise object to JSON bytes with Decimal support.
+
+    Args:
+        obj: Object to serialise
+
+    Returns:
+        JSON bytes with Decimal values as strings
+
+    Example:
+        >>> from decimal import Decimal
+        >>> dumps({"price": Decimal("19.99"), "quantity": 5})
+        b'{"price":"19.99","quantity":5}'
+    """
+    return orjson.dumps(obj, default=default_handler)
+
+
+def dumps_str(obj: Any) -> str:
+    """Serialise object to JSON string with Decimal support.
+
+    Args:
+        obj: Object to serialise
+
+    Returns:
+        JSON string with Decimal values as strings
+
+    Example:
+        >>> from decimal import Decimal
+        >>> dumps_str({"price": Decimal("19.99")})
+        '{"price":"19.99"}'
+    """
+    return dumps(obj).decode('utf-8')
+
+
+def decimal_to_json_safe(obj: Any) -> Any:
+    """Recursively convert non-JSON-serialisable objects to safe types.
+
+    This function provides an alternative approach when you need
+    Python dict/list structures rather than JSON bytes.
+
+    Converts:
+    - Decimal objects → string (preserves full precision)
+    - datetime/date objects → ISO format strings
+
+    This preserves precision for financial calculations, which is critical
+    as converting Decimal to float can lose pence in large numbers.
+
+    Args:
+        obj: Any Python object (dict, list, Decimal, datetime, etc.)
+
+    Returns:
+        Same structure with non-serialisable objects converted
+
+    Example:
+        >>> from decimal import Decimal
+        >>> decimal_to_json_safe({"price": Decimal('19.99')})
+        {'price': '19.99'}
+        >>> decimal_to_json_safe([Decimal('1.23'), Decimal('4.56')])
+        ['1.23', '4.56']
+    """
+    if isinstance(obj, Decimal):
+        # CRITICAL: String preserves precision, float does not
+        return str(obj)
+    elif isinstance(obj, (datetime, date)):
+        return obj.isoformat()
+    elif isinstance(obj, dict):
+        return {k: decimal_to_json_safe(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        return [decimal_to_json_safe(item) for item in obj]
+    elif isinstance(obj, tuple):
+        return tuple(decimal_to_json_safe(item) for item in obj)
+    return obj

backend/utils/uuid_generator.py

@@ -0,0 +1,151 @@
+"""UUID generation utilities for deterministic ID conversion.
+
+This module provides client-side UUID generation to convert human-readable
+identifiers (like 'portfolio-demo', 'user-123') into deterministic UUIDs.
+This approach is faster and more portable than database-side generation.
+"""
+
+import uuid
+from typing import Union
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Use null UUID as root namespace for application IDs
+NULL_NAMESPACE = uuid.UUID('00000000-0000-0000-0000-000000000000')
+
+
+class DeterministicUUIDGenerator:
+    """Generates deterministic UUIDs for consistent ID conversion.
+
+    This class provides a standardised way to convert human-readable
+    identifiers into deterministic UUIDs that remain consistent across
+    sessions using UUID v5 (SHA-1 based).
+
+    Args:
+        namespace: Base UUID for the namespace. Defaults to null UUID.
+
+    Example:
+        >>> generator = DeterministicUUIDGenerator()
+        >>> portfolio_id = generator.generate('portfolio-demo')
+        >>> same_id = generator.generate('portfolio-demo')
+        >>> portfolio_id == same_id
+        True
+    """
+
+    def __init__(self, namespace: Union[str, uuid.UUID, None] = None) -> None:
+        """Initialise the UUID generator with a namespace.
+
+        Args:
+            namespace: Base UUID for generation. If None, uses null UUID.
+        """
+        if namespace is None:
+            self.namespace = NULL_NAMESPACE
+        elif isinstance(namespace, str):
+            self.namespace = uuid.UUID(namespace)
+        else:
+            self.namespace = namespace
+
+        logger.debug(f"UUID generator initialised with namespace: {self.namespace}")
+
+    def generate(self, identifier: str) -> uuid.UUID:
+        """Generate deterministic UUID from identifier.
+
+        Uses UUID v5 (SHA-1 hashing) to create a deterministic UUID.
+        Same input always produces the same output.
+
+        Args:
+            identifier: Human-readable identifier string
+
+        Returns:
+            Deterministic UUID that will always be the same for this identifier
+
+        Example:
+            >>> gen = DeterministicUUIDGenerator()
+            >>> gen.generate('demo-portfolio')
+            UUID('...')
+        """
+        if not identifier:
+            raise ValueError("Identifier cannot be empty")
+
+        return uuid.uuid5(namespace=self.namespace, name=identifier)
+
+    def generate_str(self, identifier: str) -> str:
+        """Generate deterministic UUID as string.
+
+        Args:
+            identifier: Human-readable identifier string
+
+        Returns:
+            UUID as string with hyphens (e.g., '550e8400-e29b-41d4-a716-446655440000')
+        """
+        return str(self.generate(identifier))
+
+    def generate_hex(self, identifier: str) -> str:
+        """Generate deterministic UUID as hex string (no hyphens).
+
+        Args:
+            identifier: Human-readable identifier string
+
+        Returns:
+            UUID as hex string without hyphens (e.g., '550e8400e29b41d4a716446655440000')
+        """
+        return self.generate(identifier).hex
+
+
+# Global instance for application use
+_generator = DeterministicUUIDGenerator()
+
+
+def string_to_uuid(identifier: str) -> str:
+    """Convert string identifier to deterministic UUID string.
+
+    Convenience function using the global UUID generator.
+
+    Args:
+        identifier: Human-readable identifier (e.g., 'user-123', 'demo-portfolio')
+
+    Returns:
+        UUID string representation
+
+    Example:
+        >>> string_to_uuid('portfolio-demo')
+        '...'
+    """
+    return _generator.generate_str(identifier)
+
+
+def validate_or_generate_uuid(provided_uuid: Union[str, None], fallback_identifier: str) -> str:
+    """Validate provided UUID or generate from identifier.
+
+    Args:
+        provided_uuid: UUID string from client (if provided)
+        fallback_identifier: Identifier to use if validation fails
+
+    Returns:
+        Valid UUID string
+
+    Example:
+        >>> validate_or_generate_uuid(None, 'user-123')
+        '...'
+        >>> validate_or_generate_uuid('invalid', 'user-123')
+        '...'
+    """
+    if provided_uuid:
+        try:
+            # Validate it's a proper UUID
+            validated = uuid.UUID(provided_uuid)
+            # Verify it matches expected deterministic value
+            expected = _generator.generate(fallback_identifier)
+            if validated == expected:
+                return str(validated)
+            else:
+                logger.warning(
+                    f"Provided UUID {provided_uuid} doesn't match expected "
+                    f"deterministic UUID for {fallback_identifier}"
+                )
+        except (ValueError, AttributeError) as e:
+            logger.warning(f"Invalid UUID provided: {provided_uuid}, error: {e}")
+
+    # Generate server-side if validation fails or no UUID provided
+    return string_to_uuid(fallback_identifier)

pyproject.toml

@@ -26,6 +26,7 @@ dependencies = [
     # Caching & Rate Limiting
     "redis>=5.0.0",
     "upstash-redis>=0.15.0",
+    "orjson>=3.10.0",
     # Quantitative Finance (P0 - Deterministic Models)
     # Research validated: Package name is 'pyportfolioopt', import as 'pypfopt'
     "pyportfolioopt>=1.5.6",

uv.lock

@@ -3167,6 +3167,7 @@ dependencies = [
     { name = "langgraph" },
     { name = "mcp" },
     { name = "numpy" },
+    { name = "orjson" },
     { name = "pandas" },
     { name = "psycopg2-binary" },
     { name = "pydantic" },
@@ -3212,6 +3213,7 @@ requires-dist = [
     { name = "langgraph", specifier = ">=0.1.0" },
     { name = "mcp", specifier = ">=1.15.0" },
     { name = "numpy", specifier = ">=2.0.0" },
+    { name = "orjson", specifier = ">=3.10.0" },
     { name = "pandas", specifier = ">=2.2.3" },
     { name = "psycopg2-binary", specifier = ">=2.9.9" },
     { name = "pydantic", specifier = ">=2.5.0" },