preprocessing bug fix

dynamix/preprocessing.py

@@ -27,8 +27,12 @@ class DataPreprocessor:
         # Parameters for inverse transformations
         self.box_cox_params_list = None
         self.detrending_params_list = None
+        self.transformation_mean = None
+        self.transformation_std = None
+
         self.context_mean = None
         self.context_std = None
+
         self.original_context = None
         self.batch_size = None
         self.feature_dim = None
@@ -46,6 +50,12 @@ class DataPreprocessor:
         """
         # Store original context for inverse transformations
         self.original_context = context.clone()
+
+        # Before transformations standardize data
+        if self.box_cox or self.detrending:
+            self.transformation_mean = torch.mean(context, dim=0)
+            self.transformation_std = torch.std(context, dim=0)
+            context = (context - self.transformation_mean.unsqueeze(0)) / self.transformation_std.unsqueeze(0)
         
         # Apply Box-Cox transformation for each batch
         if self.box_cox:
@@ -99,6 +109,10 @@ class DataPreprocessor:
                 batch_output = output[:, b, :]
                 batch_output = BoxCoxTransformer.inverse_transform(batch_output, self.box_cox_params_list[b])
                 output[:, b, :] = batch_output
+
+        # Apply inverse standardization if transformation was applied
+        if self.transformation_mean is not None and self.transformation_std is not None:
+            output = output * self.transformation_std.unsqueeze(0) + self.transformation_mean.unsqueeze(0)
                 
         return output
     
@@ -232,7 +246,7 @@ class DataPreprocessor:
         # Apply transformations (Box-Cox, detrending)
         context = self._apply_transformations(context)
         
-        # Standardize data if requested
+        # Standardize data
         context = self._standardize_data(context)
         
         # Apply embedding to reach model dimension

dynamix/preprocessing_utilities.py

@@ -6,6 +6,7 @@ import random
 from statsmodels.tsa.stattools import acf
 from scipy.ndimage import gaussian_filter1d
 from scipy import optimize
+from scipy.optimize import curve_fit
 
 
 class TimeSeriesProcessor:
@@ -434,11 +435,6 @@ class Detrending:
         """
         # Convert to numpy
         data_np, is_torch, device, dtype = TimeSeriesProcessor.to_numpy(data)
-
-        # Apply min max scaling for a more stable trend fit
-        _min = np.min(data_np)
-        _max = np.max(data_np)
-        data_scaled = (data_np - _min) / (_max - _min)
         
         seq_length, n_dims = data_np.shape
         detrended_data = np.zeros_like(data_np)
@@ -446,10 +442,10 @@ class Detrending:
         
         for dim in range(n_dims):
             # Define the objective function for this dimension
-            objective = lambda params: Detrending.fit_objective(params, data_scaled[:, dim])
+            objective = lambda params: Detrending.fit_objective(params, data_np[:, dim])
             
             # Initial parameter guess
-            initial_params = [0.0, 1.0, data_scaled[0,dim]]
+            initial_params = [0.0, 1.0, data_np[0,dim]]
             
             # Bounds for parameters
             bounds = [(None, None), (0.0, 3.0), (None, None)]
@@ -467,10 +463,7 @@ class Detrending:
                     'maxcor': 10
                 }
             )
-            optimal_params = np.round(result.x, 3)
-            #Adjust params to min max scale
-            optimal_params[0] = (_max - _min) * optimal_params[0]
-            optimal_params[2] = (_max - _min) * optimal_params[2] + _min
+            optimal_params = np.round(result.x, 10)
             
             # Calculate trend and detrend the data
             t = np.arange(1, seq_length + 1)