Use TimeSeriesDataset() separately for training data and for validation data

[fokoa]

I started handling Pytorch Lightning. I want to use the temporalfusion modeltransform model TFT getting started . I already have training and validation data ready-to-use, and I want to charge them independently with the TimeSeriesDataset module. Here is an extract from my code:

min_encoder_len = 1
    max_encoder_len = 1
    max_prediction_len = 1
    batch_size = 32

    train_dataset = TimeSeriesDataSet(
        data=train[main_cols],
        time_idx="time_idx",
        target="target",
        group_ids=["group"],
        min_encoder_length=min_encoder_len,
        max_encoder_length=max_encoder_len,
        max_prediction_length=max_prediction_len
    )
    
    dev_dataset = TimeSeriesDataSet(
        data=dev[main_cols],
        time_idx="time_idx",
        target="target",
        group_ids=["group"],
        min_encoder_length=min_encoder_len,
        max_encoder_length=max_encoder_len,
        max_prediction_length=max_prediction_len
    )
    
    train_loader = train_dataset.to_dataloader(train=True, batch_size=batch_size, 
                                               shuffle=False, num_workers=4)
    dev_loader = dev_dataset.to_dataloader(train=False, batch_size=batch_size,
                                           shuffle=False, num_workers=4)
    
    early_stop = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=3, verbose=False, mode="min")
    lr_logger = LearningRateMonitor()
    trainer = pl.Trainer(
        max_epochs=20,
        accelerator="auto",
        gradient_clip_val=0.1,
        log_every_n_steps=1,
        limit_train_batches=30,
        callbacks=[early_stop, lr_logger]
        
    )
    
    tft = TemporalFusionTransformer.from_dataset(
        dataset=train_dataset,
        learning_rate=0.01,
        hidden_size=16,
        attention_head_size=1,
        dropout=1,
        hidden_continuous_size=5,
        output_size=6,
        loss=QuantileLoss(),
        log_interval=1,
        reduce_on_plateau_patience=4
    )
    
    print(f"Number of parameters in nework: {tft.size()/1e3}k")
    
    # find optimal learning rate (set limit_train_batches to 1.0 and log_interval = -1)
    res = Tuner(trainer).lr_find(
        model=tft,
        train_dataloaders=train_loader, 
        val_dataloaders=dev_loader,
        early_stop_threshold=10.0,
        max_lr=0.2
    )

    print(f"suggested learning rate: {res.suggestion()}")
    fig = res.plot(show=True, suggest=True)
    fig.show()

    # fit the model
    trainer.fit(
        model=tft,
        train_dataloaders=train_loader,
        val_dataloaders=dev_loader,
    )

When I execute it I get this error:

GPU available: False, used: False
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/utilities/parsing.py:196: UserWarning: Attribute 'loss' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['loss'])`.
  rank_zero_warn(
/home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/utilities/parsing.py:196: UserWarning: Attribute 'logging_metrics' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['logging_metrics'])`.
  rank_zero_warn(
Number of parameters in nework: 14.23k
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /home/fogang/venv/lib/python3.10/site-packages/IPython/core/magics/execution.py:1325 in time     │
│                                                                                                  │
│   1322 │   │   else:                                                                             │
│   1323 │   │   │   st = clock2()                                                                 │
│   1324 │   │   │   try:                                                                          │
│ ❱ 1325 │   │   │   │   exec(code, glob, local_ns)                                                │
│   1326 │   │   │   │   out=None                                                                  │
│   1327 │   │   │   │   # multi-line %%time case                                                  │
│   1328 │   │   │   │   if expr_val is not None:                                                  │
│ in <module>:4                                                                                    │
│                                                                                                  │
│ in seeding_oof:8                                                                                 │
│                                                                                                  │
│    5 │   │   print(4*'+++++++++')                                                                │
│    6 │   │   print(f"SEED:{seed}")                                                               │
│    7 │   │   print(4*'+++++++++')                                                                │
│ ❱  8 │   │   tmp_preds, tmp_score = oof(name=name, clf=model, train_df=train_df.copy(),          │
│    9 │   │   │   │   │   │   │   │      test_df=test_df.copy(), main_cols=main_cols, n_fold=n    │
│   10 │   │   if oof_preds is None:                                                               │
│   11 │   │   │   oof_preds = tmp_preds / len(seeds)                                              │
│                                                                                                  │
│ in oof:84                                                                                        │
│                                                                                                  │
│    81 │   │   print(f"Number of parameters in nework: {tft.size()/1e3}k")                        │
│    82 │   │                                                                                      │
│    83 │   │   # find optimal learning rate (set limit_train_batches to 1.0 and log_interval =    │
│ ❱  84 │   │   res = Tuner(trainer).lr_find(                                                      │
│    85 │   │   │   model=tft,                                                                     │
│    86 │   │   │   train_dataloaders=train_loader,                                                │
│    87 │   │   │   val_dataloaders=dev_loader,                                                    │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/tuner/tuning.py:175 in lr_find  │
│                                                                                                  │
│   172 │   │   lr_finder_callback._early_exit = True                                              │
│   173 │   │   self._trainer.callbacks = [lr_finder_callback] + self._trainer.callbacks           │
│   174 │   │                                                                                      │
│ ❱ 175 │   │   self._trainer.fit(model, train_dataloaders, val_dataloaders, datamodule)           │
│   176 │   │                                                                                      │
│   177 │   │   self._trainer.callbacks = [cb for cb in self._trainer.callbacks if cb is not lr_   │
│   178                                                                                            │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py:529 in fit   │
│                                                                                                  │
│    526 │   │   """                                                                               │
│    527 │   │   model = _maybe_unwrap_optimized(model)                                            │
│    528 │   │   self.strategy._lightning_module = model                                           │
│ ❱  529 │   │   call._call_and_handle_interrupt(                                                  │
│    530 │   │   │   self, self._fit_impl, model, train_dataloaders, val_dataloaders, datamodule,  │
│    531 │   │   )                                                                                 │
│    532                                                                                           │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py:42 in           │
│ _call_and_handle_interrupt                                                                       │
│                                                                                                  │
│    39 │   try:                                                                                   │
│    40 │   │   if trainer.strategy.launcher is not None:                                          │
│    41 │   │   │   return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer,    │
│ ❱  42 │   │   return trainer_fn(*args, **kwargs)                                                 │
│    43 │                                                                                          │
│    44 │   except _TunerExitException:                                                            │
│    45 │   │   _call_teardown_hook(trainer)                                                       │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py:568 in       │
│ _fit_impl                                                                                        │
│                                                                                                  │
│    565 │   │   │   model_provided=True,                                                          │
│    566 │   │   │   model_connected=self.lightning_module is not None,                            │
│    567 │   │   )                                                                                 │
│ ❱  568 │   │   self._run(model, ckpt_path=ckpt_path)                                             │
│    569 │   │                                                                                     │
│    570 │   │   assert self.state.stopped                                                         │
│    571 │   │   self.training = False                                                             │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py:953 in _run  │
│                                                                                                  │
│    950 │   │                                                                                     │
│    951 │   │   # hook                                                                            │
│    952 │   │   if self.state.fn == TrainerFn.FITTING:                                            │
│ ❱  953 │   │   │   call._call_callback_hooks(self, "on_fit_start")                               │
│    954 │   │   │   call._call_lightning_module_hook(self, "on_fit_start")                        │
│    955 │   │                                                                                     │
│    956 │   │   _log_hyperparams(self)                                                            │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py:193 in          │
│ _call_callback_hooks                                                                             │
│                                                                                                  │
│   190 │   │   fn = getattr(callback, hook_name)                                                  │
│   191 │   │   if callable(fn):                                                                   │
│   192 │   │   │   with trainer.profiler.profile(f"[Callback]{callback.state_key}.{hook_name}")   │
│ ❱ 193 │   │   │   │   fn(trainer, trainer.lightning_module, *args, **kwargs)                     │
│   194 │                                                                                          │
│   195 │   if pl_module:                                                                          │
│   196 │   │   # restore current_fx when nested context                                           │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/callbacks/lr_finder.py:125 in   │
│ on_fit_start                                                                                     │
│                                                                                                  │
│   122 │   │   │   raise _TunerExitException()                                                    │
│   123 │                                                                                          │
│   124 │   def on_fit_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> No   │
│ ❱ 125 │   │   self.lr_find(trainer, pl_module)                                                   │
│   126                                                                                            │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/callbacks/lr_finder.py:109 in   │
│ lr_find                                                                                          │
│                                                                                                  │
│   106 │                                                                                          │
│   107 │   def lr_find(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") -> None:     │
│   108 │   │   with isolate_rng():                                                                │
│ ❱ 109 │   │   │   self.optimal_lr = _lr_find(                                                    │
│   110 │   │   │   │   trainer,                                                                   │
│   111 │   │   │   │   pl_module,                                                                 │
│   112 │   │   │   │   min_lr=self._min_lr,                                                       │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/tuner/lr_finder.py:269 in       │
│ _lr_find                                                                                         │
│                                                                                                  │
│   266 │   lr_finder._exchange_scheduler(trainer)                                                 │
│   267 │                                                                                          │
│   268 │   # Fit, lr & loss logged in callback                                                    │
│ ❱ 269 │   _try_loop_run(trainer, params)                                                         │
│   270 │                                                                                          │
│   271 │   # Prompt if we stopped early                                                           │
│   272 │   if trainer.global_step != num_training + start_steps:                                  │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/tuner/lr_finder.py:499 in       │
│ _try_loop_run                                                                                    │
│                                                                                                  │
│   496 │   loop = trainer.fit_loop                                                                │
│   497 │   loop.load_state_dict(deepcopy(params["loop_state_dict"]))                              │
│   498 │   loop.restarting = False                                                                │
│ ❱ 499 │   loop.run()                                                                             │
│   500                                                                                            │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py:201 in run    │
│                                                                                                  │
│   198 │   │   while not self.done:                                                               │
│   199 │   │   │   try:                                                                           │
│   200 │   │   │   │   self.on_advance_start()                                                    │
│ ❱ 201 │   │   │   │   self.advance()                                                             │
│   202 │   │   │   │   self.on_advance_end()                                                      │
│   203 │   │   │   │   self._restarting = False                                                   │
│   204 │   │   │   except StopIteration:                                                          │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py:354 in        │
│ advance                                                                                          │
│                                                                                                  │
│   351 │   │   assert self._data_fetcher is not None                                              │
│   352 │   │   self._data_fetcher.setup(combined_loader)                                          │
│   353 │   │   with self.trainer.profiler.profile("run_training_epoch"):                          │
│ ❱ 354 │   │   │   self.epoch_loop.run(self._data_fetcher)                                        │
│   355 │                                                                                          │
│   356 │   def on_advance_end(self) -> None:                                                      │
│   357 │   │   trainer = self.trainer                                                             │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:13 │
│ 4 in run                                                                                         │
│                                                                                                  │
│   131 │   │   while not self.done:                                                               │
│   132 │   │   │   try:                                                                           │
│   133 │   │   │   │   self.advance(data_fetcher)                                                 │
│ ❱ 134 │   │   │   │   self.on_advance_end()                                                      │
│   135 │   │   │   │   self._restarting = False                                                   │
│   136 │   │   │   except StopIteration:                                                          │
│   137 │   │   │   │   break                                                                      │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py:24 │
│ 8 in on_advance_end                                                                              │
│                                                                                                  │
│   245 │   │   should_check_val = self._should_check_val_fx()                                     │
│   246 │   │   if should_check_val:                                                               │
│   247 │   │   │   self.trainer.validating = True                                                 │
│ ❱ 248 │   │   │   self.val_loop.run()                                                            │
│   249 │   │   │   self.trainer.training = True                                                   │
│   250 │   │                                                                                      │
│   251 │   │   # update plateau LR scheduler after metrics are logged                             │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/loops/utilities.py:177 in       │
│ _decorator                                                                                       │
│                                                                                                  │
│   174 │   │   else:                                                                              │
│   175 │   │   │   context_manager = torch.no_grad                                                │
│   176 │   │   with context_manager():                                                            │
│ ❱ 177 │   │   │   return loop_run(self, *args, **kwargs)                                         │
│   178 │                                                                                          │
│   179 │   return _decorator                                                                      │
│   180                                                                                            │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/loops/evaluation_loop.py:122 in │
│ run                                                                                              │
│                                                                                                  │
│   119 │   │   │   finally:                                                                       │
│   120 │   │   │   │   self._restarting = False                                                   │
│   121 │   │   self._store_dataloader_outputs()                                                   │
│ ❱ 122 │   │   return self.on_run_end()                                                           │
│   123 │                                                                                          │
│   124 │   def setup_data(self) -> None:                                                          │
│   125 │   │   trainer = self.trainer                                                             │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/loops/evaluation_loop.py:244 in │
│ on_run_end                                                                                       │
│                                                                                                  │
│   241 │   │   self.trainer._logger_connector._evaluation_epoch_end()                             │
│   242 │   │                                                                                      │
│   243 │   │   # hook                                                                             │
│ ❱ 244 │   │   self._on_evaluation_epoch_end()                                                    │
│   245 │   │                                                                                      │
│   246 │   │   logged_outputs, self._logged_outputs = self._logged_outputs, []  # free memory     │
│   247 │   │   # include any logged outputs on epoch_end                                          │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/loops/evaluation_loop.py:326 in │
│ _on_evaluation_epoch_end                                                                         │
│                                                                                                  │
│   323 │   │                                                                                      │
│   324 │   │   hook_name = "on_test_epoch_end" if trainer.testing else "on_validation_epoch_end   │
│   325 │   │   call._call_callback_hooks(trainer, hook_name)                                      │
│ ❱ 326 │   │   call._call_lightning_module_hook(trainer, hook_name)                               │
│   327 │   │                                                                                      │
│   328 │   │   trainer._logger_connector.on_epoch_end()                                           │
│   329                                                                                            │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py:144 in          │
│ _call_lightning_module_hook                                                                      │
│                                                                                                  │
│   141 │   pl_module._current_fx_name = hook_name                                                 │
│   142 │                                                                                          │
│   143 │   with trainer.profiler.profile(f"[LightningModule]{pl_module.__class__.__name__}.{hoo   │
│ ❱ 144 │   │   output = fn(*args, **kwargs)                                                       │
│   145 │                                                                                          │
│   146 │   # restore current_fx when nested context                                               │
│   147 │   pl_module._current_fx_name = prev_fx_name                                              │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/pytorch_forecasting/models/base_model.py:636 in   │
│ on_validation_epoch_end                                                                          │
│                                                                                                  │
│    633 │   │   return log                                                                        │
│    634 │                                                                                         │
│    635 │   def on_validation_epoch_end(self):                                                    │
│ ❱  636 │   │   self.on_epoch_end(self.validation_step_outputs)                                   │
│    637 │   │   self.validation_step_outputs.clear()                                              │
│    638 │                                                                                         │
│    639 │   def test_step(self, batch, batch_idx):                                                │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/pytorch_forecasting/models/temporal_fusion_transf │
│ ormer/__init__.py:539 in on_epoch_end                                                            │
│                                                                                                  │
│   536 │   │   run at epoch end for training or validation                                        │
│   537 │   │   """                                                                                │
│   538 │   │   if self.log_interval > 0 and not self.training:                                    │
│ ❱ 539 │   │   │   self.log_interpretation(outputs)                                               │
│   540 │                                                                                          │
│   541 │   def interpret_output(                                                                  │
│   542 │   │   self,                                                                              │
│                                                                                                  │
│ /home/fogang/venv/lib/python3.10/site-packages/pytorch_forecasting/models/temporal_fusion_transf │
│ ormer/__init__.py:796 in log_interpretation                                                      │
│                                                                                                  │
│   793 │   │   interpretation = {                                                                 │
│   794 │   │   │   # use padded_stack because decoder length histogram can be of different leng   │
│   795 │   │   │   name: padded_stack([x["interpretation"][name].detach() for x in outputs], si   │
│ ❱ 796 │   │   │   for name in outputs[0]["interpretation"].keys()                                │
│   797 │   │   }                                                                                  │
│   798 │   │   # normalize attention with length histogram squared to account for: 1. zeros in    │
│   799 │   │   # 2. higher attention due to less values                                           │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
IndexError: list index out of range

I would like to know how to eliminate the warnings above the traceback block?.