PermissionError with ModelCheckpointing

[aaprasad]

Hi I'm trying to train a model and am getting this error:

Traceback (most recent call last):
  File "/home/jovyan/talmolab-smb/aadi/biogtr_expts/run/animal/SLAP_M74/single_run.py", line 91, in <module>
    main(cfg.cfg)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/hydra/main.py", line 83, in decorated_main
    return task_function(cfg_passthrough)
  File "/home/jovyan/talmolab-smb/aadi/biogtr_expts/src/biogtr/biogtr/training/train.py", line 101, in main
    trainer.fit(model, dataset)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 544, in fit
    call._call_and_handle_interrupt(
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py", line 44, in _call_and_handle_interrupt
    return trainer_fn(*args, **kwargs)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 580, in _fit_impl
    self._run(model, ckpt_path=ckpt_path)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 989, in _run
    results = self._run_stage()
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 1035, in _run_stage
    self.fit_loop.run()
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/loops/fit_loop.py", line 203, in run
    self.on_advance_end()
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/loops/fit_loop.py", line 374, in on_advance_end
    call._call_callback_hooks(trainer, "on_train_epoch_end", monitoring_callbacks=True)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py", line 208, in _call_callback_hooks
    fn(trainer, trainer.lightning_module, *args, **kwargs)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 314, in on_train_epoch_end
    self._save_last_checkpoint(trainer, monitor_candidates)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 679, in _save_last_checkpoint
    self._link_checkpoint(trainer, self._last_checkpoint_saved, filepath)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 397, in _link_checkpoint
    shutil.copy(filepath, linkpath)
  File "/opt/conda/envs/biogtr/lib/python3.9/shutil.py", line 428, in copy
    copymode(src, dst, follow_symlinks=follow_symlinks)
  File "/opt/conda/envs/biogtr/lib/python3.9/shutil.py", line 317, in copymode
    chmod_func(dst, stat.S_IMODE(st.st_mode))
PermissionError: [Errno 1] Operation not permitted: '/home/jovyan/talmolab-smb/aadi/biogtr_expts/run/animal/SLAP_M74/models/tests/test_chkpt/epoch=1-best-val_num_switches=36.0.ckpt'

This is how i set up my checkpoints:

def get_checkpointing(self) -> pl.callbacks.ModelCheckpoint:
        """Getter for lightning checkpointing callback.

        Returns:
            A lightning checkpointing callback with specified params
        """
        # convert to dict to enable extracting/removing params
        checkpoint_params = OmegaConf.to_container(self.cfg.checkpointing, resolve=True)
        logging_params = self.cfg.logging
        if "dirpath" not in checkpoint_params or checkpoint_params["dirpath"] is None:
            if "group" in logging_params:
                dirpath = f"./models/{logging_params.group}/{logging_params.name}"
            else:
                dirpath = f"./models/{logging_params.name}"

        else:
            dirpath = checkpoint_params["dirpath"]
        
        dirpath = Path(dirpath).resolve()
        if not Path(dirpath).exists():
            try:
                Path(dirpath).mkdir(parents=True, exist_ok=True)
            except OSError as e:
                print(
                    f"Cannot create a new folder. Check the permissions to the given Checkpoint directory. \n {e}"
                )
        
        _ = checkpoint_params.pop("dirpath")
        checkpointers = []
        monitor = checkpoint_params.pop("monitor")
        for metric in monitor:
            checkpointer = pl.callbacks.ModelCheckpoint(
                monitor=metric, dirpath=dirpath, filename=f"{{epoch}}-{{{metric}}}", **checkpoint_params
            )
            checkpointer.CHECKPOINT_NAME_LAST = f"{{epoch}}-best-{{{metric}}}"
            checkpointers.append(checkpointer)
        return checkpointers

Its quite strange because this error never used to happen before