CUDA error：invalid configuration argument #35
Description
Has anyone had this problem?
Traceback (most recent call last):
File "main.py", line 96, in
main()
File "/home/robot/miniconda3/envs/trimrf/lib/python3.8/site-packages/gin/config.py", line 1605, in gin_wrapper
utils.augment_exception_message_and_reraise(e, err_str)
File "/home/robot/miniconda3/envs/trimrf/lib/python3.8/site-packages/gin/utils.py", line 41, in augment_exception_message_and_reraise
raise proxy.with_traceback(exception.traceback) from None
File "/home/robot/miniconda3/envs/trimrf/lib/python3.8/site-packages/gin/config.py", line 1582, in gin_wrapper
return fn(*new_args, **new_kwargs)
File "main.py", line 56, in main
trainer.fit()
File "/home/robot/Project/Tri-MipRF/trainer/trainer.py", line 140, in fit
metrics, final_rb, target = self.eval_img(
File "/home/robot/miniconda3/envs/trimrf/lib/python3.8/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "/home/robot/Project/Tri-MipRF/trainer/trainer.py", line 168, in eval_img
rb = self.model(
File "/home/robot/miniconda3/envs/trimrf/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/robot/Project/Tri-MipRF/neural_field/model/trimipRF.py", line 118, in forward
return self.rendering(
File "/home/robot/Project/Tri-MipRF/neural_field/model/trimipRF.py", line 140, in rendering
rgbs, sigmas = rgb_sigma_fn(t_starts, t_ends, ray_indices.long())
File "/home/robot/Project/Tri-MipRF/neural_field/model/trimipRF.py", line 115, in rgb_sigma_fn
rgb = self.field.query_rgb(dir=t_dirs, embedding=feature)['rgb']
File "/home/robot/Project/Tri-MipRF/neural_field/field/trimipRF.py", line 97, in query_rgb
self.mlp_head(h)
File "/home/robot/miniconda3/envs/trimrf/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/robot/miniconda3/envs/trimrf/lib/python3.8/site-packages/tinycudann-1.7-py3.8-linux-x86_64.egg/tinycudann/modules.py", line 193, in forward
self.params.to(_torch_precision(self.native_tcnn_module.param_precision())).contiguous(),
RuntimeError: CUDA error: invalid configuration argument
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with TORCH_USE_CUDA_DSA to enable device-side assertions.
In call to configurable 'main' (<function main at 0x7fe41b958dc0>)