nohup 运行一段时间后显示消息: '接收到 1 个死亡信号,正在关闭工作进程'

--- Logging error --- Traceback (most recent call last): File "/home/hhh/.conda/envs/python38/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 723, in run result = self._invoke_run(role) File "/home/hhh/.conda/envs/python38/lib/python3.8/site-packages/torch/distributed/elastic/agent/server/api.py", line 864, in _invoke_run time.sleep(monitor_interval) File "/home/hhh/.conda/envs/python38/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 62, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) torch.distributed.elastic.multiprocessing.api.SignalException: Process 2890585 got signal: 1 During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/hhh/.conda/envs/python38/lib/python3.8/site-packages/rich/logging.py", line 170, in emit self.console.print(log_renderable) File "/home/hhh/.conda/envs/python38/lib/python3.8/site-packages/rich/console.py", line 1684, in print render_options = self.options.update( File "/home/hhh/.conda/envs/python38/lib/python3.8/site-packages/rich/console.py", line 982, in options max_height=self.size.height, File "/home/hhh/.conda/envs/python38/lib/python3.8/site-packages/rich/console.py", line 1002, in size if self.is_dumb_terminal: File "/home/hhh/.conda/envs/python38/lib/python3.8/site-packages/rich/console.py", line 974, in is_dumb_terminal _term = self._environ.get("TERM", "") File "/home/hhh/.conda/envs/python38/lib/python3.8/_collections_abc.py", line 660, in get return self[key] File "/home/hhh/.conda/envs/python38/lib/python3.8/os.py", line 672, in __getitem__ value = self._data[self.encodekey(key)] File "/home/hhh/.conda/envs/python38/lib/python3.8/os.py", line 748, in encode def encode(value): File "/home/hhh/.conda/envs/python38/lib/python3.8/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 62, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) torch.distributed.elastic.multiprocessing.api.SignalException: Process 2890585 got signal: 1 Call stack: File "/home/hhh/.conda/envs/python38/bin/accelerate", line 8, in <module> sys.exit(main()) File "/home/hhh/.conda/envs/python38/lib/python3.8/site-packages/accelerate/commands/accelerate_cli.py", line 45, in main args.func(args) File "/home/hhh/.conda/envs/python38/lib/python3.8/site-packages/accelerate/commands/launch.py", line 900, in launch_command distrib_run.run(args) File "/home/hhh/.conda/envs/python38/lib/python3.8/site-packages/torch/distributed/run.py", line 785, in run elastic_launch( File