|
a |
|
b/logs/191006-132912_SEResNext50.log |
|
|
1 |
2019-10-06 13:29:12,367 - root - INFO - epoch |
|
|
2 |
2019-10-06 13:29:12,368 - root - INFO - 15 |
|
|
3 |
2019-10-06 13:29:12,368 - root - INFO - schedule_step |
|
|
4 |
2019-10-06 13:29:12,368 - root - INFO - 4 |
|
|
5 |
2019-10-06 13:29:12,368 - root - INFO - batch_size |
|
|
6 |
2019-10-06 13:29:12,368 - root - INFO - 16 |
|
|
7 |
2019-10-06 13:29:12,368 - root - INFO - test_batch_size |
|
|
8 |
2019-10-06 13:29:12,368 - root - INFO - 128 |
|
|
9 |
2019-10-06 13:29:12,368 - root - INFO - num_workers |
|
|
10 |
2019-10-06 13:29:12,368 - root - INFO - 32 |
|
|
11 |
2019-10-06 13:29:12,368 - root - INFO - eval_fre |
|
|
12 |
2019-10-06 13:29:12,368 - root - INFO - 1 |
|
|
13 |
2019-10-06 13:29:12,368 - root - INFO - msg_fre |
|
|
14 |
2019-10-06 13:29:12,368 - root - INFO - 10 |
|
|
15 |
2019-10-06 13:29:12,368 - root - INFO - save_fre |
|
|
16 |
2019-10-06 13:29:12,368 - root - INFO - 1 |
|
|
17 |
2019-10-06 13:29:12,368 - root - INFO - name |
|
|
18 |
2019-10-06 13:29:12,368 - root - INFO - SEResNext50 |
|
|
19 |
2019-10-06 13:29:12,368 - root - INFO - data_dir |
|
|
20 |
2019-10-06 13:29:12,368 - root - INFO - /media/tiger/zzr/rsna |
|
|
21 |
2019-10-06 13:29:12,368 - root - INFO - log_dir |
|
|
22 |
2019-10-06 13:29:12,368 - root - INFO - ./logs |
|
|
23 |
2019-10-06 13:29:12,368 - root - INFO - tensorboard_dir |
|
|
24 |
2019-10-06 13:29:12,368 - root - INFO - ./tensorboard |
|
|
25 |
2019-10-06 13:29:12,368 - root - INFO - model_out_dir |
|
|
26 |
2019-10-06 13:29:12,369 - root - INFO - ./model_out |
|
|
27 |
2019-10-06 13:29:12,369 - root - INFO - model_out_name |
|
|
28 |
2019-10-06 13:29:12,369 - root - INFO - final_model.pth |
|
|
29 |
2019-10-06 13:29:12,369 - root - INFO - seed |
|
|
30 |
2019-10-06 13:29:12,369 - root - INFO - 5 |
|
|
31 |
2019-10-06 13:29:12,369 - root - INFO - predefinedModel |
|
|
32 |
2019-10-06 13:29:12,369 - root - INFO - /media/tiger/zzr/rsna_script/model_out/191004-003700_temp/out_1.pth |
|
|
33 |
2019-10-06 13:29:12,369 - root - INFO - unique_name |
|
|
34 |
2019-10-06 13:29:12,369 - root - INFO - 191006-132912_SEResNext50 |
|
|
35 |
2019-10-06 13:29:12,369 - root - INFO - sub_tensorboard_dir |
|
|
36 |
2019-10-06 13:29:12,369 - root - INFO - ./tensorboard/191006-132912_SEResNext50 |
|
|
37 |
2019-10-06 13:29:12,369 - root - INFO - sub_model_out_dir |
|
|
38 |
2019-10-06 13:29:12,369 - root - INFO - ./model_out/191006-132912_SEResNext50 |
|
|
39 |
2019-10-06 13:29:31,967 - root - ERROR - Exception logged |
|
|
40 |
Traceback (most recent call last): |
|
|
41 |
File "/media/tiger/zzr/rsna_script/trian_resnet.py", line 109, in main_worker |
|
|
42 |
outputs = net(img) |
|
|
43 |
File "/home/tiger/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__ |
|
|
44 |
result = self.forward(*input, **kwargs) |
|
|
45 |
File "/media/tiger/zzr/rsna_script/res_network.py", line 131, in forward |
|
|
46 |
features = self.features(x) |
|
|
47 |
File "/home/tiger/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__ |
|
|
48 |
result = self.forward(*input, **kwargs) |
|
|
49 |
File "/home/tiger/anaconda3/lib/python3.7/site-packages/torch/nn/modules/container.py", line 92, in forward |
|
|
50 |
input = module(input) |
|
|
51 |
File "/home/tiger/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__ |
|
|
52 |
result = self.forward(*input, **kwargs) |
|
|
53 |
File "/home/tiger/anaconda3/lib/python3.7/site-packages/torch/nn/modules/container.py", line 92, in forward |
|
|
54 |
input = module(input) |
|
|
55 |
File "/home/tiger/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__ |
|
|
56 |
result = self.forward(*input, **kwargs) |
|
|
57 |
File "/home/tiger/anaconda3/lib/python3.7/site-packages/pretrainedmodels-0.7.4-py3.7.egg/pretrainedmodels/models/senet.py", line 123, in forward |
|
|
58 |
out = self.bn3(out) |
|
|
59 |
File "/home/tiger/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 493, in __call__ |
|
|
60 |
result = self.forward(*input, **kwargs) |
|
|
61 |
File "/home/tiger/anaconda3/lib/python3.7/site-packages/torch/nn/modules/batchnorm.py", line 83, in forward |
|
|
62 |
exponential_average_factor, self.eps) |
|
|
63 |
File "/home/tiger/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py", line 1697, in batch_norm |
|
|
64 |
training, momentum, eps, torch.backends.cudnn.enabled |
|
|
65 |
RuntimeError: CUDA out of memory. Tried to allocate 1024.00 MiB (GPU 0; 10.92 GiB total capacity; 9.36 GiB already allocated; 956.19 MiB free; 21.71 MiB cached) |