--- a +++ b/logs/191004-233611_temp.log @@ -0,0 +1,52 @@ +2019-10-04 23:36:11,264 - root - INFO - epoch +2019-10-04 23:36:11,265 - root - INFO - 15 +2019-10-04 23:36:11,265 - root - INFO - schedule_step +2019-10-04 23:36:11,265 - root - INFO - 4 +2019-10-04 23:36:11,265 - root - INFO - batch_size +2019-10-04 23:36:11,265 - root - INFO - 64 +2019-10-04 23:36:11,265 - root - INFO - test_batch_size +2019-10-04 23:36:11,265 - root - INFO - 128 +2019-10-04 23:36:11,265 - root - INFO - num_workers +2019-10-04 23:36:11,265 - root - INFO - 32 +2019-10-04 23:36:11,265 - root - INFO - eval_fre +2019-10-04 23:36:11,265 - root - INFO - 1 +2019-10-04 23:36:11,265 - root - INFO - msg_fre +2019-10-04 23:36:11,265 - root - INFO - 10 +2019-10-04 23:36:11,265 - root - INFO - save_fre +2019-10-04 23:36:11,265 - root - INFO - 1 +2019-10-04 23:36:11,265 - root - INFO - name +2019-10-04 23:36:11,265 - root - INFO - temp +2019-10-04 23:36:11,265 - root - INFO - data_dir +2019-10-04 23:36:11,265 - root - INFO - /media/tiger/zzr/rsna +2019-10-04 23:36:11,265 - root - INFO - log_dir +2019-10-04 23:36:11,265 - root - INFO - ./logs +2019-10-04 23:36:11,266 - root - INFO - tensorboard_dir +2019-10-04 23:36:11,266 - root - INFO - ./tensorboard +2019-10-04 23:36:11,266 - root - INFO - model_out_dir +2019-10-04 23:36:11,266 - root - INFO - ./model_out +2019-10-04 23:36:11,266 - root - INFO - model_out_name +2019-10-04 23:36:11,266 - root - INFO - final_model.pth +2019-10-04 23:36:11,266 - root - INFO - seed +2019-10-04 23:36:11,266 - root - INFO - 5 +2019-10-04 23:36:11,266 - root - INFO - predefinedModel +2019-10-04 23:36:11,266 - root - INFO - /media/tiger/zzr/rsna_script/model_out/191004-003700_temp/out_1.pth +2019-10-04 23:36:11,266 - root - INFO - unique_name +2019-10-04 23:36:11,266 - root - INFO - 191004-233611_temp +2019-10-04 23:36:11,266 - root - INFO - sub_tensorboard_dir +2019-10-04 23:36:11,266 - root - INFO - ./tensorboard/191004-233611_temp +2019-10-04 23:36:11,266 - root - INFO - sub_model_out_dir +2019-10-04 23:36:11,266 - root - INFO - ./model_out/191004-233611_temp +2019-10-04 23:37:11,790 - root - INFO - epoch:0. iter/total_iter:10/158025. lr:0.00100. loss:0.6938. spend/global_spend:45.5464/0:00:45. eta:8 days, 7:55:02 +2019-10-04 23:37:16,220 - root - INFO - epoch:0. iter/total_iter:20/158025. lr:0.00100. loss:0.2876. spend/global_spend:4.4310/0:00:49. eta:4 days, 13:40:34 +2019-10-04 23:37:20,661 - root - INFO - epoch:0. iter/total_iter:30/158025. lr:0.00100. loss:0.2106. spend/global_spend:4.4403/0:00:54. eta:3 days, 7:36:30 +2019-10-04 23:37:50,976 - root - INFO - epoch:0. iter/total_iter:40/158025. lr:0.00100. loss:0.1988. spend/global_spend:30.3148/0:01:24. eta:3 days, 20:57:41 +2019-10-04 23:37:55,436 - root - INFO - epoch:0. iter/total_iter:50/158025. lr:0.00100. loss:0.2064. spend/global_spend:4.4604/0:01:29. eta:3 days, 6:16:44 +2019-10-04 23:37:59,892 - root - INFO - epoch:0. iter/total_iter:60/158025. lr:0.00100. loss:0.1786. spend/global_spend:4.4564/0:01:33. eta:2 days, 20:29:15 +2019-10-04 23:38:27,834 - root - INFO - epoch:0. iter/total_iter:70/158025. lr:0.00100. loss:0.2058. spend/global_spend:27.9411/0:02:01. eta:3 days, 4:12:48 +2019-10-04 23:38:37,229 - root - INFO - epoch:0. iter/total_iter:80/158025. lr:0.00100. loss:0.2195. spend/global_spend:9.3954/0:02:10. eta:2 days, 23:50:06 +2019-10-04 23:38:41,702 - root - INFO - epoch:0. iter/total_iter:90/158025. lr:0.00100. loss:0.2070. spend/global_spend:4.4728/0:02:15. eta:2 days, 18:01:47 +2019-10-04 23:38:44,159 - root - ERROR - Exception logged +Traceback (most recent call last): + File "/media/tiger/zzr/rsna_script/trian_resnet.py", line 114, in main_worker + running_loss.append(loss.item()) +KeyboardInterrupt