576 lines (575 with data), 85.6 kB

2019-10-04 22:01:56,049 - root - INFO - epoch
2019-10-04 22:01:56,049 - root - INFO - 15
2019-10-04 22:01:56,049 - root - INFO - schedule_step
2019-10-04 22:01:56,049 - root - INFO - 4
2019-10-04 22:01:56,049 - root - INFO - batch_size
2019-10-04 22:01:56,049 - root - INFO - 64
2019-10-04 22:01:56,049 - root - INFO - test_batch_size
2019-10-04 22:01:56,049 - root - INFO - 128
2019-10-04 22:01:56,049 - root - INFO - num_workers
2019-10-04 22:01:56,049 - root - INFO - 16
2019-10-04 22:01:56,049 - root - INFO - eval_fre
2019-10-04 22:01:56,049 - root - INFO - 1
2019-10-04 22:01:56,049 - root - INFO - msg_fre
2019-10-04 22:01:56,049 - root - INFO - 10
2019-10-04 22:01:56,049 - root - INFO - save_fre
2019-10-04 22:01:56,049 - root - INFO - 1
2019-10-04 22:01:56,049 - root - INFO - name
2019-10-04 22:01:56,049 - root - INFO - temp
2019-10-04 22:01:56,049 - root - INFO - data_dir
2019-10-04 22:01:56,050 - root - INFO - /media/tiger/zzr/rsna
2019-10-04 22:01:56,050 - root - INFO - log_dir
2019-10-04 22:01:56,050 - root - INFO - ./logs
2019-10-04 22:01:56,050 - root - INFO - tensorboard_dir
2019-10-04 22:01:56,050 - root - INFO - ./tensorboard
2019-10-04 22:01:56,050 - root - INFO - model_out_dir
2019-10-04 22:01:56,050 - root - INFO - ./model_out
2019-10-04 22:01:56,050 - root - INFO - model_out_name
2019-10-04 22:01:56,050 - root - INFO - final_model.pth
2019-10-04 22:01:56,050 - root - INFO - seed
2019-10-04 22:01:56,050 - root - INFO - 5
2019-10-04 22:01:56,050 - root - INFO - predefinedModel
2019-10-04 22:01:56,050 - root - INFO - /media/tiger/zzr/rsna_script/model_out/191004-003700_temp/out_1.pth
2019-10-04 22:01:56,050 - root - INFO - unique_name
2019-10-04 22:01:56,050 - root - INFO - 191004-220156_temp
2019-10-04 22:01:56,050 - root - INFO - sub_tensorboard_dir
2019-10-04 22:01:56,050 - root - INFO - ./tensorboard/191004-220156_temp
2019-10-04 22:01:56,050 - root - INFO - sub_model_out_dir
2019-10-04 22:01:56,050 - root - INFO - ./model_out/191004-220156_temp
2019-10-04 22:02:30,952 - root - INFO - epoch:0. iter/total_iter:10/158025. lr:0.00100. loss:0.6938. spend/global_spend:18.9490/0:00:18. eta:3 days, 11:10:23
2019-10-04 22:02:41,562 - root - INFO - epoch:0. iter/total_iter:20/158025. lr:0.00100. loss:0.2876. spend/global_spend:10.6101/0:00:29. eta:2 days, 16:52:04
2019-10-04 22:02:46,634 - root - INFO - epoch:0. iter/total_iter:30/158025. lr:0.00100. loss:0.2106. spend/global_spend:5.0731/0:00:34. eta:2 days, 2:39:50
2019-10-04 22:02:59,197 - root - INFO - epoch:0. iter/total_iter:40/158025. lr:0.00100. loss:0.1987. spend/global_spend:12.5625/0:00:47. eta:2 days, 3:46:41
2019-10-04 22:03:03,746 - root - INFO - epoch:0. iter/total_iter:50/158025. lr:0.00100. loss:0.2063. spend/global_spend:4.5488/0:00:51. eta:1 day, 21:24:43
2019-10-04 22:03:16,206 - root - INFO - epoch:0. iter/total_iter:60/158025. lr:0.00100. loss:0.1786. spend/global_spend:12.4596/0:01:04. eta:1 day, 22:57:10
2019-10-04 22:03:27,299 - root - INFO - epoch:0. iter/total_iter:70/158025. lr:0.00100. loss:0.2058. spend/global_spend:11.0931/0:01:15. eta:1 day, 23:11:45
2019-10-04 22:03:31,897 - root - INFO - epoch:0. iter/total_iter:80/158025. lr:0.00100. loss:0.2196. spend/global_spend:4.5980/0:01:19. eta:1 day, 19:48:56
2019-10-04 22:03:43,074 - root - INFO - epoch:0. iter/total_iter:90/158025. lr:0.00100. loss:0.2071. spend/global_spend:11.1768/0:01:31. eta:1 day, 20:23:34
2019-10-04 22:03:52,547 - root - INFO - epoch:0. iter/total_iter:100/158025. lr:0.00100. loss:0.1982. spend/global_spend:9.4737/0:01:40. eta:1 day, 20:06:25
2019-10-04 22:03:57,122 - root - INFO - epoch:0. iter/total_iter:110/158025. lr:0.00100. loss:0.1935. spend/global_spend:4.5748/0:01:45. eta:1 day, 17:55:08
2019-10-04 22:04:07,229 - root - INFO - epoch:0. iter/total_iter:120/158025. lr:0.00100. loss:0.1969. spend/global_spend:10.1068/0:01:55. eta:1 day, 18:07:03
2019-10-04 22:04:12,039 - root - INFO - epoch:0. iter/total_iter:130/158025. lr:0.00100. loss:0.2025. spend/global_spend:4.8107/0:02:00. eta:1 day, 16:29:54
2019-10-04 22:04:25,040 - root - INFO - epoch:0. iter/total_iter:140/158025. lr:0.00100. loss:0.1885. spend/global_spend:13.0009/0:02:13. eta:1 day, 17:40:33
2019-10-04 22:04:35,069 - root - INFO - epoch:0. iter/total_iter:150/158025. lr:0.00100. loss:0.2184. spend/global_spend:10.0283/0:02:23. eta:1 day, 17:49:37
2019-10-04 22:04:39,679 - root - INFO - epoch:0. iter/total_iter:160/158025. lr:0.00100. loss:0.2037. spend/global_spend:4.6101/0:02:27. eta:1 day, 16:28:25
2019-10-04 22:04:48,940 - root - INFO - epoch:0. iter/total_iter:170/158025. lr:0.00100. loss:0.1991. spend/global_spend:9.2606/0:02:36. eta:1 day, 16:28:45
2019-10-04 22:04:56,943 - root - INFO - epoch:0. iter/total_iter:180/158025. lr:0.00100. loss:0.1927. spend/global_spend:8.0038/0:02:44. eta:1 day, 16:10:39
2019-10-04 22:05:03,105 - root - INFO - epoch:0. iter/total_iter:190/158025. lr:0.00100. loss:0.2071. spend/global_spend:6.1619/0:02:51. eta:1 day, 15:28:56
2019-10-04 22:05:13,014 - root - INFO - epoch:0. iter/total_iter:200/158025. lr:0.00100. loss:0.1589. spend/global_spend:9.9092/0:03:01. eta:1 day, 15:40:40
2019-10-04 22:05:17,704 - root - INFO - epoch:0. iter/total_iter:210/158025. lr:0.00100. loss:0.1769. spend/global_spend:4.6895/0:03:05. eta:1 day, 14:45:54
2019-10-04 22:05:30,724 - root - INFO - epoch:0. iter/total_iter:220/158025. lr:0.00100. loss:0.1530. spend/global_spend:13.0201/0:03:18. eta:1 day, 15:35:41
2019-10-04 22:05:42,435 - root - INFO - epoch:0. iter/total_iter:230/158025. lr:0.00100. loss:0.1708. spend/global_spend:11.7105/0:03:30. eta:1 day, 16:06:09
2019-10-04 22:05:47,118 - root - INFO - epoch:0. iter/total_iter:240/158025. lr:0.00100. loss:0.1939. spend/global_spend:4.6813/0:03:35. eta:1 day, 15:17:03
2019-10-04 22:05:57,102 - root - INFO - epoch:0. iter/total_iter:250/158025. lr:0.00100. loss:0.1836. spend/global_spend:9.9865/0:03:45. eta:1 day, 15:27:40
2019-10-04 22:06:03,694 - root - INFO - epoch:0. iter/total_iter:260/158025. lr:0.00100. loss:0.1919. spend/global_spend:6.5919/0:03:51. eta:1 day, 15:03:07
2019-10-04 22:06:13,672 - root - INFO - epoch:0. iter/total_iter:270/158025. lr:0.00100. loss:0.1821. spend/global_spend:9.9778/0:04:01. eta:1 day, 15:13:22
2019-10-04 22:06:21,280 - root - INFO - epoch:0. iter/total_iter:280/158025. lr:0.00100. loss:0.1823. spend/global_spend:7.6083/0:04:09. eta:1 day, 15:00:36
2019-10-04 22:06:26,003 - root - INFO - epoch:0. iter/total_iter:290/158025. lr:0.00100. loss:0.1782. spend/global_spend:4.7230/0:04:14. eta:1 day, 14:22:34
2019-10-04 22:06:37,594 - root - INFO - epoch:0. iter/total_iter:300/158025. lr:0.00100. loss:0.1760. spend/global_spend:11.5911/0:04:25. eta:1 day, 14:47:14
2019-10-04 22:06:49,175 - root - INFO - epoch:0. iter/total_iter:310/158025. lr:0.00100. loss:0.1647. spend/global_spend:11.5804/0:04:37. eta:1 day, 15:10:13
2019-10-04 22:06:53,844 - root - INFO - epoch:0. iter/total_iter:320/158025. lr:0.00100. loss:0.1918. spend/global_spend:4.6691/0:04:41. eta:1 day, 14:34:59
2019-10-04 22:07:05,117 - root - INFO - epoch:0. iter/total_iter:330/158025. lr:0.00100. loss:0.1770. spend/global_spend:11.2735/0:04:53. eta:1 day, 14:54:28
2019-10-04 22:07:13,815 - root - INFO - epoch:0. iter/total_iter:340/158025. lr:0.00100. loss:0.1771. spend/global_spend:8.6977/0:05:01. eta:1 day, 14:52:54
2019-10-04 22:07:20,124 - root - INFO - epoch:0. iter/total_iter:350/158025. lr:0.00100. loss:0.1665. spend/global_spend:6.3096/0:05:08. eta:1 day, 14:33:28
2019-10-04 22:07:29,847 - root - INFO - epoch:0. iter/total_iter:360/158025. lr:0.00100. loss:0.1805. spend/global_spend:9.7222/0:05:17. eta:1 day, 14:40:02
2019-10-04 22:07:34,529 - root - INFO - epoch:0. iter/total_iter:370/158025. lr:0.00100. loss:0.2054. spend/global_spend:4.6821/0:05:22. eta:1 day, 14:10:26
2019-10-04 22:07:44,238 - root - INFO - epoch:0. iter/total_iter:380/158025. lr:0.00100. loss:0.1909. spend/global_spend:9.7090/0:05:32. eta:1 day, 14:17:09
2019-10-04 22:07:56,084 - root - INFO - epoch:0. iter/total_iter:390/158025. lr:0.00100. loss:0.2016. spend/global_spend:11.8467/0:05:44. eta:1 day, 14:37:55
2019-10-04 22:08:00,780 - root - INFO - epoch:0. iter/total_iter:400/158025. lr:0.00100. loss:0.1551. spend/global_spend:4.6957/0:05:48. eta:1 day, 14:10:40
2019-10-04 22:08:08,909 - root - INFO - epoch:0. iter/total_iter:410/158025. lr:0.00100. loss:0.1947. spend/global_spend:8.1295/0:05:56. eta:1 day, 14:06:44
2019-10-04 22:08:17,834 - root - INFO - epoch:0. iter/total_iter:420/158025. lr:0.00100. loss:0.1764. spend/global_spend:8.9246/0:06:05. eta:1 day, 14:07:58
2019-10-04 22:08:24,936 - root - INFO - epoch:0. iter/total_iter:430/158025. lr:0.00100. loss:0.1709. spend/global_spend:7.1006/0:06:12. eta:1 day, 13:57:59
2019-10-04 22:08:35,700 - root - INFO - epoch:0. iter/total_iter:440/158025. lr:0.00100. loss:0.1382. spend/global_spend:10.7648/0:06:23. eta:1 day, 14:10:20
2019-10-04 22:08:40,917 - root - INFO - epoch:0. iter/total_iter:450/158025. lr:0.00100. loss:0.1867. spend/global_spend:5.2178/0:06:28. eta:1 day, 13:49:45
2019-10-04 22:08:52,385 - root - INFO - epoch:0. iter/total_iter:460/158025. lr:0.00100. loss:0.1598. spend/global_spend:11.4675/0:06:40. eta:1 day, 14:05:44
2019-10-04 22:09:04,976 - root - INFO - epoch:0. iter/total_iter:470/158025. lr:0.00100. loss:0.2141. spend/global_spend:12.5908/0:06:52. eta:1 day, 14:27:18
2019-10-04 22:09:09,667 - root - INFO - epoch:0. iter/total_iter:480/158025. lr:0.00100. loss:0.1760. spend/global_spend:4.6909/0:06:57. eta:1 day, 14:04:45
2019-10-04 22:09:17,576 - root - INFO - epoch:0. iter/total_iter:490/158025. lr:0.00100. loss:0.1588. spend/global_spend:7.9094/0:07:05. eta:1 day, 14:00:21
2019-10-04 22:09:27,398 - root - INFO - epoch:0. iter/total_iter:500/158025. lr:0.00100. loss:0.1591. spend/global_spend:9.8222/0:07:15. eta:1 day, 14:06:11
2019-10-04 22:09:35,080 - root - INFO - epoch:0. iter/total_iter:510/158025. lr:0.00100. loss:0.1665. spend/global_spend:7.6818/0:07:23. eta:1 day, 14:00:45
2019-10-04 22:09:45,880 - root - INFO - epoch:0. iter/total_iter:520/158025. lr:0.00100. loss:0.1632. spend/global_spend:10.8001/0:07:33. eta:1 day, 14:11:16
2019-10-04 22:09:50,573 - root - INFO - epoch:0. iter/total_iter:530/158025. lr:0.00100. loss:0.1646. spend/global_spend:4.6928/0:07:38. eta:1 day, 13:51:08
2019-10-04 22:10:02,387 - root - INFO - epoch:0. iter/total_iter:540/158025. lr:0.00100. loss:0.1681. spend/global_spend:11.8141/0:07:50. eta:1 day, 14:06:22
2019-10-04 22:10:12,900 - root - INFO - epoch:0. iter/total_iter:550/158025. lr:0.00100. loss:0.1792. spend/global_spend:10.5129/0:08:00. eta:1 day, 14:14:49
2019-10-04 22:10:17,590 - root - INFO - epoch:0. iter/total_iter:560/158025. lr:0.00100. loss:0.1586. spend/global_spend:4.6897/0:08:05. eta:1 day, 13:55:41
2019-10-04 22:10:26,490 - root - INFO - epoch:0. iter/total_iter:570/158025. lr:0.00100. loss:0.1663. spend/global_spend:8.9003/0:08:14. eta:1 day, 13:56:35
2019-10-04 22:10:35,307 - root - INFO - epoch:0. iter/total_iter:580/158025. lr:0.00100. loss:0.1847. spend/global_spend:8.8171/0:08:23. eta:1 day, 13:57:05
2019-10-04 22:10:41,557 - root - INFO - epoch:0. iter/total_iter:590/158025. lr:0.00100. loss:0.1732. spend/global_spend:6.2498/0:08:29. eta:1 day, 13:46:09
2019-10-04 22:10:52,703 - root - INFO - epoch:0. iter/total_iter:600/158025. lr:0.00100. loss:0.1638. spend/global_spend:11.1465/0:08:40. eta:1 day, 13:56:58
2019-10-04 22:10:59,890 - root - INFO - epoch:0. iter/total_iter:610/158025. lr:0.00100. loss:0.1601. spend/global_spend:7.1836/0:08:47. eta:1 day, 13:50:24
2019-10-04 22:11:10,157 - root - INFO - epoch:0. iter/total_iter:620/158025. lr:0.00100. loss:0.1601. spend/global_spend:10.2702/0:08:58. eta:1 day, 13:57:06
2019-10-04 22:11:17,560 - root - INFO - epoch:0. iter/total_iter:630/158025. lr:0.00100. loss:0.1619. spend/global_spend:7.4028/0:09:05. eta:1 day, 13:51:38
2019-10-04 22:11:22,228 - root - INFO - epoch:0. iter/total_iter:640/158025. lr:0.00100. loss:0.1633. spend/global_spend:4.6675/0:09:10. eta:1 day, 13:35:08
2019-10-04 22:11:33,315 - root - INFO - epoch:0. iter/total_iter:650/158025. lr:0.00100. loss:0.1607. spend/global_spend:11.0874/0:09:21. eta:1 day, 13:45:02
2019-10-04 22:11:39,730 - root - INFO - epoch:0. iter/total_iter:660/158025. lr:0.00100. loss:0.1617. spend/global_spend:6.4151/0:09:27. eta:1 day, 13:36:04
2019-10-04 22:11:47,003 - root - INFO - epoch:0. iter/total_iter:670/158025. lr:0.00100. loss:0.1837. spend/global_spend:7.2734/0:09:35. eta:1 day, 13:30:43
2019-10-04 22:11:59,925 - root - INFO - epoch:0. iter/total_iter:680/158025. lr:0.00100. loss:0.1663. spend/global_spend:12.9216/0:09:47. eta:1 day, 13:47:19
2019-10-04 22:12:06,859 - root - INFO - epoch:0. iter/total_iter:690/158025. lr:0.00100. loss:0.1998. spend/global_spend:6.9339/0:09:54. eta:1 day, 13:40:40
2019-10-04 22:12:14,893 - root - INFO - epoch:0. iter/total_iter:700/158025. lr:0.00100. loss:0.1521. spend/global_spend:8.0347/0:10:02. eta:1 day, 13:38:19
2019-10-04 22:12:28,404 - root - INFO - epoch:0. iter/total_iter:710/158025. lr:0.00100. loss:0.1756. spend/global_spend:13.5101/0:10:16. eta:1 day, 13:56:16
2019-10-04 22:12:33,028 - root - INFO - epoch:0. iter/total_iter:720/158025. lr:0.00100. loss:0.1868. spend/global_spend:4.6249/0:10:21. eta:1 day, 13:41:21
2019-10-04 22:12:42,711 - root - INFO - epoch:0. iter/total_iter:730/158025. lr:0.00100. loss:0.1509. spend/global_spend:9.6827/0:10:30. eta:1 day, 13:45:00
2019-10-04 22:12:52,536 - root - INFO - epoch:0. iter/total_iter:740/158025. lr:0.00100. loss:0.1646. spend/global_spend:9.8251/0:10:40. eta:1 day, 13:49:03
2019-10-04 22:12:57,218 - root - INFO - epoch:0. iter/total_iter:750/158025. lr:0.00100. loss:0.1582. spend/global_spend:4.6820/0:10:45. eta:1 day, 13:35:01
2019-10-04 22:13:08,638 - root - INFO - epoch:0. iter/total_iter:760/158025. lr:0.00100. loss:0.1719. spend/global_spend:11.4203/0:10:56. eta:1 day, 13:44:36
2019-10-04 22:13:17,853 - root - INFO - epoch:0. iter/total_iter:770/158025. lr:0.00100. loss:0.1752. spend/global_spend:9.2150/0:11:05. eta:1 day, 13:46:24
2019-10-04 22:13:25,814 - root - INFO - epoch:0. iter/total_iter:780/158025. lr:0.00100. loss:0.1656. spend/global_spend:7.9600/0:11:13. eta:1 day, 13:43:57
2019-10-04 22:13:34,598 - root - INFO - epoch:0. iter/total_iter:790/158025. lr:0.00100. loss:0.1574. spend/global_spend:8.7848/0:11:22. eta:1 day, 13:44:18
2019-10-04 22:13:43,012 - root - INFO - epoch:0. iter/total_iter:800/158025. lr:0.00100. loss:0.1687. spend/global_spend:8.4134/0:11:31. eta:1 day, 13:43:24
2019-10-04 22:13:56,176 - root - INFO - epoch:0. iter/total_iter:810/158025. lr:0.00100. loss:0.1688. spend/global_spend:13.1636/0:11:44. eta:1 day, 13:57:54
2019-10-04 22:14:01,668 - root - INFO - epoch:0. iter/total_iter:820/158025. lr:0.00100. loss:0.1842. spend/global_spend:5.4925/0:11:49. eta:1 day, 13:47:32
2019-10-04 22:14:13,054 - root - INFO - epoch:0. iter/total_iter:830/158025. lr:0.00100. loss:0.1559. spend/global_spend:11.3863/0:12:01. eta:1 day, 13:56:01
2019-10-04 22:14:21,061 - root - INFO - epoch:0. iter/total_iter:840/158025. lr:0.00100. loss:0.1627. spend/global_spend:8.0068/0:12:09. eta:1 day, 13:53:45
2019-10-04 22:14:33,602 - root - INFO - epoch:0. iter/total_iter:850/158025. lr:0.00100. loss:0.1803. spend/global_spend:12.5415/0:12:21. eta:1 day, 14:05:30
2019-10-04 22:14:38,260 - root - INFO - epoch:0. iter/total_iter:860/158025. lr:0.00100. loss:0.1729. spend/global_spend:4.6573/0:12:26. eta:1 day, 13:52:58
2019-10-04 22:14:49,389 - root - INFO - epoch:0. iter/total_iter:870/158025. lr:0.00100. loss:0.1748. spend/global_spend:11.1292/0:12:37. eta:1 day, 14:00:12
2019-10-04 22:14:54,076 - root - INFO - epoch:0. iter/total_iter:880/158025. lr:0.00100. loss:0.1504. spend/global_spend:4.6874/0:12:42. eta:1 day, 13:48:06
2019-10-04 22:15:05,306 - root - INFO - epoch:0. iter/total_iter:890/158025. lr:0.00100. loss:0.1940. spend/global_spend:11.2291/0:12:53. eta:1 day, 13:55:31
2019-10-04 22:15:16,607 - root - INFO - epoch:0. iter/total_iter:900/158025. lr:0.00100. loss:0.1757. spend/global_spend:11.3019/0:13:04. eta:1 day, 14:02:58
2019-10-04 22:15:23,300 - root - INFO - epoch:0. iter/total_iter:910/158025. lr:0.00100. loss:0.1756. spend/global_spend:6.6923/0:13:11. eta:1 day, 13:57:00
2019-10-04 22:15:31,963 - root - INFO - epoch:0. iter/total_iter:920/158025. lr:0.00100. loss:0.1589. spend/global_spend:8.6633/0:13:19. eta:1 day, 13:56:46
2019-10-04 22:15:46,922 - root - INFO - epoch:0. iter/total_iter:930/158025. lr:0.00100. loss:0.1664. spend/global_spend:14.9586/0:13:34. eta:1 day, 14:14:15
2019-10-04 22:15:52,879 - root - INFO - epoch:0. iter/total_iter:940/158025. lr:0.00100. loss:0.1696. spend/global_spend:5.9576/0:13:40. eta:1 day, 14:06:18
2019-10-04 22:16:02,916 - root - INFO - epoch:0. iter/total_iter:950/158025. lr:0.00100. loss:0.1750. spend/global_spend:10.0368/0:13:50. eta:1 day, 14:09:45
2019-10-04 22:16:08,980 - root - INFO - epoch:0. iter/total_iter:960/158025. lr:0.00100. loss:0.1814. spend/global_spend:6.0639/0:13:56. eta:1 day, 14:02:17
2019-10-04 22:16:21,116 - root - INFO - epoch:0. iter/total_iter:970/158025. lr:0.00100. loss:0.1513. spend/global_spend:12.1356/0:14:09. eta:1 day, 14:11:21
2019-10-04 22:16:32,553 - root - INFO - epoch:0. iter/total_iter:980/158025. lr:0.00100. loss:0.1739. spend/global_spend:11.4378/0:14:20. eta:1 day, 14:18:23
2019-10-04 22:16:37,245 - root - INFO - epoch:0. iter/total_iter:990/158025. lr:0.00100. loss:0.1509. spend/global_spend:4.6913/0:14:25. eta:1 day, 14:07:25
2019-10-04 22:16:46,568 - root - INFO - epoch:0. iter/total_iter:1000/158025. lr:0.00100. loss:0.1852. spend/global_spend:9.3236/0:14:34. eta:1 day, 14:08:48
2019-10-04 22:16:57,772 - root - INFO - epoch:0. iter/total_iter:1010/158025. lr:0.00100. loss:0.1794. spend/global_spend:11.2042/0:14:45. eta:1 day, 14:15:02
2019-10-04 22:17:03,263 - root - INFO - epoch:0. iter/total_iter:1020/158025. lr:0.00100. loss:0.1597. spend/global_spend:5.4906/0:14:51. eta:1 day, 14:06:28
2019-10-04 22:17:12,686 - root - INFO - epoch:0. iter/total_iter:1030/158025. lr:0.00100. loss:0.1783. spend/global_spend:9.4225/0:15:00. eta:1 day, 14:08:04
2019-10-04 22:17:17,744 - root - INFO - epoch:0. iter/total_iter:1040/158025. lr:0.00100. loss:0.1686. spend/global_spend:5.0582/0:15:05. eta:1 day, 13:58:39
2019-10-04 22:17:30,545 - root - INFO - epoch:0. iter/total_iter:1050/158025. lr:0.00100. loss:0.1894. spend/global_spend:12.8008/0:15:18. eta:1 day, 14:08:42
2019-10-04 22:17:36,782 - root - INFO - epoch:0. iter/total_iter:1060/158025. lr:0.00100. loss:0.1500. spend/global_spend:6.2369/0:15:24. eta:1 day, 14:02:21
2019-10-04 22:17:47,562 - root - INFO - epoch:0. iter/total_iter:1070/158025. lr:0.00100. loss:0.1622. spend/global_spend:10.7802/0:15:35. eta:1 day, 14:07:14
2019-10-04 22:17:54,938 - root - INFO - epoch:0. iter/total_iter:1080/158025. lr:0.00100. loss:0.1398. spend/global_spend:7.3768/0:15:42. eta:1 day, 14:03:46
2019-10-04 22:18:06,466 - root - INFO - epoch:0. iter/total_iter:1090/158025. lr:0.00100. loss:0.1581. spend/global_spend:11.5272/0:15:54. eta:1 day, 14:10:20
2019-10-04 22:18:13,491 - root - INFO - epoch:0. iter/total_iter:1100/158025. lr:0.00100. loss:0.1641. spend/global_spend:7.0252/0:16:01. eta:1 day, 14:06:05
2019-10-04 22:18:22,729 - root - INFO - epoch:0. iter/total_iter:1110/158025. lr:0.00100. loss:0.1623. spend/global_spend:9.2380/0:16:10. eta:1 day, 14:07:06
2019-10-04 22:18:28,082 - root - INFO - epoch:0. iter/total_iter:1120/158025. lr:0.00100. loss:0.1598. spend/global_spend:5.3527/0:16:16. eta:1 day, 13:59:02
2019-10-04 22:18:40,655 - root - INFO - epoch:0. iter/total_iter:1130/158025. lr:0.00100. loss:0.1698. spend/global_spend:12.5731/0:16:28. eta:1 day, 14:07:49
2019-10-04 22:18:52,819 - root - INFO - epoch:0. iter/total_iter:1140/158025. lr:0.00100. loss:0.1585. spend/global_spend:12.1646/0:16:40. eta:1 day, 14:15:30
2019-10-04 22:18:57,501 - root - INFO - epoch:0. iter/total_iter:1150/158025. lr:0.00100. loss:0.1730. spend/global_spend:4.6821/0:16:45. eta:1 day, 14:06:03
2019-10-04 22:19:09,259 - root - INFO - epoch:0. iter/total_iter:1160/158025. lr:0.00100. loss:0.1595. spend/global_spend:11.7575/0:16:57. eta:1 day, 14:12:42
2019-10-04 22:19:19,544 - root - INFO - epoch:0. iter/total_iter:1170/158025. lr:0.00100. loss:0.1366. spend/global_spend:10.2854/0:17:07. eta:1 day, 14:15:56
2019-10-04 22:19:26,963 - root - INFO - epoch:0. iter/total_iter:1180/158025. lr:0.00100. loss:0.1884. spend/global_spend:7.4183/0:17:14. eta:1 day, 14:12:46
2019-10-04 22:19:35,392 - root - INFO - epoch:0. iter/total_iter:1190/158025. lr:0.00100. loss:0.1409. spend/global_spend:8.4294/0:17:23. eta:1 day, 14:11:52
2019-10-04 22:19:43,724 - root - INFO - epoch:0. iter/total_iter:1200/158025. lr:0.00100. loss:0.1644. spend/global_spend:8.3320/0:17:31. eta:1 day, 14:10:46
2019-10-04 22:19:54,622 - root - INFO - epoch:0. iter/total_iter:1210/158025. lr:0.00100. loss:0.1827. spend/global_spend:10.8985/0:17:42. eta:1 day, 14:15:14
2019-10-04 22:20:05,547 - root - INFO - epoch:0. iter/total_iter:1220/158025. lr:0.00100. loss:0.1584. spend/global_spend:10.9241/0:17:53. eta:1 day, 14:19:41
2019-10-04 22:20:11,938 - root - INFO - epoch:0. iter/total_iter:1230/158025. lr:0.00100. loss:0.1829. spend/global_spend:6.3911/0:17:59. eta:1 day, 14:14:25
2019-10-04 22:20:24,099 - root - INFO - epoch:0. iter/total_iter:1240/158025. lr:0.00100. loss:0.1620. spend/global_spend:12.1615/0:18:12. eta:1 day, 14:21:24
2019-10-04 22:20:36,809 - root - INFO - epoch:0. iter/total_iter:1250/158025. lr:0.00100. loss:0.1756. spend/global_spend:12.7096/0:18:24. eta:1 day, 14:29:24
2019-10-04 22:20:41,464 - root - INFO - epoch:0. iter/total_iter:1260/158025. lr:0.00100. loss:0.1558. spend/global_spend:4.6552/0:18:29. eta:1 day, 14:20:35
2019-10-04 22:20:55,693 - root - INFO - epoch:0. iter/total_iter:1270/158025. lr:0.00100. loss:0.1854. spend/global_spend:14.2293/0:18:43. eta:1 day, 14:31:36
2019-10-04 22:21:00,350 - root - INFO - epoch:0. iter/total_iter:1280/158025. lr:0.00100. loss:0.1648. spend/global_spend:4.6569/0:18:48. eta:1 day, 14:22:54
2019-10-04 22:21:12,574 - root - INFO - epoch:0. iter/total_iter:1290/158025. lr:0.00100. loss:0.1617. spend/global_spend:12.2237/0:19:00. eta:1 day, 14:29:39
2019-10-04 22:21:25,807 - root - INFO - epoch:0. iter/total_iter:1300/158025. lr:0.00100. loss:0.1892. spend/global_spend:13.2331/0:19:13. eta:1 day, 14:38:20
2019-10-04 22:21:30,583 - root - INFO - epoch:0. iter/total_iter:1310/158025. lr:0.00100. loss:0.1504. spend/global_spend:4.7766/0:19:18. eta:1 day, 14:30:00
2019-10-04 22:21:42,834 - root - INFO - epoch:0. iter/total_iter:1320/158025. lr:0.00100. loss:0.1799. spend/global_spend:12.2506/0:19:30. eta:1 day, 14:36:36
2019-10-04 22:21:53,419 - root - INFO - epoch:0. iter/total_iter:1330/158025. lr:0.00100. loss:0.1583. spend/global_spend:10.5852/0:19:41. eta:1 day, 14:39:49
2019-10-04 22:21:58,090 - root - INFO - epoch:0. iter/total_iter:1340/158025. lr:0.00100. loss:0.1454. spend/global_spend:4.6707/0:19:46. eta:1 day, 14:31:28
2019-10-04 22:22:08,281 - root - INFO - epoch:0. iter/total_iter:1350/158025. lr:0.00100. loss:0.1693. spend/global_spend:10.1907/0:19:56. eta:1 day, 14:33:54
2019-10-04 22:22:12,956 - root - INFO - epoch:0. iter/total_iter:1360/158025. lr:0.00100. loss:0.1377. spend/global_spend:4.6747/0:20:00. eta:1 day, 14:25:43
2019-10-04 22:22:24,850 - root - INFO - epoch:0. iter/total_iter:1370/158025. lr:0.00100. loss:0.1635. spend/global_spend:11.8948/0:20:12. eta:1 day, 14:31:25
2019-10-04 22:22:40,156 - root - INFO - epoch:0. iter/total_iter:1380/158025. lr:0.00100. loss:0.1706. spend/global_spend:15.3056/0:20:28. eta:1 day, 14:43:28
2019-10-04 22:22:44,832 - root - INFO - epoch:0. iter/total_iter:1390/158025. lr:0.00100. loss:0.1407. spend/global_spend:4.6757/0:20:32. eta:1 day, 14:35:23
2019-10-04 22:22:54,581 - root - INFO - epoch:0. iter/total_iter:1400/158025. lr:0.00100. loss:0.1470. spend/global_spend:9.7490/0:20:42. eta:1 day, 14:36:53
2019-10-04 22:23:05,969 - root - INFO - epoch:0. iter/total_iter:1410/158025. lr:0.00100. loss:0.1514. spend/global_spend:11.3882/0:20:53. eta:1 day, 14:41:23
2019-10-04 22:23:10,637 - root - INFO - epoch:0. iter/total_iter:1420/158025. lr:0.00100. loss:0.1515. spend/global_spend:4.6679/0:20:58. eta:1 day, 14:33:28
2019-10-04 22:23:22,694 - root - INFO - epoch:0. iter/total_iter:1430/158025. lr:0.00100. loss:0.1388. spend/global_spend:12.0577/0:21:10. eta:1 day, 14:39:09
2019-10-04 22:23:27,411 - root - INFO - epoch:0. iter/total_iter:1440/158025. lr:0.00100. loss:0.1793. spend/global_spend:4.7168/0:21:15. eta:1 day, 14:31:27
2019-10-04 22:23:40,825 - root - INFO - epoch:0. iter/total_iter:1450/158025. lr:0.00100. loss:0.1511. spend/global_spend:13.4139/0:21:28. eta:1 day, 14:39:30
2019-10-04 22:23:54,210 - root - INFO - epoch:0. iter/total_iter:1460/158025. lr:0.00100. loss:0.1590. spend/global_spend:13.3851/0:21:42. eta:1 day, 14:47:23
2019-10-04 22:23:58,884 - root - INFO - epoch:0. iter/total_iter:1470/158025. lr:0.00100. loss:0.1827. spend/global_spend:4.6737/0:21:46. eta:1 day, 14:39:42
2019-10-04 22:24:08,760 - root - INFO - epoch:0. iter/total_iter:1480/158025. lr:0.00100. loss:0.1689. spend/global_spend:9.8760/0:21:56. eta:1 day, 14:41:18
2019-10-04 22:24:18,732 - root - INFO - epoch:0. iter/total_iter:1490/158025. lr:0.00100. loss:0.1737. spend/global_spend:9.9723/0:22:06. eta:1 day, 14:43:02
2019-10-04 22:24:25,557 - root - INFO - epoch:0. iter/total_iter:1500/158025. lr:0.00100. loss:0.1546. spend/global_spend:6.8249/0:22:13. eta:1 day, 14:39:16
2019-10-04 22:24:38,703 - root - INFO - epoch:0. iter/total_iter:1510/158025. lr:0.00100. loss:0.1496. spend/global_spend:13.1461/0:22:26. eta:1 day, 14:46:28
2019-10-04 22:24:43,413 - root - INFO - epoch:0. iter/total_iter:1520/158025. lr:0.00100. loss:0.1650. spend/global_spend:4.7101/0:22:31. eta:1 day, 14:39:06
2019-10-04 22:24:54,718 - root - INFO - epoch:0. iter/total_iter:1530/158025. lr:0.00100. loss:0.1403. spend/global_spend:11.3043/0:22:42. eta:1 day, 14:43:04
2019-10-04 22:25:06,826 - root - INFO - epoch:0. iter/total_iter:1540/158025. lr:0.00100. loss:0.1539. spend/global_spend:12.1082/0:22:54. eta:1 day, 14:48:20
2019-10-04 22:25:12,415 - root - INFO - epoch:0. iter/total_iter:1550/158025. lr:0.00100. loss:0.1729. spend/global_spend:5.5893/0:23:00. eta:1 day, 14:42:34
2019-10-04 22:25:25,451 - root - INFO - epoch:0. iter/total_iter:1560/158025. lr:0.00100. loss:0.1874. spend/global_spend:13.0361/0:23:13. eta:1 day, 14:49:20
2019-10-04 22:25:30,444 - root - INFO - epoch:0. iter/total_iter:1570/158025. lr:0.00100. loss:0.1639. spend/global_spend:4.9922/0:23:18. eta:1 day, 14:42:38
2019-10-04 22:25:42,229 - root - INFO - epoch:0. iter/total_iter:1580/158025. lr:0.00100. loss:0.1522. spend/global_spend:11.7860/0:23:30. eta:1 day, 14:47:14
2019-10-04 22:25:55,944 - root - INFO - epoch:0. iter/total_iter:1590/158025. lr:0.00100. loss:0.1620. spend/global_spend:13.7147/0:23:43. eta:1 day, 14:54:57
2019-10-04 22:26:00,638 - root - INFO - epoch:0. iter/total_iter:1600/158025. lr:0.00100. loss:0.1616. spend/global_spend:4.6941/0:23:48. eta:1 day, 14:47:51
2019-10-04 22:26:13,101 - root - INFO - epoch:0. iter/total_iter:1610/158025. lr:0.00100. loss:0.1606. spend/global_spend:12.4634/0:24:01. eta:1 day, 14:53:25
2019-10-04 22:26:23,163 - root - INFO - epoch:0. iter/total_iter:1620/158025. lr:0.00100. loss:0.1727. spend/global_spend:10.0618/0:24:11. eta:1 day, 14:55:04
2019-10-04 22:26:28,892 - root - INFO - epoch:0. iter/total_iter:1630/158025. lr:0.00100. loss:0.1910. spend/global_spend:5.7289/0:24:16. eta:1 day, 14:49:45
2019-10-04 22:26:42,519 - root - INFO - epoch:0. iter/total_iter:1640/158025. lr:0.00100. loss:0.1605. spend/global_spend:13.6268/0:24:30. eta:1 day, 14:57:03
2019-10-04 22:26:47,529 - root - INFO - epoch:0. iter/total_iter:1650/158025. lr:0.00100. loss:0.1617. spend/global_spend:5.0102/0:24:35. eta:1 day, 14:50:39
2019-10-04 22:26:58,777 - root - INFO - epoch:0. iter/total_iter:1660/158025. lr:0.00100. loss:0.1763. spend/global_spend:11.2475/0:24:46. eta:1 day, 14:54:07
2019-10-04 22:27:10,625 - root - INFO - epoch:0. iter/total_iter:1670/158025. lr:0.00100. loss:0.1578. spend/global_spend:11.8469/0:24:58. eta:1 day, 14:58:29
2019-10-04 22:27:15,903 - root - INFO - epoch:0. iter/total_iter:1680/158025. lr:0.00100. loss:0.1500. spend/global_spend:5.2791/0:25:03. eta:1 day, 14:52:36
2019-10-04 22:27:28,460 - root - INFO - epoch:0. iter/total_iter:1690/158025. lr:0.00100. loss:0.1524. spend/global_spend:12.5574/0:25:16. eta:1 day, 14:58:01
2019-10-04 22:27:33,145 - root - INFO - epoch:0. iter/total_iter:1700/158025. lr:0.00100. loss:0.1827. spend/global_spend:4.6844/0:25:21. eta:1 day, 14:51:17
2019-10-04 22:27:46,582 - root - INFO - epoch:0. iter/total_iter:1710/158025. lr:0.00100. loss:0.1745. spend/global_spend:13.4374/0:25:34. eta:1 day, 14:57:59
2019-10-04 22:28:01,942 - root - INFO - epoch:0. iter/total_iter:1720/158025. lr:0.00100. loss:0.1689. spend/global_spend:15.3599/0:25:49. eta:1 day, 15:07:30
2019-10-04 22:28:06,637 - root - INFO - epoch:0. iter/total_iter:1730/158025. lr:0.00100. loss:0.1794. spend/global_spend:4.6946/0:25:54. eta:1 day, 15:00:51
2019-10-04 22:28:22,538 - root - INFO - epoch:0. iter/total_iter:1740/158025. lr:0.00100. loss:0.1722. spend/global_spend:15.9012/0:26:10. eta:1 day, 15:11:03
2019-10-04 22:28:27,539 - root - INFO - epoch:0. iter/total_iter:1750/158025. lr:0.00100. loss:0.1615. spend/global_spend:5.0017/0:26:15. eta:1 day, 15:04:55
2019-10-04 22:28:37,693 - root - INFO - epoch:0. iter/total_iter:1760/158025. lr:0.00100. loss:0.1657. spend/global_spend:10.1533/0:26:25. eta:1 day, 15:06:28
2019-10-04 22:28:52,489 - root - INFO - epoch:0. iter/total_iter:1770/158025. lr:0.00100. loss:0.1692. spend/global_spend:14.7960/0:26:40. eta:1 day, 15:14:50
2019-10-04 22:28:57,142 - root - INFO - epoch:0. iter/total_iter:1780/158025. lr:0.00100. loss:0.1744. spend/global_spend:4.6534/0:26:45. eta:1 day, 15:08:16
2019-10-04 22:29:09,327 - root - INFO - epoch:0. iter/total_iter:1790/158025. lr:0.00100. loss:0.1523. spend/global_spend:12.1843/0:26:57. eta:1 day, 15:12:43
2019-10-04 22:29:20,156 - root - INFO - epoch:0. iter/total_iter:1800/158025. lr:0.00100. loss:0.1744. spend/global_spend:10.8292/0:27:08. eta:1 day, 15:15:10
2019-10-04 22:29:24,817 - root - INFO - epoch:0. iter/total_iter:1810/158025. lr:0.00100. loss:0.1648. spend/global_spend:4.6614/0:27:12. eta:1 day, 15:08:42
2019-10-04 22:29:38,658 - root - INFO - epoch:0. iter/total_iter:1820/158025. lr:0.00100. loss:0.1795. spend/global_spend:13.8405/0:27:26. eta:1 day, 15:15:27
2019-10-04 22:29:43,397 - root - INFO - epoch:0. iter/total_iter:1830/158025. lr:0.00100. loss:0.1619. spend/global_spend:4.7393/0:27:31. eta:1 day, 15:09:10
2019-10-04 22:29:58,358 - root - INFO - epoch:0. iter/total_iter:1840/158025. lr:0.00100. loss:0.1611. spend/global_spend:14.9608/0:27:46. eta:1 day, 15:17:25
2019-10-04 22:30:11,921 - root - INFO - epoch:0. iter/total_iter:1850/158025. lr:0.00100. loss:0.1725. spend/global_spend:13.5632/0:27:59. eta:1 day, 15:23:36
2019-10-04 22:30:16,549 - root - INFO - epoch:0. iter/total_iter:1860/158025. lr:0.00100. loss:0.1416. spend/global_spend:4.6287/0:28:04. eta:1 day, 15:17:14
2019-10-04 22:30:27,905 - root - INFO - epoch:0. iter/total_iter:1870/158025. lr:0.00100. loss:0.1610. spend/global_spend:11.3554/0:28:15. eta:1 day, 15:20:16
2019-10-04 22:30:41,559 - root - INFO - epoch:0. iter/total_iter:1880/158025. lr:0.00100. loss:0.1505. spend/global_spend:13.6542/0:28:29. eta:1 day, 15:26:28
2019-10-04 22:30:46,197 - root - INFO - epoch:0. iter/total_iter:1890/158025. lr:0.00100. loss:0.1521. spend/global_spend:4.6374/0:28:34. eta:1 day, 15:20:11
2019-10-04 22:31:01,595 - root - INFO - epoch:0. iter/total_iter:1900/158025. lr:0.00100. loss:0.1718. spend/global_spend:15.3986/0:28:49. eta:1 day, 15:28:42
2019-10-04 22:31:06,258 - root - INFO - epoch:0. iter/total_iter:1910/158025. lr:0.00100. loss:0.1658. spend/global_spend:4.6625/0:28:54. eta:1 day, 15:22:30
2019-10-04 22:31:19,028 - root - INFO - epoch:0. iter/total_iter:1920/158025. lr:0.00100. loss:0.1858. spend/global_spend:12.7705/0:29:07. eta:1 day, 15:27:21
2019-10-04 22:31:31,861 - root - INFO - epoch:0. iter/total_iter:1930/158025. lr:0.00100. loss:0.1639. spend/global_spend:12.8322/0:29:19. eta:1 day, 15:32:14
2019-10-04 22:31:36,568 - root - INFO - epoch:0. iter/total_iter:1940/158025. lr:0.00100. loss:0.1432. spend/global_spend:4.7074/0:29:24. eta:1 day, 15:26:10
2019-10-04 22:31:49,300 - root - INFO - epoch:0. iter/total_iter:1950/158025. lr:0.00100. loss:0.1827. spend/global_spend:12.7324/0:29:37. eta:1 day, 15:30:52
2019-10-04 22:32:03,919 - root - INFO - epoch:0. iter/total_iter:1960/158025. lr:0.00100. loss:0.1623. spend/global_spend:14.6187/0:29:51. eta:1 day, 15:38:01
2019-10-04 22:32:08,589 - root - INFO - epoch:0. iter/total_iter:1970/158025. lr:0.00100. loss:0.1516. spend/global_spend:4.6697/0:29:56. eta:1 day, 15:31:57
2019-10-04 22:32:20,888 - root - INFO - epoch:0. iter/total_iter:1980/158025. lr:0.00100. loss:0.1705. spend/global_spend:12.2992/0:30:08. eta:1 day, 15:35:59
2019-10-04 22:32:25,571 - root - INFO - epoch:0. iter/total_iter:1990/158025. lr:0.00100. loss:0.1592. spend/global_spend:4.6831/0:30:13. eta:1 day, 15:30:01
2019-10-04 22:32:42,109 - root - INFO - epoch:0. iter/total_iter:2000/158025. lr:0.00100. loss:0.1670. spend/global_spend:16.5378/0:30:30. eta:1 day, 15:39:31
2019-10-04 22:32:54,620 - root - INFO - epoch:0. iter/total_iter:2010/158025. lr:0.00100. loss:0.1611. spend/global_spend:12.5102/0:30:42. eta:1 day, 15:43:42
2019-10-04 22:32:59,306 - root - INFO - epoch:0. iter/total_iter:2020/158025. lr:0.00100. loss:0.1457. spend/global_spend:4.6875/0:30:47. eta:1 day, 15:37:47
2019-10-04 22:33:10,984 - root - INFO - epoch:0. iter/total_iter:2030/158025. lr:0.00100. loss:0.1695. spend/global_spend:11.6777/0:30:58. eta:1 day, 15:40:53
2019-10-04 22:33:21,761 - root - INFO - epoch:0. iter/total_iter:2040/158025. lr:0.00100. loss:0.1579. spend/global_spend:10.7769/0:31:09. eta:1 day, 15:42:47
2019-10-04 22:33:26,450 - root - INFO - epoch:0. iter/total_iter:2050/158025. lr:0.00100. loss:0.1812. spend/global_spend:4.6896/0:31:14. eta:1 day, 15:36:58
2019-10-04 22:33:40,464 - root - INFO - epoch:0. iter/total_iter:2060/158025. lr:0.00100. loss:0.1524. spend/global_spend:14.0134/0:31:28. eta:1 day, 15:42:57
2019-10-04 22:33:45,097 - root - INFO - epoch:0. iter/total_iter:2070/158025. lr:0.00100. loss:0.1738. spend/global_spend:4.6333/0:31:33. eta:1 day, 15:37:06
2019-10-04 22:33:58,455 - root - INFO - epoch:0. iter/total_iter:2080/158025. lr:0.00100. loss:0.1533. spend/global_spend:13.3577/0:31:46. eta:1 day, 15:42:13
2019-10-04 22:34:12,171 - root - INFO - epoch:0. iter/total_iter:2090/158025. lr:0.00100. loss:0.1575. spend/global_spend:13.7157/0:32:00. eta:1 day, 15:47:43
2019-10-04 22:34:16,859 - root - INFO - epoch:0. iter/total_iter:2100/158025. lr:0.00100. loss:0.1632. spend/global_spend:4.6886/0:32:04. eta:1 day, 15:42:00
2019-10-04 22:34:30,103 - root - INFO - epoch:0. iter/total_iter:2110/158025. lr:0.00100. loss:0.1502. spend/global_spend:13.2439/0:32:18. eta:1 day, 15:46:52
2019-10-04 22:34:40,952 - root - INFO - epoch:0. iter/total_iter:2120/158025. lr:0.00100. loss:0.1793. spend/global_spend:10.8492/0:32:28. eta:1 day, 15:48:45
2019-10-04 22:34:45,595 - root - INFO - epoch:0. iter/total_iter:2130/158025. lr:0.00100. loss:0.1597. spend/global_spend:4.6424/0:32:33. eta:1 day, 15:43:03
2019-10-04 22:34:58,700 - root - INFO - epoch:0. iter/total_iter:2140/158025. lr:0.00100. loss:0.1442. spend/global_spend:13.1054/0:32:46. eta:1 day, 15:47:41
2019-10-04 22:35:03,988 - root - INFO - epoch:0. iter/total_iter:2150/158025. lr:0.00100. loss:0.1472. spend/global_spend:5.2883/0:32:51. eta:1 day, 15:42:48
2019-10-04 22:35:20,226 - root - INFO - epoch:0. iter/total_iter:2160/158025. lr:0.00100. loss:0.1459. spend/global_spend:16.2373/0:33:08. eta:1 day, 15:51:09
2019-10-04 22:35:28,820 - root - INFO - epoch:0. iter/total_iter:2170/158025. lr:0.00100. loss:0.1747. spend/global_spend:8.5944/0:33:16. eta:1 day, 15:50:16
2019-10-04 22:35:36,078 - root - INFO - epoch:0. iter/total_iter:2180/158025. lr:0.00100. loss:0.1847. spend/global_spend:7.2582/0:33:24. eta:1 day, 15:47:48
2019-10-04 22:35:51,410 - root - INFO - epoch:0. iter/total_iter:2190/158025. lr:0.00100. loss:0.1608. spend/global_spend:15.3314/0:33:39. eta:1 day, 15:54:56
2019-10-04 22:36:04,287 - root - INFO - epoch:0. iter/total_iter:2200/158025. lr:0.00100. loss:0.1681. spend/global_spend:12.8771/0:33:52. eta:1 day, 15:59:05
2019-10-04 22:36:09,735 - root - INFO - epoch:0. iter/total_iter:2210/158025. lr:0.00100. loss:0.1887. spend/global_spend:5.4484/0:33:57. eta:1 day, 15:54:29
2019-10-04 22:36:22,130 - root - INFO - epoch:0. iter/total_iter:2220/158025. lr:0.00100. loss:0.1649. spend/global_spend:12.3946/0:34:10. eta:1 day, 15:58:02
2019-10-04 22:36:29,730 - root - INFO - epoch:0. iter/total_iter:2230/158025. lr:0.00100. loss:0.1527. spend/global_spend:7.6005/0:34:17. eta:1 day, 15:55:59
2019-10-04 22:36:44,181 - root - INFO - epoch:0. iter/total_iter:2240/158025. lr:0.00100. loss:0.1643. spend/global_spend:14.4513/0:34:32. eta:1 day, 16:01:53
2019-10-04 22:36:56,022 - root - INFO - epoch:0. iter/total_iter:2250/158025. lr:0.00100. loss:0.1622. spend/global_spend:11.8366/0:34:44. eta:1 day, 16:04:43
2019-10-04 22:37:03,861 - root - INFO - epoch:0. iter/total_iter:2260/158025. lr:0.00100. loss:0.1702. spend/global_spend:7.8424/0:34:51. eta:1 day, 16:02:56
2019-10-04 22:37:18,507 - root - INFO - epoch:0. iter/total_iter:2270/158025. lr:0.00100. loss:0.1609. spend/global_spend:14.6464/0:35:06. eta:1 day, 16:08:56
2019-10-04 22:37:24,040 - root - INFO - epoch:0. iter/total_iter:2280/158025. lr:0.00100. loss:0.1557. spend/global_spend:5.5335/0:35:12. eta:1 day, 16:04:31
2019-10-04 22:37:39,326 - root - INFO - epoch:0. iter/total_iter:2290/158025. lr:0.00100. loss:0.1617. spend/global_spend:15.2858/0:35:27. eta:1 day, 16:11:11
2019-10-04 22:37:48,474 - root - INFO - epoch:0. iter/total_iter:2300/158025. lr:0.00100. loss:0.1584. spend/global_spend:9.1481/0:35:36. eta:1 day, 16:10:53
2019-10-04 22:37:57,611 - root - INFO - epoch:0. iter/total_iter:2310/158025. lr:0.00100. loss:0.1448. spend/global_spend:9.1366/0:35:45. eta:1 day, 16:10:33
2019-10-04 22:38:10,977 - root - INFO - epoch:0. iter/total_iter:2320/158025. lr:0.00100. loss:0.1559. spend/global_spend:13.3662/0:35:58. eta:1 day, 16:14:57
2019-10-04 22:38:15,636 - root - INFO - epoch:0. iter/total_iter:2330/158025. lr:0.00100. loss:0.1607. spend/global_spend:4.6584/0:36:03. eta:1 day, 16:09:38
2019-10-04 22:38:28,886 - root - INFO - epoch:0. iter/total_iter:2340/158025. lr:0.00100. loss:0.1644. spend/global_spend:13.2499/0:36:16. eta:1 day, 16:13:52
2019-10-04 22:38:43,016 - root - INFO - epoch:0. iter/total_iter:2350/158025. lr:0.00100. loss:0.1643. spend/global_spend:14.1300/0:36:31. eta:1 day, 16:19:02
2019-10-04 22:38:47,739 - root - INFO - epoch:0. iter/total_iter:2360/158025. lr:0.00100. loss:0.1445. spend/global_spend:4.7230/0:36:35. eta:1 day, 16:13:50
2019-10-04 22:39:04,269 - root - INFO - epoch:0. iter/total_iter:2370/158025. lr:0.00100. loss:0.1634. spend/global_spend:16.5310/0:36:52. eta:1 day, 16:21:35
2019-10-04 22:39:14,071 - root - INFO - epoch:0. iter/total_iter:2380/158025. lr:0.00100. loss:0.1964. spend/global_spend:9.8014/0:37:02. eta:1 day, 16:21:56
2019-10-04 22:39:18,758 - root - INFO - epoch:0. iter/total_iter:2390/158025. lr:0.00100. loss:0.1650. spend/global_spend:4.6874/0:37:06. eta:1 day, 16:16:44
2019-10-04 22:39:34,755 - root - INFO - epoch:0. iter/total_iter:2400/158025. lr:0.00100. loss:0.1703. spend/global_spend:15.9970/0:37:22. eta:1 day, 16:23:48
2019-10-04 22:39:39,437 - root - INFO - epoch:0. iter/total_iter:2410/158025. lr:0.00100. loss:0.1587. spend/global_spend:4.6813/0:37:27. eta:1 day, 16:18:38
2019-10-04 22:39:54,825 - root - INFO - epoch:0. iter/total_iter:2420/158025. lr:0.00100. loss:0.1810. spend/global_spend:15.3883/0:37:42. eta:1 day, 16:24:58
2019-10-04 22:40:07,193 - root - INFO - epoch:0. iter/total_iter:2430/158025. lr:0.00100. loss:0.1467. spend/global_spend:12.3684/0:37:55. eta:1 day, 16:28:02
2019-10-04 22:40:11,839 - root - INFO - epoch:0. iter/total_iter:2440/158025. lr:0.00100. loss:0.1435. spend/global_spend:4.6453/0:37:59. eta:1 day, 16:22:52
2019-10-04 22:40:23,747 - root - INFO - epoch:0. iter/total_iter:2450/158025. lr:0.00100. loss:0.1659. spend/global_spend:11.9082/0:38:11. eta:1 day, 16:25:25
2019-10-04 22:40:39,084 - root - INFO - epoch:0. iter/total_iter:2460/158025. lr:0.00100. loss:0.1796. spend/global_spend:15.3367/0:38:27. eta:1 day, 16:31:34
2019-10-04 22:40:43,758 - root - INFO - epoch:0. iter/total_iter:2470/158025. lr:0.00100. loss:0.1630. spend/global_spend:4.6747/0:38:31. eta:1 day, 16:26:29
2019-10-04 22:40:57,462 - root - INFO - epoch:0. iter/total_iter:2480/158025. lr:0.00100. loss:0.1737. spend/global_spend:13.7033/0:38:45. eta:1 day, 16:30:52
2019-10-04 22:41:02,121 - root - INFO - epoch:0. iter/total_iter:2490/158025. lr:0.00100. loss:0.1423. spend/global_spend:4.6597/0:38:50. eta:1 day, 16:25:48
2019-10-04 22:41:17,684 - root - INFO - epoch:0. iter/total_iter:2500/158025. lr:0.00100. loss:0.1698. spend/global_spend:15.5624/0:39:05. eta:1 day, 16:32:04
2019-10-04 22:41:32,335 - root - INFO - epoch:0. iter/total_iter:2510/158025. lr:0.00100. loss:0.1517. spend/global_spend:14.6512/0:39:20. eta:1 day, 16:37:21
2019-10-04 22:41:36,989 - root - INFO - epoch:0. iter/total_iter:2520/158025. lr:0.00100. loss:0.1669. spend/global_spend:4.6543/0:39:24. eta:1 day, 16:32:19
2019-10-04 22:41:51,977 - root - INFO - epoch:0. iter/total_iter:2530/158025. lr:0.00100. loss:0.1458. spend/global_spend:14.9880/0:39:39. eta:1 day, 16:37:54
2019-10-04 22:42:03,749 - root - INFO - epoch:0. iter/total_iter:2540/158025. lr:0.00100. loss:0.1648. spend/global_spend:11.7715/0:39:51. eta:1 day, 16:40:09
2019-10-04 22:42:08,450 - root - INFO - epoch:0. iter/total_iter:2550/158025. lr:0.00100. loss:0.1628. spend/global_spend:4.7017/0:39:56. eta:1 day, 16:35:12
2019-10-04 22:42:24,511 - root - INFO - epoch:0. iter/total_iter:2560/158025. lr:0.00100. loss:0.1688. spend/global_spend:16.0593/0:40:12. eta:1 day, 16:41:47
2019-10-04 22:42:29,163 - root - INFO - epoch:0. iter/total_iter:2570/158025. lr:0.00100. loss:0.1616. spend/global_spend:4.6536/0:40:17. eta:1 day, 16:36:50
2019-10-04 22:42:45,175 - root - INFO - epoch:0. iter/total_iter:2580/158025. lr:0.00100. loss:0.1531. spend/global_spend:16.0120/0:40:33. eta:1 day, 16:43:18
2019-10-04 22:43:02,055 - root - INFO - epoch:0. iter/total_iter:2590/158025. lr:0.00100. loss:0.1677. spend/global_spend:16.8798/0:40:50. eta:1 day, 16:50:36
2019-10-04 22:43:06,688 - root - INFO - epoch:0. iter/total_iter:2600/158025. lr:0.00100. loss:0.1781. spend/global_spend:4.6325/0:40:54. eta:1 day, 16:45:38
2019-10-04 22:43:20,990 - root - INFO - epoch:0. iter/total_iter:2610/158025. lr:0.00100. loss:0.1473. spend/global_spend:14.3026/0:41:08. eta:1 day, 16:50:18
2019-10-04 22:43:25,650 - root - INFO - epoch:0. iter/total_iter:2620/158025. lr:0.00100. loss:0.1638. spend/global_spend:4.6592/0:41:13. eta:1 day, 16:45:24
2019-10-04 22:43:39,834 - root - INFO - epoch:0. iter/total_iter:2630/158025. lr:0.00100. loss:0.1694. spend/global_spend:14.1850/0:41:27. eta:1 day, 16:49:54
2019-10-04 22:43:54,772 - root - INFO - epoch:0. iter/total_iter:2640/158025. lr:0.00100. loss:0.1696. spend/global_spend:14.9376/0:41:42. eta:1 day, 16:55:07
2019-10-04 22:43:59,448 - root - INFO - epoch:0. iter/total_iter:2650/158025. lr:0.00100. loss:0.1489. spend/global_spend:4.6761/0:41:47. eta:1 day, 16:50:16
2019-10-04 22:44:16,289 - root - INFO - epoch:0. iter/total_iter:2660/158025. lr:0.00100. loss:0.1545. spend/global_spend:16.8412/0:42:04. eta:1 day, 16:57:18
2019-10-04 22:44:29,754 - root - INFO - epoch:0. iter/total_iter:2670/158025. lr:0.00100. loss:0.1431. spend/global_spend:13.4650/0:42:17. eta:1 day, 17:01:00
2019-10-04 22:44:34,432 - root - INFO - epoch:0. iter/total_iter:2680/158025. lr:0.00100. loss:0.1819. spend/global_spend:4.6779/0:42:22. eta:1 day, 16:56:10
2019-10-04 22:44:48,428 - root - INFO - epoch:0. iter/total_iter:2690/158025. lr:0.00100. loss:0.1747. spend/global_spend:13.9956/0:42:36. eta:1 day, 17:00:21
2019-10-04 22:44:53,141 - root - INFO - epoch:0. iter/total_iter:2700/158025. lr:0.00100. loss:0.1390. spend/global_spend:4.7130/0:42:41. eta:1 day, 16:55:36
2019-10-04 22:45:06,442 - root - INFO - epoch:0. iter/total_iter:2710/158025. lr:0.00100. loss:0.1659. spend/global_spend:13.3010/0:42:54. eta:1 day, 16:59:05
2019-10-04 22:45:21,076 - root - INFO - epoch:0. iter/total_iter:2720/158025. lr:0.00100. loss:0.1667. spend/global_spend:14.6337/0:43:09. eta:1 day, 17:03:49
2019-10-04 22:45:25,809 - root - INFO - epoch:0. iter/total_iter:2730/158025. lr:0.00100. loss:0.1395. spend/global_spend:4.7338/0:43:13. eta:1 day, 16:59:07
2019-10-04 22:45:37,550 - root - INFO - epoch:0. iter/total_iter:2740/158025. lr:0.00100. loss:0.1617. spend/global_spend:11.7405/0:43:25. eta:1 day, 17:01:05
2019-10-04 22:45:53,291 - root - INFO - epoch:0. iter/total_iter:2750/158025. lr:0.00100. loss:0.1506. spend/global_spend:15.7411/0:43:41. eta:1 day, 17:06:47
2019-10-04 22:45:57,934 - root - INFO - epoch:0. iter/total_iter:2760/158025. lr:0.00100. loss:0.1368. spend/global_spend:4.6430/0:43:45. eta:1 day, 17:02:02
2019-10-04 22:46:12,683 - root - INFO - epoch:0. iter/total_iter:2770/158025. lr:0.00100. loss:0.1478. spend/global_spend:14.7488/0:44:00. eta:1 day, 17:06:46
2019-10-04 22:46:24,670 - root - INFO - epoch:0. iter/total_iter:2780/158025. lr:0.00100. loss:0.1623. spend/global_spend:11.9872/0:44:12. eta:1 day, 17:08:54
2019-10-04 22:46:31,329 - root - INFO - epoch:0. iter/total_iter:2790/158025. lr:0.00100. loss:0.1456. spend/global_spend:6.6591/0:44:19. eta:1 day, 17:06:04
2019-10-04 22:46:45,999 - root - INFO - epoch:0. iter/total_iter:2800/158025. lr:0.00100. loss:0.1532. spend/global_spend:14.6699/0:44:33. eta:1 day, 17:10:39
2019-10-04 22:46:50,653 - root - INFO - epoch:0. iter/total_iter:2810/158025. lr:0.00100. loss:0.1433. spend/global_spend:4.6538/0:44:38. eta:1 day, 17:05:59
2019-10-04 22:47:06,725 - root - INFO - epoch:0. iter/total_iter:2820/158025. lr:0.00100. loss:0.1645. spend/global_spend:16.0720/0:44:54. eta:1 day, 17:11:50
2019-10-04 22:47:22,476 - root - INFO - epoch:0. iter/total_iter:2830/158025. lr:0.00100. loss:0.1494. spend/global_spend:15.7513/0:45:10. eta:1 day, 17:17:20
2019-10-04 22:47:27,124 - root - INFO - epoch:0. iter/total_iter:2840/158025. lr:0.00100. loss:0.1510. spend/global_spend:4.6477/0:45:15. eta:1 day, 17:12:41
2019-10-04 22:47:40,967 - root - INFO - epoch:0. iter/total_iter:2850/158025. lr:0.00100. loss:0.1443. spend/global_spend:13.8432/0:45:28. eta:1 day, 17:16:24
2019-10-04 22:47:55,598 - root - INFO - epoch:0. iter/total_iter:2860/158025. lr:0.00100. loss:0.1645. spend/global_spend:14.6311/0:45:43. eta:1 day, 17:20:49
2019-10-04 22:48:00,226 - root - INFO - epoch:0. iter/total_iter:2870/158025. lr:0.00100. loss:0.1607. spend/global_spend:4.6276/0:45:48. eta:1 day, 17:16:11
2019-10-04 22:48:17,048 - root - INFO - epoch:0. iter/total_iter:2880/158025. lr:0.00100. loss:0.1603. spend/global_spend:16.8227/0:46:05. eta:1 day, 17:22:32
2019-10-04 22:48:21,717 - root - INFO - epoch:0. iter/total_iter:2890/158025. lr:0.00100. loss:0.1630. spend/global_spend:4.6685/0:46:09. eta:1 day, 17:17:58
2019-10-04 22:48:36,982 - root - INFO - epoch:0. iter/total_iter:2900/158025. lr:0.00100. loss:0.1473. spend/global_spend:15.2653/0:46:24. eta:1 day, 17:22:52
2019-10-04 22:48:51,620 - root - INFO - epoch:0. iter/total_iter:2910/158025. lr:0.00100. loss:0.1575. spend/global_spend:14.6380/0:46:39. eta:1 day, 17:27:11
2019-10-04 22:48:56,267 - root - INFO - epoch:0. iter/total_iter:2920/158025. lr:0.00100. loss:0.1412. spend/global_spend:4.6467/0:46:44. eta:1 day, 17:22:37
2019-10-04 22:49:11,935 - root - INFO - epoch:0. iter/total_iter:2930/158025. lr:0.00100. loss:0.1425. spend/global_spend:15.6684/0:46:59. eta:1 day, 17:27:48
2019-10-04 22:49:27,965 - root - INFO - epoch:0. iter/total_iter:2940/158025. lr:0.00100. loss:0.1437. spend/global_spend:16.0303/0:47:15. eta:1 day, 17:33:17
2019-10-04 22:49:32,608 - root - INFO - epoch:0. iter/total_iter:2950/158025. lr:0.00100. loss:0.1621. spend/global_spend:4.6426/0:47:20. eta:1 day, 17:28:44
2019-10-04 22:49:49,668 - root - INFO - epoch:0. iter/total_iter:2960/158025. lr:0.00100. loss:0.1380. spend/global_spend:17.0602/0:47:37. eta:1 day, 17:35:04
2019-10-04 22:49:54,301 - root - INFO - epoch:0. iter/total_iter:2970/158025. lr:0.00100. loss:0.1266. spend/global_spend:4.6323/0:47:42. eta:1 day, 17:30:32
2019-10-04 22:50:12,295 - root - INFO - epoch:0. iter/total_iter:2980/158025. lr:0.00100. loss:0.1372. spend/global_spend:17.9888/0:48:00. eta:1 day, 17:37:37
2019-10-04 22:50:24,421 - root - INFO - epoch:0. iter/total_iter:2990/158025. lr:0.00100. loss:0.1723. spend/global_spend:12.1316/0:48:12. eta:1 day, 17:39:35
2019-10-04 22:50:29,047 - root - INFO - epoch:0. iter/total_iter:3000/158025. lr:0.00100. loss:0.1861. spend/global_spend:4.6255/0:48:17. eta:1 day, 17:35:04
2019-10-04 22:50:45,809 - root - INFO - epoch:0. iter/total_iter:3010/158025. lr:0.00100. loss:0.1544. spend/global_spend:16.7622/0:48:33. eta:1 day, 17:41:01
2019-10-04 22:50:59,833 - root - INFO - epoch:0. iter/total_iter:3020/158025. lr:0.00100. loss:0.1616. spend/global_spend:14.0243/0:48:47. eta:1 day, 17:44:34
2019-10-04 22:51:04,434 - root - INFO - epoch:0. iter/total_iter:3030/158025. lr:0.00100. loss:0.1515. spend/global_spend:4.6005/0:48:52. eta:1 day, 17:40:04
2019-10-04 22:51:21,877 - root - INFO - epoch:0. iter/total_iter:3040/158025. lr:0.00100. loss:0.1510. spend/global_spend:17.4432/0:49:09. eta:1 day, 17:46:30
2019-10-04 22:51:26,508 - root - INFO - epoch:0. iter/total_iter:3050/158025. lr:0.00100. loss:0.1646. spend/global_spend:4.6317/0:49:14. eta:1 day, 17:42:02
2019-10-04 22:51:40,670 - root - INFO - epoch:0. iter/total_iter:3060/158025. lr:0.00100. loss:0.1408. spend/global_spend:14.1612/0:49:28. eta:1 day, 17:45:39
2019-10-04 22:51:55,979 - root - INFO - epoch:0. iter/total_iter:3070/158025. lr:0.00100. loss:0.1617. spend/global_spend:15.3093/0:49:43. eta:1 day, 17:50:13
2019-10-04 22:52:00,586 - root - INFO - epoch:0. iter/total_iter:3080/158025. lr:0.00100. loss:0.1419. spend/global_spend:4.6072/0:49:48. eta:1 day, 17:45:46
2019-10-04 22:52:17,156 - root - INFO - epoch:0. iter/total_iter:3090/158025. lr:0.00100. loss:0.1459. spend/global_spend:16.5683/0:50:05. eta:1 day, 17:51:20
2019-10-04 22:52:34,254 - root - INFO - epoch:0. iter/total_iter:3100/158025. lr:0.00100. loss:0.1690. spend/global_spend:17.0991/0:50:22. eta:1 day, 17:57:19
2019-10-04 22:52:38,869 - root - INFO - epoch:0. iter/total_iter:3110/158025. lr:0.00100. loss:0.1401. spend/global_spend:4.6159/0:50:26. eta:1 day, 17:52:53
2019-10-04 22:52:56,942 - root - INFO - epoch:0. iter/total_iter:3120/158025. lr:0.00100. loss:0.1639. spend/global_spend:18.0725/0:50:44. eta:1 day, 17:59:38
2019-10-04 22:53:01,552 - root - INFO - epoch:0. iter/total_iter:3130/158025. lr:0.00100. loss:0.1533. spend/global_spend:4.6102/0:50:49. eta:1 day, 17:55:13
2019-10-04 22:53:16,820 - root - INFO - epoch:0. iter/total_iter:3140/158025. lr:0.00100. loss:0.1487. spend/global_spend:15.2675/0:51:04. eta:1 day, 17:59:36
2019-10-04 22:53:31,366 - root - INFO - epoch:0. iter/total_iter:3150/158025. lr:0.00100. loss:0.1867. spend/global_spend:14.5459/0:51:19. eta:1 day, 18:03:22
2019-10-04 22:53:35,993 - root - INFO - epoch:0. iter/total_iter:3160/158025. lr:0.00100. loss:0.1677. spend/global_spend:4.6271/0:51:23. eta:1 day, 17:58:59
2019-10-04 22:53:49,229 - root - INFO - epoch:0. iter/total_iter:3170/158025. lr:0.00100. loss:0.1286. spend/global_spend:13.2359/0:51:37. eta:1 day, 18:01:39
2019-10-04 22:54:07,824 - root - INFO - epoch:0. iter/total_iter:3180/158025. lr:0.00100. loss:0.1430. spend/global_spend:18.5953/0:51:55. eta:1 day, 18:08:39
2019-10-04 22:54:12,433 - root - INFO - epoch:0. iter/total_iter:3190/158025. lr:0.00100. loss:0.1311. spend/global_spend:4.6089/0:52:00. eta:1 day, 18:04:18
2019-10-04 22:54:26,626 - root - INFO - epoch:0. iter/total_iter:3200/158025. lr:0.00100. loss:0.1722. spend/global_spend:14.1933/0:52:14. eta:1 day, 18:07:41
2019-10-04 22:54:31,279 - root - INFO - epoch:0. iter/total_iter:3210/158025. lr:0.00100. loss:0.1524. spend/global_spend:4.6527/0:52:19. eta:1 day, 18:03:24
2019-10-04 22:54:48,782 - root - INFO - epoch:0. iter/total_iter:3220/158025. lr:0.00100. loss:0.1575. spend/global_spend:17.5033/0:52:36. eta:1 day, 18:09:25
2019-10-04 22:55:03,332 - root - INFO - epoch:0. iter/total_iter:3230/158025. lr:0.00100. loss:0.1656. spend/global_spend:14.5496/0:52:51. eta:1 day, 18:13:03
2019-10-04 22:55:07,947 - root - INFO - epoch:0. iter/total_iter:3240/158025. lr:0.00100. loss:0.1527. spend/global_spend:4.6149/0:52:55. eta:1 day, 18:08:44
2019-10-04 22:55:23,848 - root - INFO - epoch:0. iter/total_iter:3250/158025. lr:0.00100. loss:0.1416. spend/global_spend:15.9015/0:53:11. eta:1 day, 18:13:25
2019-10-04 22:55:40,202 - root - INFO - epoch:0. iter/total_iter:3260/158025. lr:0.00100. loss:0.1455. spend/global_spend:16.3534/0:53:28. eta:1 day, 18:18:25
2019-10-04 22:55:44,867 - root - INFO - epoch:0. iter/total_iter:3270/158025. lr:0.00100. loss:0.1903. spend/global_spend:4.6658/0:53:32. eta:1 day, 18:14:11
2019-10-04 22:56:03,830 - root - INFO - epoch:0. iter/total_iter:3280/158025. lr:0.00100. loss:0.1653. spend/global_spend:18.9623/0:53:51. eta:1 day, 18:21:12
2019-10-04 22:56:08,461 - root - INFO - epoch:0. iter/total_iter:3290/158025. lr:0.00100. loss:0.1542. spend/global_spend:4.6317/0:53:56. eta:1 day, 18:16:56
2019-10-04 22:56:24,416 - root - INFO - epoch:0. iter/total_iter:3300/158025. lr:0.00100. loss:0.1608. spend/global_spend:15.9547/0:54:12. eta:1 day, 18:21:33
2019-10-04 22:56:42,447 - root - INFO - epoch:0. iter/total_iter:3310/158025. lr:0.00100. loss:0.1461. spend/global_spend:18.0307/0:54:30. eta:1 day, 18:27:46
2019-10-04 22:56:47,127 - root - INFO - epoch:0. iter/total_iter:3320/158025. lr:0.00100. loss:0.1376. spend/global_spend:4.6799/0:54:35. eta:1 day, 18:23:33
2019-10-04 22:57:00,285 - root - INFO - epoch:0. iter/total_iter:3330/158025. lr:0.00100. loss:0.1655. spend/global_spend:13.1585/0:54:48. eta:1 day, 18:25:57
2019-10-04 22:57:14,534 - root - INFO - epoch:0. iter/total_iter:3340/158025. lr:0.00100. loss:0.1589. spend/global_spend:14.2490/0:55:02. eta:1 day, 18:29:09
2019-10-04 22:57:19,181 - root - INFO - epoch:0. iter/total_iter:3350/158025. lr:0.00100. loss:0.1579. spend/global_spend:4.6466/0:55:07. eta:1 day, 18:24:57
2019-10-04 22:57:33,024 - root - INFO - epoch:0. iter/total_iter:3360/158025. lr:0.00100. loss:0.1500. spend/global_spend:13.8434/0:55:21. eta:1 day, 18:27:50
2019-10-04 22:57:37,805 - root - INFO - epoch:0. iter/total_iter:3370/158025. lr:0.00100. loss:0.1618. spend/global_spend:4.7806/0:55:25. eta:1 day, 18:23:46
2019-10-04 22:57:56,456 - root - INFO - epoch:0. iter/total_iter:3380/158025. lr:0.00100. loss:0.1563. spend/global_spend:18.6506/0:55:44. eta:1 day, 18:30:18
2019-10-04 22:58:15,366 - root - INFO - epoch:0. iter/total_iter:3390/158025. lr:0.00100. loss:0.1596. spend/global_spend:18.9112/0:56:03. eta:1 day, 18:37:00
2019-10-04 22:58:20,000 - root - INFO - epoch:0. iter/total_iter:3400/158025. lr:0.00100. loss:0.1540. spend/global_spend:4.6308/0:56:07. eta:1 day, 18:32:49
2019-10-04 22:58:33,325 - root - INFO - epoch:0. iter/total_iter:3410/158025. lr:0.00100. loss:0.1544. spend/global_spend:13.3281/0:56:21. eta:1 day, 18:35:14
2019-10-04 22:58:49,644 - root - INFO - epoch:0. iter/total_iter:3420/158025. lr:0.00100. loss:0.1598. spend/global_spend:16.3185/0:56:37. eta:1 day, 18:39:54
2019-10-04 22:58:54,252 - root - INFO - epoch:0. iter/total_iter:3430/158025. lr:0.00100. loss:0.1532. spend/global_spend:4.6075/0:56:42. eta:1 day, 18:35:44
2019-10-04 22:59:10,646 - root - INFO - epoch:0. iter/total_iter:3440/158025. lr:0.00100. loss:0.1439. spend/global_spend:16.3941/0:56:58. eta:1 day, 18:40:25
2019-10-04 22:59:15,300 - root - INFO - epoch:0. iter/total_iter:3450/158025. lr:0.00100. loss:0.1609. spend/global_spend:4.6544/0:57:03. eta:1 day, 18:36:18
2019-10-04 22:59:31,009 - root - INFO - epoch:0. iter/total_iter:3460/158025. lr:0.00100. loss:0.1408. spend/global_spend:15.7095/0:57:19. eta:1 day, 18:40:27
2019-10-04 22:59:42,337 - root - INFO - epoch:0. iter/total_iter:3470/158025. lr:0.00100. loss:0.1579. spend/global_spend:11.3278/0:57:30. eta:1 day, 18:41:19
2019-10-04 22:59:50,365 - root - INFO - epoch:0. iter/total_iter:3480/158025. lr:0.00100. loss:0.1586. spend/global_spend:8.0282/0:57:38. eta:1 day, 18:39:44
2019-10-04 23:00:04,166 - root - INFO - epoch:0. iter/total_iter:3490/158025. lr:0.00100. loss:0.1675. spend/global_spend:13.8001/0:57:52. eta:1 day, 18:42:25
2019-10-04 23:00:15,959 - root - INFO - epoch:0. iter/total_iter:3500/158025. lr:0.00100. loss:0.1753. spend/global_spend:11.7933/0:58:03. eta:1 day, 18:43:36
2019-10-04 23:00:26,859 - root - INFO - epoch:0. iter/total_iter:3510/158025. lr:0.00100. loss:0.1527. spend/global_spend:10.8997/0:58:14. eta:1 day, 18:44:08
2019-10-04 23:00:43,521 - root - INFO - epoch:0. iter/total_iter:3520/158025. lr:0.00100. loss:0.1385. spend/global_spend:16.6627/0:58:31. eta:1 day, 18:48:52
2019-10-04 23:00:48,144 - root - INFO - epoch:0. iter/total_iter:3530/158025. lr:0.00100. loss:0.1491. spend/global_spend:4.6232/0:58:36. eta:1 day, 18:44:48
2019-10-04 23:01:04,938 - root - INFO - epoch:0. iter/total_iter:3540/158025. lr:0.00100. loss:0.1720. spend/global_spend:16.7938/0:58:52. eta:1 day, 18:49:36
2019-10-04 23:01:22,537 - root - INFO - epoch:0. iter/total_iter:3550/158025. lr:0.00100. loss:0.1541. spend/global_spend:17.5989/0:59:10. eta:1 day, 18:54:58
2019-10-04 23:01:27,150 - root - INFO - epoch:0. iter/total_iter:3560/158025. lr:0.00100. loss:0.1531. spend/global_spend:4.6129/0:59:15. eta:1 day, 18:50:54
2019-10-04 23:01:43,584 - root - INFO - epoch:0. iter/total_iter:3570/158025. lr:0.00100. loss:0.1611. spend/global_spend:16.4344/0:59:31. eta:1 day, 18:55:23
2019-10-04 23:02:01,116 - root - INFO - epoch:0. iter/total_iter:3580/158025. lr:0.00100. loss:0.1464. spend/global_spend:17.5311/0:59:49. eta:1 day, 19:00:38
2019-10-04 23:02:06,664 - root - INFO - epoch:0. iter/total_iter:3590/158025. lr:0.00100. loss:0.1530. spend/global_spend:5.5483/0:59:54. eta:1 day, 18:57:15
2019-10-04 23:02:22,407 - root - INFO - epoch:0. iter/total_iter:3600/158025. lr:0.00100. loss:0.1745. spend/global_spend:15.7436/1:00:10. eta:1 day, 19:01:11
2019-10-04 23:02:32,744 - root - INFO - epoch:0. iter/total_iter:3610/158025. lr:0.00100. loss:0.1494. spend/global_spend:10.3366/1:00:20. eta:1 day, 19:01:14
2019-10-04 23:02:47,552 - root - INFO - epoch:0. iter/total_iter:3620/158025. lr:0.00100. loss:0.1610. spend/global_spend:14.8075/1:00:35. eta:1 day, 19:04:28
2019-10-04 23:03:02,905 - root - INFO - epoch:0. iter/total_iter:3630/158025. lr:0.00100. loss:0.1437. spend/global_spend:15.3537/1:00:50. eta:1 day, 19:08:04
2019-10-04 23:03:09,092 - root - INFO - epoch:0. iter/total_iter:3640/158025. lr:0.00100. loss:0.1318. spend/global_spend:6.1867/1:00:57. eta:1 day, 19:05:09
2019-10-04 23:03:24,710 - root - INFO - epoch:0. iter/total_iter:3650/158025. lr:0.00100. loss:0.1593. spend/global_spend:15.6185/1:01:12. eta:1 day, 19:08:55
2019-10-04 23:03:42,475 - root - INFO - epoch:0. iter/total_iter:3660/158025. lr:0.00100. loss:0.1444. spend/global_spend:17.7650/1:01:30. eta:1 day, 19:14:10
2019-10-04 23:03:47,289 - root - INFO - epoch:0. iter/total_iter:3670/158025. lr:0.00100. loss:0.1514. spend/global_spend:4.8136/1:01:35. eta:1 day, 19:10:18
2019-10-04 23:04:02,796 - root - INFO - epoch:0. iter/total_iter:3680/158025. lr:0.00100. loss:0.1483. spend/global_spend:15.5067/1:01:50. eta:1 day, 19:13:56
2019-10-04 23:04:09,024 - root - INFO - epoch:0. iter/total_iter:3690/158025. lr:0.00100. loss:0.1766. spend/global_spend:6.2288/1:01:57. eta:1 day, 19:11:05
2019-10-04 23:04:26,047 - root - INFO - epoch:0. iter/total_iter:3700/158025. lr:0.00100. loss:0.1388. spend/global_spend:17.0227/1:02:14. eta:1 day, 19:15:44
2019-10-04 23:04:38,573 - root - INFO - epoch:0. iter/total_iter:3710/158025. lr:0.00100. loss:0.1687. spend/global_spend:12.5259/1:02:26. eta:1 day, 19:17:16
2019-10-04 23:04:47,921 - root - INFO - epoch:0. iter/total_iter:3720/158025. lr:0.00100. loss:0.1607. spend/global_spend:9.3478/1:02:35. eta:1 day, 19:16:34
2019-10-04 23:05:02,625 - root - INFO - epoch:0. iter/total_iter:3730/158025. lr:0.00100. loss:0.1543. spend/global_spend:14.7040/1:02:50. eta:1 day, 19:19:35
2019-10-04 23:05:16,813 - root - INFO - epoch:0. iter/total_iter:3740/158025. lr:0.00100. loss:0.1465. spend/global_spend:14.1876/1:03:04. eta:1 day, 19:22:13
2019-10-04 23:05:27,089 - root - INFO - epoch:0. iter/total_iter:3750/158025. lr:0.00100. loss:0.1497. spend/global_spend:10.2762/1:03:15. eta:1 day, 19:22:09
2019-10-04 23:05:46,025 - root - INFO - epoch:0. iter/total_iter:3760/158025. lr:0.00100. loss:0.1646. spend/global_spend:18.9360/1:03:34. eta:1 day, 19:28:01
2019-10-04 23:05:50,631 - root - INFO - epoch:0. iter/total_iter:3770/158025. lr:0.00100. loss:0.1456. spend/global_spend:4.6060/1:03:38. eta:1 day, 19:24:04
2019-10-04 23:06:05,998 - root - INFO - epoch:0. iter/total_iter:3780/158025. lr:0.00100. loss:0.1483. spend/global_spend:15.3674/1:03:53. eta:1 day, 19:27:28
2019-10-04 23:06:15,102 - root - INFO - epoch:0. iter/total_iter:3790/158025. lr:0.00100. loss:0.1519. spend/global_spend:9.1037/1:04:03. eta:1 day, 19:26:35
2019-10-04 23:06:29,174 - root - INFO - epoch:0. iter/total_iter:3800/158025. lr:0.00100. loss:0.1525. spend/global_spend:14.0725/1:04:17. eta:1 day, 19:29:05
2019-10-04 23:06:47,206 - root - INFO - epoch:0. iter/total_iter:3810/158025. lr:0.00100. loss:0.1492. spend/global_spend:18.0317/1:04:35. eta:1 day, 19:34:14
2019-10-04 23:06:51,802 - root - INFO - epoch:0. iter/total_iter:3820/158025. lr:0.00100. loss:0.1449. spend/global_spend:4.5960/1:04:39. eta:1 day, 19:30:18
2019-10-04 23:07:11,567 - root - INFO - epoch:0. iter/total_iter:3830/158025. lr:0.00100. loss:0.1486. spend/global_spend:19.7653/1:04:59. eta:1 day, 19:36:35
2019-10-04 23:07:31,988 - root - INFO - epoch:0. iter/total_iter:3840/158025. lr:0.00100. loss:0.1518. spend/global_spend:20.4206/1:05:19. eta:1 day, 19:43:16
2019-10-04 23:07:36,586 - root - INFO - epoch:0. iter/total_iter:3850/158025. lr:0.00100. loss:0.1539. spend/global_spend:4.5983/1:05:24. eta:1 day, 19:39:21
2019-10-04 23:07:52,394 - root - INFO - epoch:0. iter/total_iter:3860/158025. lr:0.00100. loss:0.1546. spend/global_spend:15.8062/1:05:40. eta:1 day, 19:42:55
2019-10-04 23:07:57,046 - root - INFO - epoch:0. iter/total_iter:3870/158025. lr:0.00100. loss:0.1761. spend/global_spend:4.6541/1:05:45. eta:1 day, 19:39:04
2019-10-04 23:08:13,319 - root - INFO - epoch:0. iter/total_iter:3880/158025. lr:0.00100. loss:0.1511. spend/global_spend:16.2720/1:06:01. eta:1 day, 19:42:55
2019-10-04 23:08:28,812 - root - INFO - epoch:0. iter/total_iter:3890/158025. lr:0.00100. loss:0.1650. spend/global_spend:15.4933/1:06:16. eta:1 day, 19:46:14
2019-10-04 23:08:33,411 - root - INFO - epoch:0. iter/total_iter:3900/158025. lr:0.00100. loss:0.1558. spend/global_spend:4.5993/1:06:21. eta:1 day, 19:42:22
2019-10-04 23:08:54,298 - root - INFO - epoch:0. iter/total_iter:3910/158025. lr:0.00100. loss:0.1474. spend/global_spend:20.8872/1:06:42. eta:1 day, 19:49:12
2019-10-04 23:09:12,219 - root - INFO - epoch:0. iter/total_iter:3920/158025. lr:0.00100. loss:0.1397. spend/global_spend:17.9209/1:07:00. eta:1 day, 19:54:04
2019-10-04 23:09:16,828 - root - INFO - epoch:0. iter/total_iter:3930/158025. lr:0.00100. loss:0.1418. spend/global_spend:4.6085/1:07:04. eta:1 day, 19:50:13
2019-10-04 23:09:31,367 - root - INFO - epoch:0. iter/total_iter:3940/158025. lr:0.00100. loss:0.1511. spend/global_spend:14.5394/1:07:19. eta:1 day, 19:52:50
2019-10-04 23:09:35,999 - root - INFO - epoch:0. iter/total_iter:3950/158025. lr:0.00100. loss:0.1441. spend/global_spend:4.6318/1:07:23. eta:1 day, 19:49:01
2019-10-04 23:09:54,316 - root - INFO - epoch:0. iter/total_iter:3960/158025. lr:0.00100. loss:0.1562. spend/global_spend:18.3163/1:07:42. eta:1 day, 19:54:05
2019-10-04 23:10:10,969 - root - INFO - epoch:0. iter/total_iter:3970/158025. lr:0.00100. loss:0.1601. spend/global_spend:16.6537/1:07:58. eta:1 day, 19:58:03
2019-10-04 23:10:15,600 - root - INFO - epoch:0. iter/total_iter:3980/158025. lr:0.00100. loss:0.1551. spend/global_spend:4.6304/1:08:03. eta:1 day, 19:54:14
2019-10-04 23:10:30,735 - root - INFO - epoch:0. iter/total_iter:3990/158025. lr:0.00100. loss:0.1640. spend/global_spend:15.1356/1:08:18. eta:1 day, 19:57:12
2019-10-04 23:10:50,405 - root - INFO - epoch:0. iter/total_iter:4000/158025. lr:0.00100. loss:0.1519. spend/global_spend:19.6705/1:08:38. eta:1 day, 20:03:04
2019-10-04 23:10:55,005 - root - INFO - epoch:0. iter/total_iter:4010/158025. lr:0.00100. loss:0.1487. spend/global_spend:4.5999/1:08:43. eta:1 day, 19:59:15
2019-10-04 23:11:10,154 - root - INFO - epoch:0. iter/total_iter:4020/158025. lr:0.00100. loss:0.1543. spend/global_spend:15.1486/1:08:58. eta:1 day, 20:02:11
2019-10-04 23:11:14,796 - root - INFO - epoch:0. iter/total_iter:4030/158025. lr:0.00100. loss:0.1598. spend/global_spend:4.6421/1:09:02. eta:1 day, 19:58:25
2019-10-04 23:11:31,080 - root - INFO - epoch:0. iter/total_iter:4040/158025. lr:0.00100. loss:0.1490. spend/global_spend:16.2835/1:09:19. eta:1 day, 20:02:03
2019-10-04 23:11:49,360 - root - INFO - epoch:0. iter/total_iter:4050/158025. lr:0.00100. loss:0.1725. spend/global_spend:18.2803/1:09:37. eta:1 day, 20:06:56
2019-10-04 23:11:53,989 - root - INFO - epoch:0. iter/total_iter:4060/158025. lr:0.00100. loss:0.1528. spend/global_spend:4.6291/1:09:41. eta:1 day, 20:03:11
2019-10-04 23:12:10,719 - root - INFO - epoch:0. iter/total_iter:4070/158025. lr:0.00100. loss:0.1524. spend/global_spend:16.7295/1:09:58. eta:1 day, 20:07:03
2019-10-04 23:12:30,203 - root - INFO - epoch:0. iter/total_iter:4080/158025. lr:0.00100. loss:0.1493. spend/global_spend:19.4837/1:10:18. eta:1 day, 20:12:39
2019-10-04 23:12:34,796 - root - INFO - epoch:0. iter/total_iter:4090/158025. lr:0.00100. loss:0.1439. spend/global_spend:4.5939/1:10:22. eta:1 day, 20:08:52
2019-10-04 23:12:52,158 - root - INFO - epoch:0. iter/total_iter:4100/158025. lr:0.00100. loss:0.1432. spend/global_spend:17.3620/1:10:40. eta:1 day, 20:13:06
2019-10-04 23:12:56,772 - root - INFO - epoch:0. iter/total_iter:4110/158025. lr:0.00100. loss:0.1567. spend/global_spend:4.6143/1:10:44. eta:1 day, 20:09:21
2019-10-04 23:13:11,136 - root - INFO - epoch:0. iter/total_iter:4120/158025. lr:0.00100. loss:0.1395. spend/global_spend:14.3633/1:10:59. eta:1 day, 20:11:42
2019-10-04 23:13:28,298 - root - INFO - epoch:0. iter/total_iter:4130/158025. lr:0.00100. loss:0.1487. spend/global_spend:17.1614/1:11:16. eta:1 day, 20:15:46
2019-10-04 23:13:32,913 - root - INFO - epoch:0. iter/total_iter:4140/158025. lr:0.00100. loss:0.1455. spend/global_spend:4.6153/1:11:20. eta:1 day, 20:12:02
2019-10-04 23:13:48,078 - root - INFO - epoch:0. iter/total_iter:4150/158025. lr:0.00100. loss:0.1367. spend/global_spend:15.1656/1:11:36. eta:1 day, 20:14:51
2019-10-04 23:14:02,198 - root - INFO - epoch:0. iter/total_iter:4160/158025. lr:0.00100. loss:0.1612. spend/global_spend:14.1202/1:11:50. eta:1 day, 20:17:00
2019-10-04 23:14:06,836 - root - INFO - epoch:0. iter/total_iter:4170/158025. lr:0.00100. loss:0.1394. spend/global_spend:4.6377/1:11:54. eta:1 day, 20:13:18
2019-10-04 23:14:26,038 - root - INFO - epoch:0. iter/total_iter:4180/158025. lr:0.00100. loss:0.1763. spend/global_spend:19.2026/1:12:14. eta:1 day, 20:18:34
2019-10-04 23:14:30,843 - root - INFO - epoch:0. iter/total_iter:4190/158025. lr:0.00100. loss:0.1690. spend/global_spend:4.8044/1:12:18. eta:1 day, 20:14:59
2019-10-04 23:14:45,258 - root - INFO - epoch:0. iter/total_iter:4200/158025. lr:0.00100. loss:0.1409. spend/global_spend:14.4154/1:12:33. eta:1 day, 20:17:17
2019-10-04 23:15:02,877 - root - INFO - epoch:0. iter/total_iter:4210/158025. lr:0.00100. loss:0.1671. spend/global_spend:17.6187/1:12:50. eta:1 day, 20:21:32
2019-10-04 23:15:07,480 - root - INFO - epoch:0. iter/total_iter:4220/158025. lr:0.00100. loss:0.1800. spend/global_spend:4.6032/1:12:55. eta:1 day, 20:17:51
2019-10-04 23:15:24,033 - root - INFO - epoch:0. iter/total_iter:4230/158025. lr:0.00100. loss:0.1483. spend/global_spend:16.5526/1:13:12. eta:1 day, 20:21:26
2019-10-04 23:15:39,711 - root - INFO - epoch:0. iter/total_iter:4240/158025. lr:0.00100. loss:0.1524. spend/global_spend:15.6777/1:13:27. eta:1 day, 20:24:27
2019-10-04 23:15:44,317 - root - INFO - epoch:0. iter/total_iter:4250/158025. lr:0.00100. loss:0.1561. spend/global_spend:4.6063/1:13:32. eta:1 day, 20:20:47
2019-10-04 23:16:02,475 - root - INFO - epoch:0. iter/total_iter:4260/158025. lr:0.00100. loss:0.1505. spend/global_spend:18.1583/1:13:50. eta:1 day, 20:25:18
2019-10-04 23:16:07,089 - root - INFO - epoch:0. iter/total_iter:4270/158025. lr:0.00100. loss:0.1780. spend/global_spend:4.6138/1:13:55. eta:1 day, 20:21:39
2019-10-04 23:16:22,624 - root - INFO - epoch:0. iter/total_iter:4280/158025. lr:0.00100. loss:0.1543. spend/global_spend:15.5357/1:14:10. eta:1 day, 20:24:34
2019-10-04 23:16:39,370 - root - INFO - epoch:0. iter/total_iter:4290/158025. lr:0.00100. loss:0.1607. spend/global_spend:16.7454/1:14:27. eta:1 day, 20:28:11
2019-10-04 23:16:44,321 - root - INFO - epoch:0. iter/total_iter:4300/158025. lr:0.00100. loss:0.1695. spend/global_spend:4.9508/1:14:32. eta:1 day, 20:24:45
2019-10-04 23:16:59,147 - root - INFO - epoch:0. iter/total_iter:4310/158025. lr:0.00100. loss:0.1532. spend/global_spend:14.8265/1:14:47. eta:1 day, 20:27:12
2019-10-04 23:17:16,185 - root - INFO - epoch:0. iter/total_iter:4320/158025. lr:0.00100. loss:0.1507. spend/global_spend:17.0376/1:15:04. eta:1 day, 20:30:58
2019-10-04 23:17:21,075 - root - INFO - epoch:0. iter/total_iter:4330/158025. lr:0.00100. loss:0.1651. spend/global_spend:4.8903/1:15:09. eta:1 day, 20:27:31
2019-10-04 23:17:40,229 - root - INFO - epoch:0. iter/total_iter:4340/158025. lr:0.00100. loss:0.1504. spend/global_spend:19.1534/1:15:28. eta:1 day, 20:32:30
2019-10-04 23:17:46,613 - root - INFO - epoch:0. iter/total_iter:4350/158025. lr:0.00100. loss:0.1511. spend/global_spend:6.3840/1:15:34. eta:1 day, 20:29:56
2019-10-04 23:18:02,805 - root - INFO - epoch:0. iter/total_iter:4360/158025. lr:0.00100. loss:0.1680. spend/global_spend:16.1920/1:15:50. eta:1 day, 20:33:09
2019-10-04 23:18:21,758 - root - INFO - epoch:0. iter/total_iter:4370/158025. lr:0.00100. loss:0.1599. spend/global_spend:18.9532/1:16:09. eta:1 day, 20:37:58
2019-10-04 23:18:26,356 - root - INFO - epoch:0. iter/total_iter:4380/158025. lr:0.00100. loss:0.1511. spend/global_spend:4.5979/1:16:14. eta:1 day, 20:34:22
2019-10-04 23:18:44,653 - root - INFO - epoch:0. iter/total_iter:4390/158025. lr:0.00100. loss:0.1652. spend/global_spend:18.2970/1:16:32. eta:1 day, 20:38:47
2019-10-04 23:19:00,070 - root - INFO - epoch:0. iter/total_iter:4400/158025. lr:0.00100. loss:0.1666. spend/global_spend:15.4176/1:16:48. eta:1 day, 20:41:29
2019-10-04 23:19:04,666 - root - INFO - epoch:0. iter/total_iter:4410/158025. lr:0.00100. loss:0.1554. spend/global_spend:4.5955/1:16:52. eta:1 day, 20:37:54
2019-10-04 23:19:19,792 - root - INFO - epoch:0. iter/total_iter:4420/158025. lr:0.00100. loss:0.1517. spend/global_spend:15.1268/1:17:07. eta:1 day, 20:40:26
2019-10-04 23:19:24,412 - root - INFO - epoch:0. iter/total_iter:4430/158025. lr:0.00100. loss:0.1652. spend/global_spend:4.6198/1:17:12. eta:1 day, 20:36:52
2019-10-04 23:19:42,690 - root - INFO - epoch:0. iter/total_iter:4440/158025. lr:0.00100. loss:0.1567. spend/global_spend:18.2775/1:17:30. eta:1 day, 20:41:12
2019-10-04 23:19:57,931 - root - INFO - epoch:0. iter/total_iter:4450/158025. lr:0.00100. loss:0.1750. spend/global_spend:15.2408/1:17:45. eta:1 day, 20:43:46
2019-10-04 23:20:03,861 - root - INFO - epoch:0. iter/total_iter:4460/158025. lr:0.00100. loss:0.1221. spend/global_spend:5.9304/1:17:51. eta:1 day, 20:40:59
2019-10-04 23:20:20,413 - root - INFO - epoch:0. iter/total_iter:4470/158025. lr:0.00100. loss:0.1443. spend/global_spend:16.5503/1:18:08. eta:1 day, 20:44:17
2019-10-04 23:20:36,366 - root - INFO - epoch:0. iter/total_iter:4480/158025. lr:0.00100. loss:0.1470. spend/global_spend:15.9548/1:18:24. eta:1 day, 20:47:14
2019-10-04 23:20:40,963 - root - INFO - epoch:0. iter/total_iter:4490/158025. lr:0.00100. loss:0.1646. spend/global_spend:4.5966/1:18:28. eta:1 day, 20:43:42
2019-10-04 23:20:56,625 - root - INFO - epoch:0. iter/total_iter:4500/158025. lr:0.00100. loss:0.1531. spend/global_spend:15.6620/1:18:44. eta:1 day, 20:46:28
2019-10-04 23:21:06,338 - root - INFO - epoch:0. iter/total_iter:4510/158025. lr:0.00100. loss:0.1562. spend/global_spend:9.7129/1:18:54. eta:1 day, 20:45:51
2019-10-04 23:21:19,110 - root - INFO - epoch:0. iter/total_iter:4520/158025. lr:0.00100. loss:0.1516. spend/global_spend:12.7722/1:19:07. eta:1 day, 20:46:57
2019-10-04 23:21:34,891 - root - INFO - epoch:0. iter/total_iter:4530/158025. lr:0.00100. loss:0.1559. spend/global_spend:15.7809/1:19:22. eta:1 day, 20:49:46
2019-10-04 23:21:43,256 - root - INFO - epoch:0. iter/total_iter:4540/158025. lr:0.00100. loss:0.1630. spend/global_spend:8.3656/1:19:31. eta:1 day, 20:48:23
2019-10-04 23:21:56,079 - root - INFO - epoch:0. iter/total_iter:4550/158025. lr:0.00100. loss:0.1554. spend/global_spend:12.8227/1:19:44. eta:1 day, 20:49:30
2019-10-04 23:22:12,057 - root - INFO - epoch:0. iter/total_iter:4560/158025. lr:0.00100. loss:0.1512. spend/global_spend:15.9783/1:20:00. eta:1 day, 20:52:23
2019-10-04 23:22:17,545 - root - INFO - epoch:0. iter/total_iter:4570/158025. lr:0.00100. loss:0.1480. spend/global_spend:5.4882/1:20:05. eta:1 day, 20:49:24
2019-10-04 23:22:32,297 - root - INFO - epoch:0. iter/total_iter:4580/158025. lr:0.00100. loss:0.1686. spend/global_spend:14.7513/1:20:20. eta:1 day, 20:51:35
2019-10-04 23:22:45,346 - root - INFO - epoch:0. iter/total_iter:4590/158025. lr:0.00100. loss:0.1559. spend/global_spend:13.0488/1:20:33. eta:1 day, 20:52:49
2019-10-04 23:22:54,708 - root - INFO - epoch:0. iter/total_iter:4600/158025. lr:0.00100. loss:0.1554. spend/global_spend:9.3625/1:20:42. eta:1 day, 20:52:00
2019-10-04 23:23:10,814 - root - INFO - epoch:0. iter/total_iter:4610/158025. lr:0.00100. loss:0.1464. spend/global_spend:16.1062/1:20:58. eta:1 day, 20:54:55
2019-10-04 23:23:21,641 - root - INFO - epoch:0. iter/total_iter:4620/158025. lr:0.00100. loss:0.1574. spend/global_spend:10.8272/1:21:09. eta:1 day, 20:54:54
2019-10-04 23:23:34,970 - root - INFO - epoch:0. iter/total_iter:4630/158025. lr:0.00100. loss:0.1397. spend/global_spend:13.3287/1:21:22. eta:1 day, 20:56:15
2019-10-04 23:23:52,510 - root - INFO - epoch:0. iter/total_iter:4640/158025. lr:0.00100. loss:0.1663. spend/global_spend:17.5401/1:21:40. eta:1 day, 20:59:56
2019-10-04 23:23:59,416 - root - INFO - epoch:0. iter/total_iter:4650/158025. lr:0.00100. loss:0.1486. spend/global_spend:6.9051/1:21:47. eta:1 day, 20:57:45
2019-10-04 23:24:12,423 - root - INFO - epoch:0. iter/total_iter:4660/158025. lr:0.00100. loss:0.1585. spend/global_spend:13.0072/1:22:00. eta:1 day, 20:58:55
2019-10-04 23:24:23,895 - root - INFO - epoch:0. iter/total_iter:4670/158025. lr:0.00100. loss:0.1372. spend/global_spend:11.4723/1:22:11. eta:1 day, 20:59:15
2019-10-04 23:24:35,947 - root - INFO - epoch:0. iter/total_iter:4680/158025. lr:0.00100. loss:0.1638. spend/global_spend:12.0515/1:22:23. eta:1 day, 20:59:53
2019-10-04 23:24:50,200 - root - INFO - epoch:0. iter/total_iter:4690/158025. lr:0.00100. loss:0.1558. spend/global_spend:14.2531/1:22:38. eta:1 day, 21:01:43
2019-10-04 23:25:04,194 - root - INFO - epoch:0. iter/total_iter:4700/158025. lr:0.00100. loss:0.1572. spend/global_spend:13.9944/1:22:52. eta:1 day, 21:03:24
2019-10-04 23:25:12,310 - root - INFO - epoch:0. iter/total_iter:4710/158025. lr:0.00100. loss:0.1527. spend/global_spend:8.1157/1:23:00. eta:1 day, 21:01:53
2019-10-04 23:25:27,388 - root - INFO - epoch:0. iter/total_iter:4720/158025. lr:0.00100. loss:0.1583. spend/global_spend:15.0782/1:23:15. eta:1 day, 21:04:09
2019-10-04 23:25:36,096 - root - INFO - epoch:0. iter/total_iter:4730/158025. lr:0.00100. loss:0.1362. spend/global_spend:8.7080/1:23:24. eta:1 day, 21:02:58
2019-10-04 23:25:52,142 - root - INFO - epoch:0. iter/total_iter:4740/158025. lr:0.00100. loss:0.1456. spend/global_spend:16.0456/1:23:40. eta:1 day, 21:05:44
2019-10-04 23:26:00,356 - root - INFO - epoch:0. iter/total_iter:4750/158025. lr:0.00100. loss:0.1586. spend/global_spend:8.2145/1:23:48. eta:1 day, 21:04:17
2019-10-04 23:26:15,642 - root - INFO - epoch:0. iter/total_iter:4760/158025. lr:0.00100. loss:0.1473. spend/global_spend:15.2859/1:24:03. eta:1 day, 21:06:37
2019-10-04 23:26:33,903 - root - INFO - epoch:0. iter/total_iter:4770/158025. lr:0.00100. loss:0.1709. spend/global_spend:18.2613/1:24:21. eta:1 day, 21:10:33
2019-10-04 23:26:38,516 - root - INFO - epoch:0. iter/total_iter:4780/158025. lr:0.00100. loss:0.1568. spend/global_spend:4.6130/1:24:26. eta:1 day, 21:07:10
2019-10-04 23:26:52,608 - root - INFO - epoch:0. iter/total_iter:4790/158025. lr:0.00100. loss:0.1694. spend/global_spend:14.0923/1:24:40. eta:1 day, 21:08:51
2019-10-04 23:26:58,711 - root - INFO - epoch:0. iter/total_iter:4800/158025. lr:0.00100. loss:0.1631. spend/global_spend:6.1030/1:24:46. eta:1 day, 21:06:17
2019-10-04 23:27:14,461 - root - INFO - epoch:0. iter/total_iter:4810/158025. lr:0.00100. loss:0.1860. spend/global_spend:15.7494/1:25:02. eta:1 day, 21:08:50
2019-10-04 23:27:29,718 - root - INFO - epoch:0. iter/total_iter:4820/158025. lr:0.00100. loss:0.1698. spend/global_spend:15.2575/1:25:17. eta:1 day, 21:11:07
2019-10-04 23:27:36,584 - root - INFO - epoch:0. iter/total_iter:4830/158025. lr:0.00100. loss:0.1627. spend/global_spend:6.8651/1:25:24. eta:1 day, 21:08:58
2019-10-04 23:27:52,528 - root - INFO - epoch:0. iter/total_iter:4840/158025. lr:0.00100. loss:0.1539. spend/global_spend:15.9448/1:25:40. eta:1 day, 21:11:36
2019-10-04 23:28:07,621 - root - INFO - epoch:0. iter/total_iter:4850/158025. lr:0.00100. loss:0.1520. spend/global_spend:15.0924/1:25:55. eta:1 day, 21:13:47
2019-10-04 23:28:12,335 - root - INFO - epoch:0. iter/total_iter:4860/158025. lr:0.00100. loss:0.1514. spend/global_spend:4.7145/1:26:00. eta:1 day, 21:10:30
2019-10-04 23:28:29,300 - root - INFO - epoch:0. iter/total_iter:4870/158025. lr:0.00100. loss:0.1426. spend/global_spend:16.9646/1:26:17. eta:1 day, 21:13:39
2019-10-04 23:28:45,859 - root - INFO - epoch:0. iter/total_iter:4880/158025. lr:0.00100. loss:0.1679. spend/global_spend:16.5591/1:26:33. eta:1 day, 21:16:34
2019-10-04 23:28:52,277 - root - INFO - epoch:0. iter/total_iter:4890/158025. lr:0.00100. loss:0.1568. spend/global_spend:6.4177/1:26:40. eta:1 day, 21:14:11
2019-10-04 23:29:10,754 - root - INFO - epoch:0. iter/total_iter:4900/158025. lr:0.00100. loss:0.1604. spend/global_spend:18.4772/1:26:58. eta:1 day, 21:18:05
2019-10-04 23:29:15,358 - root - INFO - epoch:0. iter/total_iter:4910/158025. lr:0.00100. loss:0.1406. spend/global_spend:4.6045/1:27:03. eta:1 day, 21:14:46
2019-10-04 23:29:31,758 - root - INFO - epoch:0. iter/total_iter:4920/158025. lr:0.00100. loss:0.1633. spend/global_spend:16.3999/1:27:19. eta:1 day, 21:17:35
2019-10-04 23:29:50,060 - root - INFO - epoch:0. iter/total_iter:4930/158025. lr:0.00100. loss:0.1405. spend/global_spend:18.3015/1:27:38. eta:1 day, 21:21:22
2019-10-04 23:29:54,675 - root - INFO - epoch:0. iter/total_iter:4940/158025. lr:0.00100. loss:0.1389. spend/global_spend:4.6156/1:27:42. eta:1 day, 21:18:04
2019-10-04 23:30:12,724 - root - INFO - epoch:0. iter/total_iter:4950/158025. lr:0.00100. loss:0.1500. spend/global_spend:18.0492/1:28:00. eta:1 day, 21:21:42
2019-10-04 23:30:27,618 - root - INFO - epoch:0. iter/total_iter:4960/158025. lr:0.00100. loss:0.1525. spend/global_spend:14.8938/1:28:15. eta:1 day, 21:23:42
2019-10-04 23:30:36,521 - root - INFO - epoch:0. iter/total_iter:4970/158025. lr:0.00100. loss:0.1563. spend/global_spend:8.9029/1:28:24. eta:1 day, 21:22:36
2019-10-04 23:30:50,781 - root - INFO - epoch:0. iter/total_iter:4980/158025. lr:0.00100. loss:0.1615. spend/global_spend:14.2594/1:28:38. eta:1 day, 21:24:16
2019-10-04 23:30:56,747 - root - INFO - epoch:0. iter/total_iter:4990/158025. lr:0.00100. loss:0.1468. spend/global_spend:5.9667/1:28:44. eta:1 day, 21:21:41
2019-10-04 23:31:15,267 - root - INFO - epoch:0. iter/total_iter:5000/158025. lr:0.00100. loss:0.1461. spend/global_spend:18.5193/1:29:03. eta:1 day, 21:25:30
2019-10-04 23:31:28,420 - root - INFO - epoch:0. iter/total_iter:5010/158025. lr:0.00100. loss:0.1502. spend/global_spend:13.1529/1:29:16. eta:1 day, 21:26:35
2019-10-04 23:31:36,035 - root - INFO - epoch:0. iter/total_iter:5020/158025. lr:0.00100. loss:0.1807. spend/global_spend:7.6157/1:29:24. eta:1 day, 21:24:50
2019-10-04 23:31:53,817 - root - INFO - epoch:0. iter/total_iter:5030/158025. lr:0.00100. loss:0.1443. spend/global_spend:17.7815/1:29:41. eta:1 day, 21:28:15
2019-10-04 23:32:07,268 - root - INFO - epoch:0. iter/total_iter:5040/158025. lr:0.00100. loss:0.1766. spend/global_spend:13.4510/1:29:55. eta:1 day, 21:29:28
2019-10-04 23:32:17,816 - root - INFO - epoch:0. iter/total_iter:5050/158025. lr:0.00100. loss:0.1704. spend/global_spend:10.5483/1:30:05. eta:1 day, 21:29:13
2019-10-04 23:32:26,144 - root - INFO - epoch:0. iter/total_iter:5060/158025. lr:0.00100. loss:0.1463. spend/global_spend:8.3286/1:30:14. eta:1 day, 21:27:50
2019-10-04 23:32:40,424 - root - INFO - epoch:0. iter/total_iter:5070/158025. lr:0.00100. loss:0.1617. spend/global_spend:14.2797/1:30:28. eta:1 day, 21:29:28
2019-10-04 23:32:54,244 - root - INFO - epoch:0. iter/total_iter:5080/158025. lr:0.00100. loss:0.1546. spend/global_spend:13.8195/1:30:42. eta:1 day, 21:30:51
2019-10-04 23:33:03,603 - root - INFO - epoch:0. iter/total_iter:5090/158025. lr:0.00100. loss:0.1575. spend/global_spend:9.3589/1:30:51. eta:1 day, 21:29:59
2019-10-04 23:33:14,562 - root - INFO - epoch:0. iter/total_iter:5100/158025. lr:0.00100. loss:0.1507. spend/global_spend:10.9590/1:31:02. eta:1 day, 21:29:56
2019-10-04 23:33:29,978 - root - INFO - epoch:0. iter/total_iter:5110/158025. lr:0.00100. loss:0.1430. spend/global_spend:15.4163/1:31:17. eta:1 day, 21:32:06
2019-10-04 23:33:39,413 - root - INFO - epoch:0. iter/total_iter:5120/158025. lr:0.00100. loss:0.1446. spend/global_spend:9.4352/1:31:27. eta:1 day, 21:31:17
2019-10-04 23:33:52,915 - root - INFO - epoch:0. iter/total_iter:5130/158025. lr:0.00100. loss:0.1888. spend/global_spend:13.5023/1:31:40. eta:1 day, 21:32:29
2019-10-04 23:34:02,828 - root - INFO - epoch:0. iter/total_iter:5140/158025. lr:0.00100. loss:0.1428. spend/global_spend:9.9129/1:31:50. eta:1 day, 21:31:54
2019-10-04 23:34:16,611 - root - INFO - epoch:0. iter/total_iter:5150/158025. lr:0.00100. loss:0.1805. spend/global_spend:13.7827/1:32:04. eta:1 day, 21:33:15
2019-10-04 23:34:36,114 - root - INFO - epoch:0. iter/total_iter:5160/158025. lr:0.00100. loss:0.1464. spend/global_spend:19.5031/1:32:24. eta:1 day, 21:37:24
2019-10-04 23:34:44,696 - root - INFO - epoch:0. iter/total_iter:5170/158025. lr:0.00100. loss:0.1636. spend/global_spend:8.5820/1:32:32. eta:1 day, 21:36:09
2019-10-04 23:34:58,035 - root - INFO - epoch:0. iter/total_iter:5180/158025. lr:0.00100. loss:0.1524. spend/global_spend:13.3391/1:32:46. eta:1 day, 21:37:15
2019-10-04 23:35:13,310 - root - INFO - epoch:0. iter/total_iter:5190/158025. lr:0.00100. loss:0.1673. spend/global_spend:15.2753/1:33:01. eta:1 day, 21:39:18
2019-10-04 23:35:22,359 - root - INFO - epoch:0. iter/total_iter:5200/158025. lr:0.00100. loss:0.1732. spend/global_spend:9.0481/1:33:10. eta:1 day, 21:38:17
2019-10-04 23:35:33,533 - root - INFO - epoch:0. iter/total_iter:5210/158025. lr:0.00100. loss:0.1697. spend/global_spend:11.1747/1:33:21. eta:1 day, 21:38:19
2019-10-04 23:35:43,410 - root - INFO - epoch:0. iter/total_iter:5220/158025. lr:0.00100. loss:0.1421. spend/global_spend:9.8767/1:33:31. eta:1 day, 21:37:42
2019-10-04 23:35:46,515 - root - ERROR - Exception logged
Traceback (most recent call last):
  File "/media/tiger/zzr/rsna_script/trian_resnet.py", line 104, in main_worker
    for img, lb in train_loader:
  File "/home/tiger/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 576, in __next__
    idx, batch = self._get_batch()
  File "/home/tiger/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 543, in _get_batch
    success, data = self._try_get_batch()
  File "/home/tiger/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 511, in _try_get_batch
    data = self.data_queue.get(timeout=timeout)
  File "/home/tiger/anaconda3/lib/python3.7/queue.py", line 179, in get
    self.not_empty.wait(remaining)
  File "/home/tiger/anaconda3/lib/python3.7/threading.py", line 300, in wait
    gotit = waiter.acquire(True, timeout)
KeyboardInterrupt