from pathlib import Path
from deep_t2i.trainer_GAN import BirdsTrainer

from deep_t2i.model import Birds_Export
from deep_t2i.inference_birds import pred_and_show

data_dir = Path('/root/data/birds')
model_dir = Path('../models/large_birds/gan')
result_dir = Path('../result_jpgs/large_birds/gan')
pretrained_damsm_path = Path('../models/large_birds/damsm/damsm_export.pt')
data_dir, model_dir, result_dir, pretrained_damsm_path

(Path('/root/data/birds'),
 Path('../models/large_birds/gan'),
 Path('../result_jpgs/large_birds/gan'),
 Path('../models/large_birds/damsm/damsm_export.pt'))

Train

bs = 24
data_pct = 1
g_lr = 2e-4
d_lr = 2e-4
smooth_lambda = 2.0
noise_sz = 512
trainer = BirdsTrainer(
    data_dir, 
    bs=bs, 
    data_pct=data_pct,
    g_lr=g_lr, 
    d_lr=d_lr,
    device='cuda', 
    pretrained_damsm_path=pretrained_damsm_path,
    smooth_lambda=smooth_lambda,
    noise_sz=noise_sz,
)

len(trainer.dls.train)

491

ema_decay = 0.999
n_gradient_acc = 1
step_per_epoch = len(trainer.dls.train) // (n_gradient_acc*1)
step_per_epoch

491

trainer.train(
    n_step=step_per_epoch*52, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'0'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*4, 
    ck_path=str(model_dir/'0'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 662.3s, g_loss: 26.1808, d_loss: 2.0535
2, time: 663.1s, g_loss: 16.7359, d_loss: 1.9039
3, time: 662.7s, g_loss: 11.0985, d_loss: 2.0067
4, time: 663.8s, g_loss: 9.9363, d_loss: 1.9280
5, time: 661.9s, g_loss: 8.2628, d_loss: 1.9365
6, time: 660.8s, g_loss: 7.7386, d_loss: 1.9360
7, time: 661.0s, g_loss: 7.8178, d_loss: 1.7912
8, time: 665.1s, g_loss: 7.7765, d_loss: 1.8367
9, time: 661.4s, g_loss: 7.3426, d_loss: 1.8761
10, time: 661.2s, g_loss: 7.3601, d_loss: 1.8224
11, time: 661.1s, g_loss: 7.7167, d_loss: 1.7039
12, time: 666.6s, g_loss: 7.9582, d_loss: 1.5925
13, time: 666.9s, g_loss: 7.5993, d_loss: 1.7343
14, time: 667.6s, g_loss: 7.7882, d_loss: 1.6077
15, time: 667.8s, g_loss: 8.1628, d_loss: 1.5298
16, time: 665.7s, g_loss: 7.7974, d_loss: 1.5822
17, time: 662.3s, g_loss: 8.2015, d_loss: 1.4696
18, time: 662.8s, g_loss: 8.1217, d_loss: 1.5690
19, time: 664.4s, g_loss: 8.4231, d_loss: 1.5511
20, time: 667.8s, g_loss: 7.8205, d_loss: 1.5815
21, time: 664.0s, g_loss: 7.7237, d_loss: 1.5739
22, time: 664.6s, g_loss: 8.0886, d_loss: 1.4993
23, time: 663.4s, g_loss: 8.1500, d_loss: 1.4448
24, time: 666.3s, g_loss: 7.8979, d_loss: 1.4931
25, time: 664.1s, g_loss: 8.0555, d_loss: 1.4431
26, time: 665.0s, g_loss: 8.4709, d_loss: 1.3695
27, time: 665.0s, g_loss: 8.3905, d_loss: 1.3499
28, time: 667.7s, g_loss: 7.9773, d_loss: 1.4489
29, time: 665.0s, g_loss: 8.4223, d_loss: 1.3517
30, time: 664.7s, g_loss: 8.2091, d_loss: 1.3624
31, time: 665.1s, g_loss: 8.1561, d_loss: 1.3932
32, time: 668.9s, g_loss: 8.2115, d_loss: 1.3727
33, time: 668.3s, g_loss: 8.5612, d_loss: 1.2643
34, time: 666.7s, g_loss: 8.9506, d_loss: 1.2108
35, time: 666.0s, g_loss: 8.4319, d_loss: 1.3144
36, time: 668.6s, g_loss: 9.2934, d_loss: 1.0988
37, time: 666.2s, g_loss: 8.8076, d_loss: 1.2723
38, time: 665.9s, g_loss: 8.7109, d_loss: 1.2657
39, time: 665.8s, g_loss: 8.6048, d_loss: 1.2467
40, time: 669.9s, g_loss: 9.2404, d_loss: 1.1267
41, time: 667.4s, g_loss: 8.8656, d_loss: 1.1897
42, time: 667.5s, g_loss: 8.7047, d_loss: 1.1803
43, time: 666.1s, g_loss: 9.8666, d_loss: 0.9963
44, time: 669.2s, g_loss: 8.6520, d_loss: 1.1440
45, time: 667.4s, g_loss: 8.2634, d_loss: 1.2948
46, time: 667.2s, g_loss: 8.6375, d_loss: 1.1930
47, time: 665.5s, g_loss: 9.0466, d_loss: 1.1034
48, time: 668.9s, g_loss: 8.6816, d_loss: 1.1340
49, time: 665.8s, g_loss: 8.9117, d_loss: 1.1039
50, time: 664.3s, g_loss: 9.2388, d_loss: 1.0627
51, time: 663.3s, g_loss: 8.9735, d_loss: 1.0790
52, time: 666.5s, g_loss: 9.4553, d_loss: 1.0016

total_time: 576.6min

trainer.load_checkpoint(model_dir/'0-13.pt')

trainer.train(
    n_step=step_per_epoch*49, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'1'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*7, 
    ck_path=str(model_dir/'1'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 661.5s, g_loss: 8.7466, d_loss: 1.1275
2, time: 661.9s, g_loss: 8.5862, d_loss: 1.1242
3, time: 661.8s, g_loss: 8.7477, d_loss: 1.0892
4, time: 662.0s, g_loss: 9.6717, d_loss: 0.9944
5, time: 661.2s, g_loss: 8.9648, d_loss: 1.0953
6, time: 661.5s, g_loss: 9.0704, d_loss: 1.0702
7, time: 664.2s, g_loss: 9.0275, d_loss: 1.0390
8, time: 661.6s, g_loss: 10.4165, d_loss: 0.9587
9, time: 662.1s, g_loss: 9.0731, d_loss: 1.0351
10, time: 661.6s, g_loss: 8.9915, d_loss: 1.0329
11, time: 661.5s, g_loss: 9.0452, d_loss: 1.0247
12, time: 663.8s, g_loss: 9.1218, d_loss: 1.0255
13, time: 664.2s, g_loss: 8.7774, d_loss: 1.0514
14, time: 664.4s, g_loss: 10.1808, d_loss: 0.8727
15, time: 661.9s, g_loss: 9.2812, d_loss: 0.9633
16, time: 661.6s, g_loss: 9.5064, d_loss: 0.9702
17, time: 662.6s, g_loss: 9.1953, d_loss: 0.9329
18, time: 661.7s, g_loss: 10.7745, d_loss: 0.8654
19, time: 661.4s, g_loss: 9.2038, d_loss: 0.9806
20, time: 661.1s, g_loss: 10.3891, d_loss: 0.9443
21, time: 664.1s, g_loss: 8.9366, d_loss: 1.0224
22, time: 664.3s, g_loss: 9.0898, d_loss: 0.9847
23, time: 664.9s, g_loss: 9.3276, d_loss: 0.9523
24, time: 665.1s, g_loss: 9.1363, d_loss: 0.9584
25, time: 665.2s, g_loss: 9.1507, d_loss: 0.9645
26, time: 664.6s, g_loss: 10.1319, d_loss: 0.8693
27, time: 663.7s, g_loss: 9.3936, d_loss: 0.9316
28, time: 667.3s, g_loss: 9.9116, d_loss: 0.8456
29, time: 665.1s, g_loss: 9.7269, d_loss: 0.9423
30, time: 665.6s, g_loss: 9.2008, d_loss: 0.9635
31, time: 665.0s, g_loss: 10.8543, d_loss: 0.7758
32, time: 663.8s, g_loss: 9.3452, d_loss: 0.9113
33, time: 663.0s, g_loss: 11.1231, d_loss: 0.6881
34, time: 661.9s, g_loss: 8.5159, d_loss: 1.1515
35, time: 664.7s, g_loss: 9.3357, d_loss: 0.9191
36, time: 664.2s, g_loss: 10.6048, d_loss: 0.7857
37, time: 662.4s, g_loss: 9.2132, d_loss: 0.9506
38, time: 662.0s, g_loss: 9.3864, d_loss: 0.9033
39, time: 662.4s, g_loss: 12.1219, d_loss: 0.6791
40, time: 663.3s, g_loss: 9.5699, d_loss: 0.9173
41, time: 663.8s, g_loss: 9.6216, d_loss: 0.8354
42, time: 667.9s, g_loss: 9.6432, d_loss: 0.8583
43, time: 664.7s, g_loss: 9.7375, d_loss: 0.8485
44, time: 665.4s, g_loss: 9.9404, d_loss: 0.8260
45, time: 666.0s, g_loss: 12.1783, d_loss: 0.6826
46, time: 668.5s, g_loss: 9.7123, d_loss: 0.8450
47, time: 666.0s, g_loss: 9.5701, d_loss: 0.8835
48, time: 666.2s, g_loss: 9.6563, d_loss: 0.8398
49, time: 669.2s, g_loss: 9.6111, d_loss: 0.8613

total_time: 542.1min

trainer.load_checkpoint(model_dir/'1-7.pt')

trainer.train(
    n_step=step_per_epoch*52, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'2'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*4, 
    ck_path=str(model_dir/'2'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 660.9s, g_loss: 12.5668, d_loss: 0.6289
2, time: 660.8s, g_loss: 9.6962, d_loss: 0.8377
3, time: 661.2s, g_loss: 11.3199, d_loss: 0.7667
4, time: 663.9s, g_loss: 9.4183, d_loss: 0.8841
5, time: 661.2s, g_loss: 9.7041, d_loss: 0.8232
6, time: 662.2s, g_loss: 9.7759, d_loss: 0.8379
7, time: 661.2s, g_loss: 9.7768, d_loss: 0.8065
8, time: 664.6s, g_loss: 11.4438, d_loss: 0.6956
9, time: 661.7s, g_loss: 9.6321, d_loss: 0.8517
10, time: 661.6s, g_loss: 9.6149, d_loss: 0.8254
11, time: 662.6s, g_loss: 9.8325, d_loss: 0.8057
12, time: 664.7s, g_loss: 11.5121, d_loss: 0.6947
13, time: 662.1s, g_loss: 9.7628, d_loss: 0.8179
14, time: 661.9s, g_loss: 9.7743, d_loss: 0.8000
15, time: 662.3s, g_loss: 9.9023, d_loss: 0.7722
16, time: 665.0s, g_loss: 10.8859, d_loss: 0.6701
17, time: 662.6s, g_loss: 10.9614, d_loss: 0.7663
18, time: 661.9s, g_loss: 9.7243, d_loss: 0.7970
19, time: 662.4s, g_loss: 9.8713, d_loss: 0.7907
20, time: 666.6s, g_loss: 9.6167, d_loss: 0.8298
21, time: 662.4s, g_loss: 11.8195, d_loss: 0.7114
22, time: 662.6s, g_loss: 9.4124, d_loss: 0.8406
23, time: 662.6s, g_loss: 9.9694, d_loss: 0.7467
24, time: 665.3s, g_loss: 9.7869, d_loss: 0.7942
25, time: 662.9s, g_loss: 9.8400, d_loss: 0.7778
26, time: 664.1s, g_loss: 10.9518, d_loss: 0.6481
27, time: 663.1s, g_loss: 11.3822, d_loss: 0.7339
28, time: 666.2s, g_loss: 9.9515, d_loss: 0.7783
29, time: 662.6s, g_loss: 9.7240, d_loss: 0.7820
30, time: 662.7s, g_loss: 10.0081, d_loss: 0.7600
31, time: 662.4s, g_loss: 9.7448, d_loss: 0.7983
32, time: 665.7s, g_loss: 9.9579, d_loss: 0.7784
33, time: 663.1s, g_loss: 10.7489, d_loss: 0.6650
34, time: 664.6s, g_loss: 11.4100, d_loss: 0.7142
35, time: 663.0s, g_loss: 10.0999, d_loss: 0.7880
36, time: 666.0s, g_loss: 11.5418, d_loss: 0.6151
37, time: 663.2s, g_loss: 10.3133, d_loss: 0.8635
38, time: 662.8s, g_loss: 10.0887, d_loss: 0.7341
39, time: 663.1s, g_loss: 10.0434, d_loss: 0.7557
40, time: 666.2s, g_loss: 9.9511, d_loss: 0.7661
41, time: 663.1s, g_loss: 9.9893, d_loss: 0.7683
42, time: 662.7s, g_loss: 10.1264, d_loss: 0.7478
43, time: 664.6s, g_loss: 10.0426, d_loss: 0.7588
44, time: 665.5s, g_loss: 9.9762, d_loss: 0.7660
45, time: 663.1s, g_loss: 9.9782, d_loss: 0.7784
46, time: 662.8s, g_loss: 10.1847, d_loss: 0.7435
47, time: 662.6s, g_loss: 10.5394, d_loss: 0.6926
48, time: 665.4s, g_loss: 11.5074, d_loss: 0.7258
49, time: 663.2s, g_loss: 10.0084, d_loss: 0.7869
50, time: 663.4s, g_loss: 10.1235, d_loss: 0.7219
51, time: 662.8s, g_loss: 10.3327, d_loss: 0.7336
52, time: 665.2s, g_loss: 10.3309, d_loss: 0.7357

total_time: 574.8min

trainer.load_checkpoint(model_dir/'2-13.pt')

trainer.train(
    n_step=step_per_epoch*49, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'3'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*7, 
    ck_path=str(model_dir/'3'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 661.9s, g_loss: 10.3665, d_loss: 0.7411
2, time: 662.3s, g_loss: 13.4599, d_loss: 0.6343
3, time: 662.8s, g_loss: 9.9035, d_loss: 0.7441
4, time: 663.0s, g_loss: 10.2646, d_loss: 0.7487
5, time: 664.0s, g_loss: 10.1947, d_loss: 0.7464
6, time: 664.3s, g_loss: 10.0973, d_loss: 0.7571
7, time: 668.5s, g_loss: 12.4784, d_loss: 0.6640
8, time: 665.2s, g_loss: 9.8020, d_loss: 0.8123
9, time: 665.3s, g_loss: 10.1040, d_loss: 0.7383
10, time: 665.6s, g_loss: 10.0272, d_loss: 0.7892
11, time: 665.4s, g_loss: 10.2702, d_loss: 0.7192
12, time: 665.4s, g_loss: 10.5007, d_loss: 0.7194
13, time: 665.2s, g_loss: 11.1229, d_loss: 0.6553
14, time: 668.7s, g_loss: 11.5417, d_loss: 0.7241
15, time: 665.7s, g_loss: 10.2801, d_loss: 0.7333
16, time: 665.7s, g_loss: 10.1887, d_loss: 0.7407
17, time: 665.2s, g_loss: 12.8826, d_loss: 0.5633
18, time: 664.9s, g_loss: 10.2145, d_loss: 0.7561
19, time: 665.2s, g_loss: 10.2748, d_loss: 0.7598
20, time: 664.5s, g_loss: 10.3417, d_loss: 0.7263
21, time: 667.3s, g_loss: 12.8701, d_loss: 0.6271
22, time: 665.0s, g_loss: 9.8276, d_loss: 0.7942
23, time: 664.2s, g_loss: 10.2423, d_loss: 0.7618
24, time: 665.4s, g_loss: 10.4548, d_loss: 0.7143
25, time: 665.0s, g_loss: 10.4053, d_loss: 0.7339
26, time: 666.5s, g_loss: 10.2495, d_loss: 0.7612
27, time: 666.6s, g_loss: 10.2513, d_loss: 0.7390
28, time: 669.4s, g_loss: 10.8616, d_loss: 0.6759
29, time: 666.7s, g_loss: 13.4223, d_loss: 0.6190
30, time: 666.1s, g_loss: 10.3856, d_loss: 0.7338
31, time: 667.2s, g_loss: 10.4976, d_loss: 0.7396
32, time: 664.7s, g_loss: 10.4171, d_loss: 0.7428
33, time: 664.9s, g_loss: 10.5074, d_loss: 0.7292
34, time: 665.0s, g_loss: 10.3985, d_loss: 0.7291
35, time: 667.6s, g_loss: 10.5930, d_loss: 0.7070
36, time: 665.8s, g_loss: 10.5119, d_loss: 0.7185
37, time: 665.1s, g_loss: 13.3083, d_loss: 0.6172
38, time: 664.2s, g_loss: 10.2522, d_loss: 0.7535
39, time: 664.2s, g_loss: 10.3990, d_loss: 0.7493
40, time: 665.8s, g_loss: 10.4500, d_loss: 0.7068
41, time: 663.8s, g_loss: 10.5844, d_loss: 0.7005
42, time: 666.7s, g_loss: 10.3263, d_loss: 0.7499
43, time: 663.9s, g_loss: 10.4109, d_loss: 0.7486
44, time: 663.3s, g_loss: 11.7550, d_loss: 0.5994
45, time: 662.7s, g_loss: 11.6583, d_loss: 0.7176
46, time: 662.5s, g_loss: 10.5523, d_loss: 0.7350
47, time: 662.3s, g_loss: 10.4013, d_loss: 0.7483
48, time: 662.4s, g_loss: 11.6804, d_loss: 0.5981
49, time: 664.9s, g_loss: 12.7388, d_loss: 0.6517

total_time: 543.1min

trainer.load_checkpoint(model_dir/'3-7.pt')

trainer.train(
    n_step=step_per_epoch*52, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'4'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*4, 
    ck_path=str(model_dir/'4'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 660.1s, g_loss: 10.6144, d_loss: 0.7167
2, time: 657.8s, g_loss: 13.1756, d_loss: 0.5397
3, time: 657.5s, g_loss: 10.8086, d_loss: 0.7152
4, time: 660.0s, g_loss: 10.5580, d_loss: 0.7287
5, time: 658.0s, g_loss: 10.6487, d_loss: 0.7086
6, time: 657.6s, g_loss: 10.5173, d_loss: 0.7164
7, time: 657.5s, g_loss: 13.4342, d_loss: 0.4645
8, time: 661.1s, g_loss: 9.8209, d_loss: 0.9296
9, time: 659.0s, g_loss: 10.4797, d_loss: 0.7215
10, time: 658.8s, g_loss: 10.7327, d_loss: 0.6827
11, time: 660.1s, g_loss: 11.8652, d_loss: 0.6096
12, time: 662.7s, g_loss: 13.2407, d_loss: 0.6414
13, time: 660.9s, g_loss: 10.6358, d_loss: 0.7096
14, time: 660.5s, g_loss: 10.7413, d_loss: 0.7143
15, time: 661.5s, g_loss: 10.7148, d_loss: 0.7011
16, time: 663.9s, g_loss: 13.3595, d_loss: 0.5418
17, time: 661.2s, g_loss: 10.4305, d_loss: 0.7325
18, time: 660.0s, g_loss: 10.5847, d_loss: 0.7405
19, time: 657.7s, g_loss: 10.6040, d_loss: 0.7192
20, time: 663.4s, g_loss: 12.3810, d_loss: 0.5364
21, time: 661.7s, g_loss: 12.2793, d_loss: 0.6685
22, time: 661.5s, g_loss: 10.6479, d_loss: 0.7189
23, time: 657.9s, g_loss: 13.6067, d_loss: 0.4841
24, time: 662.3s, g_loss: 10.9304, d_loss: 0.7585
25, time: 661.5s, g_loss: 12.7721, d_loss: 0.5263
26, time: 660.9s, g_loss: 10.7879, d_loss: 0.7955
27, time: 659.7s, g_loss: 10.7359, d_loss: 0.6941
28, time: 663.5s, g_loss: 10.6512, d_loss: 0.7193
29, time: 659.2s, g_loss: 12.5093, d_loss: 0.5041
30, time: 657.4s, g_loss: 13.2365, d_loss: 0.6204
31, time: 657.2s, g_loss: 10.8611, d_loss: 0.7262
32, time: 659.6s, g_loss: 10.7865, d_loss: 0.6857
33, time: 658.8s, g_loss: 10.8417, d_loss: 0.6827
34, time: 657.3s, g_loss: 10.9336, d_loss: 0.6665
35, time: 657.3s, g_loss: 10.9855, d_loss: 0.6737
36, time: 660.1s, g_loss: 10.9127, d_loss: 0.6772
37, time: 657.3s, g_loss: 10.7681, d_loss: 0.6917
38, time: 657.3s, g_loss: 13.3697, d_loss: 0.4989
39, time: 657.3s, g_loss: 10.7982, d_loss: 0.7241
40, time: 660.0s, g_loss: 13.1481, d_loss: 0.4890
41, time: 657.4s, g_loss: 10.5382, d_loss: 0.8440
42, time: 659.9s, g_loss: 10.8890, d_loss: 0.6888
43, time: 657.4s, g_loss: 13.6902, d_loss: 0.4535
44, time: 659.9s, g_loss: 13.6583, d_loss: 0.5041
45, time: 658.1s, g_loss: 12.4491, d_loss: 0.6383
46, time: 658.2s, g_loss: 10.8054, d_loss: 0.6941
47, time: 657.7s, g_loss: 12.0216, d_loss: 0.5500
48, time: 663.3s, g_loss: 13.1830, d_loss: 0.6227
49, time: 663.6s, g_loss: 10.8435, d_loss: 0.6629
50, time: 663.1s, g_loss: 13.4215, d_loss: 0.5320
51, time: 663.1s, g_loss: 10.5733, d_loss: 0.6972
52, time: 666.3s, g_loss: 10.7864, d_loss: 0.6905

total_time: 571.9min

trainer.load_checkpoint(model_dir/'4-13.pt')

trainer.train(
    n_step=step_per_epoch*50, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'5'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*5, 
    ck_path=str(model_dir/'5'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 662.9s, g_loss: 10.9000, d_loss: 0.6774
2, time: 662.1s, g_loss: 12.8238, d_loss: 0.5018
3, time: 662.6s, g_loss: 11.4208, d_loss: 0.8370
4, time: 662.7s, g_loss: 10.7139, d_loss: 0.6856
5, time: 665.6s, g_loss: 10.9827, d_loss: 0.6715
6, time: 663.6s, g_loss: 10.9773, d_loss: 0.6865
7, time: 662.4s, g_loss: 11.1836, d_loss: 0.6365
8, time: 662.7s, g_loss: 15.2781, d_loss: 0.4389
9, time: 663.4s, g_loss: 11.0210, d_loss: 0.7131
10, time: 666.4s, g_loss: 10.8267, d_loss: 0.6927
11, time: 663.3s, g_loss: 10.9509, d_loss: 0.6797
12, time: 663.2s, g_loss: 14.9854, d_loss: 0.3806
13, time: 660.6s, g_loss: 11.2673, d_loss: 0.7289
14, time: 660.7s, g_loss: 10.8894, d_loss: 0.6945
15, time: 663.3s, g_loss: 11.0332, d_loss: 0.6742
16, time: 661.5s, g_loss: 11.2900, d_loss: 0.6530
17, time: 661.8s, g_loss: 12.5692, d_loss: 0.5229
18, time: 663.6s, g_loss: 12.6079, d_loss: 0.6294
19, time: 661.2s, g_loss: 10.9919, d_loss: 0.7023
20, time: 663.5s, g_loss: 11.1688, d_loss: 0.6395
21, time: 662.1s, g_loss: 11.2613, d_loss: 0.6420
22, time: 661.0s, g_loss: 11.2523, d_loss: 0.6330
23, time: 660.8s, g_loss: 13.6837, d_loss: 0.5765
24, time: 661.1s, g_loss: 14.2986, d_loss: 0.4142
25, time: 663.3s, g_loss: 11.0907, d_loss: 0.7316
26, time: 662.3s, g_loss: 11.2258, d_loss: 0.6622
27, time: 661.5s, g_loss: 11.2672, d_loss: 0.6467
28, time: 663.5s, g_loss: 11.1313, d_loss: 0.6700
29, time: 662.2s, g_loss: 11.1652, d_loss: 0.6832
30, time: 664.3s, g_loss: 11.0767, d_loss: 0.6486
31, time: 661.6s, g_loss: 11.3750, d_loss: 0.6434
32, time: 661.5s, g_loss: 14.5869, d_loss: 0.4591
33, time: 661.3s, g_loss: 11.1043, d_loss: 0.6653
34, time: 661.2s, g_loss: 13.7658, d_loss: 0.4475
35, time: 666.1s, g_loss: 13.0329, d_loss: 0.6696
36, time: 662.3s, g_loss: 11.5940, d_loss: 0.6172
37, time: 661.5s, g_loss: 15.8315, d_loss: 0.4462
38, time: 662.2s, g_loss: 11.2353, d_loss: 0.6819
39, time: 662.7s, g_loss: 12.8559, d_loss: 0.4912
40, time: 665.7s, g_loss: 13.8362, d_loss: 0.5517
41, time: 661.4s, g_loss: 11.8107, d_loss: 0.5674
42, time: 661.4s, g_loss: 13.6914, d_loss: 0.5778
43, time: 661.6s, g_loss: 11.0788, d_loss: 0.6635
44, time: 661.4s, g_loss: 11.1740, d_loss: 0.6669
45, time: 666.6s, g_loss: 11.0419, d_loss: 0.6573
46, time: 662.2s, g_loss: 11.0634, d_loss: 0.6723
47, time: 661.3s, g_loss: 11.0740, d_loss: 0.6488
48, time: 661.5s, g_loss: 11.4087, d_loss: 0.6096
49, time: 661.0s, g_loss: 14.3253, d_loss: 0.5045
50, time: 663.4s, g_loss: 12.3398, d_loss: 0.5206

total_time: 552.1min

trainer.load_checkpoint(model_dir/'5-10.pt')

trainer.train(
    n_step=step_per_epoch*50, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'6'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*5, 
    ck_path=str(model_dir/'6'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 687.2s, g_loss: 14.8383, d_loss: 0.4984
2, time: 687.3s, g_loss: 11.4639, d_loss: 0.6284
3, time: 687.7s, g_loss: 11.1014, d_loss: 0.6632
4, time: 687.8s, g_loss: 11.1577, d_loss: 0.6415
5, time: 692.2s, g_loss: 12.7882, d_loss: 0.4822
6, time: 689.1s, g_loss: 12.9703, d_loss: 0.7199
7, time: 688.5s, g_loss: 10.8942, d_loss: 0.6660
8, time: 688.4s, g_loss: 11.2068, d_loss: 0.6355
9, time: 687.9s, g_loss: 11.1832, d_loss: 0.6476
10, time: 691.1s, g_loss: 11.3356, d_loss: 0.6135
11, time: 689.5s, g_loss: 12.5973, d_loss: 0.5098
12, time: 688.3s, g_loss: 13.1049, d_loss: 0.5946
13, time: 688.0s, g_loss: 10.9973, d_loss: 0.6651
14, time: 688.7s, g_loss: 11.2522, d_loss: 0.6529
15, time: 692.8s, g_loss: 11.3126, d_loss: 0.6378
16, time: 689.1s, g_loss: 11.1799, d_loss: 0.6584
17, time: 688.7s, g_loss: 11.4504, d_loss: 0.6268
18, time: 688.5s, g_loss: 16.1067, d_loss: 0.2993
19, time: 688.6s, g_loss: 11.5342, d_loss: 0.7347
20, time: 692.7s, g_loss: 11.5894, d_loss: 0.6257
21, time: 689.8s, g_loss: 15.7947, d_loss: 0.3898
22, time: 689.2s, g_loss: 11.5764, d_loss: 0.6511
23, time: 689.4s, g_loss: 14.4714, d_loss: 0.3834
24, time: 689.1s, g_loss: 12.8385, d_loss: 0.6368
25, time: 692.3s, g_loss: 11.3392, d_loss: 0.6304
26, time: 691.1s, g_loss: 11.3349, d_loss: 0.6577
27, time: 689.5s, g_loss: 11.3804, d_loss: 0.6233
28, time: 689.9s, g_loss: 11.4033, d_loss: 0.6276
29, time: 689.6s, g_loss: 13.5698, d_loss: 0.4519
30, time: 692.4s, g_loss: 12.3061, d_loss: 0.6657
31, time: 689.7s, g_loss: 11.6967, d_loss: 0.6024
32, time: 689.8s, g_loss: 13.0214, d_loss: 0.4668
33, time: 690.7s, g_loss: 12.4119, d_loss: 0.6520
34, time: 689.6s, g_loss: 11.1712, d_loss: 0.6533
35, time: 692.5s, g_loss: 11.5794, d_loss: 0.5933
36, time: 689.8s, g_loss: 11.4316, d_loss: 0.6455
37, time: 689.5s, g_loss: 11.3986, d_loss: 0.6136
38, time: 689.9s, g_loss: 11.7107, d_loss: 0.6112
39, time: 689.5s, g_loss: 11.4201, d_loss: 0.6164
40, time: 693.3s, g_loss: 11.6237, d_loss: 0.6173
41, time: 689.8s, g_loss: 12.1552, d_loss: 0.5407
48, time: 690.4s, g_loss: 11.5056, d_loss: 0.6526
49, time: 689.3s, g_loss: 11.3961, d_loss: 0.6203
50, time: 692.7s, g_loss: 12.2105, d_loss: 0.5307

total_time: 574.9min

trainer.load_checkpoint(model_dir/'6-10.pt')

trainer.train(
    n_step=step_per_epoch*50, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'7'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*5, 
    ck_path=str(model_dir/'7'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 668.3s, g_loss: 14.2941, d_loss: 0.5257
2, time: 667.6s, g_loss: 11.6797, d_loss: 0.6098
3, time: 666.6s, g_loss: 14.7720, d_loss: 0.4342
4, time: 666.0s, g_loss: 11.2871, d_loss: 0.6431
5, time: 668.1s, g_loss: 11.4885, d_loss: 0.6332
6, time: 665.8s, g_loss: 11.5050, d_loss: 0.6090
7, time: 665.3s, g_loss: 11.8176, d_loss: 0.6046
8, time: 665.1s, g_loss: 11.7254, d_loss: 0.6137
9, time: 665.6s, g_loss: 13.6745, d_loss: 0.4411
10, time: 668.1s, g_loss: 15.2268, d_loss: 0.5011
11, time: 665.1s, g_loss: 11.6732, d_loss: 0.6122
12, time: 665.3s, g_loss: 11.7245, d_loss: 0.6192
13, time: 666.2s, g_loss: 12.8176, d_loss: 0.4758
14, time: 666.4s, g_loss: 16.7009, d_loss: 0.3842
15, time: 670.8s, g_loss: 11.8341, d_loss: 0.6029
16, time: 667.9s, g_loss: 11.6263, d_loss: 0.6046
17, time: 668.3s, g_loss: 11.6385, d_loss: 0.6055
18, time: 669.1s, g_loss: 14.0296, d_loss: 0.4065
19, time: 669.2s, g_loss: 11.4700, d_loss: 0.7336
20, time: 672.6s, g_loss: 11.6926, d_loss: 0.5901
21, time: 669.5s, g_loss: 16.9572, d_loss: 0.3543
22, time: 669.2s, g_loss: 11.4144, d_loss: 0.6423
23, time: 669.7s, g_loss: 11.6471, d_loss: 0.6125
24, time: 668.9s, g_loss: 11.5413, d_loss: 0.6040
25, time: 671.8s, g_loss: 14.1510, d_loss: 0.4059
26, time: 668.9s, g_loss: 15.7551, d_loss: 0.4759
27, time: 668.0s, g_loss: 11.8027, d_loss: 0.6067
28, time: 668.3s, g_loss: 11.9278, d_loss: 0.5834
29, time: 670.4s, g_loss: 13.9039, d_loss: 0.4148
30, time: 673.1s, g_loss: 12.6136, d_loss: 0.6245
31, time: 670.1s, g_loss: 11.5814, d_loss: 0.6054
32, time: 668.2s, g_loss: 15.6994, d_loss: 0.4913
33, time: 667.3s, g_loss: 11.0086, d_loss: 0.6578
34, time: 667.4s, g_loss: 11.6518, d_loss: 0.5920
35, time: 670.1s, g_loss: 11.8663, d_loss: 0.5704
36, time: 667.5s, g_loss: 11.9794, d_loss: 0.5618
37, time: 668.6s, g_loss: 11.7308, d_loss: 0.6158
38, time: 666.8s, g_loss: 11.6776, d_loss: 0.6053
39, time: 667.1s, g_loss: 11.6234, d_loss: 0.6132
40, time: 669.3s, g_loss: 11.8880, d_loss: 0.5922
41, time: 666.7s, g_loss: 16.2449, d_loss: 0.2998
42, time: 667.7s, g_loss: 11.3546, d_loss: 0.8363
43, time: 668.7s, g_loss: 11.8099, d_loss: 0.5959
44, time: 666.9s, g_loss: 11.8193, d_loss: 0.5972
45, time: 669.3s, g_loss: 17.2025, d_loss: 0.2627
46, time: 666.1s, g_loss: 11.9952, d_loss: 0.6633
47, time: 670.0s, g_loss: 12.0271, d_loss: 0.5840
48, time: 669.7s, g_loss: 11.7700, d_loss: 0.6116
49, time: 669.9s, g_loss: 11.9250, d_loss: 0.6059
50, time: 673.0s, g_loss: 11.7799, d_loss: 0.6006

total_time: 556.9min

trainer.load_checkpoint(model_dir/'7-10.pt')

trainer.train(
    n_step=step_per_epoch*52, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'8'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*4, 
    ck_path=str(model_dir/'8'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 661.6s, g_loss: 12.4176, d_loss: 0.5562
2, time: 661.0s, g_loss: 11.7152, d_loss: 0.6020
3, time: 661.7s, g_loss: 14.3518, d_loss: 0.4045
4, time: 663.8s, g_loss: 12.7276, d_loss: 0.6494
5, time: 661.4s, g_loss: 12.1342, d_loss: 0.5695
6, time: 661.0s, g_loss: 12.2305, d_loss: 0.5530
7, time: 660.8s, g_loss: 11.9931, d_loss: 0.5836
8, time: 664.7s, g_loss: 13.8700, d_loss: 0.4191
9, time: 661.6s, g_loss: 13.0945, d_loss: 0.6131
10, time: 661.1s, g_loss: 12.4843, d_loss: 0.5573
11, time: 661.6s, g_loss: 11.7039, d_loss: 0.6139
12, time: 664.3s, g_loss: 14.8622, d_loss: 0.4051
13, time: 662.3s, g_loss: 12.0824, d_loss: 0.6061
14, time: 662.0s, g_loss: 11.7924, d_loss: 0.5766
15, time: 662.3s, g_loss: 14.8617, d_loss: 0.4536
16, time: 664.5s, g_loss: 11.8951, d_loss: 0.6071
17, time: 662.3s, g_loss: 12.0398, d_loss: 0.5356
18, time: 662.1s, g_loss: 14.5674, d_loss: 0.4665
19, time: 662.4s, g_loss: 12.2435, d_loss: 0.5241
20, time: 665.8s, g_loss: 14.8917, d_loss: 0.4207
21, time: 662.5s, g_loss: 11.8254, d_loss: 0.6093
22, time: 662.1s, g_loss: 15.2556, d_loss: 0.3576
23, time: 661.8s, g_loss: 14.6061, d_loss: 0.5081
24, time: 664.8s, g_loss: 12.1907, d_loss: 0.5879
25, time: 662.6s, g_loss: 11.8101, d_loss: 0.5812
26, time: 663.2s, g_loss: 12.2175, d_loss: 0.5933
27, time: 662.7s, g_loss: 11.7853, d_loss: 0.5895
28, time: 665.4s, g_loss: 12.2060, d_loss: 0.5415
29, time: 662.9s, g_loss: 16.5229, d_loss: 0.2860
30, time: 662.5s, g_loss: 13.7685, d_loss: 0.5991
31, time: 662.1s, g_loss: 12.0732, d_loss: 0.5697
32, time: 665.0s, g_loss: 11.9596, d_loss: 0.5683
33, time: 664.1s, g_loss: 14.7582, d_loss: 0.3937
34, time: 662.0s, g_loss: 17.7234, d_loss: 0.3960
35, time: 662.1s, g_loss: 12.6332, d_loss: 0.4986
36, time: 664.5s, g_loss: 12.1452, d_loss: 0.5833
37, time: 662.7s, g_loss: 12.9985, d_loss: 0.4575
38, time: 663.8s, g_loss: 18.3054, d_loss: 0.2982
39, time: 663.3s, g_loss: 13.0686, d_loss: 0.5086
40, time: 665.2s, g_loss: 14.6538, d_loss: 0.4822
41, time: 663.2s, g_loss: 12.3859, d_loss: 0.5344
42, time: 664.0s, g_loss: 11.9287, d_loss: 0.6070
43, time: 662.3s, g_loss: 12.2015, d_loss: 0.5186
44, time: 664.6s, g_loss: 12.2152, d_loss: 0.5514
45, time: 662.9s, g_loss: 15.4685, d_loss: 0.3816
46, time: 662.4s, g_loss: 11.9844, d_loss: 0.6013
47, time: 661.9s, g_loss: 12.0977, d_loss: 0.5376
48, time: 665.3s, g_loss: 12.0841, d_loss: 0.5733
49, time: 663.1s, g_loss: 12.4076, d_loss: 0.5340
50, time: 662.0s, g_loss: 15.8019, d_loss: 0.3718
51, time: 662.2s, g_loss: 12.2289, d_loss: 0.5701
52, time: 664.8s, g_loss: 12.0025, d_loss: 0.5706

total_time: 574.5min

trainer.load_checkpoint(model_dir/'8-13.pt')

trainer.train(
    n_step=step_per_epoch*50, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'9'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*5, 
    ck_path=str(model_dir/'9'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 687.0s, g_loss: 12.3093, d_loss: 0.5419
2, time: 687.5s, g_loss: 12.3291, d_loss: 0.5415
3, time: 687.6s, g_loss: 13.4712, d_loss: 0.5116
4, time: 687.5s, g_loss: 12.6042, d_loss: 0.4901
5, time: 691.9s, g_loss: 16.1012, d_loss: 0.4532
6, time: 688.0s, g_loss: 12.0456, d_loss: 0.5767
7, time: 688.2s, g_loss: 12.4961, d_loss: 0.5377
8, time: 688.5s, g_loss: 12.7256, d_loss: 0.4963
9, time: 688.3s, g_loss: 17.7709, d_loss: 0.3896
10, time: 691.7s, g_loss: 12.0674, d_loss: 0.5848
11, time: 689.7s, g_loss: 12.1577, d_loss: 0.5499
12, time: 688.6s, g_loss: 11.9990, d_loss: 0.5881
13, time: 688.2s, g_loss: 12.3411, d_loss: 0.5074
14, time: 687.7s, g_loss: 15.4673, d_loss: 0.4328
15, time: 692.3s, g_loss: 12.1270, d_loss: 0.5664
16, time: 688.6s, g_loss: 12.0260, d_loss: 0.5720
17, time: 688.3s, g_loss: 12.2703, d_loss: 0.5521
18, time: 688.4s, g_loss: 12.1143, d_loss: 0.5715
19, time: 687.9s, g_loss: 12.1667, d_loss: 0.5715
20, time: 692.8s, g_loss: 12.4428, d_loss: 0.5528
21, time: 689.3s, g_loss: 12.1243, d_loss: 0.5533
22, time: 688.7s, g_loss: 16.5320, d_loss: 0.2692
23, time: 689.2s, g_loss: 12.9731, d_loss: 0.5597
24, time: 688.8s, g_loss: 12.3776, d_loss: 0.5548
25, time: 692.0s, g_loss: 13.1826, d_loss: 0.4720
26, time: 690.9s, g_loss: 16.9569, d_loss: 0.4575
27, time: 689.2s, g_loss: 12.5438, d_loss: 0.5241
28, time: 689.0s, g_loss: 12.2447, d_loss: 0.5591
29, time: 688.7s, g_loss: 14.0217, d_loss: 0.4355
30, time: 691.5s, g_loss: 12.4296, d_loss: 0.5601
31, time: 690.0s, g_loss: 12.5309, d_loss: 0.5423
32, time: 688.9s, g_loss: 12.4092, d_loss: 0.5194
33, time: 690.7s, g_loss: 17.5897, d_loss: 0.2829
34, time: 689.1s, g_loss: 12.8679, d_loss: 0.5424
35, time: 692.3s, g_loss: 17.4180, d_loss: 0.3013
36, time: 689.8s, g_loss: 12.6412, d_loss: 0.5648
37, time: 689.1s, g_loss: 12.1967, d_loss: 0.5551
38, time: 689.4s, g_loss: 12.3252, d_loss: 0.5463
39, time: 689.0s, g_loss: 12.3869, d_loss: 0.5180
40, time: 692.9s, g_loss: 13.5074, d_loss: 0.4535
41, time: 689.9s, g_loss: 14.5832, d_loss: 0.4742
42, time: 691.3s, g_loss: 12.1930, d_loss: 0.5714
43, time: 689.6s, g_loss: 12.4536, d_loss: 0.5286
44, time: 689.2s, g_loss: 14.0465, d_loss: 0.3927
45, time: 692.4s, g_loss: 14.2170, d_loss: 0.5541
46, time: 689.6s, g_loss: 12.1928, d_loss: 0.5457
47, time: 689.3s, g_loss: 12.2389, d_loss: 0.5429
48, time: 689.2s, g_loss: 12.4169, d_loss: 0.5381
49, time: 689.1s, g_loss: 12.4997, d_loss: 0.5360
50, time: 692.7s, g_loss: 12.3229, d_loss: 0.5349

total_time: 574.7min

trainer.load_checkpoint(model_dir/'9-10.pt')

trainer.train(
    n_step=step_per_epoch*50, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'10'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*5, 
    ck_path=str(model_dir/'10'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 689.8s, g_loss: 12.1969, d_loss: 0.5583
2, time: 689.8s, g_loss: 14.5376, d_loss: 0.4470
3, time: 690.4s, g_loss: 13.4742, d_loss: 0.4489
4, time: 690.1s, g_loss: 17.7438, d_loss: 0.3396
5, time: 694.2s, g_loss: 14.2166, d_loss: 0.4110
6, time: 690.9s, g_loss: 12.3758, d_loss: 0.5792
7, time: 690.6s, g_loss: 12.4170, d_loss: 0.5503
8, time: 690.4s, g_loss: 12.4631, d_loss: 0.5397
9, time: 690.1s, g_loss: 12.4539, d_loss: 0.5310
10, time: 693.1s, g_loss: 14.9015, d_loss: 0.3611
11, time: 692.0s, g_loss: 13.5653, d_loss: 0.5483
12, time: 690.7s, g_loss: 15.3929, d_loss: 0.4293
13, time: 690.2s, g_loss: 12.3490, d_loss: 0.5537
14, time: 690.8s, g_loss: 12.6296, d_loss: 0.5064
15, time: 695.1s, g_loss: 12.5186, d_loss: 0.5257
16, time: 691.7s, g_loss: 15.3580, d_loss: 0.4174
17, time: 691.5s, g_loss: 12.4160, d_loss: 0.5620
18, time: 690.8s, g_loss: 12.5330, d_loss: 0.5257
19, time: 690.2s, g_loss: 12.3499, d_loss: 0.5317
20, time: 694.8s, g_loss: 12.2848, d_loss: 0.5481
21, time: 692.3s, g_loss: 16.4683, d_loss: 0.2760
22, time: 691.4s, g_loss: 16.5173, d_loss: 0.4632
23, time: 690.8s, g_loss: 12.6141, d_loss: 0.5332
24, time: 691.9s, g_loss: 12.4730, d_loss: 0.5471
25, time: 694.5s, g_loss: 13.3428, d_loss: 0.4989
26, time: 693.7s, g_loss: 12.3172, d_loss: 0.5349
27, time: 693.2s, g_loss: 12.7361, d_loss: 0.5272
28, time: 692.0s, g_loss: 12.6638, d_loss: 0.5016
29, time: 692.4s, g_loss: 12.5369, d_loss: 0.5113
30, time: 695.4s, g_loss: 12.4756, d_loss: 0.5431
31, time: 693.3s, g_loss: 12.5018, d_loss: 0.5374
32, time: 692.2s, g_loss: 12.8052, d_loss: 0.5008
33, time: 694.5s, g_loss: 14.7454, d_loss: 0.4094
34, time: 690.4s, g_loss: 12.6355, d_loss: 0.5494
35, time: 693.8s, g_loss: 14.3613, d_loss: 0.4307
36, time: 691.4s, g_loss: 12.6973, d_loss: 0.5246
37, time: 690.0s, g_loss: 12.3823, d_loss: 0.5368
38, time: 689.8s, g_loss: 12.6371, d_loss: 0.5359
39, time: 690.7s, g_loss: 14.6821, d_loss: 0.4075
40, time: 693.1s, g_loss: 12.5482, d_loss: 0.5453
41, time: 691.2s, g_loss: 14.9251, d_loss: 0.4142
42, time: 692.0s, g_loss: 12.5638, d_loss: 0.5376
43, time: 690.5s, g_loss: 12.5986, d_loss: 0.5381
44, time: 690.4s, g_loss: 14.3872, d_loss: 0.3864
45, time: 693.2s, g_loss: 12.8223, d_loss: 0.5194
46, time: 690.9s, g_loss: 12.5656, d_loss: 0.5605
47, time: 690.3s, g_loss: 12.6150, d_loss: 0.5039
48, time: 690.2s, g_loss: 16.6606, d_loss: 0.2849
49, time: 690.4s, g_loss: 14.1622, d_loss: 0.4935
50, time: 694.4s, g_loss: 12.3275, d_loss: 0.5736

total_time: 576.5min

trainer.load_checkpoint(model_dir/'10-10.pt')

trainer.train(
    n_step=step_per_epoch*52, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'11'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*4, 
    ck_path=str(model_dir/'11'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 661.4s, g_loss: 15.4043, d_loss: 0.3407
2, time: 662.6s, g_loss: 12.9295, d_loss: 0.5591
3, time: 663.6s, g_loss: 12.8585, d_loss: 0.5231
4, time: 665.2s, g_loss: 13.8509, d_loss: 0.4147
5, time: 662.7s, g_loss: 16.4387, d_loss: 0.4351
6, time: 662.6s, g_loss: 12.5847, d_loss: 0.5615
7, time: 663.3s, g_loss: 12.5656, d_loss: 0.5388
8, time: 666.2s, g_loss: 12.9412, d_loss: 0.5088
9, time: 662.9s, g_loss: 12.6478, d_loss: 0.5196
10, time: 662.9s, g_loss: 12.7834, d_loss: 0.5202
11, time: 663.5s, g_loss: 15.4064, d_loss: 0.3960
12, time: 665.7s, g_loss: 12.6201, d_loss: 0.5511
13, time: 662.0s, g_loss: 18.3127, d_loss: 0.2372
14, time: 661.9s, g_loss: 14.0958, d_loss: 0.5927
15, time: 661.3s, g_loss: 12.8709, d_loss: 0.4972
16, time: 663.7s, g_loss: 12.8732, d_loss: 0.5274
17, time: 661.4s, g_loss: 12.7705, d_loss: 0.5157
18, time: 661.2s, g_loss: 17.6336, d_loss: 0.2359
19, time: 661.1s, g_loss: 15.9788, d_loss: 0.4641
20, time: 664.5s, g_loss: 12.7417, d_loss: 0.5544
21, time: 661.7s, g_loss: 12.6769, d_loss: 0.5080
22, time: 662.0s, g_loss: 12.7779, d_loss: 0.5074
23, time: 663.4s, g_loss: 12.7699, d_loss: 0.5252
24, time: 665.9s, g_loss: 12.6055, d_loss: 0.5105
25, time: 662.2s, g_loss: 13.0327, d_loss: 0.4936
26, time: 662.1s, g_loss: 12.8062, d_loss: 0.5198
27, time: 660.9s, g_loss: 13.6479, d_loss: 0.4301
28, time: 663.6s, g_loss: 13.0036, d_loss: 0.5586
29, time: 661.3s, g_loss: 12.8356, d_loss: 0.4985
30, time: 661.4s, g_loss: 12.8833, d_loss: 0.5108
31, time: 661.1s, g_loss: 12.8910, d_loss: 0.4890
32, time: 663.7s, g_loss: 13.1243, d_loss: 0.5004
33, time: 662.9s, g_loss: 12.9337, d_loss: 0.5428
34, time: 661.1s, g_loss: 12.7949, d_loss: 0.5238
35, time: 661.5s, g_loss: 15.9996, d_loss: 0.5236
36, time: 663.6s, g_loss: 11.7859, d_loss: 0.5887

IOPub message rate exceeded.
The notebook server will temporarily stop sending output
to the client in order to avoid crashing it.
To change this limit, set the config variable
`--NotebookApp.iopub_msg_rate_limit`.

Current values:
NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)
NotebookApp.rate_limit_window=3.0 (secs)

40, time: 664.1s, g_loss: 13.8696, d_loss: 0.4345
41, time: 662.0s, g_loss: 13.6853, d_loss: 0.5272
42, time: 662.9s, g_loss: 13.1624, d_loss: 0.4770
43, time: 661.2s, g_loss: 13.1909, d_loss: 0.4929
44, time: 664.3s, g_loss: 13.2227, d_loss: 0.4768
45, time: 661.8s, g_loss: 13.1867, d_loss: 0.4971
46, time: 661.7s, g_loss: 14.7388, d_loss: 0.3869
47, time: 661.3s, g_loss: 15.4949, d_loss: 0.4488
48, time: 663.8s, g_loss: 12.9681, d_loss: 0.5257
49, time: 661.3s, g_loss: 13.0346, d_loss: 0.5003
50, time: 660.9s, g_loss: 13.1155, d_loss: 0.4926
51, time: 661.2s, g_loss: 12.9111, d_loss: 0.5139
52, time: 664.0s, g_loss: 13.1594, d_loss: 0.4675

total_time: 574.2min

trainer.load_checkpoint(model_dir/'11-13.pt')

trainer.train(
    n_step=step_per_epoch*52, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'12'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*4, 
    ck_path=str(model_dir/'12'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 665.4s, g_loss: 14.7451, d_loss: 0.3716
2, time: 666.5s, g_loss: 14.4521, d_loss: 0.4705
3, time: 665.7s, g_loss: 16.0901, d_loss: 0.4410
4, time: 669.5s, g_loss: 12.9452, d_loss: 0.5086
5, time: 665.8s, g_loss: 13.1977, d_loss: 0.4998
6, time: 666.2s, g_loss: 15.6391, d_loss: 0.3279
7, time: 667.7s, g_loss: 15.0243, d_loss: 0.4601
8, time: 670.3s, g_loss: 13.0215, d_loss: 0.5178
9, time: 666.6s, g_loss: 13.2136, d_loss: 0.4887
10, time: 665.6s, g_loss: 12.9522, d_loss: 0.5087
11, time: 666.2s, g_loss: 12.9962, d_loss: 0.5020
12, time: 669.8s, g_loss: 13.2016, d_loss: 0.4873
13, time: 666.9s, g_loss: 13.1891, d_loss: 0.4741
14, time: 667.0s, g_loss: 14.1780, d_loss: 0.4752
15, time: 669.2s, g_loss: 13.2045, d_loss: 0.4915
16, time: 671.1s, g_loss: 12.9136, d_loss: 0.4946
17, time: 667.6s, g_loss: 13.2719, d_loss: 0.4880
18, time: 667.5s, g_loss: 16.9829, d_loss: 0.2651
19, time: 667.5s, g_loss: 13.3947, d_loss: 0.5707
20, time: 672.6s, g_loss: 13.0867, d_loss: 0.5160
21, time: 669.7s, g_loss: 13.4214, d_loss: 0.4698
22, time: 668.1s, g_loss: 13.1677, d_loss: 0.5074
23, time: 668.1s, g_loss: 13.0369, d_loss: 0.4869
24, time: 671.6s, g_loss: 13.2335, d_loss: 0.4864
25, time: 667.5s, g_loss: 13.3639, d_loss: 0.4902
26, time: 670.0s, g_loss: 18.5767, d_loss: 0.2244
27, time: 668.5s, g_loss: 16.7510, d_loss: 0.4855
28, time: 667.8s, g_loss: 13.1323, d_loss: 0.5227
29, time: 663.1s, g_loss: 13.3979, d_loss: 0.4727
30, time: 662.4s, g_loss: 14.0235, d_loss: 0.4295
31, time: 663.0s, g_loss: 17.4161, d_loss: 0.3685
32, time: 665.8s, g_loss: 13.4657, d_loss: 0.5338
33, time: 663.8s, g_loss: 15.5342, d_loss: 0.3023
34, time: 663.1s, g_loss: 15.3475, d_loss: 0.5394
35, time: 665.4s, g_loss: 13.1502, d_loss: 0.4953
36, time: 669.7s, g_loss: 13.3567, d_loss: 0.4713
37, time: 667.7s, g_loss: 13.4918, d_loss: 0.4738
38, time: 666.6s, g_loss: 13.6377, d_loss: 0.4410
40, time: 669.6s, g_loss: 13.3742, d_loss: 0.4748
41, time: 666.6s, g_loss: 13.2604, d_loss: 0.4838
42, time: 668.7s, g_loss: 13.3075, d_loss: 0.4781
43, time: 666.2s, g_loss: 15.0885, d_loss: 0.3941
44, time: 665.9s, g_loss: 12.8900, d_loss: 0.5256
46, time: 666.9s, g_loss: 14.6219, d_loss: 0.4040
47, time: 667.0s, g_loss: 13.0551, d_loss: 0.5011
48, time: 669.8s, g_loss: 13.2255, d_loss: 0.4754
49, time: 667.8s, g_loss: 13.1236, d_loss: 0.4984
50, time: 668.4s, g_loss: 14.7449, d_loss: 0.3545
51, time: 667.9s, g_loss: 15.2094, d_loss: 0.4709
52, time: 670.3s, g_loss: 15.1095, d_loss: 0.3687

total_time: 578.4min

trainer.load_checkpoint(model_dir/'12-13.pt')

trainer.train(
    n_step=step_per_epoch*50, 
    step_per_epoch=step_per_epoch, 
    savejpg_every=step_per_epoch, 
    jpg_path=str(result_dir/'13'), 
    is_jpg_ema=False,
    saveck_every=step_per_epoch*5, 
    ck_path=str(model_dir/'13'), 
    n_gradient_acc=n_gradient_acc,
    ema_decay=ema_decay, 
)

1, time: 690.5s, g_loss: 12.9867, d_loss: 0.5244
2, time: 692.0s, g_loss: 13.1771, d_loss: 0.4954
3, time: 692.2s, g_loss: 13.3623, d_loss: 0.4474
4, time: 692.2s, g_loss: 13.4736, d_loss: 0.4648
5, time: 696.8s, g_loss: 14.9285, d_loss: 0.3535
6, time: 692.4s, g_loss: 15.3078, d_loss: 0.4657
7, time: 691.7s, g_loss: 13.2814, d_loss: 0.4809
8, time: 692.6s, g_loss: 13.5386, d_loss: 0.4492
9, time: 693.4s, g_loss: 13.5026, d_loss: 0.4612
10, time: 696.4s, g_loss: 13.4142, d_loss: 0.4545
11, time: 694.5s, g_loss: 13.4682, d_loss: 0.4980
12, time: 694.1s, g_loss: 13.6706, d_loss: 0.4436
13, time: 694.4s, g_loss: 13.4621, d_loss: 0.4660
14, time: 695.4s, g_loss: 13.5334, d_loss: 0.4811
15, time: 700.5s, g_loss: 13.4867, d_loss: 0.4643
16, time: 696.2s, g_loss: 13.3662, d_loss: 0.4999
17, time: 695.4s, g_loss: 15.3404, d_loss: 0.3710
18, time: 696.1s, g_loss: 13.6538, d_loss: 0.4859
19, time: 695.2s, g_loss: 14.7924, d_loss: 0.4459
20, time: 698.7s, g_loss: 13.3377, d_loss: 0.4709
21, time: 696.1s, g_loss: 13.3787, d_loss: 0.4720
22, time: 696.6s, g_loss: 13.1744, d_loss: 0.4742
23, time: 696.4s, g_loss: 13.6698, d_loss: 0.4642
24, time: 696.9s, g_loss: 13.5476, d_loss: 0.4663
25, time: 698.7s, g_loss: 13.6370, d_loss: 0.4344
26, time: 695.5s, g_loss: 16.9201, d_loss: 0.2609
27, time: 692.0s, g_loss: 14.2433, d_loss: 0.5049
28, time: 691.1s, g_loss: 13.4104, d_loss: 0.5000
29, time: 690.8s, g_loss: 14.7010, d_loss: 0.3791
30, time: 695.5s, g_loss: 13.5693, d_loss: 0.4373
31, time: 692.6s, g_loss: 13.5972, d_loss: 0.4619
32, time: 692.4s, g_loss: 13.4868, d_loss: 0.4787
33, time: 694.0s, g_loss: 15.8557, d_loss: 0.3019
34, time: 691.0s, g_loss: 13.6897, d_loss: 0.5532
35, time: 694.6s, g_loss: 13.7998, d_loss: 0.4566
36, time: 693.0s, g_loss: 13.4727, d_loss: 0.4647
37, time: 692.5s, g_loss: 13.5771, d_loss: 0.4586
38, time: 693.1s, g_loss: 13.9112, d_loss: 0.4574
39, time: 692.5s, g_loss: 15.2125, d_loss: 0.3690
40, time: 696.3s, g_loss: 13.8087, d_loss: 0.4443
41, time: 694.3s, g_loss: 13.6905, d_loss: 0.4604
42, time: 694.7s, g_loss: 15.3351, d_loss: 0.4045
43, time: 692.8s, g_loss: 13.5103, d_loss: 0.4691
44, time: 692.7s, g_loss: 15.1885, d_loss: 0.3940
45, time: 696.3s, g_loss: 13.6981, d_loss: 0.4653
46, time: 694.5s, g_loss: 13.7914, d_loss: 0.4245
47, time: 693.5s, g_loss: 18.3415, d_loss: 0.2132
48, time: 693.2s, g_loss: 18.4351, d_loss: 0.3747
49, time: 693.9s, g_loss: 14.0452, d_loss: 0.4515
50, time: 697.1s, g_loss: 13.9353, d_loss: 0.4506

total_time: 578.6min

trainer.load_checkpoint(model_dir/'13-10.pt')

# 662, 52, 49

# 690, 50, 

Check

trainer.show(is_ema=False)

trainer.show(is_ema=True)

check = trainer.check_d(is_ema=True)

check[1]

(tensor([[0.9997],
         [0.9852],
         [0.9949],
         [0.9760],
         [0.6487],
         [0.9350],
         [0.7138],
         [0.7514],
         [0.9572],
         [0.9368],
         [0.8783],
         [0.9369],
         [0.9689],
         [0.9989],
         [0.9651],
         [0.9895],
         [0.9056],
         [0.9646],
         [0.9761],
         [0.9910],
         [0.9704],
         [0.9866],
         [0.8979],
         [0.9673]], device='cuda:0'), tensor([[1.0217e-03],
         [5.9101e-04],
         [2.2663e-04],
         [9.2072e-09],
         [1.5338e-03],
         [6.9317e-04],
         [8.8911e-05],
         [2.1451e-04],
         [3.2855e-03],
         [7.3951e-04],
         [1.7794e-02],
         [6.9268e-04],
         [7.5821e-03],
         [2.7447e-04],
         [1.7766e-06],
         [4.8616e-08],
         [4.3875e-06],
         [2.5007e-02],
         [4.4294e-03],
         [1.0148e-01],
         [2.0785e-04],
         [3.3165e-04],
         [2.9234e-04],
         [4.2753e-03]], device='cuda:0'))

Export and Inference

simple_caps = [
    'a small red bird',
    'a small orange bird',
    'a small blue bird',
    'a small yellow bird',
    'a small black bird',
]
medium_caps = [
    'a small white bird with orange bill',
    'a large red bird with black beak',
    'a small black bird with a yellow head',
    'a large yellow bird with long black beak',
    'this bird has a green crown, black wings and a yellow belly',
]
complex_caps = [
    'this bird has a blue crown green primaries and a red belly',
]

trainer.export(model_dir/'gan_export.pt', is_ema=True)

model = Birds_Export.from_pretrained(model_dir/'gan_export.pt')

cap = simple_caps[0]
pred_and_show(model, cap)

cap = medium_caps[4]
pred_and_show(model, cap)

cap = complex_caps[0]
pred_and_show(model, cap)

cap = 'this bird is red with white and has a very short beak'
pred_and_show(model, cap)

cap = 'the bird has a yellow crown and a black eyering that is round'
pred_and_show(model, cap)