Diff of /network_motion.py [000000] .. [721c7a]

Switch to unified view

a b/network_motion.py
1
import torch
2
from torch import nn
3
import torch.nn.functional as F
4
5
def relu():
6
    return nn.ReLU(inplace=True)
7
8
9
def conv(in_channels, out_channels, kernel_size=(3,3,3), stride=(1,1,1), padding = 1, nonlinearity = relu):
10
    conv_layer = nn.Conv3d(in_channels = in_channels, out_channels= out_channels, kernel_size = kernel_size, stride = stride, padding = padding, bias = False)
11
12
    nll_layer = nonlinearity()
13
    bn_layer = nn.BatchNorm3d(out_channels)
14
15
    layers = [conv_layer, bn_layer, nll_layer]
16
    return nn.Sequential(*layers)
17
18
def deconv(in_channels, out_channels, kernel_size=(3,3,3), stride=(1,1,1), padding = 1, nonlinearity = relu):
19
    conv_layer = nn.ConvTranspose3d(in_channels = in_channels, out_channels= out_channels, kernel_size = kernel_size, stride = stride, padding = padding, output_padding = 1, bias = False)
20
21
    nll_layer = nonlinearity()
22
    bn_layer = nn.BatchNorm3d(out_channels)
23
24
    layers = [conv_layer, bn_layer, nll_layer]
25
    return nn.Sequential(*layers)
26
27
28
def conv_blocks_2(in_channels, out_channels, strides=(1,1,1)):
29
    conv1 = conv(in_channels, out_channels, stride = strides)
30
    conv2 = conv(out_channels, out_channels, stride=(1,1,1))
31
    layers = [conv1, conv2]
32
    return nn.Sequential(*layers)
33
34
35
def conv_blocks_3(in_channels, out_channels, strides=(1,1,1)):
36
    conv1 = conv(in_channels, out_channels, stride = strides)
37
    conv2 = conv(out_channels, out_channels, stride=(1,1,1))
38
    conv3 = conv(out_channels, out_channels, stride=(1,1,1))
39
    layers = [conv1, conv2, conv3]
40
    return nn.Sequential(*layers)
41
42
def fullyconnect(in_features, out_features, out_channels, nonlinearity = relu):
43
    fc_layer = nn.Linear(in_features = in_features, out_features= out_features, bias = False)
44
45
    nll_layer = nonlinearity()
46
    bn_layer = nn.BatchNorm1d(out_channels)
47
48
    layers = [fc_layer, bn_layer, nll_layer]
49
    return nn.Sequential(*layers)
50
51
def conv_2D(in_channels, out_channels, kernel_size=3, stride=1, padding = 1, nonlinearity = relu):
52
    conv_layer = nn.Conv2d(in_channels = in_channels, out_channels= out_channels, kernel_size = kernel_size, stride = stride, padding = padding, bias = False)
53
54
    nll_layer = nonlinearity()
55
    bn_layer = nn.BatchNorm2d(out_channels)
56
57
    layers = [conv_layer, bn_layer, nll_layer]
58
    return nn.Sequential(*layers)
59
60
def conv_1D(in_channels, out_channels, kernel_size=3, stride=1, padding = 1, nonlinearity = relu):
61
    conv_layer = nn.Conv1d(in_channels = in_channels, out_channels= out_channels, kernel_size = kernel_size, stride = stride, padding = padding, bias = False)
62
63
    nll_layer = nonlinearity()
64
    bn_layer = nn.BatchNorm2d(out_channels)
65
66
    layers = [conv_layer, bn_layer, nll_layer]
67
    return nn.Sequential(*layers)
68
69
def deconv_2D(in_channels, out_channels, kernel_size=3, stride=1, padding = 1, nonlinearity = relu):
70
    conv_layer = nn.ConvTranspose2d(in_channels = in_channels, out_channels= out_channels, kernel_size = kernel_size, stride = stride, padding = padding, output_padding = 1, bias = False)
71
72
    nll_layer = nonlinearity()
73
    bn_layer = nn.BatchNorm2d(out_channels)
74
75
    layers = [conv_layer, bn_layer, nll_layer]
76
    return nn.Sequential(*layers)
77
78
79
def conv_blocks_2_2D(in_channels, out_channels, strides=1):
80
    conv1 = conv_2D(in_channels, out_channels, stride = strides)
81
    conv2 = conv_2D(out_channels, out_channels, stride=1)
82
    layers = [conv1, conv2]
83
    return nn.Sequential(*layers)
84
85
86
def conv_blocks_3_2D(in_channels, out_channels, strides=1):
87
    conv1 = conv_2D(in_channels, out_channels, stride = strides)
88
    conv2 = conv_2D(out_channels, out_channels, stride=1)
89
    conv3 = conv_2D(out_channels, out_channels, stride=1)
90
    layers = [conv1, conv2, conv3]
91
    return nn.Sequential(*layers)
92
93
# Flatten layer
94
class Flatten(nn.Module):
95
    def forward(self, input):
96
        return input.view(input.size(0), -1)
97
98
def generate_grid(x, offset):
99
    x_shape = x.size()
100
    grid_d, grid_w, grid_h = torch.meshgrid([torch.linspace(-1, 1, x_shape[2]), torch.linspace(-1, 1, x_shape[3]), torch.linspace(-1, 1, x_shape[4])])  # (h, w, h)
101
    grid_d = grid_d.cuda().float()
102
    grid_w = grid_w.cuda().float()
103
    grid_h = grid_h.cuda().float()
104
105
    grid_d = nn.Parameter(grid_d, requires_grad=False)
106
    grid_w = nn.Parameter(grid_w, requires_grad=False)
107
    grid_h = nn.Parameter(grid_h, requires_grad=False)
108
109
    offset_h, offset_w, offset_d = torch.split(offset, 1, 1)
110
    offset_d = offset_d.contiguous().view(-1, int(x_shape[2]), int(x_shape[3]), int(x_shape[4]))  # (b*c, d, w, h)
111
    offset_w = offset_w.contiguous().view(-1, int(x_shape[2]), int(x_shape[3]), int(x_shape[4]))  # (b*c, d, w, h)
112
    offset_h = offset_h.contiguous().view(-1, int(x_shape[2]), int(x_shape[3]), int(x_shape[4]))  # (b*c, d, w, h)
113
114
    offset_d = grid_d + offset_d
115
    offset_w = grid_w + offset_w
116
    offset_h = grid_h + offset_h
117
118
    offsets = torch.stack((offset_h, offset_w, offset_d), 4) # should have the same order as offset
119
    return offsets
120
121
def transform(seg_source, loc, mode='bilinear'):
122
    grid = generate_grid(seg_source, loc)
123
    # seg_source: NCDHW
124
    # grid: NDHW3
125
    out = F.grid_sample(seg_source, grid, mode=mode, align_corners=True)
126
    return out
127
128
129
130
131
132
class MotionMesh_25d(nn.Module):
133
    """Deformable registration network with input from image space """
134
    def __init__(self, n_ch=64, mesh_dim=10000):
135
        super(MotionMesh_25d, self).__init__()
136
137
        self.conv_blocks_2D = [conv_blocks_2_2D(n_ch, 64), conv_blocks_2_2D(64, 128, 2), conv_blocks_3_2D(128, 256, 2),
138
                            conv_blocks_3_2D(256, 512, 2), conv_blocks_3_2D(512, 512, 2)]
139
        self.conv_list_2D = []
140
        for in_filters in [128, 256, 512, 1024, 1024]:
141
            self.conv_list_2D += [conv_2D(in_filters, 64)]
142
143
        self.conv_blocks_2D = nn.Sequential(*self.conv_blocks_2D)
144
        self.conv_list_2D = nn.Sequential(*self.conv_list_2D)
145
146
        self.conv6 = conv_2D(64 * 15, 64, 1, 1, 0)
147
        self.conv7 = conv_2D(64, 64, 1, 1, 0)
148
        self.conv3d9 = conv(1, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1))
149
        self.conv3d10 = conv(32, 64, kernel_size=(3, 3, 3), stride=(2, 2, 2))
150
        self.conv3d10_1 = conv(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1))
151
        self.conv3d11 = conv(64, 128, kernel_size=(3, 3, 3), stride=(2, 2, 2))
152
        self.conv3d11_1 = conv(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1))
153
        self.conv3d12 = conv(128, 256, kernel_size=(3, 3, 3), stride=(2, 2, 2))
154
        self.conv3d12_1 = conv(256, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1))
155
        self.conv3d13 = deconv(256, 128, kernel_size=(3, 3, 3), stride=(2, 2, 2))
156
        self.conv3d13_1 = conv(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1))
157
        self.conv3d14 = deconv(128, 64, kernel_size=(3, 3, 3), stride=(2, 2, 2))
158
        self.conv3d14_1 = conv(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1))
159
        self.conv3d15 = deconv(64, 32, kernel_size=(3, 3, 3), stride=(2, 2, 2))
160
        self.conv3d15_1 = conv(32, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1))
161
        self.conv3d16 = nn.Conv3d(32, 3, 1, stride=(1, 1, 1))
162
163
164
165
    def forward(self, x_sa, x_saed, x_2ch, x_2ched, x_4ch, x_4ched):
166
        # x: source image; x_pred: target image;
167
        net = {}
168
        net['conv0_sa'] = x_sa
169
        net['conv0_sa_ed'] = x_saed
170
        net['conv0_2ch'] = x_2ch
171
        net['conv0_2ch_ed'] = x_2ched
172
        net['conv0_4ch'] = x_4ch
173
        net['conv0_4ch_ed'] = x_4ched
174
        # 5 refers to 5 output or 5 blocks
175
        for i in range(5):
176
            net['conv%d_sa' % (i + 1)] = self.conv_blocks_2D[i](net['conv%d_sa' % i])
177
            net['conv%d_sa_ed' % (i + 1)] = self.conv_blocks_2D[i](net['conv%d_sa_ed' % i])
178
            net['conv%d_2ch' % (i + 1)] = self.conv_blocks_2D[i](net['conv%d_2ch' % i])
179
            net['conv%d_2ch_ed' % (i + 1)] = self.conv_blocks_2D[i](net['conv%d_2ch_ed' % i])
180
            net['conv%d_4ch' % (i + 1)] = self.conv_blocks_2D[i](net['conv%d_4ch' % i])
181
            net['conv%d_4ch_ed' % (i + 1)] = self.conv_blocks_2D[i](net['conv%d_4ch_ed' % i])
182
183
            net['concat%d_sa' % (i + 1)] = torch.cat((net['conv%d_sa' % (i + 1)], net['conv%d_sa_ed' % (i + 1)]), 1)
184
            net['concat%d_2ch' % (i + 1)] = torch.cat((net['conv%d_2ch' % (i + 1)], net['conv%d_2ch_ed' % (i + 1)]), 1)
185
            net['concat%d_4ch' % (i + 1)] = torch.cat((net['conv%d_4ch' % (i + 1)], net['conv%d_4ch_ed' % (i + 1)]), 1)
186
187
            net['out%d_sa' % (i + 1)] = self.conv_list_2D[i](net['concat%d_sa' % (i + 1)])
188
            net['out%d_2ch' % (i + 1)] = self.conv_list_2D[i](net['concat%d_2ch' % (i + 1)])
189
            net['out%d_4ch' % (i + 1)] = self.conv_list_2D[i](net['concat%d_4ch' % (i + 1)])
190
            if i > 0:
191
                # upsample
192
                # only upsample HW dimension
193
                # net['out%d_sa_up' % (i + 1)] = F.interpolate(net['out%d_sa' % (i + 1)],
194
                #                                              scale_factor=(1, 2 ** i, 2 ** i), mode='trilinear',
195
                #                                              align_corners=True)
196
                # upsample DHW dimension
197
                net['out%d_sa_up' % (i + 1)] = F.interpolate(net['out%d_sa' % (i + 1)], scale_factor=2 ** i, mode='bilinear', align_corners=True)
198
                net['out%d_2ch_up' % (i + 1)] = F.interpolate(net['out%d_2ch' % (i + 1)], scale_factor=2 ** i, mode='bilinear', align_corners=True)
199
                net['out%d_4ch_up' % (i + 1)] = F.interpolate(net['out%d_4ch' % (i + 1)], scale_factor=2 ** i, mode='bilinear', align_corners=True)
200
201
202
203
                # output: net['out1_sa'], net['out2_sa_up'], net['out3_sa_up'], net['out4_sa_up'], net['out5_sa_up'] are used for multiscale fusion
204
        net['concat_sa'] = torch.cat(
205
                (net['out1_sa'], net['out2_sa_up'], net['out3_sa_up'], net['out4_sa_up'], net['out5_sa_up']), 1)
206
        net['concat_2ch'] = torch.cat(
207
                (net['out1_2ch'], net['out2_2ch_up'], net['out3_2ch_up'], net['out4_2ch_up'], net['out5_2ch_up']), 1)
208
        net['concat_4ch'] = torch.cat(
209
                (net['out1_4ch'], net['out2_4ch_up'], net['out3_4ch_up'], net['out4_4ch_up'], net['out5_4ch_up']), 1)
210
211
        net['concat'] = torch.cat((net['concat_sa'], net['concat_2ch'], net['concat_4ch']), 1)
212
        net['comb_1'] = self.conv6(net['concat'])
213
        net['comb_2'] = self.conv7(net['comb_1'])
214
215
        net['conv3d0f'] = net['comb_2'].unsqueeze(1)
216
217
        net['conv3d_1'] = self.conv3d9(net['conv3d0f'])
218
        net['conv3d_2'] = self.conv3d10_1(self.conv3d10(net['conv3d_1']))
219
        net['conv3d_3'] = self.conv3d11_1(self.conv3d11(net['conv3d_2']))
220
        net['conv3d_4'] = self.conv3d12_1(self.conv3d12(net['conv3d_3']))
221
        net['conv3d_5'] = self.conv3d13_1(self.conv3d13(net['conv3d_4']))
222
        net['conv3d_6'] = self.conv3d14_1(self.conv3d14(net['conv3d_5']))
223
        net['conv3d_7'] = self.conv3d15_1(self.conv3d15(net['conv3d_6']))
224
        net['out'] = torch.tanh(self.conv3d16(net['conv3d_7']))
225
226
227
        return net
228
229
230
class Mesh_2d(nn.Module):
231
    """Deformable registration network with input from image space """
232
    def __init__(self, n_ch=1):
233
        super(Mesh_2d, self).__init__()
234
235
        self.conv1 = conv_2D(n_ch, 32)
236
        self.conv2 = conv_2D(32, 64)
237
238
    def forward(self, x_2ch, x_2ched, x_4ch, x_4ched):
239
        # x: source image; x_pred: target image;
240
        net = {}
241
242
        net['conv1_2ch'] = self.conv1(x_2ch)
243
        net['conv1_4ch'] = self.conv1(x_4ch)
244
        net['conv1s_2ch'] = self.conv1(x_2ched)
245
        net['conv1s_4ch'] = self.conv1(x_4ched)
246
247
        net['conv2_2ch'] = self.conv2(net['conv1_2ch'])
248
        net['conv2_4ch'] = self.conv2(net['conv1_4ch'])
249
        net['conv2s_2ch'] = self.conv2(net['conv1s_2ch'])
250
        net['conv2s_4ch'] = self.conv2(net['conv1s_4ch'])
251
252
253
        return net