|
a |
|
b/network_motion.py |
|
|
1 |
import torch |
|
|
2 |
from torch import nn |
|
|
3 |
import torch.nn.functional as F |
|
|
4 |
|
|
|
5 |
def relu(): |
|
|
6 |
return nn.ReLU(inplace=True) |
|
|
7 |
|
|
|
8 |
|
|
|
9 |
def conv(in_channels, out_channels, kernel_size=(3,3,3), stride=(1,1,1), padding = 1, nonlinearity = relu): |
|
|
10 |
conv_layer = nn.Conv3d(in_channels = in_channels, out_channels= out_channels, kernel_size = kernel_size, stride = stride, padding = padding, bias = False) |
|
|
11 |
|
|
|
12 |
nll_layer = nonlinearity() |
|
|
13 |
bn_layer = nn.BatchNorm3d(out_channels) |
|
|
14 |
|
|
|
15 |
layers = [conv_layer, bn_layer, nll_layer] |
|
|
16 |
return nn.Sequential(*layers) |
|
|
17 |
|
|
|
18 |
def deconv(in_channels, out_channels, kernel_size=(3,3,3), stride=(1,1,1), padding = 1, nonlinearity = relu): |
|
|
19 |
conv_layer = nn.ConvTranspose3d(in_channels = in_channels, out_channels= out_channels, kernel_size = kernel_size, stride = stride, padding = padding, output_padding = 1, bias = False) |
|
|
20 |
|
|
|
21 |
nll_layer = nonlinearity() |
|
|
22 |
bn_layer = nn.BatchNorm3d(out_channels) |
|
|
23 |
|
|
|
24 |
layers = [conv_layer, bn_layer, nll_layer] |
|
|
25 |
return nn.Sequential(*layers) |
|
|
26 |
|
|
|
27 |
|
|
|
28 |
def conv_blocks_2(in_channels, out_channels, strides=(1,1,1)): |
|
|
29 |
conv1 = conv(in_channels, out_channels, stride = strides) |
|
|
30 |
conv2 = conv(out_channels, out_channels, stride=(1,1,1)) |
|
|
31 |
layers = [conv1, conv2] |
|
|
32 |
return nn.Sequential(*layers) |
|
|
33 |
|
|
|
34 |
|
|
|
35 |
def conv_blocks_3(in_channels, out_channels, strides=(1,1,1)): |
|
|
36 |
conv1 = conv(in_channels, out_channels, stride = strides) |
|
|
37 |
conv2 = conv(out_channels, out_channels, stride=(1,1,1)) |
|
|
38 |
conv3 = conv(out_channels, out_channels, stride=(1,1,1)) |
|
|
39 |
layers = [conv1, conv2, conv3] |
|
|
40 |
return nn.Sequential(*layers) |
|
|
41 |
|
|
|
42 |
def fullyconnect(in_features, out_features, out_channels, nonlinearity = relu): |
|
|
43 |
fc_layer = nn.Linear(in_features = in_features, out_features= out_features, bias = False) |
|
|
44 |
|
|
|
45 |
nll_layer = nonlinearity() |
|
|
46 |
bn_layer = nn.BatchNorm1d(out_channels) |
|
|
47 |
|
|
|
48 |
layers = [fc_layer, bn_layer, nll_layer] |
|
|
49 |
return nn.Sequential(*layers) |
|
|
50 |
|
|
|
51 |
def conv_2D(in_channels, out_channels, kernel_size=3, stride=1, padding = 1, nonlinearity = relu): |
|
|
52 |
conv_layer = nn.Conv2d(in_channels = in_channels, out_channels= out_channels, kernel_size = kernel_size, stride = stride, padding = padding, bias = False) |
|
|
53 |
|
|
|
54 |
nll_layer = nonlinearity() |
|
|
55 |
bn_layer = nn.BatchNorm2d(out_channels) |
|
|
56 |
|
|
|
57 |
layers = [conv_layer, bn_layer, nll_layer] |
|
|
58 |
return nn.Sequential(*layers) |
|
|
59 |
|
|
|
60 |
def conv_1D(in_channels, out_channels, kernel_size=3, stride=1, padding = 1, nonlinearity = relu): |
|
|
61 |
conv_layer = nn.Conv1d(in_channels = in_channels, out_channels= out_channels, kernel_size = kernel_size, stride = stride, padding = padding, bias = False) |
|
|
62 |
|
|
|
63 |
nll_layer = nonlinearity() |
|
|
64 |
bn_layer = nn.BatchNorm2d(out_channels) |
|
|
65 |
|
|
|
66 |
layers = [conv_layer, bn_layer, nll_layer] |
|
|
67 |
return nn.Sequential(*layers) |
|
|
68 |
|
|
|
69 |
def deconv_2D(in_channels, out_channels, kernel_size=3, stride=1, padding = 1, nonlinearity = relu): |
|
|
70 |
conv_layer = nn.ConvTranspose2d(in_channels = in_channels, out_channels= out_channels, kernel_size = kernel_size, stride = stride, padding = padding, output_padding = 1, bias = False) |
|
|
71 |
|
|
|
72 |
nll_layer = nonlinearity() |
|
|
73 |
bn_layer = nn.BatchNorm2d(out_channels) |
|
|
74 |
|
|
|
75 |
layers = [conv_layer, bn_layer, nll_layer] |
|
|
76 |
return nn.Sequential(*layers) |
|
|
77 |
|
|
|
78 |
|
|
|
79 |
def conv_blocks_2_2D(in_channels, out_channels, strides=1): |
|
|
80 |
conv1 = conv_2D(in_channels, out_channels, stride = strides) |
|
|
81 |
conv2 = conv_2D(out_channels, out_channels, stride=1) |
|
|
82 |
layers = [conv1, conv2] |
|
|
83 |
return nn.Sequential(*layers) |
|
|
84 |
|
|
|
85 |
|
|
|
86 |
def conv_blocks_3_2D(in_channels, out_channels, strides=1): |
|
|
87 |
conv1 = conv_2D(in_channels, out_channels, stride = strides) |
|
|
88 |
conv2 = conv_2D(out_channels, out_channels, stride=1) |
|
|
89 |
conv3 = conv_2D(out_channels, out_channels, stride=1) |
|
|
90 |
layers = [conv1, conv2, conv3] |
|
|
91 |
return nn.Sequential(*layers) |
|
|
92 |
|
|
|
93 |
# Flatten layer |
|
|
94 |
class Flatten(nn.Module): |
|
|
95 |
def forward(self, input): |
|
|
96 |
return input.view(input.size(0), -1) |
|
|
97 |
|
|
|
98 |
def generate_grid(x, offset): |
|
|
99 |
x_shape = x.size() |
|
|
100 |
grid_d, grid_w, grid_h = torch.meshgrid([torch.linspace(-1, 1, x_shape[2]), torch.linspace(-1, 1, x_shape[3]), torch.linspace(-1, 1, x_shape[4])]) # (h, w, h) |
|
|
101 |
grid_d = grid_d.cuda().float() |
|
|
102 |
grid_w = grid_w.cuda().float() |
|
|
103 |
grid_h = grid_h.cuda().float() |
|
|
104 |
|
|
|
105 |
grid_d = nn.Parameter(grid_d, requires_grad=False) |
|
|
106 |
grid_w = nn.Parameter(grid_w, requires_grad=False) |
|
|
107 |
grid_h = nn.Parameter(grid_h, requires_grad=False) |
|
|
108 |
|
|
|
109 |
offset_h, offset_w, offset_d = torch.split(offset, 1, 1) |
|
|
110 |
offset_d = offset_d.contiguous().view(-1, int(x_shape[2]), int(x_shape[3]), int(x_shape[4])) # (b*c, d, w, h) |
|
|
111 |
offset_w = offset_w.contiguous().view(-1, int(x_shape[2]), int(x_shape[3]), int(x_shape[4])) # (b*c, d, w, h) |
|
|
112 |
offset_h = offset_h.contiguous().view(-1, int(x_shape[2]), int(x_shape[3]), int(x_shape[4])) # (b*c, d, w, h) |
|
|
113 |
|
|
|
114 |
offset_d = grid_d + offset_d |
|
|
115 |
offset_w = grid_w + offset_w |
|
|
116 |
offset_h = grid_h + offset_h |
|
|
117 |
|
|
|
118 |
offsets = torch.stack((offset_h, offset_w, offset_d), 4) # should have the same order as offset |
|
|
119 |
return offsets |
|
|
120 |
|
|
|
121 |
def transform(seg_source, loc, mode='bilinear'): |
|
|
122 |
grid = generate_grid(seg_source, loc) |
|
|
123 |
# seg_source: NCDHW |
|
|
124 |
# grid: NDHW3 |
|
|
125 |
out = F.grid_sample(seg_source, grid, mode=mode, align_corners=True) |
|
|
126 |
return out |
|
|
127 |
|
|
|
128 |
|
|
|
129 |
|
|
|
130 |
|
|
|
131 |
|
|
|
132 |
class MotionMesh_25d(nn.Module): |
|
|
133 |
"""Deformable registration network with input from image space """ |
|
|
134 |
def __init__(self, n_ch=64, mesh_dim=10000): |
|
|
135 |
super(MotionMesh_25d, self).__init__() |
|
|
136 |
|
|
|
137 |
self.conv_blocks_2D = [conv_blocks_2_2D(n_ch, 64), conv_blocks_2_2D(64, 128, 2), conv_blocks_3_2D(128, 256, 2), |
|
|
138 |
conv_blocks_3_2D(256, 512, 2), conv_blocks_3_2D(512, 512, 2)] |
|
|
139 |
self.conv_list_2D = [] |
|
|
140 |
for in_filters in [128, 256, 512, 1024, 1024]: |
|
|
141 |
self.conv_list_2D += [conv_2D(in_filters, 64)] |
|
|
142 |
|
|
|
143 |
self.conv_blocks_2D = nn.Sequential(*self.conv_blocks_2D) |
|
|
144 |
self.conv_list_2D = nn.Sequential(*self.conv_list_2D) |
|
|
145 |
|
|
|
146 |
self.conv6 = conv_2D(64 * 15, 64, 1, 1, 0) |
|
|
147 |
self.conv7 = conv_2D(64, 64, 1, 1, 0) |
|
|
148 |
self.conv3d9 = conv(1, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1)) |
|
|
149 |
self.conv3d10 = conv(32, 64, kernel_size=(3, 3, 3), stride=(2, 2, 2)) |
|
|
150 |
self.conv3d10_1 = conv(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1)) |
|
|
151 |
self.conv3d11 = conv(64, 128, kernel_size=(3, 3, 3), stride=(2, 2, 2)) |
|
|
152 |
self.conv3d11_1 = conv(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1)) |
|
|
153 |
self.conv3d12 = conv(128, 256, kernel_size=(3, 3, 3), stride=(2, 2, 2)) |
|
|
154 |
self.conv3d12_1 = conv(256, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1)) |
|
|
155 |
self.conv3d13 = deconv(256, 128, kernel_size=(3, 3, 3), stride=(2, 2, 2)) |
|
|
156 |
self.conv3d13_1 = conv(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1)) |
|
|
157 |
self.conv3d14 = deconv(128, 64, kernel_size=(3, 3, 3), stride=(2, 2, 2)) |
|
|
158 |
self.conv3d14_1 = conv(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1)) |
|
|
159 |
self.conv3d15 = deconv(64, 32, kernel_size=(3, 3, 3), stride=(2, 2, 2)) |
|
|
160 |
self.conv3d15_1 = conv(32, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1)) |
|
|
161 |
self.conv3d16 = nn.Conv3d(32, 3, 1, stride=(1, 1, 1)) |
|
|
162 |
|
|
|
163 |
|
|
|
164 |
|
|
|
165 |
def forward(self, x_sa, x_saed, x_2ch, x_2ched, x_4ch, x_4ched): |
|
|
166 |
# x: source image; x_pred: target image; |
|
|
167 |
net = {} |
|
|
168 |
net['conv0_sa'] = x_sa |
|
|
169 |
net['conv0_sa_ed'] = x_saed |
|
|
170 |
net['conv0_2ch'] = x_2ch |
|
|
171 |
net['conv0_2ch_ed'] = x_2ched |
|
|
172 |
net['conv0_4ch'] = x_4ch |
|
|
173 |
net['conv0_4ch_ed'] = x_4ched |
|
|
174 |
# 5 refers to 5 output or 5 blocks |
|
|
175 |
for i in range(5): |
|
|
176 |
net['conv%d_sa' % (i + 1)] = self.conv_blocks_2D[i](net['conv%d_sa' % i]) |
|
|
177 |
net['conv%d_sa_ed' % (i + 1)] = self.conv_blocks_2D[i](net['conv%d_sa_ed' % i]) |
|
|
178 |
net['conv%d_2ch' % (i + 1)] = self.conv_blocks_2D[i](net['conv%d_2ch' % i]) |
|
|
179 |
net['conv%d_2ch_ed' % (i + 1)] = self.conv_blocks_2D[i](net['conv%d_2ch_ed' % i]) |
|
|
180 |
net['conv%d_4ch' % (i + 1)] = self.conv_blocks_2D[i](net['conv%d_4ch' % i]) |
|
|
181 |
net['conv%d_4ch_ed' % (i + 1)] = self.conv_blocks_2D[i](net['conv%d_4ch_ed' % i]) |
|
|
182 |
|
|
|
183 |
net['concat%d_sa' % (i + 1)] = torch.cat((net['conv%d_sa' % (i + 1)], net['conv%d_sa_ed' % (i + 1)]), 1) |
|
|
184 |
net['concat%d_2ch' % (i + 1)] = torch.cat((net['conv%d_2ch' % (i + 1)], net['conv%d_2ch_ed' % (i + 1)]), 1) |
|
|
185 |
net['concat%d_4ch' % (i + 1)] = torch.cat((net['conv%d_4ch' % (i + 1)], net['conv%d_4ch_ed' % (i + 1)]), 1) |
|
|
186 |
|
|
|
187 |
net['out%d_sa' % (i + 1)] = self.conv_list_2D[i](net['concat%d_sa' % (i + 1)]) |
|
|
188 |
net['out%d_2ch' % (i + 1)] = self.conv_list_2D[i](net['concat%d_2ch' % (i + 1)]) |
|
|
189 |
net['out%d_4ch' % (i + 1)] = self.conv_list_2D[i](net['concat%d_4ch' % (i + 1)]) |
|
|
190 |
if i > 0: |
|
|
191 |
# upsample |
|
|
192 |
# only upsample HW dimension |
|
|
193 |
# net['out%d_sa_up' % (i + 1)] = F.interpolate(net['out%d_sa' % (i + 1)], |
|
|
194 |
# scale_factor=(1, 2 ** i, 2 ** i), mode='trilinear', |
|
|
195 |
# align_corners=True) |
|
|
196 |
# upsample DHW dimension |
|
|
197 |
net['out%d_sa_up' % (i + 1)] = F.interpolate(net['out%d_sa' % (i + 1)], scale_factor=2 ** i, mode='bilinear', align_corners=True) |
|
|
198 |
net['out%d_2ch_up' % (i + 1)] = F.interpolate(net['out%d_2ch' % (i + 1)], scale_factor=2 ** i, mode='bilinear', align_corners=True) |
|
|
199 |
net['out%d_4ch_up' % (i + 1)] = F.interpolate(net['out%d_4ch' % (i + 1)], scale_factor=2 ** i, mode='bilinear', align_corners=True) |
|
|
200 |
|
|
|
201 |
|
|
|
202 |
|
|
|
203 |
# output: net['out1_sa'], net['out2_sa_up'], net['out3_sa_up'], net['out4_sa_up'], net['out5_sa_up'] are used for multiscale fusion |
|
|
204 |
net['concat_sa'] = torch.cat( |
|
|
205 |
(net['out1_sa'], net['out2_sa_up'], net['out3_sa_up'], net['out4_sa_up'], net['out5_sa_up']), 1) |
|
|
206 |
net['concat_2ch'] = torch.cat( |
|
|
207 |
(net['out1_2ch'], net['out2_2ch_up'], net['out3_2ch_up'], net['out4_2ch_up'], net['out5_2ch_up']), 1) |
|
|
208 |
net['concat_4ch'] = torch.cat( |
|
|
209 |
(net['out1_4ch'], net['out2_4ch_up'], net['out3_4ch_up'], net['out4_4ch_up'], net['out5_4ch_up']), 1) |
|
|
210 |
|
|
|
211 |
net['concat'] = torch.cat((net['concat_sa'], net['concat_2ch'], net['concat_4ch']), 1) |
|
|
212 |
net['comb_1'] = self.conv6(net['concat']) |
|
|
213 |
net['comb_2'] = self.conv7(net['comb_1']) |
|
|
214 |
|
|
|
215 |
net['conv3d0f'] = net['comb_2'].unsqueeze(1) |
|
|
216 |
|
|
|
217 |
net['conv3d_1'] = self.conv3d9(net['conv3d0f']) |
|
|
218 |
net['conv3d_2'] = self.conv3d10_1(self.conv3d10(net['conv3d_1'])) |
|
|
219 |
net['conv3d_3'] = self.conv3d11_1(self.conv3d11(net['conv3d_2'])) |
|
|
220 |
net['conv3d_4'] = self.conv3d12_1(self.conv3d12(net['conv3d_3'])) |
|
|
221 |
net['conv3d_5'] = self.conv3d13_1(self.conv3d13(net['conv3d_4'])) |
|
|
222 |
net['conv3d_6'] = self.conv3d14_1(self.conv3d14(net['conv3d_5'])) |
|
|
223 |
net['conv3d_7'] = self.conv3d15_1(self.conv3d15(net['conv3d_6'])) |
|
|
224 |
net['out'] = torch.tanh(self.conv3d16(net['conv3d_7'])) |
|
|
225 |
|
|
|
226 |
|
|
|
227 |
return net |
|
|
228 |
|
|
|
229 |
|
|
|
230 |
class Mesh_2d(nn.Module): |
|
|
231 |
"""Deformable registration network with input from image space """ |
|
|
232 |
def __init__(self, n_ch=1): |
|
|
233 |
super(Mesh_2d, self).__init__() |
|
|
234 |
|
|
|
235 |
self.conv1 = conv_2D(n_ch, 32) |
|
|
236 |
self.conv2 = conv_2D(32, 64) |
|
|
237 |
|
|
|
238 |
def forward(self, x_2ch, x_2ched, x_4ch, x_4ched): |
|
|
239 |
# x: source image; x_pred: target image; |
|
|
240 |
net = {} |
|
|
241 |
|
|
|
242 |
net['conv1_2ch'] = self.conv1(x_2ch) |
|
|
243 |
net['conv1_4ch'] = self.conv1(x_4ch) |
|
|
244 |
net['conv1s_2ch'] = self.conv1(x_2ched) |
|
|
245 |
net['conv1s_4ch'] = self.conv1(x_4ched) |
|
|
246 |
|
|
|
247 |
net['conv2_2ch'] = self.conv2(net['conv1_2ch']) |
|
|
248 |
net['conv2_4ch'] = self.conv2(net['conv1_4ch']) |
|
|
249 |
net['conv2s_2ch'] = self.conv2(net['conv1s_2ch']) |
|
|
250 |
net['conv2s_4ch'] = self.conv2(net['conv1s_4ch']) |
|
|
251 |
|
|
|
252 |
|
|
|
253 |
return net |