Open3D (C++ API)  0.16.1
ContinuousConvTransposeBackpropFilter.h
Go to the documentation of this file.
1// ----------------------------------------------------------------------------
2// - Open3D: www.open3d.org -
3// ----------------------------------------------------------------------------
4// The MIT License (MIT)
5//
6// Copyright (c) 2018-2021 www.open3d.org
7//
8// Permission is hereby granted, free of charge, to any person obtaining a copy
9// of this software and associated documentation files (the "Software"), to deal
10// in the Software without restriction, including without limitation the rights
11// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12// copies of the Software, and to permit persons to whom the Software is
13// furnished to do so, subject to the following conditions:
14//
15// The above copyright notice and this permission notice shall be included in
16// all copies or substantial portions of the Software.
17//
18// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24// IN THE SOFTWARE.
25// ----------------------------------------------------------------------------
26
27#pragma once
28
29#include <tbb/parallel_for.h>
30
31#include <mutex>
32
34
35namespace open3d {
36namespace ml {
37namespace impl {
38
41template <class TFeat,
42 class TOut,
43 class TReal,
44 class TIndex,
45 InterpolationMode INTERPOLATION,
46 CoordinateMapping MAPPING,
47 bool ALIGN_CORNERS,
48 bool INDIVIDUAL_EXTENT,
49 bool ISOTROPIC_EXTENT,
50 bool NORMALIZE>
51void _CConvTransposeBackpropFilterCPU(TOut* filter_backprop,
52 const std::vector<int>& filter_dims,
53 size_t num_out,
54 const TReal* out_positions,
55 const TFeat* out_importance,
56 size_t num_inp,
57 const TReal* inp_positions,
58 const TFeat* inp_features,
59 const TFeat* inp_neighbors_importance_sum,
60 const int64_t* inp_neighbors_row_splits,
61 size_t neighbors_index_size,
62 const TIndex* neighbors_index,
63 const TFeat* neighbors_importance,
64 const int64_t* neighbors_row_splits,
65 const TReal* extents,
66 const TReal* offsets,
67 const TFeat* out_features_gradient) {
68 const bool NEIGHBOR_IMPORTANCE = neighbors_importance;
69 const int VECSIZE = 32;
70 typedef Eigen::Array<TReal, VECSIZE, 1> Vec_t;
71 typedef InterpolationVec<TReal, VECSIZE, INTERPOLATION> InterpolationVec_t;
72 InterpolationVec_t interpolation;
73
74 const int in_channels = filter_dims[filter_dims.size() - 2];
75 const int out_channels = filter_dims[filter_dims.size() - 1];
76
77 int spatial_filter_size = 1;
78 for (int i = 0; i < 3; ++i) spatial_filter_size *= filter_dims[i];
79 Eigen::Array<int, 3, 1> filter_size_xyz(filter_dims[2], filter_dims[1],
80 filter_dims[0]);
81
82 memset(filter_backprop, 0,
83 sizeof(TOut) * spatial_filter_size * in_channels * out_channels);
84 std::mutex filter_backprop_mutex;
85
86 tbb::parallel_for(
87 tbb::blocked_range<size_t>(0, num_out, 32),
88 [&](const tbb::blocked_range<size_t>& r) {
89 int range_length = r.end() - r.begin();
90
91 Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> B(
92 in_channels * spatial_filter_size, range_length);
93 B.setZero();
94 Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> C(
95 out_channels, range_length);
96
97 typedef Eigen::Array<TFeat, VECSIZE, Eigen::Dynamic> Matrix;
98 Matrix infeat(VECSIZE, in_channels);
99
100 Eigen::Array<TReal, 3, 1> offsets_(offsets[0], offsets[1],
101 offsets[2]);
102
103 Eigen::Array<TReal, VECSIZE, 3> inv_extents;
104 if (INDIVIDUAL_EXTENT == false) {
105 if (ISOTROPIC_EXTENT) {
106 inv_extents = 1 / extents[0];
107 } else {
108 inv_extents.col(0) = 1 / extents[0];
109 inv_extents.col(1) = 1 / extents[1];
110 inv_extents.col(2) = 1 / extents[2];
111 }
112 }
113
114 for (size_t out_idx = r.begin(); out_idx != r.end();
115 ++out_idx) {
116 const int out_col = out_idx - r.begin();
117 const size_t neighbor_start = neighbors_row_splits[out_idx];
118 const size_t neighbor_end =
119 neighbors_row_splits[out_idx + 1];
120
121 C.col(out_col) = Eigen::Map<
122 const Eigen::Array<TFeat, Eigen::Dynamic, 1>>(
123 out_features_gradient + out_idx * out_channels,
124 out_channels, 1);
125
126 typename InterpolationVec_t::Weight_t interp_weights;
127 typename InterpolationVec_t::Idx_t interp_indices;
128
129 int vec_valid_count = 0;
130 Vec_t x, y, z;
131
132 // set to zero to avoid problems with vectors with less than
133 // VECSIZE valid entries
134 x.setZero();
135 y.setZero();
136 z.setZero();
137 for (size_t n = neighbor_start; n < neighbor_end; ++n) {
138 const size_t inp_idx = neighbors_index[n];
139
140 const int i = vec_valid_count;
141 x(i) = out_positions[out_idx * 3 + 0] -
142 inp_positions[inp_idx * 3 + 0];
143 y(i) = out_positions[out_idx * 3 + 1] -
144 inp_positions[inp_idx * 3 + 1];
145 z(i) = out_positions[out_idx * 3 + 2] -
146 inp_positions[inp_idx * 3 + 2];
147
148 if (INDIVIDUAL_EXTENT) {
149 if (ISOTROPIC_EXTENT) {
150 inv_extents.row(i) = 1 / extents[inp_idx];
151 } else {
152 inv_extents(i, 0) =
153 1 / extents[3 * inp_idx + 0];
154 inv_extents(i, 1) =
155 1 / extents[3 * inp_idx + 1];
156 inv_extents(i, 2) =
157 1 / extents[3 * inp_idx + 2];
158 }
159 }
160
161 TFeat n_importance = NEIGHBOR_IMPORTANCE
162 ? neighbors_importance[n]
163 : TFeat(1);
164 for (int ic = 0; ic < in_channels; ++ic)
165 infeat(i, ic) =
166 inp_features[inp_idx * in_channels + ic] *
167 n_importance;
168
169 if (NORMALIZE) {
170 TFeat normalizer(1);
171 if (NEIGHBOR_IMPORTANCE) {
172 if (inp_neighbors_importance_sum[inp_idx] !=
173 TFeat(0))
174 normalizer /= inp_neighbors_importance_sum
175 [inp_idx];
176 } else {
177 size_t num_inp_neighbors;
178 const size_t inp_neighbor_start =
179 inp_neighbors_row_splits[inp_idx];
180 const size_t inp_neighbor_end =
181 inp_neighbors_row_splits[inp_idx + 1];
182 num_inp_neighbors =
183 inp_neighbor_end - inp_neighbor_start;
184 if (num_inp_neighbors > 0)
185 normalizer /= TFeat(num_inp_neighbors);
186 }
187 for (int ic = 0; ic < in_channels; ++ic)
188 infeat(i, ic) *= normalizer;
189 }
190
191 ++vec_valid_count;
192 if (vec_valid_count == VECSIZE ||
193 n + 1 == neighbor_end) {
194 ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
195 x, y, z, filter_size_xyz, inv_extents,
196 offsets_);
197 interpolation.Interpolate(
198 interp_weights, interp_indices, x, y, z,
199 filter_size_xyz, in_channels);
200 for (int k = 0; k < vec_valid_count; ++k) {
201 for (int j = 0; j < InterpolationVec_t::Size();
202 ++j) {
203 for (int ic = 0; ic < in_channels; ++ic)
204 B(interp_indices(j, k) + ic, out_col) +=
205 TFeat(interp_weights(j, k)) *
206 infeat(k, ic);
207 }
208 }
209 vec_valid_count = 0;
210 }
211 }
212
213 } // out_idx
214
215 if (out_importance) {
216 for (size_t out_idx = r.begin(); out_idx != r.end();
217 ++out_idx) {
218 const int out_col = out_idx - r.begin();
219 C.col(out_col) *= out_importance[out_idx];
220 }
221 }
222
223 Eigen::Matrix<TOut, Eigen::Dynamic, Eigen::Dynamic> A(
224 out_channels, spatial_filter_size * in_channels);
225
226 A = (C * B.transpose()).template cast<TOut>();
227
228 {
229 std::lock_guard<std::mutex> lock(filter_backprop_mutex);
230 int linear_i = 0;
231 for (int j = 0; j < spatial_filter_size * in_channels; ++j)
232 for (int i = 0; i < out_channels; ++i, ++linear_i) {
233 filter_backprop[linear_i] += A(i, j);
234 }
235 }
236 });
237}
238
321template <class TFeat, class TOut, class TReal, class TIndex>
322void CConvTransposeBackpropFilterCPU(TOut* filter_backprop,
323 const std::vector<int>& filter_dims,
324 size_t num_out,
325 const TReal* out_positions,
326 const TFeat* out_importance,
327 size_t num_inp,
328 const TReal* inp_positions,
329 const TFeat* inp_features,
330 const TFeat* inp_neighbors_importance_sum,
331 const int64_t* inp_neighbors_row_splits,
332 size_t neighbors_index_size,
333 const TIndex* neighbors_index,
334 const TFeat* neighbors_importance,
335 const int64_t* neighbors_row_splits,
336 const TReal* extents,
337 const TReal* offsets,
338 const TFeat* out_features_gradient,
339 InterpolationMode interpolation,
340 CoordinateMapping coordinate_mapping,
341 bool align_corners,
342 bool individual_extent,
343 bool isotropic_extent,
344 bool normalize) {
345#define FN_PARAMETERS \
346 filter_backprop, filter_dims, num_out, out_positions, out_importance, \
347 num_inp, inp_positions, inp_features, \
348 inp_neighbors_importance_sum, inp_neighbors_row_splits, \
349 neighbors_index_size, neighbors_index, neighbors_importance, \
350 neighbors_row_splits, extents, offsets, out_features_gradient
351
352#define CALL_TEMPLATE(INTERPOLATION, MAPPING, ALIGN_CORNERS, \
353 INDIVIDUAL_EXTENT, ISOTROPIC_EXTENT, NORMALIZE) \
354 if (INTERPOLATION == interpolation && MAPPING == coordinate_mapping && \
355 ALIGN_CORNERS == align_corners && \
356 INDIVIDUAL_EXTENT == individual_extent && \
357 ISOTROPIC_EXTENT == isotropic_extent && NORMALIZE == normalize) \
358 _CConvTransposeBackpropFilterCPU<TFeat, TOut, TReal, TIndex, \
359 INTERPOLATION, MAPPING, \
360 ALIGN_CORNERS, INDIVIDUAL_EXTENT, \
361 ISOTROPIC_EXTENT, NORMALIZE>( \
362 FN_PARAMETERS);
363
364#define CALL_TEMPLATE2(INTERPOLATION, MAPPING) \
365 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, true) \
366 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, false) \
367 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, true) \
368 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, false) \
369 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, true) \
370 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, false) \
371 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, true) \
372 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, false) \
373 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, true) \
374 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, false) \
375 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, true) \
376 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, false) \
377 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, true) \
378 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, false) \
379 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, true) \
380 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, false)
381
382#define CALL_TEMPLATE3(INTERPOLATION) \
383 CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::BALL_TO_CUBE_RADIAL) \
384 CALL_TEMPLATE2(INTERPOLATION, \
385 CoordinateMapping::BALL_TO_CUBE_VOLUME_PRESERVING) \
386 CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::IDENTITY)
387
388#define CALL_TEMPLATE4 \
389 CALL_TEMPLATE3(InterpolationMode::LINEAR) \
390 CALL_TEMPLATE3(InterpolationMode::LINEAR_BORDER) \
391 CALL_TEMPLATE3(InterpolationMode::NEAREST_NEIGHBOR)
392
394
395#undef CALL_TEMPLATE
396#undef CALL_TEMPLATE2
397#undef CALL_TEMPLATE3
398#undef CALL_TEMPLATE4
399
400#undef FN_PARAMETERS
401}
402
403} // namespace impl
404} // namespace ml
405} // namespace open3d
#define CALL_TEMPLATE4
#define VECSIZE
void _CConvTransposeBackpropFilterCPU(TOut *filter_backprop, const std::vector< int > &filter_dims, size_t num_out, const TReal *out_positions, const TFeat *out_importance, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_neighbors_importance_sum, const int64_t *inp_neighbors_row_splits, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, const TFeat *out_features_gradient)
Definition: ContinuousConvTransposeBackpropFilter.h:51
InterpolationMode
Definition: ContinuousConvTypes.h:37
void CConvTransposeBackpropFilterCPU(TOut *filter_backprop, const std::vector< int > &filter_dims, size_t num_out, const TReal *out_positions, const TFeat *out_importance, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_neighbors_importance_sum, const int64_t *inp_neighbors_row_splits, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, const TFeat *out_features_gradient, InterpolationMode interpolation, CoordinateMapping coordinate_mapping, bool align_corners, bool individual_extent, bool isotropic_extent, bool normalize)
Definition: ContinuousConvTransposeBackpropFilter.h:322
CoordinateMapping
Definition: ContinuousConvTypes.h:45
Definition: PinholeCameraIntrinsic.cpp:35
Class for computing interpolation weights.
Definition: CoordinateTransformation.h:204