Open3D (C++ API)  0.16.0
ContinuousConvBackpropFilter.h
Go to the documentation of this file.
1// ----------------------------------------------------------------------------
2// - Open3D: www.open3d.org -
3// ----------------------------------------------------------------------------
4// The MIT License (MIT)
5//
6// Copyright (c) 2018-2021 www.open3d.org
7//
8// Permission is hereby granted, free of charge, to any person obtaining a copy
9// of this software and associated documentation files (the "Software"), to deal
10// in the Software without restriction, including without limitation the rights
11// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12// copies of the Software, and to permit persons to whom the Software is
13// furnished to do so, subject to the following conditions:
14//
15// The above copyright notice and this permission notice shall be included in
16// all copies or substantial portions of the Software.
17//
18// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24// IN THE SOFTWARE.
25// ----------------------------------------------------------------------------
26
27#pragma once
28
29#include <tbb/parallel_for.h>
30
31#include <mutex>
32
34
35namespace open3d {
36namespace ml {
37namespace impl {
38
39// Implementation of CConvBackropFilterCPU
40template <class TFeat,
41 class TOut,
42 class TReal,
43 class TIndex,
44 InterpolationMode INTERPOLATION,
45 CoordinateMapping MAPPING,
46 bool ALIGN_CORNERS,
47 bool INDIVIDUAL_EXTENT,
48 bool ISOTROPIC_EXTENT,
49 bool POINT_IMPORTANCE>
50void _CConvBackropFilterCPU(TOut* filter_backprop,
51 const std::vector<int>& filter_dims,
52 size_t num_out,
53 const TReal* out_positions,
54 size_t num_inp,
55 const TReal* inp_positions,
56 const TFeat* inp_features,
57 const TFeat* inp_importance,
58 size_t neighbors_index_size,
59 const TIndex* neighbors_index,
60 const TFeat* neighbors_importance,
61 const int64_t* neighbors_row_splits,
62 const TReal* extents,
63 const TReal* offsets,
64 const TFeat* out_features_gradient,
65 bool normalize) {
66 const bool NEIGHBORS_IMPORTANCE = neighbors_importance;
67 const int VECSIZE = 32;
68 typedef Eigen::Array<TReal, VECSIZE, 1> Vec_t;
69 typedef InterpolationVec<TReal, VECSIZE, INTERPOLATION> InterpolationVec_t;
70 InterpolationVec_t interpolation;
71
72 const int in_channels = filter_dims[filter_dims.size() - 2];
73 const int out_channels = filter_dims[filter_dims.size() - 1];
74
75 int spatial_filter_size = 1;
76 for (int i = 0; i < 3; ++i) spatial_filter_size *= filter_dims[i];
77 const int total_filter_size =
78 spatial_filter_size * in_channels * out_channels;
79 Eigen::Array<int, 3, 1> filter_size_xyz(filter_dims[2], filter_dims[1],
80 filter_dims[0]);
81
82 memset(filter_backprop, 0, sizeof(TOut) * total_filter_size);
83 std::mutex filter_backprop_mutex;
84
85 tbb::parallel_for(
86 tbb::blocked_range<size_t>(0, num_out, 32),
87 [&](const tbb::blocked_range<size_t>& r) {
88 int range_length = r.end() - r.begin();
89
90 Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> B(
91 in_channels * spatial_filter_size, range_length);
92 B.setZero();
93 Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> C(
94 out_channels, range_length);
95
96 typedef Eigen::Array<TFeat, VECSIZE, Eigen::Dynamic> Matrix;
97 Matrix infeat(VECSIZE, in_channels);
98
99 Eigen::Array<TReal, 3, 1> offsets_(offsets[0], offsets[1],
100 offsets[2]);
101
102 Eigen::Array<TReal, VECSIZE, 3> inv_extents;
103 if (INDIVIDUAL_EXTENT == false) {
104 if (ISOTROPIC_EXTENT) {
105 inv_extents = 1 / extents[0];
106 } else {
107 inv_extents.col(0) = 1 / extents[0];
108 inv_extents.col(1) = 1 / extents[1];
109 inv_extents.col(2) = 1 / extents[2];
110 }
111 }
112
113 for (size_t out_idx = r.begin(); out_idx != r.end();
114 ++out_idx) {
115 const int out_col = out_idx - r.begin();
116 const size_t neighbor_start = neighbors_row_splits[out_idx];
117 const size_t neighbor_end =
118 neighbors_row_splits[out_idx + 1];
119 TOut normalizer(0);
120
121 if (INDIVIDUAL_EXTENT) {
122 if (ISOTROPIC_EXTENT) {
123 inv_extents = 1 / extents[out_idx];
124 } else {
125 inv_extents.col(0) = 1 / extents[3 * out_idx + 0];
126 inv_extents.col(1) = 1 / extents[3 * out_idx + 1];
127 inv_extents.col(2) = 1 / extents[3 * out_idx + 2];
128 }
129 }
130
131 typename InterpolationVec_t::Weight_t interp_weights;
132 typename InterpolationVec_t::Idx_t interp_indices;
133
134 int vec_valid_count = 0;
135 Vec_t x, y, z;
136
137 // set to zero to avoid problems with vectors with less than
138 // VECSIZE valid entries
139 x.setZero();
140 y.setZero();
141 z.setZero();
142 for (size_t n = neighbor_start; n < neighbor_end; ++n) {
143 const size_t inp_idx = neighbors_index[n];
144 const int i = vec_valid_count;
145 x(i) = inp_positions[inp_idx * 3 + 0] -
146 out_positions[out_idx * 3 + 0];
147 y(i) = inp_positions[inp_idx * 3 + 1] -
148 out_positions[out_idx * 3 + 1];
149 z(i) = inp_positions[inp_idx * 3 + 2] -
150 out_positions[out_idx * 3 + 2];
151
152 const TFeat n_importance =
153 (NEIGHBORS_IMPORTANCE ? neighbors_importance[n]
154 : TFeat(1));
155 normalizer += TOut(n_importance);
156
157 for (int ic = 0; ic < in_channels; ++ic)
158 infeat(i, ic) =
159 inp_features[inp_idx * in_channels + ic];
160
161 TFeat importance = TFeat(1);
162 if (POINT_IMPORTANCE)
163 importance = inp_importance[inp_idx];
164 if (NEIGHBORS_IMPORTANCE) importance *= n_importance;
165
166 if (POINT_IMPORTANCE || NEIGHBORS_IMPORTANCE) {
167 for (int ic = 0; ic < in_channels; ++ic)
168 infeat(i, ic) *= importance;
169 }
170
171 ++vec_valid_count;
172 if (vec_valid_count == VECSIZE) {
173 ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
174 x, y, z, filter_size_xyz, inv_extents,
175 offsets_);
176 interpolation.Interpolate(
177 interp_weights, interp_indices, x, y, z,
178 filter_size_xyz, in_channels);
179 for (int k = 0; k < VECSIZE; ++k)
180 for (int j = 0; j < InterpolationVec_t::Size();
181 ++j) {
182 for (int ic = 0; ic < in_channels; ++ic)
183 B(interp_indices(j, k) + ic, out_col) +=
184 TFeat(interp_weights(j, k)) *
185 infeat(k, ic);
186 }
187 vec_valid_count = 0;
188 }
189 }
190 if (vec_valid_count) {
191 ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
192 x, y, z, filter_size_xyz, inv_extents,
193 offsets_);
194 interpolation.Interpolate(interp_weights,
195 interp_indices, x, y, z,
196 filter_size_xyz, in_channels);
197 for (int k = 0; k < vec_valid_count; ++k)
198 for (int j = 0; j < InterpolationVec_t::Size();
199 ++j) {
200 for (int ic = 0; ic < in_channels; ++ic)
201 B(interp_indices(j, k) + ic, out_col) +=
202 TFeat(interp_weights(j, k)) *
203 infeat(k, ic);
204 }
205 }
206
207 C.col(out_col) = Eigen::Map<
208 const Eigen::Array<TFeat, Eigen::Dynamic, 1>>(
209 out_features_gradient + out_idx * out_channels,
210 out_channels, 1);
211
212 if (normalize && normalizer != TOut(0))
213 C.col(out_col) /= TFeat(normalizer);
214
215 } // out_idx
216
217 Eigen::Matrix<TOut, Eigen::Dynamic, Eigen::Dynamic> A(
218 out_channels, spatial_filter_size * in_channels);
219
220 A = (C * B.transpose()).template cast<TOut>();
221
222 {
223 std::lock_guard<std::mutex> lock(filter_backprop_mutex);
224 int linear_i = 0;
225 for (int j = 0; j < spatial_filter_size * in_channels; ++j)
226 for (int i = 0; i < out_channels; ++i, ++linear_i) {
227 filter_backprop[linear_i] += A(i, j);
228 }
229 }
230 });
231}
232
305template <class TFeat, class TOut, class TReal, class TIndex>
306void CConvBackpropFilterCPU(TOut* filter_backprop,
307 const std::vector<int>& filter_dims,
308 size_t num_out,
309 const TReal* out_positions,
310 size_t num_inp,
311 const TReal* inp_positions,
312 const TFeat* inp_features,
313 const TFeat* inp_importance,
314 size_t neighbors_index_size,
315 const TIndex* neighbors_index,
316 const TFeat* neighbors_importance,
317 const int64_t* neighbors_row_splits,
318 const TReal* extents,
319 const TReal* offsets,
320 const TFeat* out_features_gradient,
321 InterpolationMode interpolation,
322 CoordinateMapping coordinate_mapping,
323 bool align_corners,
324 bool individual_extent,
325 bool isotropic_extent,
326 bool normalize) {
327 bool has_importance = inp_importance;
328
329#define FN_PARAMETERS \
330 filter_backprop, filter_dims, num_out, out_positions, num_inp, \
331 inp_positions, inp_features, inp_importance, neighbors_index_size, \
332 neighbors_index, neighbors_importance, neighbors_row_splits, \
333 extents, offsets, out_features_gradient, normalize
334
335#define CALL_TEMPLATE(INTERPOLATION, MAPPING, ALIGN_CORNERS, \
336 INDIVIDUAL_EXTENT, ISOTROPIC_EXTENT, HAS_IMPORTANCE) \
337 if (INTERPOLATION == interpolation && MAPPING == coordinate_mapping && \
338 ALIGN_CORNERS == align_corners && \
339 INDIVIDUAL_EXTENT == individual_extent && \
340 ISOTROPIC_EXTENT == isotropic_extent && \
341 HAS_IMPORTANCE == has_importance) \
342 _CConvBackropFilterCPU<TFeat, TOut, TReal, TIndex, INTERPOLATION, \
343 MAPPING, ALIGN_CORNERS, INDIVIDUAL_EXTENT, \
344 ISOTROPIC_EXTENT, HAS_IMPORTANCE>( \
345 FN_PARAMETERS);
346
347#define CALL_TEMPLATE2(INTERPOLATION, MAPPING) \
348 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, true) \
349 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, false) \
350 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, true) \
351 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, false) \
352 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, true) \
353 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, false) \
354 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, true) \
355 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, false) \
356 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, true) \
357 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, false) \
358 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, true) \
359 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, false) \
360 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, true) \
361 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, false) \
362 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, true) \
363 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, false)
364
365#define CALL_TEMPLATE3(INTERPOLATION) \
366 CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::BALL_TO_CUBE_RADIAL) \
367 CALL_TEMPLATE2(INTERPOLATION, \
368 CoordinateMapping::BALL_TO_CUBE_VOLUME_PRESERVING) \
369 CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::IDENTITY)
370
371#define CALL_TEMPLATE4 \
372 CALL_TEMPLATE3(InterpolationMode::LINEAR) \
373 CALL_TEMPLATE3(InterpolationMode::LINEAR_BORDER) \
374 CALL_TEMPLATE3(InterpolationMode::NEAREST_NEIGHBOR)
375
377
378#undef CALL_TEMPLATE
379#undef CALL_TEMPLATE2
380#undef CALL_TEMPLATE3
381#undef CALL_TEMPLATE4
382
383#undef FN_PARAMETERS
384}
385
386} // namespace impl
387} // namespace ml
388} // namespace open3d
#define CALL_TEMPLATE4
#define VECSIZE
InterpolationMode
Definition: ContinuousConvTypes.h:37
void _CConvBackropFilterCPU(TOut *filter_backprop, const std::vector< int > &filter_dims, size_t num_out, const TReal *out_positions, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_importance, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, const TFeat *out_features_gradient, bool normalize)
Definition: ContinuousConvBackpropFilter.h:50
void CConvBackpropFilterCPU(TOut *filter_backprop, const std::vector< int > &filter_dims, size_t num_out, const TReal *out_positions, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_importance, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, const TFeat *out_features_gradient, InterpolationMode interpolation, CoordinateMapping coordinate_mapping, bool align_corners, bool individual_extent, bool isotropic_extent, bool normalize)
Definition: ContinuousConvBackpropFilter.h:306
CoordinateMapping
Definition: ContinuousConvTypes.h:45
Definition: PinholeCameraIntrinsic.cpp:35
Class for computing interpolation weights.
Definition: CoordinateTransformation.h:204