Python keras.backend 模块，conv2d() 实例源码

def call(self, inputs):
        binary_kernel = binarize(self.kernel, H=self.H) 
        outputs = K.conv2d(
            inputs,
            binary_kernel,
            strides=self.strides,
            padding=self.padding,
            data_format=self.data_format,
            dilation_rate=self.dilation_rate)

        if self.use_bias:
            outputs = K.bias_add(
                outputs,
                self.bias,
                data_format=self.data_format)

        if self.activation is not None:
            return self.activation(outputs)
        return outputs

def call(self, x):
        # input shape: (nb_samples, time (padded with zeros), input_dim)
        # note that the .build() method of subclasses MUST define
        # self.input_spec with a complete input shape.
        input_shape = self.input_spec[0].shape

        if self.window_size > 1:
            x = K.temporal_padding(x, (self.window_size-1, 0))
        x = K.expand_dims(x, 2)  # add a dummy dimension

        # z, g
        output = K.conv2d(x, self.kernel, strides=self.strides,
                          padding='valid',
                          data_format='channels_last')
        output = K.squeeze(output, 2)  # remove the dummy dimension
        if self.use_bias:
            output = K.bias_add(output, self.bias, data_format='channels_last')
        z  = output[:, :, :self.output_dim]
        g = output[:, :, self.output_dim:]

        return self.activation(z) * K.sigmoid(g)

def preprocess_input(self, inputs, training=None):
        if self.window_size > 1:
            inputs = K.temporal_padding(inputs, (self.window_size-1, 0))
        inputs = K.expand_dims(inputs, 2)  # add a dummy dimension

        output = K.conv2d(inputs, self.kernel, strides=self.strides,
                          padding='valid',
                          data_format='channels_last')
        output = K.squeeze(output, 2)  # remove the dummy dimension
        if self.use_bias:
            output = K.bias_add(output, self.bias, data_format='channels_last')

        if self.dropout is not None and 0. < self.dropout < 1.:
            z = output[:, :, :self.units]
            f = output[:, :, self.units:2 * self.units]
            o = output[:, :, 2 * self.units:]
            f = K.in_train_phase(1 - _dropout(1 - f, self.dropout), f, training=training)
            return K.concatenate([z, f, o], -1)
        else:
            return output

def call(self, inputs):
        ternary_kernel = ternarize(self.kernel, H=self.H) 
        outputs = K.conv2d(
            inputs,
            ternary_kernel,
            strides=self.strides,
            padding=self.padding,
            data_format=self.data_format,
            dilation_rate=self.dilation_rate)

        if self.use_bias:
            outputs = K.bias_add(
                outputs,
                self.bias,
                data_format=self.data_format)

        if self.activation is not None:
            return self.activation(outputs)
        return outputs

def call(self, inputs, **kwargs):
        outputs = K.conv2d(
            inputs,
            self.kernel,
            strides=self.strides,
            padding=self.padding,
            data_format=self.data_format,
            dilation_rate=self.dilation_rate)
        if self.use_bias:
            outputs = K.bias_add(
                outputs,
                self.bias,
                data_format=self.data_format)
        outputs = BatchNormalization(momentum=self.momentum)(outputs)
        if self.activation is not None:
            return self.activation(outputs)
        return outputs

def call(self, x, mask=None):
        conv_out = K.conv2d(x, self.W, strides=self.strides,
                            padding=self.padding,
                            data_format=self.data_format,
                            filter_shape=self.kernel_shape)

        if self.data_format == 'channels_first':
            # Complex-cell filter operation
            conv_out1 = K.sqrt(K.square(conv_out[:, :self.filters_complex, :, :]) + K.square(conv_out[:, self.filters_complex:2*self.filters_complex, :, :]) + K.epsilon())
            # Simple-cell filter operation
            conv_out2 = K.concatenate([conv_out1, conv_out[:, 2*self.filters_complex:, :, :]], axis=1)
        elif self.data_format == 'channels_last':
            # Complex-cell filter operation
            conv_out1 = K.sqrt(K.square(conv_out[:, :, :, :self.filters_complex]) + K.square(conv_out[:, :, :, self.filters_complex:2*self.filters_complex]) + K.epsilon())
            # Simple-cell filter operation
            conv_out2 = K.concatenate([conv_out1, conv_out[:, :, :, 2*self.filters_complex:]], axis=3)

        if self.bias:
            if self.data_format == 'channels_first':
                conv_out2 += K.reshape(self.b, (1, self.filters_complex + self.filters_simple, 1, 1))
            elif self.data_format == 'channels_last':
                conv_out2 += K.reshape(self.b, (1, 1, 1, self.filters_complex + self.filters_simple))

        return self.activation(conv_out2)

def make_soft(y_true, fragment_length, nb_output_bins, train_with_soft_target_stdev, with_prints=False):
    receptive_field, _ = compute_receptive_field()
    n_outputs = fragment_length - receptive_field + 1

    # Make a gaussian kernel.
    kernel_v = scipy.signal.gaussian(9, std=train_with_soft_target_stdev)
    print(kernel_v)
    kernel_v = np.reshape(kernel_v, [1, 1, -1, 1])
    kernel = K.variable(kernel_v)

    if with_prints:
        y_true = print_t(y_true, 'y_true initial')

    # y_true: [batch, timesteps, input_dim]
    y_true = K.reshape(y_true, (-1, 1, nb_output_bins, 1))  # Same filter for all output; combine with batch.
    # y_true: [batch*timesteps, n_channels=1, input_dim, dummy]
    y_true = K.conv2d(y_true, kernel, border_mode='same')
    y_true = K.reshape(y_true, (-1, n_outputs, nb_output_bins))  # Same filter for all output; combine with batch.
    # y_true: [batch, timesteps, input_dim]
    y_true /= K.sum(y_true, axis=-1, keepdims=True)

    if with_prints:
        y_true = print_t(y_true, 'y_true after')
    return y_true

def call(self, x, mask=None):
        # compute the candidate hidden state
        transform = K.conv2d(x, self.W, strides=self.subsample,
                          border_mode=self.border_mode,
                          dim_ordering=self.dim_ordering,
                          filter_shape=self.W_shape)
        if self.bias:
            transform += K.reshape(self.b, (1, 1, 1, self.nb_filter))
        transform = self.activation(transform)

        transform_gate = K.conv2d(x, self.W_gate, strides=self.subsample,
                          border_mode=self.border_mode,
                          dim_ordering=self.dim_ordering,
                          filter_shape=self.W_shape)
        if self.bias:
            transform_gate += K.reshape(self.b_gate, (1, 1, 1, self.nb_filter))
        transform_gate = K.sigmoid(transform_gate)
        carry_gate = 1.0 - transform_gate

        return transform * transform_gate + x * carry_gate

    # Define get_config method so load_from_json can run

def conv_step(self, x, W, b=None, border_mode="valid"):
        input_shape = self.input_spec[0].shape

        conv_out = K.conv2d(x, W, strides=self.subsample,
                            border_mode=border_mode,
                            dim_ordering=self.dim_ordering,
                            image_shape=(input_shape[0],
                                         input_shape[2],
                                         input_shape[3],
                                         input_shape[4]),
                            filter_shape=self.W_shape)
        if b:
            if self.dim_ordering == 'th':
                conv_out = conv_out + K.reshape(b, (1, self.nb_filter, 1, 1))
            elif self.dim_ordering == 'tf':
                conv_out = conv_out + K.reshape(b, (1, 1, 1, self.nb_filter))
            else:
                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)

        return conv_out

def conv_step_hidden(self, x, W, border_mode="valid"):
        # This new function was defined because the
        # image shape must be hardcoded
        input_shape = self.input_spec[0].shape
        output_shape = self.get_output_shape_for(input_shape)

        if self.return_sequences:
            out_row, out_col, out_filter = output_shape[2:]
        else:
            out_row, out_col, out_filter = output_shape[1:]

        conv_out = K.conv2d(x, W, strides=(1, 1),
                            border_mode=border_mode,
                            dim_ordering=self.dim_ordering,
                            image_shape=(input_shape[0],
                                         out_row, out_col,
                                         out_filter),
                            filter_shape=self.W_shape1)

        return conv_out

def ori_loss(y_true, y_pred, lamb=1.):
    # clip
    y_pred = K.tf.clip_by_value(y_pred, K.epsilon(), 1 - K.epsilon())
    # get ROI
    label_seg = K.sum(y_true, axis=-1, keepdims=True)
    label_seg = K.tf.cast(K.tf.greater(label_seg, 0), K.tf.float32) 
    # weighted cross entropy loss
    lamb_pos, lamb_neg = 1., 1. 
    logloss = lamb_pos*y_true*K.log(y_pred)+lamb_neg*(1-y_true)*K.log(1-y_pred)
    logloss = logloss*label_seg # apply ROI
    logloss = -K.sum(logloss) / (K.sum(label_seg) + K.epsilon())
    # coherence loss, nearby ori should be as near as possible
    mean_kernal = np.reshape(np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1]], dtype=np.float32)/8, [3, 3, 1, 1])    
    sin2angle_ori, cos2angle_ori, modulus_ori = ori2angle(y_pred)
    sin2angle = K.conv2d(sin2angle_ori, mean_kernal, padding='same')
    cos2angle = K.conv2d(cos2angle_ori, mean_kernal, padding='same')
    modulus = K.conv2d(modulus_ori, mean_kernal, padding='same')
    coherence = K.sqrt(K.square(sin2angle) + K.square(cos2angle)) / (modulus + K.epsilon())
    coherenceloss = K.sum(label_seg) / (K.sum(coherence*label_seg) + K.epsilon()) - 1
    loss = logloss + lamb*coherenceloss
    return loss

def ori_loss(y_true, y_pred, lamb=1.):
    # clip
    y_pred = K.tf.clip_by_value(y_pred, K.epsilon(), 1 - K.epsilon())
    # get ROI
    label_seg = K.sum(y_true, axis=-1, keepdims=True)
    label_seg = K.tf.cast(K.tf.greater(label_seg, 0), K.tf.float32) 
    # weighted cross entropy loss
    lamb_pos, lamb_neg = 1., 1. 
    logloss = lamb_pos*y_true*K.log(y_pred)+lamb_neg*(1-y_true)*K.log(1-y_pred)
    logloss = logloss*label_seg # apply ROI
    logloss = -K.sum(logloss) / (K.sum(label_seg) + K.epsilon())
    # coherence loss, nearby ori should be as near as possible
    mean_kernal = np.reshape(np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1]], dtype=np.float32)/8, [3, 3, 1, 1])    
    sin2angle_ori, cos2angle_ori, modulus_ori = ori2angle(y_pred)
    sin2angle = K.conv2d(sin2angle_ori, mean_kernal, padding='same')
    cos2angle = K.conv2d(cos2angle_ori, mean_kernal, padding='same')
    modulus = K.conv2d(modulus_ori, mean_kernal, padding='same')
    coherence = K.sqrt(K.square(sin2angle) + K.square(cos2angle)) / (modulus + K.epsilon())
    coherenceloss = K.sum(label_seg) / (K.sum(coherence*label_seg) + K.epsilon()) - 1
    loss = logloss + lamb*coherenceloss
    return loss

def call(self, x, mask=None):
        # 1e-8 is used to prevent division by 0
        W_norm = self.W / (tf.sqrt(tf.reduce_sum(tf.square(self.W), axis=[0, 1, 2], keep_dims=True)) + 1e-8)

        output = K.conv2d(x, W_norm, strides=self.subsample,
                          border_mode=self.border_mode,
                          dim_ordering=self.dim_ordering,
                          filter_shape=self.W_shape)

        if self.bias:
            if self.dim_ordering == 'th':
                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
            elif self.dim_ordering == 'tf':
                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
            else:
                raise ValueError('Invalid dim_ordering:', self.dim_ordering)
        output = self.activation(output)
        return output

def call(self, x, mask=None):
        w = self.W
        w_rot = [w]
        for i in range(3):
            w = shift_rotate(w, shift=2)
            w_rot.append(w)

        outputs = tf.stack([K.conv2d(x, w_i, strides=self.subsample,
                                     border_mode=self.border_mode,
                                     dim_ordering=self.dim_ordering,
                                     filter_shape=self.W_shape) for w_i in w_rot])

        output = K.max(outputs, 0)

        if self.bias:
            if self.dim_ordering == 'th':
                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
            elif self.dim_ordering == 'tf':
                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
            else:
                raise ValueError('Invalid dim_ordering:', self.dim_ordering)
        output = self.activation(output)
        return output

def call(self, x, mask=None):
        w = self.W
        w_rot = [w]
        for i in range(7):
            w = shift_rotate(w)
            w_rot.append(w)

        outputs = tf.stack([K.conv2d(x, w_i, strides=self.subsample,
                                     border_mode=self.border_mode,
                                     dim_ordering=self.dim_ordering,
                                     filter_shape=self.W_shape) for w_i in w_rot])

        output = K.max(outputs, 0)

        if self.bias:
            if self.dim_ordering == 'th':
                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
            elif self.dim_ordering == 'tf':
                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
            else:
                raise ValueError('Invalid dim_ordering:', self.dim_ordering)
        output = self.activation(output)
        return output

def _spectrogram_mono(self, x):
        '''x.shape : (None, 1, len_src),
        returns 2D batch of a mono power-spectrogram'''
        x = K.permute_dimensions(x, [0, 2, 1])
        x = K.expand_dims(x, 3)  # add a dummy dimension (channel axis)
        subsample = (self.n_hop, 1)
        output_real = K.conv2d(x, self.dft_real_kernels,
                               strides=subsample,
                               padding=self.padding,
                               data_format='channels_last')
        output_imag = K.conv2d(x, self.dft_imag_kernels,
                               strides=subsample,
                               padding=self.padding,
                               data_format='channels_last')
        output = output_real ** 2 + output_imag ** 2
        # now shape is (batch_sample, n_frame, 1, freq)
        if self.image_data_format == 'channels_last':
            output = K.permute_dimensions(output, [0, 3, 1, 2])
        else:
            output = K.permute_dimensions(output, [0, 2, 3, 1])
        return output

def _compute_mask_output(self, mask_tensor):
        if self.layer.rank == 1:
            mask_output = K.conv1d(
                mask_tensor,
                self.mask_kernel,
                self.layer.strides[0],
                self.layer.padding,
                self.layer.data_format,
                self.layer.dilation_rate[0]
            )
        if self.layer.rank == 2:
            mask_output = K.conv2d(
                mask_tensor,
                self.mask_kernel,
                self.layer.strides,
                self.layer.padding,
                self.layer.data_format,
                self.layer.dilation_rate
            )
        if self.layer.rank == 3:
            mask_output = K.conv3d(
                mask_tensor,
                self.mask_kernel,
                self.layer.strides,
                self.layer.padding,
                self.layer.data_format,
                self.layer.dilation_rate
            )
        return mask_output

def call(self, x):
        kernel = self.kernel * self.g / K.sqrt(K.sum(K.square(self.kernel), axis=[0, 1, 2], keepdims=True))
        output = K.conv2d(x, kernel, strides=self.strides,
                          padding=self.padding,
                          data_format=self.data_format)
        if self.use_bias:
            output = K.bias_add(output, self.bias, data_format=self.data_format)
        if self.activation is not None:
            output = self.activation(output)
        return output

def call(self, x, mask=None):
        output = K.conv2d(x, self.W, border_mode=self.border_mode, strides=self.strides)
        output += K.reshape(self.b, (1, self.nb_filters, 1, 1))
        return output

def call(self, x, mask=None):
        output = K.conv2d(x, self.W, border_mode=self.border_mode, strides=self.strides)
        output += K.reshape(self.b, (1, self.nb_filters, 1, 1))
        return output

def call(self, x, mask=None):
        # TODO: learn what is this mask parameter and how to use it.
        output = K.conv2d(x, self.W * self.mask, strides=self.subsample,
                          border_mode=self.border_mode,
                          dim_ordering=self.dim_ordering,
                          filter_shape=self.W_shape)
        if self.bias:
            if self.dim_ordering == 'th':
                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
            elif self.dim_ordering == 'tf':
                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
            else:
                raise ValueError('Invalid dim_ordering:', self.dim_ordering)
        output = self.activation(output)
        return output

def call(self, x, mask=None):
        xshape = K.shape(x)
        output_shape = [-1] + list(self.compute_output_shape(xshape)[1:])

        if self.data_format == 'channels_first':
            x = K.reshape(x, (-1, 1, xshape[2], xshape[3]))
        elif self.data_format == 'channels_last':
            x = K.permute_dimensions(x, (0, 3, 1, 2))
            x = K.reshape(x, (-1, xshape[1], xshape[2], 1))

        conv_out = K.conv2d(x, self.kernel,
                            strides=self.strides,
                            padding=self.padding,
                            data_format=self.data_format,
                            dilation_rate=self.dilation_rate)

        if self.data_format == 'channels_first':
            # Complex-cell filter operation
            conv_out1 = K.sqrt(K.square(conv_out[:, :self.filters_complex, :, :]) + K.square(conv_out[:, self.filters_complex:2*self.filters_complex, :, :]) + self.epsilon)
            # Simple-cell filter operation
            conv_out2 = K.concatenate([conv_out1, conv_out[:, 2*self.filters_complex:, :, :]], axis=1)
        elif self.data_format == 'channels_last':
            # Complex-cell filter operation
            conv_out1 = K.sqrt(K.square(conv_out[:, :, :, :self.filters_complex]) + K.square(conv_out[:, :, :, self.filters_complex:2*self.filters_complex]) + self.epsilon)
            # Simple-cell filter operation
            conv_out2 = K.concatenate([conv_out1, conv_out[:, :, :, 2*self.filters_complex:]], axis=3)

        if self.bias:
            if self.data_format == 'channels_first':
                conv_out2 += K.reshape(self.bias, (1, self.filters_complex + self.filters_simple, 1, 1))
            elif self.data_format == 'channels_last':
                conv_out2 += K.reshape(self.bias, (1, 1, 1, self.filters_complex + self.filters_simple))
                # conv_out2 = K.reshape(conv_out2, [-1, xshape[3], output_shape[1], output_shape[2], self.filters_complex + self.filters_simple])
                # conv_out2 = K.permute_dimensions(conv_out2, (0, 2, 3, 1, 4))

        conv_out2 = self.activation(conv_out2)

        return K.reshape(conv_out2, output_shape)

def call(self, x, mask=None):
        xshape = K.shape(x)
        output_shape = [-1] + list(self.compute_output_shape(xshape)[1:])

        if self.data_format == 'channels_first':
            x = K.reshape(x, (-1, 1, xshape[2], xshape[3]))
        elif self.data_format == 'channels_last':
            x = K.permute_dimensions(x, (0, 3, 1, 2))
            x = K.reshape(x, (-1, xshape[1], xshape[2], 1))

        conv_out = K.conv2d(x, self.kernel,
                            strides=self.strides,
                            padding=self.padding,
                            data_format=self.data_format,
                            dilation_rate=self.dilation_rate)

        if self.data_format == 'channels_first':
            # Complex-cell filter operation
            conv_out1 = K.sqrt(K.square(conv_out[:, :self.filters_complex, :, :]) + K.square(conv_out[:, self.filters_complex:2*self.filters_complex, :, :]) + K.epsilon())
            # Simple-cell filter operation
            conv_out2 = K.concatenate([conv_out1, conv_out[:, 2*self.filters_complex:, :, :]], axis=1)
        elif self.data_format == 'channels_last':
            # Complex-cell filter operation
            conv_out1 = K.sqrt(K.square(conv_out[:, :, :, :self.filters_complex]) + K.square(conv_out[:, :, :, self.filters_complex:2*self.filters_complex]) + K.epsilon())
            # Simple-cell filter operation
            conv_out2 = K.concatenate([conv_out1, conv_out[:, :, :, 2*self.filters_complex:]], axis=3)

        conv_out2 = K.reshape(conv_out2, output_shape)

        if self.bias:
            if self.data_format == 'channels_first':
                conv_out2 += K.reshape(self.bias, (1, -1, 1, 1))
            elif self.data_format == 'channels_last':
                conv_out2 += K.reshape(self.bias, (1, 1, 1, -1))
                # conv_out2 = K.reshape(conv_out2, [-1, xshape[3], output_shape[1], output_shape[2], self.filters_complex + self.filters_simple])
                # conv_out2 = K.permute_dimensions(conv_out2, (0, 2, 3, 1, 4))

        return self.activation(conv_out2)

def call(self, x, mask=None):
        x = K.permute_dimensions(x, (0, 2, 1))
        x = K.reshape(x, (-1, self.input_length))
        x = K.expand_dims(x, 1)
        x = K.expand_dims(x, -1)
        if self.real_filts is not None:
            conv_out_r = K.conv2d(x, self.W_r, strides=self.subsample,
                                  border_mode=self.border_mode,
                                  dim_ordering='th')
        else:
            conv_out_r = x

        if self.complex_filts is not None:
            conv_out_c1 = K.conv2d(x, self.W_c1, strides=self.subsample,
                                   border_mode=self.border_mode,
                                   dim_ordering='th')
            conv_out_c2 = K.conv2d(x, self.W_c2, strides=self.subsample,
                                   border_mode=self.border_mode,
                                   dim_ordering='th')
            conv_out_c = K.sqrt(K.square(conv_out_c1) + K.square(conv_out_c2) + K.epsilon())
            output = K.concatenate((conv_out_r, conv_out_c), axis=1)
        else:
            output = conv_out_r

        output_shape = self.get_output_shape_for((None, self.input_length, self.input_dim))
        output = K.squeeze(output, 3)  # remove the dummy 3rd dimension
        output = K.permute_dimensions(output, (2, 1, 0))
        output = K.reshape(output, (-1, output_shape[1], output.shape[1]*output.shape[2]))
        return output

def find_patch_matches(a, a_norm, b):
    '''For each patch in A, find the best matching patch in B'''
    # we want cross-correlation here so flip the kernels
    convs = K.conv2d(a, b[:, :, ::-1, ::-1], border_mode='valid')
    argmax = K.argmax(convs / a_norm, axis=1)
    return argmax

def call(self, x, mask=None):
        # compute the candidate hidden state
        transform = K.conv2d(x, self.W, strides=self.subsample,
                          border_mode=self.border_mode,
                          dim_ordering=self.dim_ordering,
                          filter_shape=self.W_shape)
        if self.bias:
            if self.dim_ordering == 'th':
                transform += K.reshape(self.b, (1, self.nb_filter, 1, 1))
            elif self.dim_ordering == 'tf':
                transform += K.reshape(self.b, (1, 1, 1, self.nb_filter))
            else:
                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
        transform = self.activation(transform)

        transform_gate = K.conv2d(x, self.W_gate, strides=self.subsample,
                          border_mode=self.border_mode,
                          dim_ordering=self.dim_ordering,
                          filter_shape=self.W_shape)
        if self.bias:
            if self.dim_ordering == 'th':
                transform_gate += K.reshape(self.b_gate, (1, self.nb_filter, 1, 1))
            elif self.dim_ordering == 'tf':
                transform_gate += K.reshape(self.b_gate, (1, 1, 1, self.nb_filter))
            else:
                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
        transform_gate = K.sigmoid(transform_gate)
        carry_gate = 1.0 - transform_gate

        return transform * transform_gate + x * carry_gate

def find_patch_matches(a, a_norm, b):
    '''For each patch in A, find the best matching patch in B'''
    convs = None
    if K.backend() == 'theano':
        # HACK: This was not being performed on the GPU for some reason.
        from theano.sandbox.cuda import dnn
        if dnn.dnn_available():
            convs = dnn.dnn_conv(
                img=a, kerns=b[:, :, ::-1, ::-1], border_mode='valid')
    if convs is None:
        convs = K.conv2d(a, b[:, :, ::-1, ::-1], border_mode='valid')
    argmax = K.argmax(convs / a_norm, axis=1)
    return argmax

def orientation(image, stride=8, window=17):
    with K.tf.name_scope('orientation'):
        assert image.get_shape().as_list()[3] == 1, 'Images must be grayscale'
        strides = [1, stride, stride, 1]
        E = np.ones([window, window, 1, 1])
        sobelx = np.reshape(np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]], dtype=float), [3, 3, 1, 1])
        sobely = np.reshape(np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]], dtype=float), [3, 3, 1, 1])
        gaussian = np.reshape(gaussian2d((5, 5), 1), [5, 5, 1, 1])
        with K.tf.name_scope('sobel_gradient'):
            Ix = K.tf.nn.conv2d(image, sobelx, strides=[1,1,1,1], padding='SAME', name='sobel_x')
            Iy = K.tf.nn.conv2d(image, sobely, strides=[1,1,1,1], padding='SAME', name='sobel_y')
        with K.tf.name_scope('eltwise_1'):
            Ix2 = K.tf.multiply(Ix, Ix, name='IxIx')
            Iy2 = K.tf.multiply(Iy, Iy, name='IyIy')
            Ixy = K.tf.multiply(Ix, Iy, name='IxIy')
        with K.tf.name_scope('range_sum'):
            Gxx = K.tf.nn.conv2d(Ix2, E, strides=strides, padding='SAME', name='Gxx_sum')
            Gyy = K.tf.nn.conv2d(Iy2, E, strides=strides, padding='SAME', name='Gyy_sum')
            Gxy = K.tf.nn.conv2d(Ixy, E, strides=strides, padding='SAME', name='Gxy_sum')
        with K.tf.name_scope('eltwise_2'):
            Gxx_Gyy = K.tf.subtract(Gxx, Gyy, name='Gxx_Gyy')
            theta = atan2([2*Gxy, Gxx_Gyy]) + np.pi
        # two-dimensional low-pass filter: Gaussian filter here
        with K.tf.name_scope('gaussian_filter'):
            phi_x = K.tf.nn.conv2d(K.tf.cos(theta), gaussian, strides=[1,1,1,1], padding='SAME', name='gaussian_x')
            phi_y = K.tf.nn.conv2d(K.tf.sin(theta), gaussian, strides=[1,1,1,1], padding='SAME', name='gaussian_y')
            theta = atan2([phi_y, phi_x])/2
    return theta

def seg_loss(y_true, y_pred, lamb=1.):
    # clip
    y_pred = K.tf.clip_by_value(y_pred, K.epsilon(), 1 - K.epsilon())
    # weighted cross entropy loss
    total_elements = K.sum(K.tf.ones_like(y_true))
    label_pos = K.tf.cast(K.tf.greater(y_true, 0.0), K.tf.float32)   
    lamb_pos = 0.5 * total_elements / K.sum(label_pos)
    lamb_neg = 1 / (2 - 1/lamb_pos)
    logloss = lamb_pos*y_true*K.log(y_pred)+lamb_neg*(1-y_true)*K.log(1-y_pred)
    logloss = -K.mean(K.sum(logloss, axis=-1))
    # smooth loss
    smooth_kernal = np.reshape(np.array([[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]], dtype=np.float32)/8, [3, 3, 1, 1])
    smoothloss = K.mean(K.abs(K.conv2d(y_pred, smooth_kernal)))
    loss = logloss + lamb*smoothloss
    return loss

def orientation(image, stride=8, window=17):
    with K.tf.name_scope('orientation'):
        assert image.get_shape().as_list()[3] == 1, 'Images must be grayscale'
        strides = [1, stride, stride, 1]
        E = np.ones([window, window, 1, 1])
        sobelx = np.reshape(np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]], dtype=float), [3, 3, 1, 1])
        sobely = np.reshape(np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]], dtype=float), [3, 3, 1, 1])
        gaussian = np.reshape(gaussian2d((5, 5), 1), [5, 5, 1, 1])
        with K.tf.name_scope('sobel_gradient'):
            Ix = K.tf.nn.conv2d(image, sobelx, strides=[1,1,1,1], padding='SAME', name='sobel_x')
            Iy = K.tf.nn.conv2d(image, sobely, strides=[1,1,1,1], padding='SAME', name='sobel_y')
        with K.tf.name_scope('eltwise_1'):
            Ix2 = K.tf.multiply(Ix, Ix, name='IxIx')
            Iy2 = K.tf.multiply(Iy, Iy, name='IyIy')
            Ixy = K.tf.multiply(Ix, Iy, name='IxIy')
        with K.tf.name_scope('range_sum'):
            Gxx = K.tf.nn.conv2d(Ix2, E, strides=strides, padding='SAME', name='Gxx_sum')
            Gyy = K.tf.nn.conv2d(Iy2, E, strides=strides, padding='SAME', name='Gyy_sum')
            Gxy = K.tf.nn.conv2d(Ixy, E, strides=strides, padding='SAME', name='Gxy_sum')
        with K.tf.name_scope('eltwise_2'):
            Gxx_Gyy = K.tf.subtract(Gxx, Gyy, name='Gxx_Gyy')
            theta = atan2([2*Gxy, Gxx_Gyy]) + np.pi
        # two-dimensional low-pass filter: Gaussian filter here
        with K.tf.name_scope('gaussian_filter'):
            phi_x = K.tf.nn.conv2d(K.tf.cos(theta), gaussian, strides=[1,1,1,1], padding='SAME', name='gaussian_x')
            phi_y = K.tf.nn.conv2d(K.tf.sin(theta), gaussian, strides=[1,1,1,1], padding='SAME', name='gaussian_y')
            theta = atan2([phi_y, phi_x])/2
    return theta

def ori_highest_peak(y_pred, length=180):
    glabel = gausslabel(length=length,stride=2).astype(np.float32)
    ori_gau = K.conv2d(y_pred,glabel,padding='same')
    return ori_gau

def get_output(self, train=False):
        X = train
        X = K.expand_dims(X, -1)  # add a dimension of the right
        X = K.permute_dimensions(X, (0, 2, 1, 3))
        conv_out = K.conv2d(X, self.W, strides=self.subsample,
                            border_mode=self.border_mode,
                            dim_ordering='th')

        output = conv_out + K.reshape(self.b, (1, self.nb_filter, 1, 1))
        output = self.activation(output)
        output = K.squeeze(output, 3)  # remove the dummy 3rd dimension
        output = K.permute_dimensions(output, (0, 2, 1))
        return output

def call(self, inputs):
        _, kernel_b = xnorize(self.kernel, self.H)
        _, inputs_b = xnorize(inputs)
        outputs = K.conv2d(inputs_b, kernel_b, strides=self.strides,
                           padding=self.padding,
                           data_format=self.data_format,
                           dilation_rate=self.dilation_rate)

        # calculate Wa and xa

        # kernel_a
        mask = K.reshape(self.kernel, (-1, self.filters)) # self.nb_row * self.nb_col * channels, filters 
        kernel_a = K.stop_gradient(K.mean(K.abs(mask), axis=0)) # filters

        # inputs_a
        if self.data_format == 'channels_first':
            channel_axis = 1
        else:
            channel_axis = -1 
        mask = K.mean(K.abs(inputs), axis=channel_axis, keepdims=True) 
        ones = K.ones(self.kernel_size + (1, 1))
        inputs_a = K.conv2d(mask, ones, strides=self.strides,
                      padding=self.padding,
                      data_format=self.data_format,
                      dilation_rate=self.dilation_rate) # nb_sample, 1, new_nb_row, new_nb_col
        if self.data_format == 'channels_first':
            outputs = outputs * K.stop_gradient(inputs_a) * K.expand_dims(K.expand_dims(K.expand_dims(kernel_a, 0), -1), -1)
        else:
            outputs = outputs * K.stop_gradient(inputs_a) * K.expand_dims(K.expand_dims(K.expand_dims(kernel_a, 0), 0), 0)

        if self.use_bias:
            outputs = K.bias_add(
                outputs,
                self.bias,
                data_format=self.data_format)

        if self.activation is not None:
            return self.activation(outputs)
        return outputs


# Aliases

def call(self, x, mask=None):
        """
        """

        # Assuming 'th' ordering
        # Input shape is:
        #   (batch, channels, rows, cols) ; e.g. (20, 3, 32, 32)
        # Generate a filter of size, incorporate the coordinates here
        #   (batch, filter_size ** 2, rows, cols) ; (20, 7*7, 32, 32)
        # Expand each channel to (filter_size ** 2) by shifting it using the
        # dfn trick
        # Do an elementwise dot with filter and dimension reduction on channel
        # axis

        # Generated filter tensors
        filters = self.gen.get_output(x)

        # Alias
        fs = self.filter_size
        # Aux convolution to shift the images
        if self.dim_ordering == "th":
            shifter_shape = (fs**2, 1, fs, fs)
            ch_axis = 1
        elif self.dim_ordering == "tf":
            shifter_shape = (fs**2, fs, fs, 1)
            ch_axis = 3

        shifter = np.reshape(np.eye(fs**2, fs**2), shifter_shape)

        shifter = K.variable(value=shifter)

        # Use same filter in all channels and return same number of channels
        outputs = []
        for i in range(self.filters_in):
            if self.dim_ordering == "th":
                x_channel = x[:, [i], :, :]
            elif self.dim_ordering == "tf":
                x_channel = x[:, :, :, [i]]

            # This creates shifted version of x in all direction
            # When stacked together and summed after elemwise mult with
            # filters, this results in an effective convolution
            x_shifted = K.conv2d(
                x_channel,
                shifter,
                strides=self.strides,
                border_mode=self.border_mode,
                dim_ordering=self.dim_ordering)

            output = K.sum(x_shifted * filters, axis=ch_axis, keepdims=True)
            outputs.append(output)

        output = K.concatenate(outputs, axis=ch_axis)

        return output

def call(self, x, mask=None):
        """
        """

        # Assuming 'th' ordering
        # Input shape is:
        #   (batch, channels, rows, cols) ; e.g. (20, 3, 32, 32)
        # Generate a filter of size, incorporate the coordinates here
        #   (batch, filter_size ** 2, rows, cols) ; (20, 7*7, 32, 32)
        # Expand each channel to (filter_size ** 2) by shifting it using the
        # dfn trick
        # Do an elementwise dot with filter and dimension reduction on channel
        # axis

        # Generated filter tensors
        filters = self.gen.get_output(x)

        # Alias
        fs = self.filter_size
        # Aux convolution to shift the images
        if self.dim_ordering == "th":
            shifter_shape = (fs**2, 1, fs, fs)
            ch_axis = 1
        elif self.dim_ordering == "tf":
            shifter_shape = (fs**2, fs, fs, 1)
            ch_axis = 3

        shifter = np.reshape(np.eye(fs**2, fs**2), shifter_shape)

        shifter = K.variable(value=shifter)

        # Use same filter in all channels and return same number of channels
        outputs = []
        for i in range(self.filters_in):
            if self.dim_ordering == "th":
                x_channel = x[:, [i], :, :]
            elif self.dim_ordering == "tf":
                x_channel = x[:, :, :, [i]]

            # This creates shifted version of x in all direction
            # When stacked together and summed after elemwise mult with
            # filters, this results in an effective convolution
            x_shifted = K.conv2d(
                x_channel,
                shifter,
                strides=self.strides,
                border_mode=self.border_mode,
                dim_ordering=self.dim_ordering)

            output = K.sum(x_shifted * filters, axis=ch_axis, keepdims=True)
            outputs.append(output)

        output = K.concatenate(outputs, axis=ch_axis)

        return output

def call(self, x, mask=None):
        """
        """

        # Assuming 'th' ordering
        # Input shape is:
        #   (batch, channels, rows, cols) ; e.g. (20, 3, 32, 32)
        # Generate a filter of size, incorporate the coordinates here
        #   (batch, filter_size ** 2, rows, cols) ; (20, 7*7, 32, 32)
        # Expand each channel to (filter_size ** 2) by shifting it using the
        # dfn trick
        # Do an elementwise dot with filter and dimension reduction on channel
        # axis

        # Generated filter tensors
        filters = self.gen.get_output(x)

        # Alias
        fs = self.filter_size
        # Aux convolution to shift the images
        if self.dim_ordering == "th":
            shifter_shape = (fs**2, 1, fs, fs)
            ch_axis = 1
        elif self.dim_ordering == "tf":
            shifter_shape = (fs**2, fs, fs, 1)
            ch_axis = 3

        shifter = np.reshape(np.eye(fs**2, fs**2), shifter_shape)

        shifter = K.variable(value=shifter)

        # Use same filter in all channels and return same number of channels
        outputs = []
        for i in range(self.filters_in):
            if self.dim_ordering == "th":
                x_channel = x[:, [i], :, :]
            elif self.dim_ordering == "tf":
                x_channel = x[:, :, :, [i]]

            # This creates shifted version of x in all direction
            # When stacked together and summed after elemwise mult with
            # filters, this results in an effective convolution
            x_shifted = K.conv2d(
                x_channel,
                shifter,
                strides=self.strides,
                border_mode=self.border_mode,
                dim_ordering=self.dim_ordering)

            output = K.sum(x_shifted * filters, axis=ch_axis, keepdims=True)
            outputs.append(output)

        output = K.concatenate(outputs, axis=ch_axis)

        return output

def call(self, inputs):
        xshape = K.shape(inputs)
        inputs = K.reshape(inputs, (-1, xshape[2], xshape[3], xshape[4]))

        sin_step = K.reshape(K.sin(self.delays_pi[:, None, None]*self.temporal_freqs[None, :, None]*self.temporal_frequencies_scaling)*self.temporal_kernel[:, None, :], (-1, self.filters_temporal*self.temporal_frequencies))
        cos_step = K.reshape(K.cos(self.delays_pi[:, None, None]*self.temporal_freqs[None, :, None]*self.temporal_frequencies_scaling)*self.temporal_kernel[:, None, :], (-1, self.filters_temporal*self.temporal_frequencies))

        w0t = K.concatenate((cos_step, -sin_step), axis=0)
        w1t = K.concatenate((sin_step, cos_step), axis=0)

        conv_out1 = K.conv2d(
            inputs,
            self.spatial_kernel,
            strides=self.strides,
            padding=self.padding,
            data_format=self.data_format,
            dilation_rate=self.dilation_rate)

        conv_out1_shape = K.shape(conv_out1)

        conv_out1 = K.reshape(conv_out1, (-1, self.delays, conv_out1_shape[1], conv_out1_shape[2], conv_out1_shape[3]))

        if self.data_format == 'channels_first':
            # samps x delays x stack x X x Y
            conv_out1 = K.permute_dimensions(conv_out1, (0, 2, 3, 4, 1))
            # samps x stack x X x Y x delays
        elif self.data_format == 'channels_last':
            # samps x delays x X x Y x stack
            conv_out1 = K.permute_dimensions(conv_out1, (0, 4, 2, 3, 1))
            # samps x stack x X x Y x delays
        else:
            raise Exception('Invalid data_format: ' + self.data_format)

        # split out complex and simple filter pairs
        conv_out12 = K.concatenate([conv_out1[:, :self.filters_complex, :, :, :], conv_out1[:, self.filters_complex+self.filters_simple:2*self.filters_complex+self.filters_simple, :, :, :]], axis=4)
        conv_out34 = K.concatenate([conv_out1[:, self.filters_complex:self.filters_complex + self.filters_simple, :, :, :], conv_out1[:, 2*self.filters_complex + self.filters_simple:, :, :, :]], axis=4)

        # apply temporal trade-off to get temporal filter outputs and compute complex and simple outputs
        conv_out = K.sqrt(K.square(K.dot(conv_out12, w0t)) + K.square(K.dot(conv_out12, w1t)) + K.epsilon())
        conv_outlin = K.dot(conv_out34, w0t)
        # samps x stack x X x Y x temporal_filters*temporal_frequencies

        output = K.concatenate([conv_out, conv_outlin], axis=1)

        if self.data_format == 'channels_first':
            output = K.permute_dimensions(output, (0, 4, 1, 2, 3))
            # samps x temporal_filters*temporal_frequencies x stack x X x Y
        elif self.data_format == 'channels_last':
            output = K.permute_dimensions(output, (0, 4, 2, 3, 1))
            # samps x temporal_filters*temporal_frequencies x X x Y x stack

        if self.bias:
            if self.data_format == 'channels_first':
                output += K.reshape(self.bias, (1, self.filters_temporal*self.temporal_frequencies, self.filters_complex + self.filters_simple, 1, 1))
            elif self.data_format == 'channels_last':
                output += K.reshape(self.bias, (1, self.filters_temporal*self.temporal_frequencies, 1, 1, self.filters_complex + self.filters_simple))

        output = self.activation(output)
        return output