Sigmoids
File:
/src/neuralNetBlocks/activations.hppLines 42–56 /**
* @brief This calculates the following sigmoid function for all elements in a single tile.
*
* `f(x) = 1/(1 + exp(-x))`
*
* @ingroup activations
* @param x A single tile of data.
*
* @tparam numFracBits Number of fraction bits in the fixedpoint numbers.
* @tparam T The type of the data.
* @return result The sigmoid function outputs for all elements in a tile.
*/
// clang-format on
template <FracRepType numFracBits, typename T>
qVar_t<FixedPoint<T, 31>> sigmoid(qVar_t<FixedPoint<T, numFracBits>> x) {
TanH (Hyperbolic Tangents)
File:
/src/neuralNetBlocks/activations.hppLines 197–211 /**
* @brief This function performs the tanh function on the input, which is a single tile.
*
* `tanh(x) = (1- exp(-2*x))/(1 + exp(-2*x))`
*
* @ingroup activations
* @param x A single tile of data.
*
* @tparam numFracBits Number of fraction bits in the fixedpoint numbers.
* @tparam T The type of the data.
* @return result The tanh function outputs for all elements in a tile.
*/
// clang-format on
template <FracRepType numFracBits, typename T>
qVar_t<FixedPoint32<31>> tanh(qVar_t<FixedPoint<T, numFracBits>> x) {
ReLU (Rectified Linear Units)
File:
/src/neuralNetBlocks/activations.hppLines 560–573 /**
* @brief This function performs the ReLU6 function on the input, which is a single tile: `f(x) = min(max(0, x), 6)`.
*
* @param qData A single tile of data
*
* @tparam reluMethod The Relu method chosen.
* @tparam T The type of the data
* @return qOutout The ReLU6 function outputs for all elements in a tile.
*/
// clang-format on
template <ReluMethod reluMethod,
typename T,
typename std::enable_if_t<reluMethod == ReluMethod::SIX, std::int32_t> = 0>
qVar_t<T> relu(qVar_t<T> qData) {
File:
/src/neuralNetBlocks/activations.hppLines 560–573 /**
* @brief This function performs the ReLU6 function on the input, which is a single tile: `f(x) = min(max(0, x), 6)`.
*
* @param qData A single tile of data
*
* @tparam reluMethod The Relu method chosen.
* @tparam T The type of the data
* @return qOutout The ReLU6 function outputs for all elements in a tile.
*/
// clang-format on
template <ReluMethod reluMethod,
typename T,
typename std::enable_if_t<reluMethod == ReluMethod::SIX, std::int32_t> = 0>
qVar_t<T> relu(qVar_t<T> qData) {
Softmax
File:
/src/neuralNetBlocks/activations.hppLines 816–863 /**
* @brief This function performs a softmax, or normalized exponential function, on the input data buffer
* and stores the result in the output data buffer.
*
* A softmax normalizes the input data to a probability distribution proportional to the exponentials of
* the inputs. In other words, softmax takes the exponential function of each input elements and then
* normalizes by dividing each of the elements by the sum of all the exponentials. The output has the same
* shape as the input but is bound to the interval (0, 1).
*
* Additionally, there is a iteration that goes through the
* the scores and takes the maximum value for each channel. This maximum is subtracted
* from each score with a new range of (-inf, 0] so the output of exp
* is bound to the interval (0, 1]. This is done to ensure that the
* softmax has a countable sum (0, Height*Width) which is numerically stable while
* still ensuring that the maximum/largest scores are well represented in fixed-point.
*
* The equation for softmax is:
*
* ```
* softmax(x_i) = exp(x_i) / summation(exp(x_j) for all j)
* ```
*
* @tparam OcmTensorShape The shape of the OCM that we get values from.
* @tparam masked Whether or not to apply a causal mask to this softmax.
* @tparam NDArrayIn The input NDArray type.
* @tparam NDArrayOut The output NDArray type.
* @tparam Lambda The type of the lambda postprocessing function.
* @tparam reduction Which axis the softmax reduces over (Spatial for single-channel input).
* @param qInBuffer The input buffer.
* @param qOutBuffer The output buffer.
* @param maskThresh The mask threshold to be apply if masked=true.
* @param postProc The postprocessing function.
* @return
*/
template <typename OcmTensorShape,
bool masked = false,
SoftmaxReduction reduction = (OcmTensorShape::NUM_CHN == 1) ? SoftmaxReduction::Spatial
: SoftmaxReduction::Channel,
typename NDArrayIn,
typename NDArrayOut,
typename Lambda = EmptyType,
std::enable_if_t<reduction == SoftmaxReduction::Spatial || reduction == SoftmaxReduction::Width ||
reduction == SoftmaxReduction::Height,
std::int32_t> = 0>
INLINE void softmax(NDArrayIn& qInBuffer,
NDArrayOut& qOutBuffer,
const std::int32_t& maskThresh = OcmTensorShape::NUM_ELEMS_PER_BCH,
const Lambda& postProc = EmptyType()) {
Mish
File:
/src/neuralNetBlocks/activations.hppLines 588–598 /**
* @brief This function calculates mish activation: `f(x) = x * tanh(log(exp(x)+1))`.
*
* @param x A single tile of data
*
* @tparam T The type of the data
* @return result Mish activation function outputs.
*/
// clang-format on
template <typename T>
qVar_t<T> mish(qVar_t<T> x) {
