Loss

class Loss(ABC):
    """Loss Meta class
    """
    @abstractmethod
    def __init__(self):
        pass

    @abstractmethod
    def forward(self):
        """mandatory method for child class
        """
        pass

    def calculate(self, output, y):
        """Calculate mean loss

        Args:
            output : output from the layer
            y : truth value/ target/ expected outcome
        """
        # it can be individual outcome of different kind of loss functions
        sample_losses = self.forward(output, y)

        # calculating mean
        data_loss = np.mean(sample_losses)
        return data_loss

Categorical Cross Entropy

\begin{align*} L(a^{[l]},y) &= y log(a^{[l]}) + (1 - y) log(1 - a^{[l]})\\ \\ \text{derivative of loss over a --> da}\\ \frac{\partial L}{\partial a} &= \big[ \frac{y}{a} + \frac{1 -y}{1 - a}(-1) \big]\\ \frac{\partial L}{\partial a} &= \big[ \frac{y}{a} - \frac{1 -y}{1 - a} \big]\\ \\ \text{derivative of loss over z --> dz}\\ \frac{\partial L}{\partial z} &= \frac{\partial L}{\partial a} \frac{\partial a}{\partial z}\\ \\ \text{derivative of loss over w --> dw}\\ \frac{\partial L}{\partial w} &= \frac{\partial L}{\partial a} \frac{\partial a}{\partial z} \frac{\partial z}{\partial w} \\ \text{derivative of loss over b --> db}\\ \frac{\partial L}{\partial b} &= \frac{\partial L}{\partial a} \frac{\partial a}{\partial z} \frac{\partial z}{\partial b} \end{align*}

• y_pred_clipped

  • numpy.clip is used to clip the values from min and max values like bandpass filter

  • min = 1.0 * 10^-7

  • max = 1 - 1.0 * 10^-7

• correct_confidences

  • probabilities for target value that has been

  • calculated earlier

  • only for categorical variables

class LossCategoricalCrossEntropy(Loss):
    """Categorical Cross entropy loss
    """

    def forward(self, y_pred, y_true):
        """forward propogation calculation

        Args:
            y_pred (numpy.ndarray) : predictions generated
            y_true (numpy.ndarray) : actual values
        """

        # get total number of rows/samples
        samples = len(y_pred)


        y_pred_clipped = np.clip(y_pred,1e-7,1-1e-7)

        correct_confidences = None
        if len(y_true.shape) == 1:
            correct_confidences = y_pred_clipped[range(samples),y_true]

        elif len(y_true.shape) == 2:
            correct_confidences = np.sum(y_pred_clipped * y_true, axis = 1)

        else:
            pass

        # losses
        negative_log_Likelihoods = -np.log(correct_confidences)
        return negative_log_Likelihoods