我正在使用迁移学习为斯坦福汽车ResNet-18数据集构建分类模型。我想实施标签平滑来惩罚过度自信的预测并提高泛化能力。
TensorFlow在 中有一个简单的关键字参数CrossEntropyLoss。PyTorch有没有人为我可以即插即用构建了类似的功能？
27571 次
6 回答
29
多类神经网络的泛化和学习速度通常可以通过使用软目标来显着提高，这些软目标是硬目标的加权平均和标签上的均匀分布。以这种方式平滑标签可防止网络变得过于自信，标签平滑已用于许多最先进的模型，包括图像分类、语言翻译和语音识别。
标签平滑已经在Tensorflow交叉熵损失函数中实现。二元交叉熵，分类交叉熵。但是目前，在PyTorch. 但是，对此进行了积极的讨论，并希望将提供官方软件包。这是讨论线程：问题 #7455。
在这里，我们将从实践者那里带来一些可用的标签平滑（LS）PyTorch的最佳实现。基本上，有很多方法可以实现LS。请参考这个具体的讨论，一个在这里，另一个在这里。在这里，我们将以两种独特的方式实现，每种方式都有两个版本；所以总共4。
选项 1：CrossEntropyLossWithProbs
这样，它就接受了one-hot目标向量。用户必须手动平滑他们的目标向量。它可以在with torch.no_grad()范围内完成，因为它暂时将所有requires_grad标志设置为 false。
import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.nn.modules.loss import _WeightedLoss
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1, weight = None):
"""if smoothing == 0, it's one-hot method
if 0 < smoothing < 1, it's smooth method
"""
super(LabelSmoothingLoss, self).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.weight = weight
self.cls = classes
self.dim = dim
def forward(self, pred, target):
assert 0 <= self.smoothing < 1
pred = pred.log_softmax(dim=self.dim)
if self.weight is not None:
pred = pred * self.weight.unsqueeze(0)
with torch.no_grad():
true_dist = torch.zeros_like(pred)
true_dist.fill_(self.smoothing / (self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
此外，我们self. smoothing在此实现上添加了一个断言复选标记并添加了损失加权支持。
Shital 已经在这里发布了答案。这里我们要指出的是，这个实现类似于Devin Yang的上述实现。但是，在这里我们提到他的代码时将code syntax.
class SmoothCrossEntropyLoss(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
def k_one_hot(self, targets:torch.Tensor, n_classes:int, smoothing=0.0):
with torch.no_grad():
targets = torch.empty(size=(targets.size(0), n_classes),
device=targets.device) \
.fill_(smoothing /(n_classes-1)) \
.scatter_(1, targets.data.unsqueeze(1), 1.-smoothing)
return targets
def reduce_loss(self, loss):
return loss.mean() if self.reduction == 'mean' else loss.sum() \
if self.reduction == 'sum' else loss
def forward(self, inputs, targets):
assert 0 <= self.smoothing < 1
targets = self.k_one_hot(targets, inputs.size(-1), self.smoothing)
log_preds = F.log_softmax(inputs, -1)
if self.weight is not None:
log_preds = log_preds * self.weight.unsqueeze(0)
return self.reduce_loss(-(targets * log_preds).sum(dim=-1))
查看
import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.nn.modules.loss import _WeightedLoss
if __name__=="__main__":
# 1. Devin Yang
crit = LabelSmoothingLoss(classes=5, smoothing=0.5)
predict = torch.FloatTensor([[0, 0.2, 0.7, 0.1, 0],
[0, 0.9, 0.2, 0.2, 1],
[1, 0.2, 0.7, 0.9, 1]])
v = crit(Variable(predict),
Variable(torch.LongTensor([2, 1, 0])))
print(v)
# 2. Shital Shah
crit = SmoothCrossEntropyLoss(smoothing=0.5)
predict = torch.FloatTensor([[0, 0.2, 0.7, 0.1, 0],
[0, 0.9, 0.2, 0.2, 1],
[1, 0.2, 0.7, 0.9, 1]])
v = crit(Variable(predict),
Variable(torch.LongTensor([2, 1, 0])))
print(v)
tensor(1.4178)
tensor(1.4178)
选项 2：LabelSmoothingCrossEntropyLoss
这样，它接受目标向量并且使用不手动平滑目标向量，而是内置模块负责标签平滑。它允许我们根据 来实现标签平滑F.nll_loss。
（一个）。Wangleiofficial : Source - (AFAIK), Original Poster
(b)。Datasaurus :来源 - 添加了加权支持
此外，我们略微减少了编码编写以使其更简洁。
class LabelSmoothingLoss(torch.nn.Module):
def __init__(self, smoothing: float = 0.1,
reduction="mean", weight=None):
super(LabelSmoothingLoss, self).__init__()
self.smoothing = smoothing
self.reduction = reduction
self.weight = weight
def reduce_loss(self, loss):
return loss.mean() if self.reduction == 'mean' else loss.sum() \
if self.reduction == 'sum' else loss
def linear_combination(self, x, y):
return self.smoothing * x + (1 - self.smoothing) * y
def forward(self, preds, target):
assert 0 <= self.smoothing < 1
if self.weight is not None:
self.weight = self.weight.to(preds.device)
n = preds.size(-1)
log_preds = F.log_softmax(preds, dim=-1)
loss = self.reduce_loss(-log_preds.sum(dim=-1))
nll = F.nll_loss(
log_preds, target, reduction=self.reduction, weight=self.weight
)
return self.linear_combination(loss / n, nll)
class LabelSmoothing(nn.Module):
"""NLL loss with label smoothing.
"""
def __init__(self, smoothing=0.0):
"""Constructor for the LabelSmoothing module.
:param smoothing: label smoothing factor
"""
super(LabelSmoothing, self).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
def forward(self, x, target):
logprobs = torch.nn.functional.log_softmax(x, dim=-1)
nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1))
nll_loss = nll_loss.squeeze(1)
smooth_loss = -logprobs.mean(dim=-1)
loss = self.confidence * nll_loss + self.smoothing * smooth_loss
return loss.mean()
查看
if __name__=="__main__":
# Wangleiofficial
crit = LabelSmoothingLoss(smoothing=0.3, reduction="mean")
predict = torch.FloatTensor([[0, 0.2, 0.7, 0.1, 0],
[0, 0.9, 0.2, 0.2, 1],
[1, 0.2, 0.7, 0.9, 1]])
v = crit(Variable(predict),
Variable(torch.LongTensor([2, 1, 0])))
print(v)
# NVIDIA
crit = LabelSmoothing(smoothing=0.3)
predict = torch.FloatTensor([[0, 0.2, 0.7, 0.1, 0],
[0, 0.9, 0.2, 0.2, 1],
[1, 0.2, 0.7, 0.9, 1]])
v = crit(Variable(predict),
Variable(torch.LongTensor([2, 1, 0])))
print(v)
tensor(1.3883)
tensor(1.3883)
更新：正式添加
torch.nn.CrossEntropyLoss(weight=None, size_average=None,
ignore_index=- 100, reduce=None,
reduction='mean', label_smoothing=0.0)
于 2021-03-24T00:51:01.953 回答
9
我一直在寻找派生自_LossPyTorch 中其他损失类并尊重基本参数的选项，例如reduction. 不幸的是，我找不到直接的替代品，所以最终自己写了。但是，我尚未对此进行完全测试：
import torch
from torch.nn.modules.loss import _WeightedLoss
import torch.nn.functional as F
class SmoothCrossEntropyLoss(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth_one_hot(targets:torch.Tensor, n_classes:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad():
targets = torch.empty(size=(targets.size(0), n_classes),
device=targets.device) \
.fill_(smoothing /(n_classes-1)) \
.scatter_(1, targets.data.unsqueeze(1), 1.-smoothing)
return targets
def forward(self, inputs, targets):
targets = SmoothCrossEntropyLoss._smooth_one_hot(targets, inputs.size(-1),
self.smoothing)
lsm = F.log_softmax(inputs, -1)
if self.weight is not None:
lsm = lsm * self.weight.unsqueeze(0)
loss = -(targets * lsm).sum(-1)
if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
其他选项：
- utils.pytorch实现
- 深度匹配实现
于 2019-12-10T10:17:29.720 回答
6
据我所知没有。
以下是 PyTorch 实现的两个示例：
LabelSmoothingLossOpenNMT 框架中用于机器翻译的模块attention-is-all-you-need-pytorch，重新实现谷歌的Attention is all you need paper
于 2019-04-15T09:00:49.827 回答
1
标签平滑 PyTorch 实现 参考：https ://github.com/wangleiofficial/label-smoothing-pytorch
import torch.nn.functional as F
def linear_combination(x, y, epsilon):
return epsilon * x + (1 - epsilon) * y
def reduce_loss(loss, reduction='mean'):
return loss.mean() if reduction == 'mean' else loss.sum() if reduction == 'sum' else loss
class LabelSmoothingCrossEntropy(nn.Module):
def __init__(self, epsilon: float = 0.1, reduction='mean'):
super().__init__()
self.epsilon = epsilon
self.reduction = reduction
def forward(self, preds, target):
n = preds.size()[-1]
log_preds = F.log_softmax(preds, dim=-1)
loss = reduce_loss(-log_preds.sum(dim=-1), self.reduction)
nll = F.nll_loss(log_preds, target, reduction=self.reduction)
return linear_combination(loss / n, nll, self.epsilon)
于 2020-11-03T06:07:06.407 回答
1
Pytorch 从 1.10.0 版本开始正式支持torch.nn.CrossEntropyLoss.
于 2021-11-22T11:40:09.493 回答
0
这目前在 PyTorch 中没有正式实现，但已作为高优先级功能请求 #7455提出，并在 TorchVision问题 #2980中单独提出。
其他库中有许多实现：
- NJUNMT-pytorch
NMTCritierion()._smooth_label() - 浮潜
snorkel.classification.cross_entropy_with_probs() - OpenNMT
LabelSmoothingLoss()
以及一些非官方的实现/代码片段：
TensorFlow / Keras 实现
tf.keras.losses.CategoricalCrossentropy(label_smoothing)
于 2021-03-26T13:09:54.493 回答