英文:
Pytorch T5 training loss not changing
问题
你试图微调一个T5模型以获得更准确的摘要，但是你的损失非常高，每个时代都不变化。我尝试增加学习率，但是模型仍然无法训练。由于损失根本不变化，似乎代码存在一些问题。我的输入文本非常大，但考虑到T5已经可以用于摘要，我认为这应该没问题。
以下是你的代码的翻译：
# 导入所需的库
import torch
from torchtext.models import T5_BASE_GENERATION, T5Transform
from torchtext.prototype.generate import GenerationUtils
from torch.utils.data import DataLoader, Dataset
import torch.nn.functional as F
import json
import os
# 定义一些常量和参数
padding_idx = 0
eos_idx = 0
max_seq_len = 65536
t5_sp_model_path = "/t5_tokenizer_base.model"
# 定义自定义数据集类
class CustomDataset(Dataset):
def __init__(self, data):
self.data = data
def __getitem__(self, index):
example = self.data[index]
input_text = example["input_text"]
target_text = example["target_text"]
return input_text, target_text
def __len__(self):
return len(self.data)
# 加载标记的数据集
train_data = [] # 训练数据列表
valid_data = [] # 验证数据列表
labeled = json.loads(open("data.json", 'r').read())
datalen = len(labeled)
valper = 0.2
train_ind = range(datalen - int(datalen * valper))
valid_ind = range(datalen - int(datalen * valper), datalen)
for x in train_ind:
train_data.append(labeled[str(x)])
for x in valid_ind:
valid_data.append(labeled[str(x)])
# 创建T5模型和转换的实例
if os.path.exists('model.pt'):
t5_base = torch.load("model.pt")
else:
t5_base = T5_BASE_GENERATION.get_model()
transform = T5Transform(sp_model_path=t5_sp_model_path, max_seq_len=max_seq_len, eos_idx=eos_idx, padding_idx=padding_idx)
# 定义训练参数
device = torch.device("cuda")
batch_size = 1
num_epochs = 5
# 将训练和验证数据转换为张量
train_dataset = CustomDataset(train_data)
valid_dataset = CustomDataset(valid_data)
train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
valid_dataloader = DataLoader(valid_dataset, batch_size=batch_size)
# 定义损失函数和优化器
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(t5_base.parameters(), lr=0.001)
# 训练循环
t5_base.train()
t5_base.to(device)
for epoch in range(num_epochs):
total_loss = 0
total_batches = 0
for batch in train_dataloader:
input_batch = []
target_batch = []
for input_text, target_text in zip(batch[0], batch[1]):
input_batch.append('summarize: ' + input_text[:5000])
target_batch.append(target_text)
input_batch = transform(input_batch)
target_batch = transform(target_batch)
input_batch = input_batch.to(device)
target_batch = target_batch.to(device)
optimizer.zero_grad()
sequence_generator = GenerationUtils(t5_base)
sequence_generator.device = device
beam_size = 1
output = sequence_generator.generate(input_batch, eos_idx=eos_idx, num_beams=beam_size, max_length=30)
logits = output.float().squeeze()
log_size = int(logits.numel())
target = target_batch.view(-1)
tar_size = int(target.numel())
if log_size > tar_size:
target = torch.nn.functional.pad(target, (1, log_size - tar_size - 1)).float()
else:
logits = torch.nn.functional.pad(logits, (0, tar_size - log_size)).float()
loss = criterion(logits, target)
loss.requires_grad = True
loss.backward()
optimizer.step()
total_loss += loss.item()
total_batches += 1
average_loss = total_loss / total_batches
print(f"Epoch {epoch + 1}/{num_epochs}, Loss: {average_loss}")
print('------------------------------------------------------')
你的结果显示了每个时代的损失，但损失似乎非常高且没有改变。可能需要检查模型架构、数据预处理或者学习率等方面，以解决损失不变的问题。希望这可以帮助你解决问题。如果你需要进一步的帮助，请随时提问。
英文:
I am trying to fine tune a T5 model for more accurate summarization, but my loss is very high and does not change with each epoch. I have tried increasing the learning rate, but the model still does not train. It seems like there is some issue with the code since the loss doesn't change at all. My input texts are very large, but I thought this would be fine given that T5 can already be used for summarization.
training code:
import torch
from torchtext.models import T5_BASE_GENERATION, T5Transform
from torchtext.prototype.generate import GenerationUtils
from torch.utils.data import DataLoader, Dataset
import torch.nn.functional as F
import json
import os
padding_idx = 0
eos_idx = 0
max_seq_len = 65536 #16384
t5_sp_model_path = "/t5_tokenizer_base.model"
# Define your custom dataset class
class CustomDataset(Dataset):
def __init__(self, data):
self.data = data
def __getitem__(self, index):
example = self.data[index]
input_text = example["input_text"]
target_text = example["target_text"]
return input_text, target_text
def __len__(self):
return len(self.data)
# Load labeled dataset
train_data = [] # List of labeled training examples
valid_data = [] # List of labeled validation examples
labeled = json.loads(open("data.json", 'r').read())
datalen = len(labeled)
valper = 0.2
train_ind = range(datalen - int(datalen * valper))
valid_ind = range(datalen - int(datalen * valper), datalen)
for x in train_ind:
train_data.append(labeled[str(x)])
for x in valid_ind:
valid_data.append(labeled[str(x)])
# Create instances of the T5 model and transformation
if os.path.exists('model.pt'):
t5_base = torch.load("model.pt")
else:
t5_base = T5_BASE_GENERATION.get_model()
transform = T5Transform(sp_model_path=t5_sp_model_path, max_seq_len=max_seq_len, eos_idx=eos_idx, padding_idx=padding_idx)
# Define the training parameters
device = torch.device("cuda") # if torch.cuda.is_available() else "cpu")
batch_size = 1
num_epochs = 5
# Convert your training and validation data into tensors
train_dataset = CustomDataset(train_data)
valid_dataset = CustomDataset(valid_data)
train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
valid_dataloader = DataLoader(valid_dataset, batch_size=batch_size)
# Define the loss function and optimizer
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(t5_base.parameters(), lr=0.001)
# Training loop
t5_base.train()
t5_base.to(device)
for epoch in range(num_epochs):
total_loss = 0
total_batches = 0
for batch in train_dataloader:
input_batch = []
target_batch = []
for input_text, target_text in zip(batch[0], batch[1]):
input_batch.append('summarize: ' + input_text[:5000])
target_batch.append(target_text)
#print(input_batch)
#print(target_batch)
#print('\n\n\n')
input_batch = transform(input_batch)
target_batch = transform(target_batch)
input_batch = input_batch.to(device)
target_batch = target_batch.to(device)
optimizer.zero_grad()
sequence_generator = GenerationUtils(t5_base)
sequence_generator.device = device
beam_size = 1
output = sequence_generator.generate(input_batch, eos_idx=eos_idx, num_beams=beam_size, max_length=30)
#print(' ')
#print(output)
#print('OUTPUT:', transform.decode(output.tolist()))
#print('TARGET:', target_batch)
#print('\n')
logits = output.float().squeeze()
log_size = int(logits.numel())
#print(logits)
#print(logits.shape)
#print('\n\n')
target = target_batch.view(-1)
tar_size = int(target.numel())
#print(target)
#print(target.shape)
if log_size > tar_size:
#print('Target Shorter')
target = torch.nn.functional.pad(target, (1, log_size - tar_size - 1)).float()
else:
#print('Input Shorter')
logits = torch.nn.functional.pad(logits, (0, tar_size - log_size)).float()
#print('\n')
#print(logits)
#print(logits.shape)
#print(target)
#print(target.shape)
#print('\n\n')
loss = criterion(logits, target)
#print('LOSS:', loss)
loss.requires_grad = True
loss.backward()
optimizer.step()
#print(f"Completed Batch #{total_batches}")
#print('\n')
total_loss += loss.item()
total_batches += 1
#print(total_loss, total_batches)
average_loss = total_loss / total_batches
print(f"Epoch {epoch + 1}/{num_epochs}, Loss: {average_loss}")
print('------------------------------------------------------')
print('\n')
torch.save(t5_base, "model.pt")
result:
Epoch 1/5, Loss: 1139705410.5365853
------------------------------------------------------
Epoch 2/5, Loss: 1139705410.5365853
------------------------------------------------------
Epoch 3/5, Loss: 1139705410.5365853
------------------------------------------------------
Epoch 4/5, Loss: 1139705410.5365853
------------------------------------------------------
Epoch 5/5, Loss: 1139705410.5365853
------------------------------------------------------
sample data.json (news article summary):
{
"0": {
"input_text": "Ukrainian President Volodymyr Zelenskiy attended a summit of the Arab League in Saudi Arabia on Friday to canvas support for his people, while Saudi Crown Prince Mohammed bin Salman expressed his readiness to mediate in the war between Moscow and Kyiv. Also at the Jeddah gathering, Arab leaders warmly welcomed back into their fold Syria\u2019s President Bashar al-Assad \u2014 who has received heavy support from Russia in his country\u2019s civil war \u2014 following a decade of isolation. \u201cWe reaffirm the kingdom\u2019s readiness to continue mediating efforts between Russia and Ukraine, and to support all international efforts aimed at resolving the crisis politically in a way that contributes to achieving security,\u201d the Saudi Crown Prince said in his opening speech. Prince Mohammed has mediated in the conflict before. Zelenskiy, who was also due to attend a summit of the G7 leaders in the Japanese city of Hiroshima this weekend, thanked Saudi Arabia for its past help and said delegates would each receive the text of his 10-point peace plan. He asked them to work with Ukraine directly without intermediaries. Gulf states have tried to remain neutral in the Ukraine conflict despite Western pressure on Gulf oil producers to help isolate Russia, a fellow OPEC+ member. Saving people In his address to the summit, Zelenskiy said some countries including members of the Arab League preferred to \u201cturn a blind eye\u201d to Russia\u2019s illegal annexation of Ukrainian land and to its jailing of some Ukrainians during the 15-month war. \u201cI am sure we can all be united in saving people from the cages of Russian prisons,\u201d he said, speaking in English. Last year, in a diplomatic coup, Crown Prince Mohammed secured the release of 10 foreigners captured by Russia in Ukraine. The move was apparently made possible by his close ties with Russian President Vladimir Putin. \u201cThe Kingdom of Saudi Arabia plays a significant role and we are ready to take our cooperation to a new level,\u201d Zelenskiy said wrote on Twitter shortly after arriving in Jeddah. Saudi Arabia faced heavy criticism from the United States over an OPEC+ decision to cut oil production, seen as helping Russia to refill its coffers by boosting prices. Even though the October decision initially drew the ire of the United States and other Western countries, market dynamics since then have shown the cuts to be prudent. At a time when Russia\u2019s war on Ukraine has roiled global energy markets, the role the kingdom plays as the world\u2019s largest oil exporter has grown in importance to both Washington and Moscow. KYIV, Ukraine (AP) \u2014 Ukrainian President Volodymyr Zelenskyy addressed a summit of Arab leaders in Saudi Arabia on Friday before what a senior official said would be a trip to Japan for a meeting with the leaders of the world\u2019s most powerful democracies. Zelenskyy has in recent months made foreign trips to shore up diplomatic support for Ukraine\u2019s fight against Russia\u2019s full-scale invasion almost 15 months ago and solicit more military support. He earlier this week returned from a three-day trip to Italy, the Vatican, Germany, France and the United Kingdom. Ukraine and Russia are squaring up for a major and potentially decisive phase of the war as Kyiv prepares an expected counteroffensive. The conflict has been bogged down in a war of attrition in recent months amid bad weather. Zelenskyy\u2019s office said he was invited to attend the Arab League summit in Jeddah, where he met with Saudi Crown Prince Mohammed bin Salman before holding other bilateral meetings. They discussed Zelenskyy\u2019s peace plan, the security situation in Ukraine and possible investments in the reconstruction of the country, a presidential statement said. Zelenskyy also invited Prince Mohammed to visit Ukraine. Zelenskyy urged leaders at the summit to resist Moscow\u2019s influence and consider his peace proposals, which include the withdrawal of the Kremlin\u2019s forces from occupied areas of Ukraine. \u201cI\u2019m more than sure that none of you will agree to surrender a third of your country to the invaders,\u201d Zelenskyy said in English. \u201cAnother priority is the protection of the Muslim community of Ukraine,\u201d Zelenskyy said. \u201cCrimea was the first to suffer from the Russian occupation, and most of those who suffer repression in occupied Crimea are Muslims.\u201d Crimean Tatar leader Mustafa Dzhemilev accompanied Zelenskyy on the visit. Zelenskyy will later travel to a Group of Seven summit in Japan, where leaders of the world\u2019s most powerful democracies aim to step up punishment on Russia for its full-scale invasion of Ukraine, according to Oleksiy Danilov, the secretary of Ukraine\u2019s National Security and Defense Council. However, Danilov\u2019s office later posted a statement backtracking on his announcement and saying Zelenskyy would appear at the G-7 summit via video. Zelenskyy\u2019s movements are kept secret for security reasons. Meanwhile, Russian forces kept up their long-range bombardment of Ukrainian targets while drones reportedly damaged train lines behind their front line. About 130 meters (430 feet) of railway track were damaged and trains were halted for hours after an explosion derailed eight cars of a freight train carrying grain in Russia-occupied Crimea, Russian state media reported Friday. Thursday\u2019s blast prompted renewed suspicions about possible Ukrainian saboteur activity behind Russian lines. Train traffic was also halted in northern Crimea on Thursday night after a drone hit a railway track near the town of Dzhankoi, Russia\u2019s Baza Telegram channel reported. Sergei Aksyonov, the Kremlin-appointed head of Crimea, said in a separate post that four Ukrainian drones were shot down overnight in the peninsula\u2019s north. Aksyonov claimed there was no damage or casualties. Russia overnight fired cruise missiles, drones and artillery at targets across Ukraine, killing two civilians, officials said Friday. The attacks included an air assault on Kyiv for the second straight day and the 10th time in three weeks. The Kremlin\u2019s forces also took aim at central, eastern and southern Ukraine, and the western Lviv region near the border with Poland. Russia launched 22 Iranian-made Shahed drones and six Kalibr cruise missiles during the night, the Ukrainian Air Force said. It said air defenses downed 16 drones and three missiles. The Russian shelling killed two civilians and wounded nine others in Ukraine\u2019s eastern Donetsk region, said its governor, Pavlo Kyrylenko. The missile attacks that have intensified recently aim to \u201cdisrupt Ukraine\u2019s plans and preparations for active military operations during the spring-summer campaign,\u201d according to a statement from Ukraine\u2019s intelligence agency, published on Telegram. The targets are Ukraine\u2019s military control points and barracks, supply routes and the places where ammunition, equipment, fuel are stored, it said. On Friday, the United Nations said operations to ship Ukrainian grain were \u201cpartially restarting,\u201d two days after Russia gave a green light to extend the deal for two months. The U.N. also urged a swift return to the previous tempo of ship arrivals and departures from all three Black Sea ports and inspections of their cargo. U.N. associate spokesperson Stephanie Tremblay said the Joint Coordination Center, which includes representatives from the four parties involved in the deal \u2013 Russia, Ukraine, Turkey and the United Nations -- approved the registration Friday of six new vessels to participate in the grain shipments. Nine applications to participate remain pending, she said. No ships are currently loading at any of the three ports, Tremblay said, but inspection teams from the center checked and cleared three new vessels Friday to proceed to the ports of Odesa and Chornomorsk. ___ Hanna Arhirova in Kyiv and Edith M. Lederer at the United Nations contributed to this report. ___ Follow AP\u2019s coverage of the war in Ukraine at https://apnews.com/hub/russia-ukraine President Zelenskyy makes a stop in Saudi Arabia on his way to Japan to meet with the G7. At the G7 increased sanctions against Russia are on the table, but negotiations still continue. A look at the battle in Bakhmut plus Crimean Tartars; expelled by the Soviets decades ago, now looking to return. TALLINN, Estonia (AP) \u2014 While the world awaits Ukraine\u2019s spring battlefield offensive, its leader, Volodymyr Zelenskyy, has launched a diplomatic one. In the span of a week, he's dashed to Italy, the Vatican, Germany, France and Britain to shore up support for defending his country. On Friday, he was in Saudi Arabia to meet with Arab leaders, some of whom are allies with Moscow. President Vladimir Putin, meanwhile, was in the southern Russian city of Pyatigorsk, chairing a meeting with local officials, sitting at a large table at a distance from the other attendees. The Russian president has faced unprecedented international isolation, with an International Criminal Court arrest warrant hanging over his head and clouding the prospects of traveling to many destinations, including those viewed as Moscow's allies. With his invasion of Ukraine, \u201cPutin took a gamble and lost really, really big time,\u201d said Theresa Fallon, director of the Brussels-based Centre for Russia Europe Asia Studies. \u201cHe is an international pariah, really.\u201d It was only 10 years ago when Putin stood proudly among his peers at the time -\u2013 Barack Obama, Angela Merkel and Shinzo Abe \u2013 at a Group of Eight summit in Northern Ireland. Russia has since been kicked out of the group, which consists of Canada, France, Germany, Italy, Japan, Britain and the United States, for illegally annexing Crimea in 2014. Now it appears to be Ukraine\u2019s turn in the spotlight. There were conflicting messages from Kyiv whether Zelenskyy would attend the G7 in Japan on Sunday. The secretary of Ukraine\u2019s National Security and Defense Council said on national television the president would be there, but the council later walked back those remarks, saying Zelenskyy would join via video link. The president\u2019s office would not confirm either way for security reasons. But whether in person or via video, it would be of great symbolic and geopolitical significance. \u201cIt conveys the fact that the G7 continues to strongly support Ukraine,\u201d said Nigel Gould-Davies, senior fellow for Russia and Eurasia at the International Institute for Strategic Studies. \u201cIt\u2019s a visible marker of the continued commitment of the most highly industrialized and highly developed countries in the world.\u201d Story continues It also comes at a time when the optics are just not in the Kremlin\u2019s favor. There\u2019s uncertainty over whether Putin can travel to South Africa in August for a summit of the BRICS nations of Brazil, Russia, India, China and South Africa. Moscow has long showcased the alliance as an alternative to the West\u2019s global dominance, but this year it is already proving awkward for the Kremlin. South Africa, the host of the summit, is a signatory to the ICC and is obligated to comply with the arrest warrant on war crimes charges. South Africa has not announced that Putin will definitely come to the summit but has been planning for his possible arrival. South African President Cyril Ramaphosa has appointed an inter-ministerial committee, led by Deputy President Paul Mashatile, to consider South Africa\u2019s options with regard to its ICC commitment over Putin\u2019s possible trip. While it is highly unlikely the Russian president would be arrested there if he decides to go, the public debate about whether he can is in itself \u201can unwelcome development whose impact should not be underestimated,\u201d according to Gould-Davies. Then there are Moscow\u2019s complicated relations with its own neighbors. Ten days ago, Putin projected the image of solidarity, with leaders of Armenia, Belarus and Central Asian states standing beside him at a Victory Day military parade on Red Square. This week, however, the leaders of Kazakhstan, Kyrgyzstan, Tajikistan, Turkmenistan and Uzbekistan flocked to China and met with leader Xi Jinping at a summit that highlighted the erosion of Russia\u2019s influence in the region as Beijing seeks to make economic inroads into Central Asia. Xi is using the opportunity \u201cof a weakened Russia, a distracted Russia, almost a pariah-state Russia to increase (China\u2019s) influence in the region,\u201d Fallon said. Putin\u2019s effort this month to shore up more friends in the South Caucasus by scrapping visa requirements for Georgian nationals and lifting a four-year ban on direct flights to the country also didn\u2019t appear to go as smoothly as the Kremlin may have hoped. The first flight that landed Friday in Georgia was met with protests, and the country\u2019s pro-Western president has decried the move as a provocation. Zelenskyy\u2019s ongoing world tour can be seen as a success on many levels. Invitations from other world leaders is a sign they think Ukraine is \"going to come out of the war in good shape,\u201d said Phillips P. O\u2019Brien, professor of strategic studies at the University of St. Andrews in Scotland. Otherwise, \u201cit simply wouldn\u2019t be happening,\u201d he said. \"No one would want to be around a leader they think is going to be defeated and a country that\u2019s going to collapse.\u201d By contrast, the ICC warrant might make it harder for leaders even to visit Putin in Moscow because \u201cit\u2019s not a good look to visit an indicted war criminal,\u201d Gould-Davies said. European leaders promised him an arsenal of missiles, tanks and drones, and even though no commitment has been made on fighter jets \u2013 something Kyiv has wanted for months \u2013 a conversation about finding ways to do it has begun. His appearance Friday at the Arab League summit in Jeddah, a Saudi Arabian port on the Red Sea, highlighted Kyiv\u2019s effort to spread its plight for support far and wide, including in some countries whose sympathies are with Russia. In addition to Zelenskyy, Saudi Crown Prince Mohammed bin Salman also welcomed Syrian President Bashar Assad at the summit after a 12-year suspension \u2013 something analysts say aligns with Moscow\u2019s interests. Anna Borshchevskaya, a senior fellow at the Washington Institute who focuses on Russia\u2019s policy in the Middle East, called it \u201canother testament to the fact that Russia is not isolated globally for its invasion of Ukraine, that the Middle East is one part of the world where Russia is able to find avenues to avoid global isolation \u2013 both ideological isolation but also economic isolation.\u201d She added that Zelenskyy and his government deserve credit for \u201cin recognizing that they need to reach out more to improve their diplomatic efforts in this part of the world and other parts of the world where the Russian narrative resonates.\u201d Kyiv could expect that \u201cthis is the beginning of a larger shift in perception that could eventually translate into potential support,\u201d Borshchevskaya said. Similarly, the Ukrainian president\u2019s participation in the G7 summit is \u201ca message to the rest of the world, to Russia and beyond, and the so-called Global South,\u201d Gould-Davies believes. There is a concern in the West over the extent to which some major developing economies \u2013 Brazil, South Africa and, to a degree, India \u2013 \u201care not criticizing, not condemning Russia and indeed in various ways are helping to mitigate the impact of sanctions on Russia,\u201d he said. \u201cCollectively, economically, they matter. So there is, I think, this felt need for a renewed diplomatic campaign to bring some of these most important states into the kind of the Western way of looking at these things,\u201d Gould-Davies said. ___ Associated Press writers Danica Kirka in London and Gerald Imray in Cape Town, South Africa, contributed. ___ Follow AP's coverage of the war in Ukraine at https://apnews.com/hub/russia-ukraine Syrian President Bashar al-Assad (L) is welcomed in Jeddah on the eve of the Arab League summit Ukrainian President Volodymyr Zelensky said he landed Friday in Saudi Arabia, host of an Arab League summit attended by long isolated Syrian President Bashar al-Assad, a close Russian ally. The previously unannounced visit is Zelensky's first to the Middle East since Moscow's invasion in February 2022, giving the Ukrainian leader an opportunity to address leaders in the region that has been far less united in its support of Kyiv than staunch Western allies. \"Arrived in Saudi Arabia. I will speak at the Arab League summit,\" Zelensky said on Twitter, adding he plans to meet with Saudi Crown Prince Mohammed bin Salman and other leaders. He arrived in the Red Sea coastal city of Jeddah one day after Assad, whose government is being readmitted to the Arab League after its suspension in 2011 over the brutal crackdown on pro-democracy demonstrators that led to civil war. The summit in Saudi Arabia comes at a time when the world's biggest oil exporter is flexing its diplomatic muscle across the Middle East and beyond. An Arab League official told AFP Zelenky's invitation came from Saudi Arabia, not the bloc.",
"target_text": "Ukranian President Zelenskyy attends Arab League summit in Saudi Arabia"
}
}
Any help would be greatly appreciated!
答案1
得分: 1
以下是翻译好的部分：
根据您关于这个问题的其他提问，您提到您是`PyTorch`的新手，我的答案反映了我在需要使用我不熟悉的新的机器学习库/算法时个人使用的一般方法。
由于T5是一个流行的算法，我尝试找到一个“模板”，展示如何对其进行摘要微调，其中我的代码代表了我通过阅读`transformers`库上T5文档时找到的这个[来源](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb#scrollTo=932p8NhxeNw4)的**适应**。最重要的是，我修改了`CustomDataset`类，以使其与您正在处理的数据兼容；此外，我还修改了一些返回警告的代码片段。请注意，该方法使用`PyTorch`进行训练，但使用`transformers`加载T5。
如下所示，示例代码在时间上“成功”地训练了T5，以获得更低的损失。当然，您仍然需要根据需要调整代码的某些部分（例如，将`max_sequence_size`增加到您的要求，将计算迁移到`GPU`，添加验证代码等）；尽管如此，我相信您将能够使用下面的示例，前述的来源，以及`PyTorch`和`Transformers`的官方用户指南来继续进行！
from transformers import T5Tokenizer, T5ForConditionalGeneration
import torch
from torch.utils.data import Dataset, DataLoader
input_data ={
"0": {
"input_text": "乌克兰总统弗拉基米尔·泽连斯基出席了沙特阿拉伯的阿拉伯联盟峰会，以争取对他的人民提供支持，而沙特王储穆罕默德·本·萨勒曼则表示愿意在莫斯科和基辅之间的战争中进行调解。在吉达聚会期间，阿拉伯领导人热烈欢迎叙利亚总统巴沙尔·阿萨德重新加入他们的行列——在十年的孤立后，俄罗斯在该国内战中对他的支持很大。沙特王储在开幕致辞中说：“我们重申王国继续调解俄罗斯和乌克兰之间的努力，支持所有旨在以政治方式解决危机的国际努力，以有助于实现安全。”穆罕默德王储此前曾在冲突中进行过调解。 泽连斯基还将出席日本广岛市的七国领导人峰会，他在那里感谢沙特阿拉伯过去的帮助，并表示代表会议的每位代表都将收到他的10点和平计划的文本。他要求他们直接与乌克兰合作，不经中介。尽管西方曾经向海湾石油生产国施压，要求其帮助孤立俄罗斯，但海湾国家一直试图在乌克兰冲突中保持中立，俄罗斯是石油输出国组织的成员。 在峰会上发表讲话时，泽连斯基说，一些国家，包括阿拉伯联盟成员国，更愿意对俄罗斯非法吞并乌克兰领土以及在15个月战争中监禁乌克兰人的行为“视而不见”。他说：“我相信我们都可以团结起来，拯救人们免受俄罗斯监狱的囚禁。” 去年，穆罕默德王子成功地争取到俄罗斯在乌克兰俘虏的10名外国人的释放。这一举措显然得益于他与俄罗斯总统弗拉基米尔·普京的密切关系。 泽连斯基在抵达吉达后不久在推特上写道：“沙特阿拉伯王国发挥着重要的作用，我们愿意将我们的合作提升到一个新的水平。” 美国因石油输出国组织的决定削减石油产量而严厉批评沙特阿拉伯，认为这有助于提高油价，从而帮助俄罗斯重新填补其财政缺口。尽管最初引起了美国和其他西方国家的愤怒，但自那时以来的市场动态表明这些削减是明智的。在俄罗斯对乌克兰的战争搅动了全球能源市场的时候，作为世界最大石油出口国的沙特阿拉伯的作用对华盛顿和莫斯科都变得越来越重要。乌克兰总统弗拉基米尔·泽连斯基在前往日本与世界最强大的民主国家领导人会晤之前在沙特阿拉伯停留。在即将满
<details>
<summary>英文:</summary>
Based on your [other question on this problem](https://stackoverflow.com/questions/76320189/pytorch-calculate-loss-with-smaller-target-tensor), where you mentioned that you're new to `PyTorch`, my answer reflects the general approach I personally use when I need to work with a new machine-learning library/algorithm I am unfamiliar with.
Since T5 is a popular algorithm, I tried to find a "template" showing how to fine-tune it for summarization, where my code represents an **adaptation** of this [source](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb#scrollTo=932p8NhxeNw4), which I found by reading the [documentation on T5](https://huggingface.co/docs/transformers/model_doc/t5#resources) of the [`transformers`](https://huggingface.co/docs/transformers/index) library. Most importantly, I modified the `CustomDataset` class so it's compatible with the data you're working with; also, I modified some pieces of the code which returned warnings. Please note that the approach uses `PyTorch` for training but loads T5 using `transformers`.
As shown below, the example code trains T5 "successfully" in terms of obtaining a lower loss over time. Of course you still need to adapt parts of the code (such as increasing `max_sequence_size` to your requirement, moving the computations to `GPU`, add the validation code, etc); nevertheless, I'm sure you will be able to take it from here using the example below, the aforementioned source, as well as the official user guides of `PyTorch` and `Transformers`!
from transformers import T5Tokenizer, T5ForConditionalGeneration
import torch
from torch.utils.data import Dataset, DataLoader
input_data ={
"0": {
"input_text": "Ukrainian President Volodymyr Zelenskiy attended a summit of the Arab League in Saudi Arabia on Friday to canvas support for his people, while Saudi Crown Prince Mohammed bin Salman expressed his readiness to mediate in the war between Moscow and Kyiv. Also at the Jeddah gathering, Arab leaders warmly welcomed back into their fold Syria\u2019s President Bashar al-Assad \u2014 who has received heavy support from Russia in his country\u2019s civil war \u2014 following a decade of isolation. \u201cWe reaffirm the kingdom\u2019s readiness to continue mediating efforts between Russia and Ukraine, and to support all international efforts aimed at resolving the crisis politically in a way that contributes to achieving security,\u201d the Saudi Crown Prince said in his opening speech. Prince Mohammed has mediated in the conflict before. Zelenskiy, who was also due to attend a summit of the G7 leaders in the Japanese city of Hiroshima this weekend, thanked Saudi Arabia for its past help and said delegates would each receive the text of his 10-point peace plan. He asked them to work with Ukraine directly without intermediaries. Gulf states have tried to remain neutral in the Ukraine conflict despite Western pressure on Gulf oil producers to help isolate Russia, a fellow OPEC+ member. Saving people In his address to the summit, Zelenskiy said some countries including members of the Arab League preferred to \u201cturn a blind eye\u201d to Russia\u2019s illegal annexation of Ukrainian land and to its jailing of some Ukrainians during the 15-month war. \u201cI am sure we can all be united in saving people from the cages of Russian prisons,\u201d he said, speaking in English. Last year, in a diplomatic coup, Crown Prince Mohammed secured the release of 10 foreigners captured by Russia in Ukraine. The move was apparently made possible by his close ties with Russian President Vladimir Putin. \u201cThe Kingdom of Saudi Arabia plays a significant role and we are ready to take our cooperation to a new level,\u201d Zelenskiy said wrote on Twitter shortly after arriving in Jeddah. Saudi Arabia faced heavy criticism from the United States over an OPEC+ decision to cut oil production, seen as helping Russia to refill its coffers by boosting prices. Even though the October decision initially drew the ire of the United States and other Western countries, market dynamics since then have shown the cuts to be prudent. At a time when Russia\u2019s war on Ukraine has roiled global energy markets, the role the kingdom plays as the world\u2019s largest oil exporter has grown in importance to both Washington and Moscow. KYIV, Ukraine (AP) \u2014 Ukrainian President Volodymyr Zelenskyy addressed a summit of Arab leaders in Saudi Arabia on Friday before what a senior official said would be a trip to Japan for a meeting with the leaders of the world\u2019s most powerful democracies. Zelenskyy has in recent months made foreign trips to shore up diplomatic support for Ukraine\u2019s fight against Russia\u2019s full-scale invasion almost 15 months ago and solicit more military support. He earlier this week returned from a three-day trip to Italy, the Vatican, Germany, France and the United Kingdom. Ukraine and Russia are squaring up for a major and potentially decisive phase of the war as Kyiv prepares an expected counteroffensive. The conflict has been bogged down in a war of attrition in recent months amid bad weather. Zelenskyy\u2019s office said he was invited to attend the Arab League summit in Jeddah, where he met with Saudi Crown Prince Mohammed bin Salman before holding other bilateral meetings. They discussed Zelenskyy\u2019s peace plan, the security situation in Ukraine and possible investments in the reconstruction of the country, a presidential statement said. Zelenskyy also invited Prince Mohammed to visit Ukraine. Zelenskyy urged leaders at the summit to resist Moscow\u2019s influence and consider his peace proposals, which include the withdrawal of the Kremlin\u2019s forces from occupied areas of Ukraine. \u201cI\u2019m more than sure that none of you will agree to surrender a third of your country to the invaders,\u201d Zelenskyy said in English. \u201cAnother priority is the protection of the Muslim community of Ukraine,\u201d Zelenskyy said. \u201cCrimea was the first to suffer from the Russian occupation, and most of those who suffer repression in occupied Crimea are Muslims.\u201d Crimean Tatar leader Mustafa Dzhemilev accompanied Zelenskyy on the visit. Zelenskyy will later travel to a Group of Seven summit in Japan, where leaders of the world\u2019s most powerful democracies aim to step up punishment on Russia for its full-scale invasion of Ukraine, according to Oleksiy Danilov, the secretary of Ukraine\u2019s National Security and Defense Council. However, Danilov\u2019s office later posted a statement backtracking on his announcement and saying Zelenskyy would appear at the G-7 summit via video. Zelenskyy\u2019s movements are kept secret for security reasons. Meanwhile, Russian forces kept up their long-range bombardment of Ukrainian targets while drones reportedly damaged train lines behind their front line. About 130 meters (430 feet) of railway track were damaged and trains were halted for hours after an explosion derailed eight cars of a freight train carrying grain in Russia-occupied Crimea, Russian state media reported Friday. Thursday\u2019s blast prompted renewed suspicions about possible Ukrainian saboteur activity behind Russian lines. Train traffic was also halted in northern Crimea on Thursday night after a drone hit a railway track near the town of Dzhankoi, Russia\u2019s Baza Telegram channel reported. Sergei Aksyonov, the Kremlin-appointed head of Crimea, said in a separate post that four Ukrainian drones were shot down overnight in the peninsula\u2019s north. Aksyonov claimed there was no damage or casualties. Russia overnight fired cruise missiles, drones and artillery at targets across Ukraine, killing two civilians, officials said Friday. The attacks included an air assault on Kyiv for the second straight day and the 10th time in three weeks. The Kremlin\u2019s forces also took aim at central, eastern and southern Ukraine, and the western Lviv region near the border with Poland. Russia launched 22 Iranian-made Shahed drones and six Kalibr cruise missiles during the night, the Ukrainian Air Force said. It said air defenses downed 16 drones and three missiles. The Russian shelling killed two civilians and wounded nine others in Ukraine\u2019s eastern Donetsk region, said its governor, Pavlo Kyrylenko. The missile attacks that have intensified recently aim to \u201cdisrupt Ukraine\u2019s plans and preparations for active military operations during the spring-summer campaign,\u201d according to a statement from Ukraine\u2019s intelligence agency, published on Telegram. The targets are Ukraine\u2019s military control points and barracks, supply routes and the places where ammunition, equipment, fuel are stored, it said. On Friday, the United Nations said operations to ship Ukrainian grain were \u201cpartially restarting,\u201d two days after Russia gave a green light to extend the deal for two months. The U.N. also urged a swift return to the previous tempo of ship arrivals and departures from all three Black Sea ports and inspections of their cargo. U.N. associate spokesperson Stephanie Tremblay said the Joint Coordination Center, which includes representatives from the four parties involved in the deal \u2013 Russia, Ukraine, Turkey and the United Nations -- approved the registration Friday of six new vessels to participate in the grain shipments. Nine applications to participate remain pending, she said. No ships are currently loading at any of the three ports, Tremblay said, but inspection teams from the center checked and cleared three new vessels Friday to proceed to the ports of Odesa and Chornomorsk. ___ Hanna Arhirova in Kyiv and Edith M. Lederer at the United Nations contributed to this report. ___ Follow AP\u2019s coverage of the war in Ukraine at https://apnews.com/hub/russia-ukraine President Zelenskyy makes a stop in Saudi Arabia on his way to Japan to meet with the G7. At the G7 increased sanctions against Russia are on the table, but negotiations still continue. A look at the battle in Bakhmut plus Crimean Tartars; expelled by the Soviets decades ago, now looking to return. TALLINN, Estonia (AP) \u2014 While the world awaits Ukraine\u2019s spring battlefield offensive, its leader, Volodymyr Zelenskyy, has launched a diplomatic one. In the span of a week, he's dashed to Italy, the Vatican, Germany, France and Britain to shore up support for defending his country. On Friday, he was in Saudi Arabia to meet with Arab leaders, some of whom are allies with Moscow. President Vladimir Putin, meanwhile, was in the southern Russian city of Pyatigorsk, chairing a meeting with local officials, sitting at a large table at a distance from the other attendees. The Russian president has faced unprecedented international isolation, with an International Criminal Court arrest warrant hanging over his head and clouding the prospects of traveling to many destinations, including those viewed as Moscow's allies. With his invasion of Ukraine, \u201cPutin took a gamble and lost really, really big time,\u201d said Theresa Fallon, director of the Brussels-based Centre for Russia Europe Asia Studies. \u201cHe is an international pariah, really.\u201d It was only 10 years ago when Putin stood proudly among his peers at the time -\u2013 Barack Obama, Angela Merkel and Shinzo Abe \u2013 at a Group of Eight summit in Northern Ireland. Russia has since been kicked out of the group, which consists of Canada, France, Germany, Italy, Japan, Britain and the United States, for illegally annexing Crimea in 2014. Now it appears to be Ukraine\u2019s turn in the spotlight. There were conflicting messages from Kyiv whether Zelenskyy would attend the G7 in Japan on Sunday. The secretary of Ukraine\u2019s National Security and Defense Council said on national television the president would be there, but the council later walked back those remarks, saying Zelenskyy would join via video link. The president\u2019s office would not confirm either way for security reasons. But whether in person or via video, it would be of great symbolic and geopolitical significance. \u201cIt conveys the fact that the G7 continues to strongly support Ukraine,\u201d said Nigel Gould-Davies, senior fellow for Russia and Eurasia at the International Institute for Strategic Studies. \u201cIt\u2019s a visible marker of the continued commitment of the most highly industrialized and highly developed countries in the world.\u201d Story continues It also comes at a time when the optics are just not in the Kremlin\u2019s favor. There\u2019s uncertainty over whether Putin can travel to South Africa in August for a summit of the BRICS nations of Brazil, Russia, India, China and South Africa. Moscow has long showcased the alliance as an alternative to the West\u2019s global dominance, but this year it is already proving awkward for the Kremlin. South Africa, the host of the summit, is a signatory to the ICC and is obligated to comply with the arrest warrant on war crimes charges. South Africa has not announced that Putin will definitely come to the summit but has been planning for his possible arrival. South African President Cyril Ramaphosa has appointed an inter-ministerial committee, led by Deputy President Paul Mashatile, to consider South Africa\u2019s options with regard to its ICC commitment over Putin\u2019s possible trip. While it is highly unlikely the Russian president would be arrested there if he decides to go, the public debate about whether he can is in itself \u201can unwelcome development whose impact should not be underestimated,\u201d according to Gould-Davies. Then there are Moscow\u2019s complicated relations with its own neighbors. Ten days ago, Putin projected the image of solidarity, with leaders of Armenia, Belarus and Central Asian states standing beside him at a Victory Day military parade on Red Square. This week, however, the leaders of Kazakhstan, Kyrgyzstan, Tajikistan, Turkmenistan and Uzbekistan flocked to China and met with leader Xi Jinping at a summit that highlighted the erosion of Russia\u2019s influence in the region as Beijing seeks to make economic inroads into Central Asia. Xi is using the opportunity \u201cof a weakened Russia, a distracted Russia, almost a pariah-state Russia to increase (China\u2019s) influence in the region,\u201d Fallon said. Putin\u2019s effort this month to shore up more friends in the South Caucasus by scrapping visa requirements for Georgian nationals and lifting a four-year ban on direct flights to the country also didn\u2019t appear to go as smoothly as the Kremlin may have hoped. The first flight that landed Friday in Georgia was met with protests, and the country\u2019s pro-Western president has decried the move as a provocation. Zelenskyy\u2019s ongoing world tour can be seen as a success on many levels. Invitations from other world leaders is a sign they think Ukraine is \"going to come out of the war in good shape,\u201d said Phillips P. O\u2019Brien, professor of strategic studies at the University of St. Andrews in Scotland. Otherwise, \u201cit simply wouldn\u2019t be happening,\u201d he said. \"No one would want to be around a leader they think is going to be defeated and a country that\u2019s going to collapse.\u201d By contrast, the ICC warrant might make it harder for leaders even to visit Putin in Moscow because \u201cit\u2019s not a good look to visit an indicted war criminal,\u201d Gould-Davies said. European leaders promised him an arsenal of missiles, tanks and drones, and even though no commitment has been made on fighter jets \u2013 something Kyiv has wanted for months \u2013 a conversation about finding ways to do it has begun. His appearance Friday at the Arab League summit in Jeddah, a Saudi Arabian port on the Red Sea, highlighted Kyiv\u2019s effort to spread its plight for support far and wide, including in some countries whose sympathies are with Russia. In addition to Zelenskyy, Saudi Crown Prince Mohammed bin Salman also welcomed Syrian President Bashar Assad at the summit after a 12-year suspension \u2013 something analysts say aligns with Moscow\u2019s interests. Anna Borshchevskaya, a senior fellow at the Washington Institute who focuses on Russia\u2019s policy in the Middle East, called it \u201canother testament to the fact that Russia is not isolated globally for its invasion of Ukraine, that the Middle East is one part of the world where Russia is able to find avenues to avoid global isolation \u2013 both ideological isolation but also economic isolation.\u201d She added that Zelenskyy and his government deserve credit for \u201cin recognizing that they need to reach out more to improve their diplomatic efforts in this part of the world and other parts of the world where the Russian narrative resonates.\u201d Kyiv could expect that \u201cthis is the beginning of a larger shift in perception that could eventually translate into potential support,\u201d Borshchevskaya said. Similarly, the Ukrainian president\u2019s participation in the G7 summit is \u201ca message to the rest of the world, to Russia and beyond, and the so-called Global South,\u201d Gould-Davies believes. There is a concern in the West over the extent to which some major developing economies \u2013 Brazil, South Africa and, to a degree, India \u2013 \u201care not criticizing, not condemning Russia and indeed in various ways are helping to mitigate the impact of sanctions on Russia,\u201d he said. \u201cCollectively, economically, they matter. So there is, I think, this felt need for a renewed diplomatic campaign to bring some of these most important states into the kind of the Western way of looking at these things,\u201d Gould-Davies said. ___ Associated Press writers Danica Kirka in London and Gerald Imray in Cape Town, South Africa, contributed. ___ Follow AP's coverage of the war in Ukraine at https://apnews.com/hub/russia-ukraine Syrian President Bashar al-Assad (L) is welcomed in Jeddah on the eve of the Arab League summit Ukrainian President Volodymyr Zelensky said he landed Friday in Saudi Arabia, host of an Arab League summit attended by long isolated Syrian President Bashar al-Assad, a close Russian ally. The previously unannounced visit is Zelensky's first to the Middle East since Moscow's invasion in February 2022, giving the Ukrainian leader an opportunity to address leaders in the region that has been far less united in its support of Kyiv than staunch Western allies. \"Arrived in Saudi Arabia. I will speak at the Arab League summit,\" Zelensky said on Twitter, adding he plans to meet with Saudi Crown Prince Mohammed bin Salman and other leaders. He arrived in the Red Sea coastal city of Jeddah one day after Assad, whose government is being readmitted to the Arab League after its suspension in 2011 over the brutal crackdown on pro-democracy demonstrators that led to civil war. The summit in Saudi Arabia comes at a time when the world's biggest oil exporter is flexing its diplomatic muscle across the Middle East and beyond. An Arab League official told AFP Zelenky's invitation came from Saudi Arabia, not the bloc.",
"target_text": "Ukranian President Zelenskyy attends Arab League summit in Saudi Arabia"
}
}
# duplicate single entry to get some additional "examples"
for i in range(1, 5):
input_data[str(i)] = dict()
input_data[str(i)]['input_text'] = input_data['0']['input_text']
input_data[str(i)]['target_text'] = input_data['0']['target_text']
class CustomDataset(Dataset):
def __init__(self, data_dict, tokenizer, max_sequence_size, max_summary_len):
self.tokenizer = tokenizer
self.data_dict = data_dict
self.max_sequence_size = max_sequence_size
self.max_summary_len = max_summary_len
self.input_text = [
"summarize: " + self.data_dict[key]['input_text']
for key in self.data_dict.keys()
]
self.target_text = [
self.data_dict[key]['target_text']
for key in self.data_dict.keys()
]
def __len__(self):
return len(self.input_text)
def __getitem__(self, index):
input_text_at_index = self.input_text[index]
target_text_at_index = self.target_text[index]
source = self.tokenizer.batch_encode_plus(
batch_text_or_text_pairs=[input_text_at_index],
max_length= self.max_sequence_size,
padding='max_length',
return_tensors='pt',
truncation=True
)
target = self.tokenizer.batch_encode_plus(
batch_text_or_text_pairs=[target_text_at_index],
max_length= self.max_summary_len,
padding='max_length',
return_tensors='pt',
truncation=True
)
source_ids = source['input_ids'].squeeze()
source_mask = source['attention_mask'].squeeze()
target_ids = target['input_ids'].squeeze()
target_mask = target['attention_mask'].squeeze()
return {
'source_ids': source_ids.to(dtype=torch.long),
'source_mask': source_mask.to(dtype=torch.long),
'target_ids': target_ids.to(dtype=torch.long),
'target_ids_y': target_ids.to(dtype=torch.long)
}
def train(epoch, tokenizer, model, loader, optimizer):
model.train()
for i, data in enumerate(loader, 0):
y = data['target_ids'] #.to(device, dtype = torch.long)
y_ids = y[:, :-1].contiguous()
lm_labels = y[:, 1:].clone().detach()
lm_labels[y[:, 1:] == tokenizer.pad_token_id] = -100
ids = data['source_ids'] #.to(device, dtype = torch.long)
mask = data['source_mask'] #.to(device, dtype = torch.long)
outputs = model(input_ids = ids, attention_mask = mask, decoder_input_ids=y_ids, labels=lm_labels)
loss = outputs[0]
print(f'Epoch: {epoch}, Loss: {loss.item()}')
optimizer.zero_grad()
loss.backward()
optimizer.step()
MAX_SEQUENCE_SIZE = 600
tokenizer = T5Tokenizer.from_pretrained(
"t5-base",
model_max_length=MAX_SEQUENCE_SIZE
)
model = T5ForConditionalGeneration.from_pretrained("t5-base")
dataset = CustomDataset(
data_dict = input_data,
tokenizer = tokenizer,
max_sequence_size=MAX_SEQUENCE_SIZE,
max_summary_len=128
)
# Defining the parameters for creation of dataloaders
train_params = {
'batch_size': 2,
'shuffle': True,
'num_workers': 0
}
training_loader = DataLoader(dataset, **train_params)
optimizer = torch.optim.Adam(
params=model.parameters(),
lr=0.001
)
for epoch in range(2):
train(
epoch=epoch,
tokenizer=tokenizer,
model=model,
loader=training_loader,
optimizer=optimizer
)
# Epoch: 0, Loss: 6.379357814788818
# Epoch: 0, Loss: 2.551377534866333
# Epoch: 0, Loss: 2.4488468170166016
# Epoch: 1, Loss: 1.555290937423706
# Epoch: 1, Loss: 0.6974927186965942
# Epoch: 1, Loss: 0.1120450347661972
</details>
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库，让每个人都能够通过互相帮助和分享经验来进步。
评论