Skip to content
Snippets Groups Projects
Commit ac962bf2 authored by marvnsch's avatar marvnsch
Browse files

Add changes

parent 8c22e8ee
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
LSTM_without_attention.ipynb
+ 40
− 67
View file @ ac962bf2
%% Cell type:code id:initial_id tags:
``` python
import torch
import torch.nn as nn
import torch.optim as optim
import random
from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.processors import TemplateProcessing
```
%% Cell type:markdown id:2b9477923b668978 tags:
# Data Preparation
%% Cell type:code id:dbc5f26f27746098 tags:
``` python
def load_data() -> tuple[list[str], list[str]]:
with open("data/training-data/eup/europarl-v7.de-en.de", "r", encoding="utf8") as f:
data_de = [line.rstrip("\n") for line in f]
with open("data/training-data/eup/europarl-v7.de-en.en", "r", encoding="utf8") as f:
data_en = [line.rstrip("\n") for line in f]
ltd = set() # save lines to delete later
for i in range(max(len(data_de), len(data_en))):
# Move sentence to next line if line is empty other file
if data_de[i] == "":
data_en[i+1] = data_en[i] + " " + data_en[i+1]
ltd.add(i)
if data_en[i] == "":
data_de[i+1] = data_de[i] + " " + data_de[i+1]
ltd.add(i)
# Remove lines, where difference in words is > 40%
if abs(count_words(data_de[i]) - count_words(data_en[i])) / (max(count_words(data_de[i]), count_words(data_en[i])) + 1) > 0.4:
ltd.add(i)
# Remove lines < 3 words or > 10 words
if max(count_words(data_de[i]), count_words(data_en[i])) < 3 or max(count_words(data_de[i]), count_words(data_en[i])) > 10:
ltd.add(i)
temp_de = [l for i, l in enumerate(data_de) if i not in ltd]
data_de = temp_de
temp_en = [l for i, l in enumerate(data_en) if i not in ltd]
data_en = temp_en
print(len(data_de),len(data_en))
# Print 3 random sentence pairs
ix = torch.randint(low=0, high=max(len(data_de), len(data_en)), size=(3, ))
for i in ix:
print(f"Zeile: {i}\nDeutsch: {data_de[i]}\nEnglish: {data_en[i]}\n")
print(f"\nNumber of lines: {len(data_de), len(data_en)}")
return data_de, data_en
def count_words(string: str) -> int:
return len(string.split())
de, en = load_data()
# setting the unknown token (e.g. for emojis)
tokenizer_en = Tokenizer(BPE(unk_token="[UNK]"))
tokenizer_de = Tokenizer(BPE(unk_token="[UNK]"))
# adding special tokens
# [UNK] : unknown word/token
# [CLS] : starting token (new sentence sequence)
# [SEP] : separator for chaining multiple sentences
# [PAD] : padding needed for encoder input
trainer = BpeTrainer(vocab_size=10000,
special_tokens=["[UNK]", "[SOS]", "[EOS]", "[PAD]"])
tokenizer_en.pre_tokenizer = Whitespace()
tokenizer_de.pre_tokenizer = Whitespace()
tokenizer_en.train(["data/training-data/eup/europarl-v7.de-en.en"], trainer)
tokenizer_de.train(["data/training-data/eup/europarl-v7.de-en.de"], trainer)
# configure post processing
tokenizer_en.post_processor = TemplateProcessing(
single="[SOS] $A [EOS]",
special_tokens=[
("[SOS]", tokenizer_en.token_to_id("[SOS]")),
("[EOS]", tokenizer_en.token_to_id("[EOS]")),
],
)
tokenizer_de.post_processor = TemplateProcessing(
single="[SOS] $A [EOS]",
special_tokens=[
("[SOS]", tokenizer_de.token_to_id("[SOS]")),
("[EOS]", tokenizer_de.token_to_id("[EOS]")),
],
)
target_vocab_size = tokenizer_de.get_vocab_size()
source_vocab_size = tokenizer_en.get_vocab_size()
```
%% Output
219633 219633
Zeile: 48925
Deutsch: Die Sicherheit sollte dabei einen hohen Stellenwert haben.
English: Safety should assume an important place.
Zeile: 183033
Deutsch: Aber der Bericht Azzolini bringt neue und wertvolle Denkanstöße.
English: But the Azzolini report contributes new and worthwhile approaches.
Zeile: 210873
Deutsch: Wir sind da also tief in Widersprüche verstrickt.
English: We are therefore overwhelmed with contradictions.
Number of lines: (219633, 219633)
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[7], line 84
73 tokenizer_de.post_processor = TemplateProcessing(
74 single="[SOS] $A [EOS]",
75 special_tokens=[
(...)
78 ],
79 )
83 from pathlib import Path
---> 84 workdir = Path(__file__).parent.absolute()
86 tokenizer_de.save(str(workdir / "tokenizer_de.json"))
88 target_vocab_size = tokenizer_de.get_vocab_size()
NameError: name '__file__' is not defined
%% Cell type:code id:dac2a6b0b10d6bdf tags:
``` python
# Define Device
if torch.cuda.is_available():
device = torch.device("cuda")
print("device: cuda")
elif torch.backends.mps.is_available():
device = torch.device("mps")
print("device: mps")
else:
device = torch.device("cpu")
print("device: cpu")
```
%% Cell type:code id:8edfacb67dc8c527 tags:
``` python
def training_data(source: list[str],
target: list[str],
dataset_size: int,
batch_size: int = 64,
sort: bool = True) -> tuple[torch.tensor, torch.tensor]:
tokenizer_de.no_padding()
tokenizer_en.no_padding()
if dataset_size > len(source):
raise IndexError("Dataset size is larger than the source data")
# sort the training data if true
if sort:
temp = ([list(a) for a in zip(source[:dataset_size], target[:dataset_size])])
temp.sort(key=lambda s: len(s[0]) + len(s[1]))
source, target = list(zip(*temp))
# select random sentences
for i in range(0, len(source) - batch_size, batch_size):
x_training_data = source[i:i + batch_size]
y_training_data = target[i:i + batch_size]
# tokenize data
tokenizer_en.enable_padding(pad_id=3)
x_training_data = tokenizer_en.encode_batch(x_training_data)
tokenizer_de.enable_padding(pad_id=3)
y_training_data = tokenizer_de.encode_batch(y_training_data)
# extract ids for every sequence
for j in range(batch_size):
x_training_data[j] = x_training_data[j].ids
y_training_data[j] = y_training_data[j].ids
# put data into tensor
x_training_data = torch.tensor(x_training_data, device=device)
y_training_data = torch.tensor(y_training_data, device=device)
# transpose tensors to match input requirements for lstm
x_training_data = torch.transpose(x_training_data, 0, 1)
y_training_data = torch.transpose(y_training_data, 0, 1)
yield x_training_data, y_training_data
```
%% Cell type:markdown id:ca6d3d436fd31e33 tags:
### Model Definition
%% Cell type:code id:3b2c4dbc74a1f144 tags:
``` python
# Prepare model
class Encoder(nn.Module):
def __init__(self, input_size: int, embedding_size: int,
hidden_size: int, num_layers: int):
super(Encoder, self).__init__()
self.num_layers = num_layers
self.embedding = nn.Embedding(input_size, embedding_size, device=device)
self.rnn = nn.LSTM(input_size=embedding_size, hidden_size=hidden_size,
num_layers=num_layers, device=device)
def forward(self, x):
# shape x : (sequence_len, batch_size)
embedding = self.embedding(x)
# shape embedding : sequence_len, batch_size, embedding_size)
output, (hidden, cell) = self.rnn(embedding)
return hidden, cell
class Decoder(nn.Module):
def __init__(self, input_size: int, embedding_size: int,
hidden_size: int, num_layers: int, output_size: int):
super(Decoder, self).__init__()
self.num_layers = num_layers
self.embedding = nn.Embedding(input_size, embedding_size, device=device)
self.rnn = nn.LSTM(input_size=embedding_size, hidden_size=hidden_size,
num_layers=num_layers, device=device)
self.fc = nn.Linear(hidden_size, output_size, device=device)
def forward(self, x, hidden, cell):
x = x.reshape(1, -1)
# shape x : (1, batch_size)
embedding = self.embedding(x)
# embedding shape : (1, batch_size, embedding_size)
dec_output, (hidden, cell) = self.rnn(embedding, (hidden, cell))
# shape output : (1, batch_size, hidden_size)
predictions = self.fc(dec_output)
# shape predictions : (1, batch_size, vocab_len)
predictions = predictions.squeeze(0)
return predictions, hidden, cell
class Seq2Seq(nn.Module):
def __init__(self, encoder: Encoder, decoder: Decoder):
super(Seq2Seq, self).__init__()
self.encoder = encoder
self.decoder = decoder
def forward(self, source, target = None, teacher_forcing_ratio: float = 0.5):
dec_batch_size = source.shape[1]
target_len = target.shape[0]
outputs = torch.zeros(target_len, dec_batch_size, target_vocab_size, device=device)
hidden, cell = self.encoder(source)
x = target[0]
for t in range(1, target_len):
output, hidden, cell = self.decoder(x, hidden, cell)
outputs[t] = output
best_guess = output.argmax(1)
if target is not None:
x = target[t] if random.random() < teacher_forcing_ratio else best_guess
else:
x = best_guess
return outputs
# DEBUG
# training hyperparameters
num_epochs = 50
learning_rate = 0.001
batch_size = 64
dataset_size = 100000
# model hyperparameters
input_size_encoder = source_vocab_size
input_size_decoder = target_vocab_size
output_size_decoder = target_vocab_size
encoder_embedding_size = 300
decoder_embedding_size = 300
model_hidden_size = 1024
model_num_layers = 2
encoder_net = Encoder(input_size=input_size_encoder,
embedding_size=encoder_embedding_size,
hidden_size=model_hidden_size,
num_layers=model_num_layers)
decoder_net = Decoder(input_size=input_size_decoder,
embedding_size=decoder_embedding_size,
hidden_size=model_hidden_size,
num_layers=model_num_layers,
output_size=output_size_decoder)
model = Seq2Seq(encoder=encoder_net, decoder=decoder_net)
model.train()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch + 1, num_epochs))
loss_value = 0
for batch_idx, (x_train, y_train) in enumerate(training_data(source=en,
target=de,
dataset_size=dataset_size,
batch_size=batch_size)):
optimizer.zero_grad()
predict = model(x_train, y_train)
predict = predict[1:].reshape(-1, predict.shape[2])
y_train = y_train[1:].reshape(-1)
#predict = predict.reshape(-1, predict.shape[2])
#y_train = y_train.reshape(-1)
loss = criterion(predict, y_train)
loss.backward()
optimizer.step()
loss_value += loss.item()
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1)
print("loss: " + str(loss_value / (dataset_size / batch_size)))
```
%% Output
Epoch 1/50
loss: 3.251950553474426
Epoch 2/50
loss: 2.5886474338912966
Epoch 3/50
loss: 2.2590867259407044
Epoch 4/50
loss: 1.9914781001472472
Epoch 5/50
loss: 1.7620788085746766
Epoch 6/50
loss: 1.526157800474167
Epoch 7/50
loss: 1.3238139695024491
Epoch 8/50
loss: 1.1473209317588806
Epoch 9/50
loss: 0.9837613027739525
Epoch 10/50
loss: 0.8556473323225975
Epoch 11/50
loss: 0.7408420387899876
Epoch 12/50
loss: 0.6378299428713322
Epoch 13/50
loss: 0.5609761751067638
Epoch 14/50
loss: 0.5038776994109154
Epoch 15/50
loss: 0.4588668634200096
Epoch 16/50
loss: 0.41355706704318523
Epoch 17/50
loss: 0.3838378005027771
Epoch 18/50
loss: 0.3672341006922722
Epoch 19/50
loss: 0.3412713519346714
Epoch 20/50
loss: 0.3302113852745295
Epoch 21/50
loss: 0.32545686868429186
Epoch 22/50
loss: 0.31084906596302986
Epoch 23/50
loss: 0.30543883962869645
Epoch 24/50
loss: 0.3089831199032068
Epoch 25/50
loss: 0.29750473050653936
Epoch 26/50
loss: 0.2859128334259987
Epoch 27/50
loss: 0.2887848159578443
Epoch 28/50
loss: 0.2810410741758347
Epoch 29/50
loss: 0.28322928114712237
Epoch 30/50
---------------------------------------------------------------------------
KeyboardInterrupt Traceback (most recent call last)
Cell In[11], line 112
106 loss_value = 0
108 for batch_idx, (x_train, y_train) in enumerate(training_data(source=en,
109 target=de,
110 dataset_size=dataset_size,
111 batch_size=batch_size)):
--> 112 optimizer.zero_grad()
114 predict = model(x_train, y_train)
115 predict = predict[1:].reshape(-1, predict.shape[2])
File _pydevd_bundle/pydevd_cython_darwin_311_64.pyx:1187, in _pydevd_bundle.pydevd_cython_darwin_311_64.SafeCallWrapper.__call__()
File _pydevd_bundle/pydevd_cython_darwin_311_64.pyx:627, in _pydevd_bundle.pydevd_cython_darwin_311_64.PyDBFrame.trace_dispatch()
File _pydevd_bundle/pydevd_cython_darwin_311_64.pyx:937, in _pydevd_bundle.pydevd_cython_darwin_311_64.PyDBFrame.trace_dispatch()
File _pydevd_bundle/pydevd_cython_darwin_311_64.pyx:928, in _pydevd_bundle.pydevd_cython_darwin_311_64.PyDBFrame.trace_dispatch()
File _pydevd_bundle/pydevd_cython_darwin_311_64.pyx:585, in _pydevd_bundle.pydevd_cython_darwin_311_64.PyDBFrame.do_wait_suspend()
File ~/Library/Application Support/JetBrains/Toolbox/apps/PyCharm-P/ch-0/233.11799.298/PyCharm.app/Contents/plugins/python/helpers/pydev/pydevd.py:1184, in PyDB.do_wait_suspend(self, thread, frame, event, arg, send_suspend_message, is_unhandled_exception)
1181 from_this_thread.append(frame_id)
1183 with self._threads_suspended_single_notification.notify_thread_suspended(thread_id, stop_reason):
-> 1184 self._do_wait_suspend(thread, frame, event, arg, suspend_type, from_this_thread)
File ~/Library/Application Support/JetBrains/Toolbox/apps/PyCharm-P/ch-0/233.11799.298/PyCharm.app/Contents/plugins/python/helpers/pydev/pydevd.py:1199, in PyDB._do_wait_suspend(self, thread, frame, event, arg, suspend_type, from_this_thread)
1196 self._call_mpl_hook()
1198 self.process_internal_commands()
-> 1199 time.sleep(0.01)
1201 self.cancel_async_evaluation(get_current_thread_id(thread), str(id(frame)))
1203 # process any stepping instructions
KeyboardInterrupt:
%% Cell type:markdown id:9854eaee8392caa1 tags:
### Model Parameters
%% Cell type:code id:a0d73467f967ecd9 tags:
``` python
from prettytable import PrettyTable
def count_parameters(model):
table = PrettyTable(["Modules", "Parameters"])
total_params = 0
for name, parameter in model.named_parameters():
if not parameter.requires_grad:
continue
params = parameter.numel()
table.add_row([name, params])
total_params += params
print(table)
print(f"Total Trainable Params: {total_params}")
return total_params
count_parameters(model)
```
%% Cell type:markdown id:ea6107f129162137 tags:
### Test the model
%% Cell type:code id:fa8a86342abe0a97 tags:
``` python
# test the model
test_sentence_en = "Can you reach out to me?"
test_sentence_de = "Kannst du mit mir in Kontakt treten."
test_sentence_en = "Why are you here?"
test_sentence_de = "Warum sind Sie hier?"
test_sentence_en_encoded = tokenizer_en.encode(test_sentence_en)
test_sentence_de_encoded = tokenizer_de.encode(test_sentence_de)
target_vector = torch.zeros(len(test_sentence_de_encoded.ids), 1)
model.eval()
x_test = torch.transpose(torch.tensor([test_sentence_en_encoded.ids], device=device), 0, 1)
y_test = torch.transpose(torch.tensor([test_sentence_de_encoded.ids], device=device), 0, 1)
print(y_test.shape)
prediction = model(x_test, y_test, teacher_forcing_ratio=0.0)
logits = torch.nn.functional.softmax(prediction, dim=2)
result_ids = logits.argmax(dim=2)
print(tokenizer_de.decode(list(result_ids)))
```
%% Output
torch.Size([7, 1])
Warum ist dabei Erfolg ?
......
models/RNN_no_attention_unidirectional.py deleted 100644 → 0
+ 0
− 10
View file @ 8c22e8ee
import torch
import torch.nn as nn
import torch.optim as optim
import random
from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.processors import TemplateProcessing
\ No newline at end of file
models/RNN_no_attention_unidirectional/RNN_no_attention_unidirectional.py 0 → 100644
+ 148
− 0
View file @ ac962bf2
from datetime import datetime
import random
from pathlib import Path
import torch
import torch.nn as nn
import torch.optim as optim
import utils.pytorch
import utils.training
project_root = Path(__file__).parent.parent.parent.absolute()
work_dir = Path(__file__).parent.absolute()
class Encoder(nn.Module):
def __init__(self, input_size: int, embedding_size: int,
hidden_size: int, num_layers: int, torch_device: torch.device):
super(Encoder, self).__init__()
self.num_layers = num_layers
self.embedding = nn.Embedding(input_size, embedding_size, device=torch_device)
self.rnn = nn.LSTM(input_size=embedding_size, hidden_size=hidden_size,
num_layers=num_layers, device=torch_device)
def forward(self, x):
# shape x : (sequence_len, batch_size)
embedding = self.embedding(x)
# shape embedding : sequence_len, batch_size, embedding_size)
output, (hidden, cell) = self.rnn(embedding)
return hidden, cell
class Decoder(nn.Module):
def __init__(self, input_size: int, embedding_size: int,
hidden_size: int, num_layers: int, output_size: int,
torch_device: torch.device):
super(Decoder, self).__init__()
self.num_layers = num_layers
self.embedding = nn.Embedding(input_size, embedding_size, device=torch_device)
self.rnn = nn.LSTM(input_size=embedding_size, hidden_size=hidden_size,
num_layers=num_layers, device=torch_device)
self.fc = nn.Linear(hidden_size, output_size, device=torch_device)
def forward(self, x, hidden, cell):
x = x.reshape(1, -1)
# shape x : (1, batch_size)
embedding = self.embedding(x)
# embedding shape : (1, batch_size, embedding_size)
dec_output, (hidden, cell) = self.rnn(embedding, (hidden, cell))
# shape output : (1, batch_size, hidden_size)
predictions = self.fc(dec_output)
# shape predictions : (1, batch_size, vocab_len)
predictions = predictions.squeeze(0)
return predictions, hidden, cell
class Seq2Seq(nn.Module):
def __init__(self, encoder: Encoder, decoder: Decoder,
torch_device: torch.device, target_vocab_size: int,
teacher_forcing_ratio: float = 0.5):
super(Seq2Seq, self).__init__()
self.target_vocab_size = target_vocab_size
self.torch_device = torch_device
self.teacher_forcing_ratio = teacher_forcing_ratio
self.encoder = encoder
self.decoder = decoder
def forward(self, source, target=None):
dec_batch_size = source.shape[1]
target_len = target.shape[0]
outputs = torch.zeros(target_len, dec_batch_size, self.target_vocab_size, device=self.torch_device)
hidden, cell = self.encoder(source)
x = target[0]
for t in range(1, target_len):
output, hidden, cell = self.decoder(x, hidden, cell)
outputs[t] = output
best_guess = output.argmax(1)
if target is not None and self.training:
x = target[t] if random.random() < self.teacher_forcing_ratio else best_guess
else:
x = best_guess
return outputs
# setup environment
source_data_path = str(project_root / "data/training-data/news-commentary-v11.en")
target_data_path = str(project_root / "data/training-data/news-commentary-v11.de")
device = utils.pytorch.get_available_device()
date_time_now = datetime.now().strftime("%m%d%Y_%H%M")
model_output_path = str(work_dir / f"./checkpoints/{date_time_now}RNN_no_attention_unidirectional")
# define hyperparameters
vocab_size = 10000
input_size_encoder = vocab_size
input_size_decoder = vocab_size
output_size_decoder = vocab_size
encoder_embedding_size = 300
decoder_embedding_size = 300
model_hidden_size = 1024
model_num_layers = 2
num_epochs = 50
learning_rate = 0.001
batch_size = 64
dataset_size = 100000
# create model
encoder_net = Encoder(input_size=input_size_encoder,
embedding_size=encoder_embedding_size,
hidden_size=model_hidden_size,
num_layers=model_num_layers,
torch_device=device)
decoder_net = Decoder(input_size=input_size_decoder,
embedding_size=decoder_embedding_size,
hidden_size=model_hidden_size,
num_layers=model_num_layers,
output_size=output_size_decoder,
torch_device=device)
model = Seq2Seq(encoder=encoder_net, decoder=decoder_net, torch_device=device, target_vocab_size=vocab_size)
model.train()
# prepare training run
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
# get training data
# train the model
utils.training.train(model=model,
train_loader=training_loader,
val_loader=validation_loader,
criterion=criterion,
optimizer=optimizer,
num_epochs=num_epochs,
saving_interval=1000,
model_output_path=model_output_path)
\ No newline at end of file
utils/pytorch.py 0 → 100644
+ 14
− 0
View file @ ac962bf2
import pytorch
def get_available_device() -> torch.device:
if torch.cuda.is_available():
device = torch.device("cuda")
print("device: cuda")
elif torch.backends.mps.is_available():
device = torch.device("mps")
print("device: mps")
else:
device = torch.device("cpu")
print("device: cpu")
return device
\ No newline at end of file
utils/training.py 0 → 100644
+ 61
− 0
View file @ ac962bf2
import torch
from progressbar import progressbar
def train(model, train_loader, val_loader,
criterion: torch.nn.modules.loss, optimizer: torch.optim,
num_epochs: int, saving_interval: int, model_output_path: str):
"""
Train a model based on training data and validation data
:param model: the model to train (nn.Module)
:param train_loader: the generator object containing the training data
:param val_loader: the generator object containing the validation data
:param criterion: the loss function to be applied to the model output
:param optimizer: the optimizer function to be applied to the model parameters
:param num_epochs: the number of training epochs
:param saving_interval: the number of steps after the model is saved
"""
save_counter = 0
model.train()
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch + 1, num_epochs))
for batch_idx, (x_train, y_train) in enumerate(progressbar(train_loader)):
optimizer.zero_grad()
predict = model(x_train, y_train)
predict = predict[1:].reshape(-1, predict.shape[2])
y_train = y_train[1:].reshape(-1)
loss = criterion(predict, y_train)
loss.backward()
optimizer.step()
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1)
if save_counter % saving_interval == 0:
# saving the model
torch.save(model.state_dict(), model_output_path)
# checking loss with validation data
loss_value = 0
val_batch_count = 0
for batch_idx, (x_val, y_val) in enumerate(val_loader):
with torch.no_grad():
predict = model(x_val, y_val)
predict = predict[1:].reshape(-1, predict.shape[2])
y_val = y_val[1:].reshape(-1)
loss = criterion(predict, y_val)
loss_value += loss.item()
val_batch_count += 1
print("--> loss : " + str(loss_value / val_batch_count))
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment