基于深度学习的自然语言处理（Deep Learning-based Natural Language Processing）

pythonCopy codeimport tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
# 定义文本数据
texts = [
    'I love coding',
    'Coding is fun',
    'Programming is challenging'
]
# 对文本进行分词和编码
tokenizer = Tokenizer()
tokenizer.fit_on_texts(texts)
word_index = tokenizer.word_index
sequences = tokenizer.texts_to_sequences(texts)
# 对文本进行填充，使其长度一致
max_length = max([len(seq) for seq in sequences])
padded_sequences = pad_sequences(sequences, maxlen=max_length)
# 构建神经网络模型
model = keras.models.Sequential([
    keras.layers.Embedding(len(word_index)+1, 100, input_length=max_length),
    keras.layers.Bidirectional(keras.layers.LSTM(64)),
    keras.layers.Dense(1, activation='sigmoid')
])
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# 训练模型
model.fit(padded_sequences, [1, 1, 0], epochs=10)
# 使用模型进行预测
test_texts = [
    'I enjoy coding'
    'Programming is amazing'
]
test_sequences = tokenizer.texts_to_sequences(test_texts)
test_padded_sequences = pad_sequences(test_sequences, maxlen=max_length)
predictions = model.predict(test_padded_sequences)
print(predictions)

pythonCopy codeimport torch
import torch.nn as nn
import torch.optim as optim
from torchtext.datasets import AG_NEWS
from torchtext.data import Field, LabelField, BucketIterator
# 设置随机种子
torch.manual_seed(1234)
# 定义Field对象
TEXT = Field(tokenize='spacy', lower=True)
LABEL = LabelField(dtype=torch.long)
# 加载数据集
train_data, test_data = AG_NEWS.splits(TEXT, LABEL)
# 构建词汇表
TEXT.build_vocab(train_data, max_size=10000, vectors="glove.6B.100d")
LABEL.build_vocab(train_data)
# 定义模型
class TextClassifier(nn.Module):
    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, num_layers, bidirectional, dropout):
        super(TextClassifier, self).__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        self.rnn = nn.LSTM(embedding_dim, hidden_dim, num_layers=num_layers, bidirectional=bidirectional, dropout=dropout)
        self.fc = nn.Linear(hidden_dim * 2 if bidirectional else hidden_dim, output_dim)
        self.dropout = nn.Dropout(dropout)
    
    def forward(self, text):
        embedded = self.dropout(self.embedding(text))
        output, (hidden, cell) = self.rnn(embedded)
        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1)) if self.rnn.bidirectional else hidden[-1,:,:]
        return self.fc(hidden.squeeze(0))
# 初始化模型和优化器
vocab_size = len(TEXT.vocab)
embedding_dim = 100
hidden_dim = 256
output_dim = len(LABEL.vocab)
num_layers = 2
bidirectional = True
dropout = 0.5
model = TextClassifier(vocab_size, embedding_dim, hidden_dim, output_dim, num_layers, bidirectional, dropout)
optimizer = optim.Adam(model.parameters(), lr=1e-3)
criterion = nn.CrossEntropyLoss()
# 将数据分成批次
BATCH_SIZE = 64
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train_iterator, test_iterator = BucketIterator.splits(
    (train_data, test_data),
    batch_size=BATCH_SIZE,
    device=device
)
# 训练模型
model = model.to(device)
criterion = criterion.to(device)
for epoch in range(10):
    model.train()
    for batch in train_iterator:
        optimizer.zero_grad()
        text, label = batch.text, batch.label
        output = model(text).squeeze(1)
        loss = criterion(output, label)
        loss.backward()
        optimizer.step()
    print('Epoch:', epoch, 'Loss:', loss.item())
# 测试模型
model.eval()
correct = 0
total = 0
with torch.no_grad():
    for batch in test_iterator:
        text, label = batch.text, batch.label
        output = model(text).squeeze(1)
        _, predicted = torch.max(output, dim=1)
        total += label.size(0)
        correct += (predicted == label).sum().item()
print('Test Accuracy: {:.2f}%'.format(100 * correct / total))