pip install transformers

import torch 

from transformers import GPT2LMHeadModel, GPT2Tokenizer 

 

# 初始化tokenizer和model 

tokenizer = GPT2Tokenizer.from_pretrained('gpt2-medium') 

model = GPT2LMHeadModel.from_pretrained('gpt2-medium') 

 

# 设置要生成文本的起始文本 

prompt = "松山湖的扫地僧真是厉害，" 

input_ids = tokenizer.encode(prompt, return_tensors='pt') 

 

# 生成文本 

generated = 0 

num_generate = 10  # 生成文本的长度 

 

# 设置模型为评估模式 

model.eval() 

 

with torch.no_grad(): 

    while generated < num_generate: 

        output = model.generate(input_ids, max_length=len(input_ids[0]) + num_generate) 

        output = output[0, len(input_ids[0]):] 

        input_ids = torch.cat((input_ids, output), 

        text = tokenizer.decode(input_ids[0], skip_special_tokens=True) 

        generated += len(output) 

        print(text) 

 

        # 如果生成的文本包含句尾符号，可以提前结束生成 

        if tokenizer.eos_token_id in output: 

            break

pip install transformers torch

import torch 

from transformers import BertTokenizer, BertForSequenceClassification, Trainer, TrainingArguments 

from sklearn.model_selection import train_test_split 

from torch.utils.data import DataLoader, TensorDataset 

 

# 加载BERT tokenizer和预训练模型 

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') 

model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)  # 假设是二分类任务 

 

# 准备数据 

# 这里使用自定义的数据集

sentences = [ 

    "华为遥遥领先", 

    "人工智能未来的科技发展之路.", 

    "我应该在我的博客中多写一些关于大语言模型的算法吧", 

    # .

] 

labels = [1, 0, 1, # ... 对应每个句子的标签，1表示正面，0表示负面 

          # ... 更多标签 

         ] 

 

# 将文本转换为BERT模型的输入格式 

inputs = tokenizer(sentences, padding=True, truncation=True, return_tensors="pt") 

 

# 划分数据集为训练集和测试集 

inputs_train, inputs_test, labels_train, labels_test = train_test_split(inputs['input_ids'], torch.tensor(labels), test_size=0.2, random_state=42) 

 

# 创建DataLoader 

train_dataset = TensorDataset(inputs_train, labels_train) 

train_dataloader = DataLoader(train_dataset, batch_size=16, shuffle=True) 

 

# 设置训练参数 

training_args = TrainingArguments( 

    output_dir='./results',          # 输出目录 

    num_train_epochs=3,              # 训练轮数 

    per_device_train_batch_size=16, # 批量大小 

    warmup_steps=500,               # 学习率热身步数 

    weight_decay=0.01,              # 权重衰减 

    logging_dir='./logs',           # TensorBoard日志目录 

    logging_steps=10,               # 日志记录步数 

) 

 

# 定义训练函数（这里使用Transformers提供的Trainer类） 

trainer = Trainer( 

    model=model,                         # 模型 

    args=training_args,                 # 训练参数 

    train_dataset=train_dataset,        # 训练数据集 

) 

 

# 开始训练 

trainer.train() 

 

# 评估模型（这里仅作为示例，你可能需要添加自己的评估逻辑） 

trainer.evaluate() 

 

# 使用模型进行预测（示例） 

with torch.no_grad(): 

    predictions = model(**inputs_test).logits.argmax(-1) 

print(predictions)  # 输出预测的标签

import torch 

from transformers import T5Tokenizer, T5ForConditionalGeneration 

 

# 加载T5 tokenizer和预训练模型 

tokenizer = T5Tokenizer.from_pretrained('t5-small') 

model = T5ForConditionalGeneration.from_pretrained('t5-small') 

 

# 定义一个输入文本前缀 

input_text = "请说一下华为在在人工智能方面的优势：" 

 

# 使用tokenizer将输入文本转换为模型可接受的格式 

input_ids = tokenizer.encode(input_text, return_tensors='pt') 

 

# 生成文本 

output = model.generate(input_ids, max_length=50, num_beams=4, early_stopping=True) 

 

# 使用tokenizer将生成的ids解码为文本 

generated_text = tokenizer.decode(output[0], skip_special_tokens=True) 

 

print(generated_text)

import torch 

import torch.nn as nn 

import torch.optim as optim 

from transformers import AutoTokenizer, AutoModelForCausalLM 

from torch.utils.data import DataLoader, Dataset 

 

# 加载预训练模型和分词器 

tokenizer = AutoTokenizer.from_pretrained('gpt2-medium') 

model = AutoModelForCausalLM.from_pretrained('gpt2-medium') 

 

# 自定义数据集类 

class TextDataset(Dataset): 

    def __init__(self, texts): 

        self.texts = texts 

        self.encoded_texts = tokenizer(texts, padding=True, truncation=True, return_tensors='pt') 

 

    def __len__(self): 

        return len(self.texts) 

 

    def __getitem__(self, idx): 

        return self.encoded_texts['input_ids'][idx] 

 

# 加载和预处理数据集 

texts = [...] 

dataset = TextDataset(texts) 

dataloader = DataLoader(dataset, batch_size=32, shuffle=True) 

 

# 定义损失函数和优化器 

criterion = nn.CrossEntropyLoss(ignore_index=tokenizer.pad_token_id) 

optimizer = optim.Adam(model.parameters(), lr=1e-5) 

 

# 训练模型 

num_epochs = 10 

for epoch in range(num_epochs): 

    for batch in dataloader: 

        input_ids = batch 

        outputs = model(input_ids, labels=input_ids[:, 1:]) 

        loss = criterion(outputs.logits.view(-1, model.config.vocab_size), input_ids[:, 1:].view(-1)) 

         

        optimizer.zero_grad() 

        loss.backward() 

        optimizer.step() 

         

    print(f'Epoch {epoch+1}/{num_epochs}, Loss: {loss.item()}') 

 

# 保存模型 

torch.save(model.state_dict(), 'llm_model.pth')