DL Zoo 深度学习库

class Linear : public std::enable_shared_from_this<Linear> {
private:
    int in_features_;
    int out_features_;
    bool use_bias_;
    
    TensorPtr weight_;
    TensorPtr bias_;
    
public:
    Linear(int in_features, int out_features, bool use_bias = true)
        : in_features_(in_features), out_features_(out_features), use_bias_(use_bias) {
        // 使用Xavier初始化权重
        weight_ = Tensor::xavier(in_features, out_features, true);
        
        if (use_bias) {
            bias_ = Tensor::zeros(1, out_features, true);
        }
    }
    
    TensorPtr forward(TensorPtr input) {
        // input shape: [batch_size, in_features]
        // output shape: [batch_size, out_features]
        
        // y = input * weight + bias
        auto output = ops::matmul(input, weight_);
        
        if (use_bias_) {
            output = ops::add(output, bias_);
        }
        
        return output;
    }
    
    std::vector<TensorPtr> parameters() {
        std::vector<TensorPtr> params = {weight_};
        if (use_bias_) {
            params.push_back(bias_);
        }
        return params;
    }
    
    void zero_grad() {
        weight_->zero_grad();
        if (use_bias_) {
            bias_->zero_grad();
        }
    }
};

class Conv2d : public std::enable_shared_from_this<Conv2d> {
private:
    int in_channels_;
    int out_channels_;
    int kernel_size_;
    int stride_;
    int padding_;
    bool use_bias_;
    
    TensorPtr weight_;  // [out_channels, in_channels, kernel_size, kernel_size]
    TensorPtr bias_;
    
public:
    Conv2d(int in_channels, int out_channels, int kernel_size,
           int stride = 1, int padding = 0, bool use_bias = true)
        : in_channels_(in_channels), out_channels_(out_channels),
          kernel_size_(kernel_size), stride_(stride), padding_(padding),
          use_bias_(use_bias) {
        
        // 权重初始化：He初始化（适用于ReLU）
        double std = sqrt(2.0 / (in_channels * kernel_size * kernel_size));
        weight_ = Tensor::randn(out_channels, in_channels * kernel_size * kernel_size,
                                0, std, true);
        
        if (use_bias) {
            bias_ = Tensor::zeros(1, out_channels, true);
        }
    }
    
    TensorPtr forward(TensorPtr input) {
        // 这里需要实现实际的卷积操作
        // 为简化示例，使用im2col + gemm的方式
        // 完整实现需要处理4D张量 [batch, channels, height, width]
        
        // 伪代码，实际需要更复杂的实现
        auto output = /* 卷积操作 */;
        
        return output;
    }
    
    std::vector<TensorPtr> parameters() {
        std::vector<TensorPtr> params = {weight_};
        if (use_bias_) {
            params.push_back(bias_);
        }
        return params;
    }
    
    void zero_grad() {
        weight_->zero_grad();
        if (use_bias_) {
            bias_->zero_grad();
        }
    }
};

class MultiHeadAttention : public std::enable_shared_from_this<MultiHeadAttention> {
private:
    int d_model_;           // 模型维度
    int num_heads_;         // 头数
    int d_k_;               // 每头维度
    
    std::shared_ptr<Linear> w_q_;  // 查询投影
    std::shared_ptr<Linear> w_k_;  // 键投影
    std::shared_ptr<Linear> w_v_;  // 值投影
    std::shared_ptr<Linear> w_o_;  // 输出投影
    
    // 辅助函数：分割多头
    TensorPtr split_heads(TensorPtr x, int batch_size, int seq_len) {
        // x shape: [seq_len, batch_size, d_model]
        // reshape to [seq_len, batch_size, num_heads, d_k]
        // 然后转置为 [batch_size, num_heads, seq_len, d_k]
        
        // 这里需要实现reshape和转置
        return x;
    }
    
    // 辅助函数：合并多头
    TensorPtr combine_heads(TensorPtr x, int batch_size, int seq_len) {
        // x shape: [batch_size, num_heads, seq_len, d_k]
        // 转置并reshape回 [seq_len, batch_size, d_model]
        return x;
    }
    
public:
    MultiHeadAttention(int d_model, int num_heads) 
        : d_model_(d_model), num_heads_(num_heads), d_k_(d_model / num_heads) {
        
        assert(d_model % num_heads == 0);  // 必须能整除
        
        w_q_ = std::make_shared<Linear>(d_model, d_model);
        w_k_ = std::make_shared<Linear>(d_model, d_model);
        w_v_ = std::make_shared<Linear>(d_model, d_model);
        w_o_ = std::make_shared<Linear>(d_model, d_model);
    }
    
    TensorPtr forward(TensorPtr query, TensorPtr key, TensorPtr value, 
                      TensorPtr mask = nullptr) {
        // 获取形状信息
        int seq_len = query->rows();      // 序列长度
        int batch_size = query->cols();   // 批次大小
        
        // 1. 线性投影
        auto Q = w_q_->forward(query);  // [seq_len, batch_size, d_model]
        auto K = w_k_->forward(key);
        auto V = w_v_->forward(value);
        
        // 2. 分割多头
        auto Q_heads = split_heads(Q, batch_size, seq_len);  // [batch_size, num_heads, seq_len, d_k]
        auto K_heads = split_heads(K, batch_size, seq_len);
        auto V_heads = split_heads(V, batch_size, seq_len);
        
        // 3. 计算注意力分数: scores = Q * K^T / sqrt(d_k)
        // Q_heads: [batch_size, num_heads, seq_len, d_k]
        // K_heads^T: [batch_size, num_heads, d_k, seq_len]
        // scores: [batch_size, num_heads, seq_len, seq_len]
        
        // 这里需要实现批量矩阵乘法
        TensorPtr scores;  // = matmul(Q_heads, transpose(K_heads, -2, -1))
        
        // 缩放
        double scale = 1.0 / sqrt(d_k_);
        scores = ops::scalar_mul(scores, scale);
        
        // 4. 应用mask（如果有）
        if (mask != nullptr) {
            // mask shape: [batch_size, 1, 1, seq_len] 或 [batch_size, 1, seq_len, seq_len]
            // 将mask位置的分数设为负无穷
            scores = ops::add(scores, mask);
        }
        
        // 5. Softmax
        auto attn_weights = ops::softmax(scores, -1);  // 在最后一维做softmax
        
        // 6. 加权求和: output = attn_weights * V
        // attn_weights: [batch_size, num_heads, seq_len, seq_len]
        // V_heads: [batch_size, num_heads, seq_len, d_k]
        // output: [batch_size, num_heads, seq_len, d_k]
        auto output = ops::matmul(attn_weights, V_heads);
        
        // 7. 合并多头
        auto combined = combine_heads(output, batch_size, seq_len);  // [seq_len, batch_size, d_model]
        
        // 8. 输出投影
        return w_o_->forward(combined);
    }
    
    std::vector<TensorPtr> parameters() {
        auto params = w_q_->parameters();
        auto k_params = w_k_->parameters();
        auto v_params = w_v_->parameters();
        auto o_params = w_o_->parameters();
        
        params.insert(params.end(), k_params.begin(), k_params.end());
        params.insert(params.end(), v_params.begin(), v_params.end());
        params.insert(params.end(), o_params.begin(), o_params.end());
        
        return params;
    }
};

class Trainer {
private:
    std::shared_ptr<Model> model_;
    std::unique_ptr<AdamW> optimizer_;
    std::vector<std::unique_ptr<Callback>> callbacks_;
    
public:
    Trainer(std::shared_ptr<Model> model, double lr = 1e-3)
        : model_(model) {
        optimizer_ = std::make_unique<AdamW>(lr);
    }
    
    void add_callback(std::unique_ptr<Callback> callback) {
        callbacks_.push_back(std::move(callback));
    }
    
    template<typename DatasetType>
    void fit(DataLoader<DatasetType>& train_loader, int epochs) {
        // 训练开始回调
        for (auto& cb : callbacks_) {
            cb->on_train_begin();
        }
        
        for (int epoch = 0; epoch < epochs; epoch++) {
            // epoch开始回调
            for (auto& cb : callbacks_) {
                cb->on_epoch_begin(epoch);
            }
            
            double epoch_loss = 0.0;
            int batch_count = 0;
            
            // 遍历batch
            for (auto& batch : train_loader) {
                // batch开始回调
                for (auto& cb : callbacks_) {
                    cb->on_batch_begin(batch_count);
                }
                
                // 前向传播
                auto output = model_->forward(batch.input);
                
                // 计算损失
                auto loss = model_->loss(output, batch.target);
                
                // 反向传播
                optimizer_->zero_grad(model_->parameters());
                loss->backward();
                optimizer_->step(model_->parameters());
                
                epoch_loss += loss->item();
                batch_count++;
                
                // batch结束回调
                for (auto& cb : callbacks_) {
                    cb->on_batch_end(batch_count, loss->item());
                }
            }
            
            // epoch结束回调
            for (auto& cb : callbacks_) {
                cb->on_epoch_end(epoch, epoch_loss / batch_count);
            }
        }
        
        // 训练结束回调
        for (auto& cb : callbacks_) {
            cb->on_train_end();
        }
    }
};

class Callback {
public:
    virtual ~Callback() = default;
    virtual void on_epoch_begin(int epoch) {}
    virtual void on_epoch_end(int epoch, double loss) {}
    virtual void on_batch_begin(int batch) {}
    virtual void on_batch_end(int batch, double loss) {}
    virtual void on_train_begin() {}
    virtual void on_train_end() {}
};

// 日志回调
class LoggerCallback : public Callback {
private:
    std::ofstream log_file_;
    
public:
    LoggerCallback(const std::string& filename) {
        log_file_.open(filename);
    }
    
    void on_epoch_end(int epoch, double loss) override {
        std::cout << "Epoch " << epoch << " finished, loss: " << loss << std::endl;
        log_file_ << "Epoch " << epoch << ", loss: " << loss << std::endl;
    }
};

// 早停回调
class EarlyStoppingCallback : public Callback {
private:
    int patience_;
    int counter_ = 0;
    double best_loss_ = std::numeric_limits<double>::max();
    
public:
    EarlyStoppingCallback(int patience) : patience_(patience) {}
    
    void on_epoch_end(int epoch, double loss) override {
        if (loss < best_loss_) {
            best_loss_ = loss;
            counter_ = 0;
        } else {
            counter_++;
            if (counter_ >= patience_) {
                std::cout << "Early stopping triggered at epoch " << epoch << std::endl;
                // 这里可以抛出异常或设置标志位来停止训练
            }
        }
    }
};

#include "dl/models/model_zoo.hpp"

using namespace dl;

int main() {
    // 初始化模型动物园
    DLModelZoo zoo("./models");
    
    // 打印所有可用模型
    zoo.print_catalog();
    
    // 搜索特定任务的模型
    auto cv_models = zoo.search_by_task("Image Classification");
    std::cout << "找到 " << cv_models.size() << " 个图像分类模型" << std::endl;
    
    // 获取Top K模型
    auto top_nlp = zoo.get_top_k(3, "NLP");
    
    // 加载预训练模型
    auto resnet = zoo.load_model(pretrained::RESNET50);
    
    // 模型推理
    auto input = Tensor::randn(224 * 224 * 3, 1);  // 单张图片
    auto output = resnet->forward(input);
    
    // 获取预测结果
    auto probs = ops::softmax(output, 0);
    
    // 导出模型为ONNX
    zoo.export_model(pretrained::RESNET50, "onnx", "resnet50.onnx");
    
    // 导出为TensorRT引擎
    zoo.export_model(pretrained::YOLOV5S, "tensorrt", "yolov5s.engine");
    
    return 0;
}

#include "dl/core/tensor.hpp"
#include "dl/layers/conv.hpp"
#include "dl/layers/linear.hpp"
#include "dl/layers/dropout.hpp"
#include "dl/layers/sequential.hpp"
#include "dl/optimizers/adamw.hpp"
#include "dl/data/dataset.hpp"
#include "dl/training/trainer.hpp"
#include "dl/data/mnist_loader.hpp"

using namespace dl;

class MNISTCNN : public Model {
private:
    std::shared_ptr<Conv2d> conv1;
    std::shared_ptr<Conv2d> conv2;
    std::shared_ptr<Linear> fc1;
    std::shared_ptr<Linear> fc2;
    std::shared_ptr<Dropout> dropout;
    
public:
    MNISTCNN() {
        // 卷积层: 1x28x28 -> 32x14x14
        conv1 = std::make_shared<Conv2d>(1, 32, 3, 1, 1);
        
        // 卷积层: 32x14x14 -> 64x7x7
        conv2 = std::make_shared<Conv2d>(32, 64, 3, 1, 1);
        
        // 全连接层: 64*7*7 -> 128
        fc1 = std::make_shared<Linear>(64 * 7 * 7, 128);
        
        // 输出层: 128 -> 10
        fc2 = std::make_shared<Linear>(128, 10);
        
        // Dropout
        dropout = std::make_shared<Dropout>(0.5);
    }
    
    TensorPtr forward(TensorPtr x) override {
        // 输入x: [batch_size, 784] 需要reshape为 [batch_size, 1, 28, 28]
        
        // Conv1 + ReLU + MaxPool
        x = conv1->forward(x);
        x = ops::relu(x);
        x = max_pool2d(x, 2);  // 2x2池化
        
        // Conv2 + ReLU + MaxPool
        x = conv2->forward(x);
        x = ops::relu(x);
        x = max_pool2d(x, 2);
        
        // 展平
        x = flatten(x, 1);
        
        // FC1 + ReLU + Dropout
        x = fc1->forward(x);
        x = ops::relu(x);
        x = dropout->forward(x);
        
        // FC2 + Softmax
        x = fc2->forward(x);
        x = ops::softmax(x, 1);
        
        return x;
    }
    
    TensorPtr loss(TensorPtr output, TensorPtr target) override {
        // 交叉熵损失
        return ops::cross_entropy(output, target);
    }
    
    std::vector<TensorPtr> parameters() override {
        auto params = conv1->parameters();
        auto conv2_params = conv2->parameters();
        auto fc1_params = fc1->parameters();
        auto fc2_params = fc2->parameters();
        
        params.insert(params.end(), conv2_params.begin(), conv2_params.end());
        params.insert(params.end(), fc1_params.begin(), fc1_params.end());
        params.insert(params.end(), fc2_params.begin(), fc2_params.end());
        
        return params;
    }
};

int main() {
    // 加载MNIST数据集
    MNISTLoader loader("./data/mnist");
    auto train_dataset = loader.get_train_dataset();
    auto test_dataset = loader.get_test_dataset();
    
    DataLoader<MNISTDataset> train_loader(train_dataset, 64, true);  // batch_size=64, shuffle
    DataLoader<MNISTDataset> test_loader(test_dataset, 64, false);
    
    // 创建模型
    auto model = std::make_shared<MNISTCNN>();
    
    // 创建训练器
    Trainer trainer(model, 0.001);
    
    // 添加回调
    trainer.add_callback(std::make_unique<LoggerCallback>("mnist_training.log"));
    trainer.add_callback(std::make_unique<EarlyStoppingCallback>(5));
    
    // 训练
    std::cout << "开始训练..." << std::endl;
    trainer.fit(train_loader, 10);
    
    // 评估
    std::cout << "开始评估..." << std::endl;
    double accuracy = evaluate(model, test_loader);
    std::cout << "测试准确率: " << accuracy * 100 << "%" << std::endl;
    
    return 0;
}

#include "dl/models/transformer.hpp"
#include "dl/data/tokenizer.hpp"

using namespace dl;

int main() {
    // 配置Transformer
    TransformerConfig config;
    config.vocab_size = 50000;     // 词表大小
    config.d_model = 512;           // 模型维度
    config.num_heads = 8;           // 注意力头数
    config.num_encoder_layers = 6;  // 编码器层数
    config.num_decoder_layers = 6;  // 解码器层数
    config.d_ff = 2048;             // 前馈网络维度
    config.dropout = 0.1;           // Dropout概率
    config.max_seq_len = 512;        // 最大序列长度
    
    // 创建Transformer模型
    auto transformer = std::make_shared<Transformer>(config);
    
    // 创建分词器
    auto tokenizer = std::make_shared<BPETokenizer>("./models/vocab.json", "./models/merges.txt");
    
    // 准备输入文本
    std::string src_text = "Hello, how are you?";
    std::string tgt_text = "你好，你好吗？";
    
    // 编码
    auto src_ids = tokenizer->encode(src_text);  // [seq_len, 1]
    auto tgt_ids = tokenizer->encode(tgt_text);
    
    // 创建输入张量 (添加batch维度)
    auto src_tensor = Tensor::from_vector(src_ids, src_ids.size(), 1);
    auto tgt_tensor = Tensor::from_vector(tgt_ids, tgt_ids.size(), 1);
    
    // 创建mask
    auto src_mask = create_padding_mask(src_tensor);
    auto tgt_mask = create_look_ahead_mask(tgt_tensor);
    
    // 前向传播
    auto output = transformer->forward(src_tensor, tgt_tensor, src_mask, tgt_mask);
    
    // 获取预测结果
    auto logits = output;  // [seq_len, batch, vocab_size]
    
    // 解码
    auto predictions = ops::argmax(logits, -1);  // [seq_len, batch]
    
    // 将预测结果转换为文本
    std::vector<int> pred_ids;
    for (int i = 0; i < predictions->rows(); i++) {
        pred_ids.push_back(static_cast<int>(predictions->data()(i, 0)));
    }
    
    std::string translated = tokenizer->decode(pred_ids);
    std::cout << "翻译结果: " << translated << std::endl;
    
    // 训练示例
    Trainer trainer(transformer, 0.0001);
    
    // 创建翻译数据集
    auto translation_dataset = std::make_shared<TranslationDataset>(
        "data/train.src", "data/train.tgt", tokenizer, config.max_seq_len
    );
    
    DataLoader<TranslationDataset> train_loader(translation_dataset, 32, true);
    
    // 添加学习率调度器（预热+衰减）
    auto lr_scheduler = std::make_unique<TransformerLRScheduler>(0.0001, 4000);
    
    // 训练
    for (int epoch = 0; epoch < 10; epoch++) {
        double lr = lr_scheduler->get_lr(epoch * train_loader.size());
        trainer.set_learning_rate(lr);
        
        trainer.fit(train_loader, 1);
    }
    
    return 0;
}

cmake_minimum_required(VERSION 3.15)
project(DL_Zoo VERSION 2.0.0)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -O3 -march=native -flto")

# 选项
option(BUILD_EXAMPLES "Build examples" ON)
option(BUILD_TESTS "Build tests" ON)
option(USE_OPENMP "Use OpenMP parallelization" ON)
option(USE_CUDA "Use CUDA acceleration" OFF)

# 依赖
find_package(Eigen3 REQUIRED)
find_package(OpenCV QUIET)
find_package(OpenMP)

if(USE_CUDA)
    find_package(CUDAToolkit REQUIRED)
    enable_language(CUDA)
endif()

include_directories(${EIGEN3_INCLUDE_DIRS})
include_directories(include)

# 主库
file(GLOB_RECURSE LIB_SOURCES "src/*.cpp")
add_library(dl_zoo STATIC ${LIB_SOURCES})

target_include_directories(dl_zoo PUBLIC include)

target_link_libraries(dl_zoo
    ${EIGEN3_LIBRARIES}
)

if(OpenMP_FOUND AND USE_OPENMP)
    target_link_libraries(dl_zoo OpenMP::OpenMP_CXX)
endif()

# 示例
if(BUILD_EXAMPLES)
    add_subdirectory(examples)
endif()

# 测试
if(BUILD_TESTS)
    enable_testing()
    add_subdirectory(tests)
endif()

# 安装
install(TARGETS dl_zoo
    ARCHIVE DESTINATION lib
    LIBRARY DESTINATION lib
    RUNTIME DESTINATION bin
)

install(DIRECTORY include/ DESTINATION include)

# 创建构建目录
mkdir build && cd build

# 配置（Release版本）
cmake .. -DCMAKE_BUILD_TYPE=Release \
         -DBUILD_EXAMPLES=ON \
         -DUSE_OPENMP=ON

# 编译
make -j$(nproc)

# 运行示例
./examples/mnist_cnn
./examples/transformer_demo

# 安装到系统
sudo make install

// 梯度裁剪示例
void clip_gradients(std::vector<TensorPtr>& parameters, double max_norm) {
    double total_norm = 0.0;
    
    // 计算梯度范数
    for (auto& param : parameters) {
        if (param->requires_grad()) {
            total_norm += param->grad().squaredNorm();
        }
    }
    total_norm = std::sqrt(total_norm);
    
    // 裁剪
    if (total_norm > max_norm) {
        double clip_coef = max_norm / (total_norm + 1e-6);
        for (auto& param : parameters) {
            if (param->requires_grad()) {
                param->grad() *= clip_coef;
            }
        }
    }
}

# Fork仓库后
git clone https://github.com/your-username/dl_zoo.git
cd dl_zoo
git checkout -b feature/your-feature

# 开发...
# 运行测试
cd build
ctest

# 提交PR