服务器名称

CPU

内存

GPU

显存

服务器名称

系统

IP

CUDA版本

docker pull nvidia/cuda:8.0-devel-ubuntu16.04

nvidia-docker run -it --name caffe2 --net=host --hostname caffe2 --dns 8.8.8.8 –v /mnt:/mnt nvidia/cuda:8.0-devel-ubuntu16.04 bash

export http_proxy="http://192.168.5.18:3128"

export https_proxy="http://192.168.5.18:3128"

export ip_range=$(echo 192.168.79.{1..255} | sed 's/ /,/g')

export no_proxy="localhost,127.0.0.1,$ip_range,.huawei.com"

deb http://mirrors.163.com/ubuntu/ trusty main restricted universe multiverse

deb http://mirrors.163.com/ubuntu/ trusty-security main restricted universe multiverse

deb http://mirrors.163.com/ubuntu/ trusty-updates main restricted universe multiverse

deb http://mirrors.163.com/ubuntu/ trusty-proposed main restricted universe multiverse

deb http://mirrors.163.com/ubuntu/ trusty-backports main restricted universe multiverse

deb-src http://mirrors.163.com/ubuntu/ trusty main restricted universe multiverse

deb-src http://mirrors.163.com/ubuntu/ trusty-security main restricted universe multiverse

deb-src http://mirrors.163.com/ubuntu/ trusty-updates main restricted universe multiverse

deb-src http://mirrors.163.com/ubuntu/ trusty-proposed main restricted universe multiverse

deb-src http://mirrors.163.com/ubuntu/ trusty-backports main restricted universe multiverse

apt-get install openssh-server

apt-get install -y --no-install-recommends build-essential cmake git libgoogle-glog-dev libgtest-dev libiomp-dev libleveldb-dev liblmdb-dev libopencv-dev libopenmpi-dev libsnappy-dev libprotobuf-dev openmpi-bin openmpi-doc protobuf-compiler protobuf-c-compiler libgflags-dev python-dev python-pip python-setuptools graphviz

sudo pip install flask future hypothesis numpy protobuf pydot python-nvd3 pyyaml requests scikit-image scipy setuptools six tornado jupyter matplotlib pydot

echo "deb http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list

apt-get update

apt-get install -y --no-install-recommends libcudnn7=7.1.2.21-1+cuda8.0 libcudnn7-dev=7.1.2.21-1+cuda8.0

rm -rf /var/lib/apt/lists/*

tar -xzvf cudnn-8.0-linux-x64-v7.1.tgz

cd cuda/include/cudnn.h /usr/local/cuda/include/

cp cuda/lib64/libcudnn* /usr/local/cuda/lib64/

rm cudnn-8.0-linux-x64-v7.1.tgz && sudo ldconfig

export PATH=/usr/local/cuda-8.0/bin:$PATH

export LD_LIBRARY_PATH=/usr/local/cuda-8.0/lib64:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH

cd /opt/caffe2 && mkdir build && cd build

cmake ..

make install

export PYTHONPATH=/usr/local:$PYTHONPATH

export PYTHONPATH=/opt/caffe2/build:$PYTHONPATH

export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH

apt-get install nfs-kernel-server

mkdir /export

chmod 777 /export

tar –xzvf cmake-3.10.3.tar.gz

cd cmake-3.10.3

./boostrap

make

make install

export GIT_SSL_NO_VERIFY=1

sudo update-ca-certificates

echo -n | openssl s_client -showcerts -connect github.com:443 2>/dev/null | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p'

tar –xzvf protobuf-all-3.5.1.tar.gz

cd protobuf-3.5.1

./autogen.sh

./configure

make

make check

make install

ln -s /usr/local/bin/protoc /usr/bin/protoc

ln -s /usr/local/lib/libprotobuf.so /usr/lib64/libprotobuf.so

from __future__ import absolute_import

from __future__ import division

from __future__ import print_function

from __future__ import unicode_literals

from caffe2.python import core, workspace, model_helper, net_drawer, memonger, brew

from caffe2.python import data_parallel_model as dpm

from caffe2.python.models import resnet

from caffe2.proto import caffe2_pb2

import numpy as np

import time

import os

from IPython import display

workspace.GlobalInit(['caffe2', '--caffe2_log_level=2'])

# This section checks if you have the training and testing databases

current_folder = os.path.join(os.path.expanduser('~'), 'caffe2_notebooks')

data_folder = os.path.join(current_folder, 'tutorial_data', 'resnet_trainer')

# Train/test data

train_data_db = os.path.join(data_folder, "imagenet_cars_boats_train")

train_data_db_type = "lmdb"

# actually 640 cars and 640 boats = 1280

train_data_count = 1280

test_data_db = os.path.join(data_folder, "imagenet_cars_boats_val")

test_data_db_type = "lmdb"

# actually 48 cars and 48 boats = 96

test_data_count = 96

# Get the dataset if it is missing

def DownloadDataset(url, path):

    import requests, zipfile, StringIO

    print("Downloading {} ... ".format(url))

    r = requests.get(url, stream=True)

    z = zipfile.ZipFile(StringIO.StringIO(r.content))

    z.extractall(path)

    print("Done downloading to {}!".format(path))

# Make the data folder if it doesn't exist

if not os.path.exists(data_folder):

    os.makedirs(data_folder)

else:

    print("Data folder found at {}".format(data_folder))

# See if you already have to db, and if not, download it

if not os.path.exists(train_data_db):

    DownloadDataset("http://download.caffe2.ai/databases/resnet_trainer.zip", data_folder)

# Configure how you want to train the model and with how many GPUs

# This is set to use two GPUs in a single machine, but if you have more GPUs, extend the array [0, 1, 2, n]

gpus = [0]

# Batch size of 32 sums up to roughly 5GB of memory per device

batch_per_device = 32

total_batch_size = batch_per_device * len(gpus)

# This model discriminates between two labels: car or boat

num_labels = 2

# Initial learning rate (scale with total batch size)

base_learning_rate = 0.0004 * total_batch_size

# only intends to influence the learning rate after 10 epochs

stepsize = int(10 * train_data_count / total_batch_size)

# Weight decay (L2 regularization)

weight_decay = 1e-4

train_model = model_helper.ModelHelper(name="resnet_test")

workspace.ResetWorkspace()

reader = train_model.CreateDB("train_reader", db=train_data_db, db_type=train_data_db_type)

def add_image_input_ops(model):

    # utilize the ImageInput operator to prep the images

    data, label = brew.image_input(model,

        reader,

        ["data", "label"],

        batch_size=batch_per_device,

        # mean: to remove color values that are common

        mean=128.,

        # std is going to be modified randomly to influence the mean subtraction

        std=128.,

        # scale to rescale each image to a common size

        scale=256,

        # crop to the square each image to exact dimensions

        crop=224,

        # not running in test mode

        is_test=False,

        # mirroring of the images will occur randomly

        mirror=1

    )

    # prevent back-propagation: optional performance improvement; may not be observable at small scale

data = model.net.StopGradient(data, data)

def create_resnet50_model_ops(model, loss_scale=1.0):

    # Creates a residual network

    [softmax, loss] = resnet.create_resnet50(

        model,

        "data",

        num_input_channels=3,

        num_labels=num_labels,

        label="label",

    )

    prefix = model.net.Proto().name

    loss = model.net.Scale(loss, prefix + "_loss", scale=loss_scale)

    brew.accuracy(model, [softmax, "label"], prefix + "_accuracy")

return [loss]

def add_parameter_update_ops(model):

    brew.add_weight_decay(model, weight_decay)

    iter = brew.iter(model, "iter")

    lr = model.net.LearningRate(

        [iter],

        "lr",

        base_lr=base_learning_rate,

        policy="step",

        stepsize=stepsize,

        gamma=0.1,

    )

    # Momentum SGD update

    for param in model.GetParams():

        param_grad = model.param_to_grad[param]

        param_momentum = model.param_init_net.ConstantFill(

            [param], param + '_momentum', value=0.0

        )

        # Update param_grad and param_momentum in place

        model.net.MomentumSGDUpdate(

            [param_grad, param_momentum, lr, param],

            [param_grad, param_momentum, param],

            momentum=0.9,

            # Nesterov Momentum works slightly better than standard momentum

            nesterov=1,

        )

def optimize_gradient_memory(model, loss):

    model.net._net = memonger.share_grad_blobs(

        model.net,

        loss,

        set(model.param_to_grad.values()),

        # Due to memonger internals, we need a namescope here. Let's make one up; we'll need it later!

        namescope="imonaboat",

        share_activations=False)

# We need to give the network context and force it to run on the first GPU even if there are more.

device_opt = core.DeviceOption(caffe2_pb2.CUDA, gpus[0])

# Here's where that NameScope comes into play

with core.NameScope("imonaboat"):

    # Picking that one GPU

    with core.DeviceScope(device_opt):

        # Run our reader, and create the layers that transform the images

        add_image_input_ops(train_model)

        # Generate our residual network and return the losses

        losses = create_resnet50_model_ops(train_model)

        # Create gradients for each loss

        blobs_to_gradients = train_model.AddGradientOperators(losses)

        # Kick off the learning and managing of the weights

        add_parameter_update_ops(train_model)

    # Optimize memory usage by consolidating where we can

    optimize_gradient_memory(train_model, [blobs_to_gradients[losses[0]]])

# Startup the network

workspace.RunNetOnce(train_model.param_init_net)

# Load all of the initial weights; overwrite lets you run this multiple times

workspace.CreateNet(train_model.net, overwrite=True)

num_epochs = 1

for epoch in range(num_epochs):

    # Split up the images evenly: total images / batch size

    num_iters = int(train_data_count / total_batch_size)

    for iter in range(num_iters):

        # Stopwatch start!

        t1 = time.time()

        # Run this iteration!

        workspace.RunNet(train_model.net.Proto().name)

        t2 = time.time()

        dt = t2 - t1

        # Stopwatch stopped! How'd we do?

        print((

            "Finished iteration {:>" + str(len(str(num_iters))) + "}/{}" +

            " (epoch {:>" + str(len(str(num_epochs))) + "}/{})" +

            " ({:.2f} images/sec)").

            format(iter+1, num_iters, epoch+1, num_epochs, total_batch_size/dt))

192.168.133.10  caffe2

time python resnet50_trainer.py --train_data ~/caffe2_notebooks/tutorial_data/resnet_trainer/imagenet_cars_boats_train/ --test_data ~/caffe2_notebooks/tutorial_data/resnet_trainer/imagenet_cars_boats_val/ --gpus 0 --num_labels 2 --base_learning_rate 0.0384  --batch_size 32 --epoch_size 1280 --num_epochs 10 --num_shards 3 --shard_id 0 --run_id 1234 --file_store_path=/mnt/

time python resnet50_trainer.py --train_data ~/caffe2_notebooks/tutorial_data/resnet_trainer/imagenet_cars_boats_train/ --test_data ~/caffe2_notebooks/tutorial_data/resnet_trainer/imagenet_cars_boats_val/ --gpus 0 --num_labels 2 --base_learning_rate 0.0384  --batch_size 32 --epoch_size 1280 --num_epochs 10 --num_shards 3 --shard_id 1--run_id 1234 --file_store_path=/mnt/

time python resnet50_trainer.py --train_data ~/caffe2_notebooks/tutorial_data/resnet_trainer/imagenet_cars_boats_train/ --test_data ~/caffe2_notebooks/tutorial_data/resnet_trainer/imagenet_cars_boats_val/ --gpus 0 --num_labels 2 --base_learning_rate 0.0384  --batch_size 32 --epoch_size 1280 --num_epochs 10 --num_shards 3 --shard_id 2 --run_id 1234 --file_store_path=/mnt/

参数

个人理解

服务器数目

GPU/服务器

总GPU数目

单节点Batch size

时间（s）