深層学習の学習 【その5】

動くようになったと思っていた下記記事…

kuroyagi.hatenablog.com

実はちゃんと動いていませんでした。

lossは計算できていそうだったのですが、accuracyの計算が最初はnanで色々弄っても変わらず…

ということで必死に調べたら良さそうな例を見つけました。

www.iandprogram.net

これだとChainer ver.2でも動きます。ただし、細かいところでは所々書き換えないと上手く動かないので動くものを【ソースコード1】に示しておきます。

【ソースコード1】

import matplotlib.pyplot as plt
import numpy as np
from sklearn.datasets import fetch_mldata
import chainer
from chainer import cuda, Variable, optimizers, Chain
import chainer.functions  as F
import chainer.links as L
import sys

mnist = fetch_mldata("MNIST original", data_home=".")

# 学習用データを N個、検証用データを残りの個数と設定
batchsize = 100
N = 60000

# 学習の繰り返し回数
n_epoch   = 20

x_all = mnist['data'].astype(np.float32) / 255
y_all = mnist['target'].astype(np.int32)
x_train, x_test = np.split(x_all, [N])
y_train, y_test = np.split(y_all, [N])
N_test = y_test.size

class MLP(Chain):
    def __init__(self):
        super(MLP, self).__init__(
        l1=L.Linear(784, 100),
        l2=L.Linear(100, 100),
        l3=L.Linear(100, 10),
        )

    def __call__(self, x):
        h1 = F.dropout(F.relu(self.l1(x)))
        h2 = F.dropout(F.relu(self.l2(h1)))
        y = self.l3(h2)
        return y

class Classifier(Chain):
    def __init__(self, predictor):
        super(Classifier, self).__init__(predictor=predictor)
        def __call__(self, x, t):
            y = self.predictor(x)
            self.loss = F.softmax_cross_entropy(y, t)
            self.accuracy = F.accuracy(y, t)
            return self.loss

model = L.Classifier(MLP())
optimizer = optimizers.Adam()
optimizer.setup(model)



train_loss = np.array([])
train_acc = np.array([])
test_loss = np.array([])
test_acc  = np.array([])


for epoch in range(1, n_epoch+1):
    print('epoch %d' % epoch)
    indexes = np.random.permutation(N)

    # learning loop
    sum_loss = 0
    sum_acc = 0
    for i in range(0, N, batchsize):
        x = Variable(x_train[indexes[i : i + batchsize]])
        t = Variable(y_train[indexes[i : i + batchsize]])

        model.zerograds()
        loss = model(x, t)
        loss.backward()
        optimizer.update()

        train_loss = np.append(train_loss, model.loss.data)
        train_acc = np.append(train_acc, model.accuracy.data)

        sum_loss += float(cuda.to_cpu(model.loss.data)) * batchsize
        sum_acc += float(cuda.to_cpu(model.accuracy.data)) * batchsize
    # 訓練データの誤差と、正解精度を表示
    print('train mean loss={}, accuracy={}'.format(sum_loss / N, sum_acc / N))

    # test loop
    sum_loss = 0
    sum_acc = 0
    for i in range(0, N_test, batchsize):
        x = Variable(x_test[i : i + batchsize])
        t = Variable(y_test[i : i + batchsize])

        loss = model(x, t)

        test_loss = np.append(test_loss, model.loss.data)
        test_acc = np.append(test_acc, model.accuracy.data)

        sum_loss += float(cuda.to_cpu(model.loss.data)) * batchsize
        sum_acc += float(cuda.to_cpu(model.accuracy.data)) * batchsize
        # テストデータでの誤差と、正解精度を表示
    print('test  mean loss={}, accuracy={}'.format(sum_loss / N_test, sum_acc / N_test))

# 精度と誤差をグラフ描画
fig = plt.figure(1, figsize=(8,6))
ax1 = plt.subplot(2,2,1)
plt.plot(range(len(train_acc)), train_acc, color='tomato', linewidth=0.5)
plt.ylim([0.7, 1.0])
plt.title("Accuracy of digit recognition.")
ax1.patch.set_facecolor('lightgray')
plt.legend(["train_acc","train_acc"],loc=4)
ax1.grid(color='white')
plt.draw()

ax2 = plt.subplot(2,2,2)
plt.plot(range(len(test_acc)), test_acc, color='blue', linewidth=0.5)
plt.ylim([0.7, 1.0])
plt.title("Accuracy of digit recognition.")
ax2.patch.set_facecolor('lightgray')
plt.legend(["test_acc","test_acc"],loc=4)
ax2.grid(color='white')
plt.draw()

ax3 = plt.subplot(2,2,3)
plt.plot(range(len(train_loss)), train_loss, color='tomato', linewidth=0.5)
plt.ylim([0, 1.0])
plt.title("Accuracy of digit recognition.")
ax3.patch.set_facecolor('lightgray')
plt.legend(["train_loss","train_loss"],loc=2)
ax3.grid(color='white')
plt.draw()

ax4 = plt.subplot(2,2,4)
plt.plot(range(len(test_loss)), test_loss, color='blue', linewidth=0.5)
plt.ylim([0, 1.0])
plt.title("Loss of digit recognition of test.")
ax4.patch.set_facecolor('lightgray')
plt.legend(["loss_acc","loss_acc"],loc=2)
ax4.grid(color='white')
plt.show()

python慣れした方からすると稚拙なコーディングかもしれませんが、とりあえずご勘弁を。

結果は以下の通り。

epoch 1
train mean loss=0.6871868654588859, accuracy=0.786616666490833
test  mean loss=0.39174167409539223, accuracy=0.8857000035047531
epoch 2
train mean loss=0.35576792211582264, accuracy=0.89826666812102
test  mean loss=0.3195906089618802, accuracy=0.9073000049591065
epoch 3
train mean loss=0.2916684139644106, accuracy=0.9167333355545998
test  mean loss=0.2758765056356788, accuracy=0.9182000011205673
epoch 4
train mean loss=0.2618119852182766, accuracy=0.925100001692772
test  mean loss=0.2553010545670986, accuracy=0.9276000040769578
epoch 5
train mean loss=0.24032313730567695, accuracy=0.9301166690389315
test  mean loss=0.23504490087740124, accuracy=0.9305000054836273
epoch 6
train mean loss=0.2236388165752093, accuracy=0.9348666685819625
test  mean loss=0.2372814403101802, accuracy=0.9333000069856644
epoch 7
train mean loss=0.2108203659703334, accuracy=0.9385333358248075
test  mean loss=0.2233092796895653, accuracy=0.9367000049352646
epoch 8
train mean loss=0.20275122684116165, accuracy=0.9420666680733363
test  mean loss=0.22291907452046872, accuracy=0.9378000050783157
epoch 9
train mean loss=0.2000635218930741, accuracy=0.9412666672468185
test  mean loss=0.20354541193693876, accuracy=0.9415000063180924
epoch 10
train mean loss=0.18843867233023048, accuracy=0.9447833350300789
test  mean loss=0.22143425399437547, accuracy=0.9366000038385391
epoch 11
train mean loss=0.18182048660392564, accuracy=0.9462000024318695
test  mean loss=0.2139146727323532, accuracy=0.9403000020980835
epoch 12
train mean loss=0.17873456213002403, accuracy=0.9475833360354106
test  mean loss=0.21722277217544617, accuracy=0.943000003695488
epoch 13
train mean loss=0.1730892478550474, accuracy=0.948933335741361
test  mean loss=0.2017302264086902, accuracy=0.9448000031709671
epoch 14
train mean loss=0.17219340765848756, accuracy=0.9503333347042402
test  mean loss=0.21239303594455122, accuracy=0.9428000038862229
epoch 15
train mean loss=0.16510552507514756, accuracy=0.9506833347678184
test  mean loss=0.1887702170945704, accuracy=0.9479000049829484
epoch 16
train mean loss=0.1646082587291797, accuracy=0.9521000014742216
test  mean loss=0.20417385942302643, accuracy=0.9409000051021575
epoch 17
train mean loss=0.16335310517499843, accuracy=0.9516833351055781
test  mean loss=0.1972041698358953, accuracy=0.946600005030632
epoch 18
train mean loss=0.15869922854316731, accuracy=0.9528166691462199
test  mean loss=0.20000080622732638, accuracy=0.9453000068664551
epoch 19
train mean loss=0.15312283591677744, accuracy=0.9541666690508525
test  mean loss=0.2099972905218601, accuracy=0.9452000057697296
epoch 20
train mean loss=0.1568498678536465, accuracy=0.9533166686693827
test  mean loss=0.1955061467550695, accuracy=0.9453000032901764

参考にしてた元の記事と少し違うのですが何故だか分かりませんでした。ただし、ほとんど同じといえば同じ…一応出来たということで！

ちなみにグラフは以下の通りです。

【画像1:計算結果】

さて、ようやく続きを進めることが出来ます。

もともと参考にしていたのは【記事1】です。

【記事1】
qiita.com

次は答え合わせです。

ただ、やはり上手くいきません。model.l1を呼び出せないのでそこに相当する動作を探したところ以下に使える表現がありました。

【記事2】
ritsuan.com

ということでpredctionに相当するpredを定義してあげます。なんとなくフィーリングで直したらいけました。動くソースコードを載せておきます。

【ソースコード2】

import matplotlib.pyplot as plt
import numpy as np
from sklearn.datasets import fetch_mldata
import chainer
from chainer import cuda, Variable, optimizers, Chain
import chainer.functions  as F
import chainer.links as L
import sys

mnist = fetch_mldata("MNIST original", data_home=".")

# 学習用データを N個、検証用データを残りの個数と設定
batchsize = 100
N = 60000

# 学習の繰り返し回数
n_epoch   = 20

x_all = mnist['data'].astype(np.float32) / 255
y_all = mnist['target'].astype(np.int32)
x_train, x_test = np.split(x_all, [N])
y_train, y_test = np.split(y_all, [N])
N_test = y_test.size

class MLP(Chain):
    def __init__(self):
        super(MLP, self).__init__(
        l1=L.Linear(784, 100),
        l2=L.Linear(100, 100),
        l3=L.Linear(100, 10),
        )

    def __call__(self, x):
        h1 = F.dropout(F.relu(self.l1(x)))
        h2 = F.dropout(F.relu(self.l2(h1)))
        y = self.l3(h2)
        return y

class Classifier(Chain):
    def __init__(self, predictor):
        super(Classifier, self).__init__()
        with self.init_scope():
            self.predictor = predictor

    def __call__(self, x, t):
        y = self.predictor(x)
        loss = F.softmax_cross_entropy(y, t)
        accuracy = F.accuracy(y, t)
        report({'loss': loss, 'accuracy': accuracy}, self)
        return loss

model = L.Classifier(MLP())
optimizer = optimizers.Adam()
optimizer.setup(model)



train_loss = np.array([])
train_acc = np.array([])
test_loss = np.array([])
test_acc  = np.array([])


for epoch in range(1, n_epoch+1):
    print('epoch %d' % epoch)
    indexes = np.random.permutation(N)

    # learning loop
    sum_loss = 0
    sum_acc = 0
    for i in range(0, N, batchsize):
        x = Variable(x_train[indexes[i : i + batchsize]])
        t = Variable(y_train[indexes[i : i + batchsize]])

        model.zerograds()
        loss = model(x, t)
        loss.backward()
        optimizer.update()

        train_loss = np.append(train_loss, model.loss.data)
        train_acc = np.append(train_acc, model.accuracy.data)

        sum_loss += float(cuda.to_cpu(model.loss.data)) * batchsize
        sum_acc += float(cuda.to_cpu(model.accuracy.data)) * batchsize
    # 訓練データの誤差と、正解精度を表示
    print('train mean loss={}, accuracy={}'.format(sum_loss / N, sum_acc / N))

    # test loop
    sum_loss = 0
    sum_acc = 0
    for i in range(0, N_test, batchsize):
        x = Variable(x_test[i : i + batchsize])
        t = Variable(y_test[i : i + batchsize])

        loss = model(x, t)

        test_loss = np.append(test_loss, model.loss.data)
        test_acc = np.append(test_acc, model.accuracy.data)

        sum_loss += float(cuda.to_cpu(model.loss.data)) * batchsize
        sum_acc += float(cuda.to_cpu(model.accuracy.data)) * batchsize
        # テストデータでの誤差と、正解精度を表示
    print('test  mean loss={}, accuracy={}'.format(sum_loss / N_test, sum_acc / N_test))

# 精度と誤差をグラフ描画
nfig = 1
fig = plt.figure(nfig, figsize=(8,6))
ax1 = plt.subplot(2,2,1)
plt.plot(range(len(train_acc)), train_acc, color='tomato', linewidth=0.5)
plt.ylim([0.7, 1.0])
plt.title("Accuracy of digit recognition.")
ax1.patch.set_facecolor('lightgray')
plt.legend(["train_acc","train_acc"],loc=4)
ax1.grid(color='white')
plt.draw()

ax2 = plt.subplot(2,2,2)
plt.plot(range(len(test_acc)), test_acc, color='blue', linewidth=0.5)
plt.ylim([0.7, 1.0])
plt.title("Accuracy of digit recognition.")
ax2.patch.set_facecolor('lightgray')
plt.legend(["test_acc","test_acc"],loc=4)
ax2.grid(color='white')
plt.draw()

ax3 = plt.subplot(2,2,3)
plt.plot(range(len(train_loss)), train_loss, color='tomato', linewidth=0.5)
plt.ylim([0, 1.0])
plt.title("Accuracy of digit recognition.")
ax3.patch.set_facecolor('lightgray')
plt.legend(["train_loss","train_loss"],loc=2)
ax3.grid(color='white')
plt.draw()

ax4 = plt.subplot(2,2,4)
plt.plot(range(len(test_loss)), test_loss, color='blue', linewidth=0.5)
plt.ylim([0, 1.0])
plt.title("Loss of digit recognition of test.")
ax4.patch.set_facecolor('lightgray')
plt.legend(["loss_acc","loss_acc"],loc=2)
ax4.grid(color='white')
plt.draw()

# Result
# 手書き数字データを描画する関数
plt.style.use('fivethirtyeight')
def draw_digit(data, n, ans, recog):
    size = 28
    plt.subplot(10,10,n)

    X, Y = np.meshgrid(range(size),range(size))
    Z = data.reshape(size,size)   # convert from vector to 28x28 matrix
    Z = Z[::-1,:]             # flip vertical
    plt.xlim(0,27)
    plt.ylim(0,27)
    plt.title("ans=%d, recog=%d"%(ans,recog), size=8)
    plt.pcolor(X, Y, Z)
    plt.gray()
    plt.tick_params(labelbottom="off")
    plt.tick_params(labelleft="off")


nfig = nfig + 1
plt.figure(nfig,figsize=(16,16))


cnt = 0
for idx in np.random.permutation(N)[:100]:
    cnt += 1
    x = x_train[idx].astype(np.float32)
    pred = model.predictor(Variable(np.array(x.reshape((1, 784)), dtype=np.float32))).data
    draw_digit(x_train[idx], cnt, y_train[idx], pred.argmax(axis=1)[0])
plt.show()

以上を実行すると以下のグラフが得られます。

小さくて見えないかもしれないので適当に10個抜き出すと…

7か9か分からないのがありますね…

ひとまずここまでにします。

この後は各層での処理を可視化していく予定です。