1. PythonDeep LearningAkiba Shinya

2. TitleNameDateOUTLINEAbout meNeural NetworkDeep Learning- Auto Encoder- Dropout 3. TitleAbout meAkiba Shinya - M2Name- - @aki_n1wa Date 4. TitleNameDateDeep Learning(DL)slideshareDLDL(!pylearn2 5. TitleNameDateNeural Networkhttp://thinkit.co.jp/article/30/3/1!0 6. TitleNameDateNeural Network10 1210 7. TitleNameDateNeural Network()NN2828= 784from MNIST database(Mixed National Institute of Standards and Technology database)http://yann.lecun.com/exdb/mnist/ 8. TitleNameDateNeural Network3NeuralNetwork 78419610:x :h:yWx, Whb, c)MNISTNN784-800-800-10NN(NN) 9. TitleNameDateNeural Network3NeuralNetwork 784196 1100000000000000000100 10. TitleNameDateNeural NetworkNN1. 2. 11. TitleNameDateNeural NetworkNN1. 2. 3NN, Wx, b, Wh, c, Theano 12. TitleNameDateNeural NetworkTheanoPythonGPUhttp://deeplearning.net/software/theano/ 13. TitleNameDateNeural Networkimport theanoimport theano.tensor as Tfrom sklearn.datasets import fetch_mldatafrom sklearn import preprocessing!# sklearnmnist = fetch_mldata('MNIST original', data_home=".")X_digits, _, _, t_digits = mnist.values()data_size = 1000index = np.arange(70000)index = shuffle(index)[0:70000]x_digits = X_digits[index[0:data_size]]t_digits = t_digits[index[0:data_size]]!data_x = []# for A in x_digits:data_x.append(np.array([1 if x > 100 else 0 for x in A]))!# lb = preprocessing.LabelBinarizer()lb.fit([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])data_t = lb.transform(Y_digits) 14. TitleNameDateNeural Network# b, c = T.dvectors("b", "c")wx, wh = T.dmatrices("wx", "wh")t, y, x, h = T.dvectors('t', 'y', 'x', 'h')!# _wx = theano.shared(np.random.randn(196, 784), name="_wx")_wh = theano.shared(np.random.randn(10,196), name="_wh")_b = theano.shared(np.random.randn(196), name="_b")_c = theano.shared(np.random.randn(10), name=_c)!def sigmoid(x):return 1./(1+T.exp(-x))!h = sigmoid(T.dot(wx, x) + b)y = sigmoid(T.dot(wh, h) + c)!# eta = 0.01# cost_func = T.sum((t - y)**2) 15. TitleNameDateNeural Network# grad_cost = T.grad(cost=cost_func, wrt=[wx, b, wh, c])!# inputsupdatesgrad_cost_func = theano.function(inputs=[t, wh, bh, wx, bx, x],outputs=[],updates={_wx: _wx - eta * grad_cost[0],_b: _b - eta * grad_cost[1],_wh: _wh - eta * grad_cost[2],_c: _c - eta * grad_cost[3]})!train_count = 1000# for n in range(train_count):for x, t in zip(data_x, data_t):input = [t,_wh.get_value(),_bh.get_value(),_wx.get_value(),_bx.get_value(),x]grad_cost_func(input) 16. TitleNameDateNeural NetworkNNNN 100.0% 75.6%NN 17. TitleNameDateNeural NetworkNNNN 100.0% 75.6%NN Auto Encoder Dropout 18. TitleNameDateAuto EncoderAuto EncoderBackpropagationNN()!!Auto EncoderNN 19. TitleNameDateAuto Encoder196784784xxWx, b, Wx, b(Wx, bNN) 20. TitleNameDateAuto Encoder19678478419610196 21. TitleNameDateAuto Encoder# decodede_x = T.dvector(de_x")de_b = T.dvector(de_b")de_wx = T.dmatrix(de_wx)!# decodede_wx = theano.shared(np.random.randn(784, 196), name=_de_wx")de_b = theano.shared(np.random.randn(784), name=_de_b)!de_x = sigmoid(T.dot(de_wx, h) + de_b)pre1_cost_func = T.sum((de_x - x)**2)!# pre1_grad_cost = T.grad(cost=pre1_cost_func, wrt=[wx, b, de_wx, de_b])!# inputsupdatespre1_grad_cost_func = theano.function(inputs=[de_x, de_wx, de_b, wx, bx, x],outputs=[],updates={_wx: _wx - eta * pre1_grad_cost[0],_b: _b - eta * pre1_grad_cost[1],_de_wx: _wh - eta * pre1_grad_cost[2],_de_b: _c - eta * pre1_grad_cost[3]}) 22. TitleNameDateAuto EncoderAuto EncoderNN 100.0% 75.6%AE 99.7% 82.1% 23. TitleNameDateDropoutDropoutNN!NNDropout!0 24. TitleNameDate78419610020%050%0(NN)!NN()!L220%50%Dropout 25. TitleNameDateDropout# 0 or 1 [0, 1, 0, 1, 1, 1, 0, 1, , 1]bino_x = np.random.binomial(1, 0.8, 784)bino_h = np.random.binomial(1, 0.5, 196)!h = sigmoid(T.dot(wx, bino_x * x) + b)y = sigmoid(T.dot(wh, bion_h * h) + c) 26. TitleNameDateDropoutAuto Encoder + DropoutNN 100.0% 75.6%AE 99.7% 82.1%AE+Dropout 96.4% 84.2% 27. TitleNameDateMNIST60,00010,0004NN!DLAuto EncoderDropoutDL!NNDropout( 28. TitleNameDateG. E. Hinton, N. Srivastava, A. Krizhevsky, I. Sutskever and R. R. Salakhutdinov,University of Toronto2012, Improving neural networks by preventing co-adaptation of feature detectorshttp://arxiv.org/pdf/1207.0580.pdf!MarcAurelio Ranzato, Y-Lan Boureau, Yann LeCunNew York UniversitySparse Feature Learning for Deep Belief Networkshttp://yann.lecun.com/exdb/publis/pdf/ranzato-nips-07.pdf