algorithm/Python
1
0
Fork 0
mirror of https://github.com/TheAlgorithms/Python.git synced 2025-07-06 10:31:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

increment 1

Browse Source
This commit is contained in:
Alex Brown
2018-10-19 07:48:28 -05:00
parent 718b99ae39
commit 564179a0ec
131 changed files with 16252 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

327
neural_network/FCN.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

193
neural_network/bpnn.py Normal file
View File

@ -0,0 +1,193 @@
#!/usr/bin/python
# encoding=utf8
'''
A Framework of Back Propagation Neural Network（BP） model
Easy to use:
* add many layers as you want ！！！
* clearly see how the loss decreasing
Easy to expand:
* more activation functions
* more loss functions
* more optimization method
Author: Stephen Lee
Github : https://github.com/RiptideBo
Date: 2017.11.23
'''
import numpy as np
import matplotlib.pyplot as plt
def sigmoid(x):
return 1 / (1 + np.exp(-1 * x))
class DenseLayer():
'''
Layers of BP neural network
'''
def __init__(self,units,activation=None,learning_rate=None,is_input_layer=False):
'''
common connected layer of bp network
:param units: numbers of neural units
:param activation: activation function
:param learning_rate: learning rate for paras
:param is_input_layer: whether it is input layer or not
'''
self.units = units
self.weight = None
self.bias = None
self.activation = activation
if learning_rate is None:
learning_rate = 0.3
self.learn_rate = learning_rate
self.is_input_layer = is_input_layer
def initializer(self,back_units):
self.weight = np.asmatrix(np.random.normal(0,0.5,(self.units,back_units)))
self.bias = np.asmatrix(np.random.normal(0,0.5,self.units)).T
if self.activation is None:
self.activation = sigmoid
def cal_gradient(self):
if self.activation == sigmoid:
gradient_mat = np.dot(self.output ,(1- self.output).T)
gradient_activation = np.diag(np.diag(gradient_mat))
else:
gradient_activation = 1
return gradient_activation
def forward_propagation(self,xdata):
self.xdata = xdata
if self.is_input_layer:
# input layer
self.wx_plus_b = xdata
self.output = xdata
return xdata
else:
self.wx_plus_b = np.dot(self.weight,self.xdata) - self.bias
self.output = self.activation(self.wx_plus_b)
return self.output
def back_propagation(self,gradient):
gradient_activation = self.cal_gradient() # i * i 维
gradient = np.asmatrix(np.dot(gradient.T,gradient_activation))
self._gradient_weight = np.asmatrix(self.xdata)
self._gradient_bias = -1
self._gradient_x = self.weight
self.gradient_weight = np.dot(gradient.T,self._gradient_weight.T)
self.gradient_bias = gradient * self._gradient_bias
self.gradient = np.dot(gradient,self._gradient_x).T
# ----------------------upgrade
# -----------the Negative gradient direction --------
self.weight = self.weight - self.learn_rate * self.gradient_weight
self.bias = self.bias - self.learn_rate * self.gradient_bias.T
return self.gradient
class BPNN():
'''
Back Propagation Neural Network model
'''
def __init__(self):
self.layers = []
self.train_mse = []
self.fig_loss = plt.figure()
self.ax_loss = self.fig_loss.add_subplot(1,1,1)
def add_layer(self,layer):
self.layers.append(layer)
def build(self):
for i,layer in enumerate(self.layers[:]):
if i < 1:
layer.is_input_layer = True
else:
layer.initializer(self.layers[i-1].units)
def summary(self):
for i,layer in enumerate(self.layers[:]):
print('------- layer %d -------'%i)
print('weight.shape ',np.shape(layer.weight))
print('bias.shape ',np.shape(layer.bias))
def train(self,xdata,ydata,train_round,accuracy):
self.train_round = train_round
self.accuracy = accuracy
self.ax_loss.hlines(self.accuracy, 0, self.train_round * 1.1)
x_shape = np.shape(xdata)
for round_i in range(train_round):
all_loss = 0
for row in range(x_shape[0]):
_xdata = np.asmatrix(xdata[row,:]).T
_ydata = np.asmatrix(ydata[row,:]).T
# forward propagation
for layer in self.layers:
_xdata = layer.forward_propagation(_xdata)
loss, gradient = self.cal_loss(_ydata, _xdata)
all_loss = all_loss + loss
# back propagation
# the input_layer does not upgrade
for layer in self.layers[:0:-1]:
gradient = layer.back_propagation(gradient)
mse = all_loss/x_shape[0]
self.train_mse.append(mse)
self.plot_loss()
if mse < self.accuracy:
print('----达到精度----')
return mse
def cal_loss(self,ydata,ydata_):
self.loss = np.sum(np.power((ydata - ydata_),2))
self.loss_gradient = 2 * (ydata_ - ydata)
# vector (shape is the same as _ydata.shape)
return self.loss,self.loss_gradient
def plot_loss(self):
if self.ax_loss.lines:
self.ax_loss.lines.remove(self.ax_loss.lines[0])
self.ax_loss.plot(self.train_mse, 'r-')
plt.ion()
plt.show()
plt.pause(0.1)
def example():
x = np.random.randn(10,10)
y = np.asarray([[0.8,0.4],[0.4,0.3],[0.34,0.45],[0.67,0.32],
[0.88,0.67],[0.78,0.77],[0.55,0.66],[0.55,0.43],[0.54,0.1],
[0.1,0.5]])
model = BPNN()
model.add_layer(DenseLayer(10))
model.add_layer(DenseLayer(20))
model.add_layer(DenseLayer(30))
model.add_layer(DenseLayer(2))
model.build()
model.summary()
model.train(xdata=x,ydata=y,train_round=100,accuracy=0.01)
if __name__ == '__main__':
example()

306
neural_network/convolution_neural_network.py Normal file
View File

@ -0,0 +1,306 @@
#-*- coding: utf-8 -*-
'''
- - - - - -- - - - - - - - - - - - - - - - - - - - - - -
Name - - CNN - Convolution Neural Network For Photo Recognizing
Goal - - Recognize Handing Writting Word Photo
Detail：Total 5 layers neural network
* Convolution layer
* Pooling layer
* Input layer layer of BP
* Hiden layer of BP
* Output layer of BP
Author: Stephen Lee
Github: 245885195@qq.com
Date: 2017.9.20
- - - - - -- - - - - - - - - - - - - - - - - - - - - - -
'''
from __future__ import print_function
import numpy as np
import matplotlib.pyplot as plt
class CNN():
def __init__(self,conv1_get,size_p1,bp_num1,bp_num2,bp_num3,rate_w=0.2,rate_t=0.2):
'''
:param conv1_get: [a,c,d]，size, number, step of convolution kernel
:param size_p1: pooling size
:param bp_num1: units number of flatten layer
:param bp_num2: units number of hidden layer
:param bp_num3: units number of output layer
:param rate_w: rate of weight learning
:param rate_t: rate of threshold learning
'''
self.num_bp1 = bp_num1
self.num_bp2 = bp_num2
self.num_bp3 = bp_num3
self.conv1 = conv1_get[:2]
self.step_conv1 = conv1_get[2]
self.size_pooling1 = size_p1
self.rate_weight = rate_w
self.rate_thre = rate_t
self.w_conv1 = [np.mat(-1*np.random.rand(self.conv1[0],self.conv1[0])+0.5) for i in range(self.conv1[1])]
self.wkj = np.mat(-1 * np.random.rand(self.num_bp3, self.num_bp2) + 0.5)
self.vji = np.mat(-1*np.random.rand(self.num_bp2, self.num_bp1)+0.5)
self.thre_conv1 = -2*np.random.rand(self.conv1[1])+1
self.thre_bp2 = -2*np.random.rand(self.num_bp2)+1
self.thre_bp3 = -2*np.random.rand(self.num_bp3)+1
def save_model(self,save_path):
#save model dict with pickle
import pickle
model_dic = {'num_bp1':self.num_bp1,
'num_bp2':self.num_bp2,
'num_bp3':self.num_bp3,
'conv1':self.conv1,
'step_conv1':self.step_conv1,
'size_pooling1':self.size_pooling1,
'rate_weight':self.rate_weight,
'rate_thre':self.rate_thre,
'w_conv1':self.w_conv1,
'wkj':self.wkj,
'vji':self.vji,
'thre_conv1':self.thre_conv1,
'thre_bp2':self.thre_bp2,
'thre_bp3':self.thre_bp3}
with open(save_path, 'wb') as f:
pickle.dump(model_dic, f)
print('Model saved： %s'% save_path)
@classmethod
def ReadModel(cls,model_path):
#read saved model
import pickle
with open(model_path, 'rb') as f:
model_dic = pickle.load(f)
conv_get= model_dic.get('conv1')
conv_get.append(model_dic.get('step_conv1'))
size_p1 = model_dic.get('size_pooling1')
bp1 = model_dic.get('num_bp1')
bp2 = model_dic.get('num_bp2')
bp3 = model_dic.get('num_bp3')
r_w = model_dic.get('rate_weight')
r_t = model_dic.get('rate_thre')
#create model instance
conv_ins = CNN(conv_get,size_p1,bp1,bp2,bp3,r_w,r_t)
#modify model parameter
conv_ins.w_conv1 = model_dic.get('w_conv1')
conv_ins.wkj = model_dic.get('wkj')
conv_ins.vji = model_dic.get('vji')
conv_ins.thre_conv1 = model_dic.get('thre_conv1')
conv_ins.thre_bp2 = model_dic.get('thre_bp2')
conv_ins.thre_bp3 = model_dic.get('thre_bp3')
return conv_ins
def sig(self,x):
return 1 / (1 + np.exp(-1*x))
def do_round(self,x):
return round(x, 3)
def convolute(self,data,convs,w_convs,thre_convs,conv_step):
#convolution process
size_conv = convs[0]
num_conv =convs[1]
size_data = np.shape(data)[0]
#get the data slice of original image data, data_focus
data_focus = []
for i_focus in range(0, size_data - size_conv + 1, conv_step):
for j_focus in range(0, size_data - size_conv + 1, conv_step):
focus = data[i_focus:i_focus + size_conv, j_focus:j_focus + size_conv]
data_focus.append(focus)
#caculate the feature map of every single kernel, and saved as list of matrix
data_featuremap = []
Size_FeatureMap = int((size_data - size_conv) / conv_step + 1)
for i_map in range(num_conv):
featuremap = []
for i_focus in range(len(data_focus)):
net_focus = np.sum(np.multiply(data_focus[i_focus], w_convs[i_map])) - thre_convs[i_map]
featuremap.append(self.sig(net_focus))
featuremap = np.asmatrix(featuremap).reshape(Size_FeatureMap, Size_FeatureMap)
data_featuremap.append(featuremap)
#expanding the data slice to One dimenssion
focus1_list = []
for each_focus in data_focus:
focus1_list.extend(self.Expand_Mat(each_focus))
focus_list = np.asarray(focus1_list)
return focus_list,data_featuremap
def pooling(self,featuremaps,size_pooling,type='average_pool'):
#pooling process
size_map = len(featuremaps[0])
size_pooled = int(size_map/size_pooling)
featuremap_pooled = []
for i_map in range(len(featuremaps)):
map = featuremaps[i_map]
map_pooled = []
for i_focus in range(0,size_map,size_pooling):
for j_focus in range(0, size_map, size_pooling):
focus = map[i_focus:i_focus + size_pooling, j_focus:j_focus + size_pooling]
if type == 'average_pool':
#average pooling
map_pooled.append(np.average(focus))
elif type == 'max_pooling':
#max pooling
map_pooled.append(np.max(focus))
map_pooled = np.asmatrix(map_pooled).reshape(size_pooled,size_pooled)
featuremap_pooled.append(map_pooled)
return featuremap_pooled
def _expand(self,datas):
#expanding three dimension data to one dimension list
data_expanded = []
for i in range(len(datas)):
shapes = np.shape(datas[i])
data_listed = datas[i].reshape(1,shapes[0]*shapes[1])
data_listed = data_listed.getA().tolist()[0]
data_expanded.extend(data_listed)
data_expanded = np.asarray(data_expanded)
return data_expanded
def _expand_mat(self,data_mat):
#expanding matrix to one dimension list
data_mat = np.asarray(data_mat)
shapes = np.shape(data_mat)
data_expanded = data_mat.reshape(1,shapes[0]*shapes[1])
return data_expanded
def _calculate_gradient_from_pool(self,out_map,pd_pool,num_map,size_map,size_pooling):
'''
calcluate the gradient from the data slice of pool layer
pd_pool: list of matrix
out_map: the shape of data slice(size_map*size_map)
return: pd_all: list of matrix, [num, size_map, size_map]
'''
pd_all = []
i_pool = 0
for i_map in range(num_map):
pd_conv1 = np.ones((size_map, size_map))
for i in range(0, size_map, size_pooling):
for j in range(0, size_map, size_pooling):
pd_conv1[i:i + size_pooling, j:j + size_pooling] = pd_pool[i_pool]
i_pool = i_pool + 1
pd_conv2 = np.multiply(pd_conv1,np.multiply(out_map[i_map],(1-out_map[i_map])))
pd_all.append(pd_conv2)
return pd_all
def trian(self,patterns,datas_train, datas_teach, n_repeat, error_accuracy,draw_e = bool):
#model traning
print('----------------------Start Training-------------------------')
print((' - - Shape: Train_Data ',np.shape(datas_train)))
print((' - - Shape: Teach_Data ',np.shape(datas_teach)))
rp = 0
all_mse = []
mse = 10000
while rp < n_repeat and mse >= error_accuracy:
alle = 0
print('-------------Learning Time %d--------------'%rp)
for p in range(len(datas_train)):
#print('------------Learning Image: %d--------------'%p)
data_train = np.asmatrix(datas_train[p])
data_teach = np.asarray(datas_teach[p])
data_focus1,data_conved1 = self.convolute(data_train,self.conv1,self.w_conv1,
self.thre_conv1,conv_step=self.step_conv1)
data_pooled1 = self.pooling(data_conved1,self.size_pooling1)
shape_featuremap1 = np.shape(data_conved1)
'''
print(' -----original shape ', np.shape(data_train))
print(' ---- after convolution ',np.shape(data_conv1))
print(' -----after pooling ',np.shape(data_pooled1))
'''
data_bp_input = self._expand(data_pooled1)
bp_out1 = data_bp_input
bp_net_j = np.dot(bp_out1,self.vji.T) - self.thre_bp2
bp_out2 = self.sig(bp_net_j)
bp_net_k = np.dot(bp_out2 ,self.wkj.T) - self.thre_bp3
bp_out3 = self.sig(bp_net_k)
#--------------Model Leaning ------------------------
# calcluate error and gradient---------------
pd_k_all = np.multiply((data_teach - bp_out3), np.multiply(bp_out3, (1 - bp_out3)))
pd_j_all = np.multiply(np.dot(pd_k_all,self.wkj), np.multiply(bp_out2, (1 - bp_out2)))
pd_i_all = np.dot(pd_j_all,self.vji)
pd_conv1_pooled = pd_i_all / (self.size_pooling1*self.size_pooling1)
pd_conv1_pooled = pd_conv1_pooled.T.getA().tolist()
pd_conv1_all = self._calculate_gradient_from_pool(data_conved1,pd_conv1_pooled,shape_featuremap1[0],
shape_featuremap1[1],self.size_pooling1)
#weight and threshold learning process---------
#convolution layer
for k_conv in range(self.conv1[1]):
pd_conv_list = self._expand_mat(pd_conv1_all[k_conv])
delta_w = self.rate_weight * np.dot(pd_conv_list,data_focus1)
self.w_conv1[k_conv] = self.w_conv1[k_conv] + delta_w.reshape((self.conv1[0],self.conv1[0]))
self.thre_conv1[k_conv] = self.thre_conv1[k_conv] - np.sum(pd_conv1_all[k_conv]) * self.rate_thre
#all connected layer
self.wkj = self.wkj + pd_k_all.T * bp_out2 * self.rate_weight
self.vji = self.vji + pd_j_all.T * bp_out1 * self.rate_weight
self.thre_bp3 = self.thre_bp3 - pd_k_all * self.rate_thre
self.thre_bp2 = self.thre_bp2 - pd_j_all * self.rate_thre
# calculate the sum error of all single image
errors = np.sum(abs((data_teach - bp_out3)))
alle = alle + errors
#print(' ----Teach ',data_teach)
#print(' ----BP_output ',bp_out3)
rp = rp + 1
mse = alle/patterns
all_mse.append(mse)
def draw_error():
yplot = [error_accuracy for i in range(int(n_repeat * 1.2))]
plt.plot(all_mse, '+-')
plt.plot(yplot, 'r--')
plt.xlabel('Learning Times')
plt.ylabel('All_mse')
plt.grid(True, alpha=0.5)
plt.show()
print('------------------Training Complished---------------------')
print((' - - Training epoch: ', rp, ' - - Mse: %.6f' % mse))
if draw_e:
draw_error()
return mse
def predict(self,datas_test):
#model predict
produce_out = []
print('-------------------Start Testing-------------------------')
print((' - - Shape: Test_Data ',np.shape(datas_test)))
for p in range(len(datas_test)):
data_test = np.asmatrix(datas_test[p])
data_focus1, data_conved1 = self.convolute(data_test, self.conv1, self.w_conv1,
self.thre_conv1, conv_step=self.step_conv1)
data_pooled1 = self.pooling(data_conved1, self.size_pooling1)
data_bp_input = self._expand(data_pooled1)
bp_out1 = data_bp_input
bp_net_j = bp_out1 * self.vji.T - self.thre_bp2
bp_out2 = self.sig(bp_net_j)
bp_net_k = bp_out2 * self.wkj.T - self.thre_bp3
bp_out3 = self.sig(bp_net_k)
produce_out.extend(bp_out3.getA().tolist())
res = [list(map(self.do_round,each)) for each in produce_out]
return np.asarray(res)
def convolution(self,data):
#return the data of image after convoluting process so we can check it out
data_test = np.asmatrix(data)
data_focus1, data_conved1 = self.convolute(data_test, self.conv1, self.w_conv1,
self.thre_conv1, conv_step=self.step_conv1)
data_pooled1 = self.pooling(data_conved1, self.size_pooling1)
return data_conved1,data_pooled1
if __name__ == '__main__':
pass
'''
I will put the example on other file
'''

124
neural_network/perceptron.py Normal file
View File

@ -0,0 +1,124 @@
'''
Perceptron
w = w + N * (d(k) - y) * x(k)
Using perceptron network for oil analysis,
with Measuring of 3 parameters that represent chemical characteristics we can classify the oil, in p1 or p2
p1 = -1
p2 = 1
'''
from __future__ import print_function
import random
class Perceptron:
def __init__(self, sample, exit, learn_rate=0.01, epoch_number=1000, bias=-1):
self.sample = sample
self.exit = exit
self.learn_rate = learn_rate
self.epoch_number = epoch_number
self.bias = bias
self.number_sample = len(sample)
self.col_sample = len(sample[0])
self.weight = []
def training(self):
for sample in self.sample:
sample.insert(0, self.bias)
for i in range(self.col_sample):
self.weight.append(random.random())
self.weight.insert(0, self.bias)
epoch_count = 0
while True:
erro = False
for i in range(self.number_sample):
u = 0
for j in range(self.col_sample + 1):
u = u + self.weight[j] * self.sample[i][j]
y = self.sign(u)
if y != self.exit[i]:
for j in range(self.col_sample + 1):
self.weight[j] = self.weight[j] + self.learn_rate * (self.exit[i] - y) * self.sample[i][j]
erro = True
#print('Epoch: \n',epoch_count)
epoch_count = epoch_count + 1
# if you want controle the epoch or just by erro
if erro == False:
print(('\nEpoch:\n',epoch_count))
print('------------------------\n')
#if epoch_count > self.epoch_number or not erro:
break
def sort(self, sample):
sample.insert(0, self.bias)
u = 0
for i in range(self.col_sample + 1):
u = u + self.weight[i] * sample[i]
y = self.sign(u)
if y == -1:
print(('Sample: ', sample))
print('classification: P1')
else:
print(('Sample: ', sample))
print('classification: P2')
def sign(self, u):
return 1 if u >= 0 else -1
samples = [
[-0.6508, 0.1097, 4.0009],
[-1.4492, 0.8896, 4.4005],
[2.0850, 0.6876, 12.0710],
[0.2626, 1.1476, 7.7985],
[0.6418, 1.0234, 7.0427],
[0.2569, 0.6730, 8.3265],
[1.1155, 0.6043, 7.4446],
[0.0914, 0.3399, 7.0677],
[0.0121, 0.5256, 4.6316],
[-0.0429, 0.4660, 5.4323],
[0.4340, 0.6870, 8.2287],
[0.2735, 1.0287, 7.1934],
[0.4839, 0.4851, 7.4850],
[0.4089, -0.1267, 5.5019],
[1.4391, 0.1614, 8.5843],
[-0.9115, -0.1973, 2.1962],
[0.3654, 1.0475, 7.4858],
[0.2144, 0.7515, 7.1699],
[0.2013, 1.0014, 6.5489],
[0.6483, 0.2183, 5.8991],
[-0.1147, 0.2242, 7.2435],
[-0.7970, 0.8795, 3.8762],
[-1.0625, 0.6366, 2.4707],
[0.5307, 0.1285, 5.6883],
[-1.2200, 0.7777, 1.7252],
[0.3957, 0.1076, 5.6623],
[-0.1013, 0.5989, 7.1812],
[2.4482, 0.9455, 11.2095],
[2.0149, 0.6192, 10.9263],
[0.2012, 0.2611, 5.4631]
]
exit = [-1, -1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, -1, -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1]
network = Perceptron(sample=samples, exit = exit, learn_rate=0.01, epoch_number=1000, bias=-1)
network.trannig()
while True:
sample = []
for i in range(3):
sample.insert(i, float(raw_input('value: ')))
network.sort(sample)