Source Code for Module Bio.NeuralNetwork.BackPropagation.Layer

  1  """Model a single layer in a nueral network. 
  2   
  3  These classes deal with a layers in the neural network (ie. the input layer, 
  4  hidden layers and the output layer). 
  5  """ 
  6  # standard library 
  7  import math 
  8  import random 
  9   
 10   
 11 -def logistic_function(value): 
 12      """Transform the value with the logistic function. 
 13   
 14      XXX This is in the wrong place -- I need to find a place to put it 
 15      that makes sense. 
 16      """ 
 17      return 1.0 / (1.0 + math.exp(-value)) 
 18   
 19   
 20 -class AbstractLayer(object): 
 21      """Abstract base class for all layers. 
 22      """ 
 23 -    def __init__(self, num_nodes, has_bias_node): 
 24          """Initialize the layer. 
 25   
 26          Arguments: 
 27   
 28          o num_nodes -- The number of nodes that are contained in this layer. 
 29   
 30          o has_bias_node -- Specify whether or not this node has a bias 
 31          node. This node is not included in the number of nodes in the network, 
 32          but is used in constructing and dealing with the network. 
 33          """ 
 34          # specify all of the nodes in the network 
 35          if has_bias_node: 
 36              lower_range = 0 
 37          else: 
 38              lower_range = 1 
 39   
 40          self.nodes = range(lower_range, num_nodes + 1) 
 41   
 42          self.weights = {} 
 43   
 44 -    def __str__(self): 
 45          """Debugging output. 
 46          """ 
 47          return "weights: %s" % self.weights 
 48   
 49 -    def set_weight(self, this_node, next_node, value): 
 50          """Set a weight value from one node to the next. 
 51   
 52          If weights are not explicitly set, they will be initialized to 
 53          random values to start with. 
 54          """ 
 55          if (this_node, next_node) not in self.weights: 
 56              raise ValueError("Invalid node values passed.") 
 57   
 58          self.weights[(this_node, next_node)] = value 
 59   
 60   
 61 -class InputLayer(AbstractLayer): 
 62 -    def __init__(self, num_nodes, next_layer): 
 63          """Initialize the input layer. 
 64   
 65          Arguments: 
 66   
 67          o num_nodes -- The number of nodes in the input layer. 
 68   
 69          o next_layer -- The next layer in the neural network this is 
 70          connected to. 
 71          """ 
 72          AbstractLayer.__init__(self, num_nodes, 1) 
 73   
 74          self._next_layer = next_layer 
 75   
 76          # set up the weights 
 77          self.weights = {} 
 78          for own_node in self.nodes: 
 79              for other_node in self._next_layer.nodes: 
 80                  self.weights[(own_node, other_node)] = \ 
 81                                          random.randrange(-2.0, 2.0) 
 82   
 83          # set up the weight changes 
 84          self.weight_changes = {} 
 85          for own_node in self.nodes: 
 86              for other_node in self._next_layer.nodes: 
 87                  self.weight_changes[(own_node, other_node)] = 0.0 
 88   
 89          # set up the calculated values for each node -- these will 
 90          # actually just be set from inputs into the network. 
 91          self.values = {} 
 92          for node in self.nodes: 
 93              # set the bias node -- always has a value of 1 
 94              if node == 0: 
 95                  self.values[0] = 1 
 96              else: 
 97                  self.values[node] = 0 
 98   
 99 -    def update(self, inputs): 
100          """Update the values of the nodes using given inputs. 
101   
102          Arguments: 
103   
104          o inputs -- A list of inputs into the network -- this must be 
105          equal to the number of nodes in the layer. 
106          """ 
107          if len(inputs) != len(self.values.keys()) - 1: 
108              raise ValueError("Inputs do not match input layer nodes.") 
109   
110          # set the node values from the inputs 
111          for input_num in range(len(inputs)): 
112              self.values[input_num + 1] = inputs[input_num] 
113   
114          # propagate the update to the next layer 
115          self._next_layer.update(self) 
116   
117 -    def backpropagate(self, outputs, learning_rate, momentum): 
118          """Recalculate all weights based on the last round of prediction. 
119   
120          Arguments: 
121   
122          o learning_rate -- The learning rate of the network 
123   
124          o momentum - The amount of weight to place on the previous weight 
125          change. 
126   
127          o outputs - The output info we are using to calculate error. 
128          """ 
129          # first backpropagate to the next layers 
130          next_errors = self._next_layer.backpropagate(outputs, learning_rate, 
131                                                       momentum) 
132   
133          for this_node in self.nodes: 
134              for next_node in self._next_layer.nodes: 
135                  error_deriv = (next_errors[next_node] * 
136                                 self.values[this_node]) 
137   
138                  delta = (learning_rate * error_deriv + 
139                          momentum * self.weight_changes[(this_node, next_node)]) 
140   
141                  # apply the change to the weight 
142                  self.weights[(this_node, next_node)] += delta 
143   
144                  # remember the weight change for next time 
145                  self.weight_changes[(this_node, next_node)] = delta 
146   
147   
148 -class HiddenLayer(AbstractLayer): 
149 -    def __init__(self, num_nodes, next_layer, activation=logistic_function): 
150          """Initialize a hidden layer. 
151   
152          Arguments: 
153   
154          o num_nodes -- The number of nodes in this hidden layer. 
155   
156          o next_layer -- The next layer in the neural network that this 
157          is connected to. 
158   
159          o activation -- The transformation function used to transform 
160          predicted values. 
161          """ 
162          AbstractLayer.__init__(self, num_nodes, 1) 
163   
164          self._next_layer = next_layer 
165          self._activation = activation 
166   
167          # set up the weights 
168          self.weights = {} 
169          for own_node in self.nodes: 
170              for other_node in self._next_layer.nodes: 
171                  self.weights[(own_node, other_node)] = \ 
172                                          random.randrange(-2.0, 2.0) 
173   
174          # set up the weight changes 
175          self.weight_changes = {} 
176          for own_node in self.nodes: 
177              for other_node in self._next_layer.nodes: 
178                  self.weight_changes[(own_node, other_node)] = 0.0 
179   
180          # set up the calculated values for each node 
181          self.values = {} 
182          for node in self.nodes: 
183              # bias node 
184              if node == 0: 
185                  self.values[node] = 1 
186              else: 
187                  self.values[node] = 0 
188   
189 -    def update(self, previous_layer): 
190          """Update the values of nodes from the previous layer info. 
191   
192          Arguments: 
193   
194          o previous_layer -- The previous layer in the network. 
195          """ 
196          # update each node in this network 
197          for update_node in self.nodes[1:]: 
198              # sum up the weighted inputs from the previous network 
199              sum = 0.0 
200              for node in previous_layer.nodes: 
201                  sum += (previous_layer.values[node] * 
202                          previous_layer.weights[(node, update_node)]) 
203   
204              self.values[update_node] = self._activation(sum) 
205   
206          # propagate the update to the next layer 
207          self._next_layer.update(self) 
208   
209 -    def backpropagate(self, outputs, learning_rate, momentum): 
210          """Recalculate all weights based on the last round of prediction. 
211   
212          Arguments: 
213   
214          o learning_rate -- The learning rate of the network 
215   
216          o momentum - The amount of weight to place on the previous weight 
217          change. 
218   
219          o outputs - The output values we are using to see how good our 
220          network is at predicting things. 
221          """ 
222          # first backpropagate to the next layers 
223          next_errors = self._next_layer.backpropagate(outputs, learning_rate, 
224                                                       momentum) 
225   
226          # --- update the weights 
227          for this_node in self.nodes: 
228              for next_node in self._next_layer.nodes: 
229                  error_deriv = (next_errors[next_node] * 
230                                 self.values[this_node]) 
231   
232                  delta = (learning_rate * error_deriv + 
233                          momentum * self.weight_changes[(this_node, next_node)]) 
234   
235                  # apply the change to the weight 
236                  self.weights[(this_node, next_node)] += delta 
237   
238                  # remember the weight change for next time 
239                  self.weight_changes[(this_node, next_node)] = delta 
240   
241          # --- calculate error terms 
242          errors = {} 
243          for error_node in self.nodes: 
244              # get the error info propagated from the next layer 
245              previous_error = 0.0 
246              for next_node in self._next_layer.nodes: 
247                  previous_error += (next_errors[next_node] * 
248                                     self.weights[(error_node, next_node)]) 
249   
250              # get the correction factor 
251              corr_factor = (self.values[error_node] * 
252                             (1 - self.values[error_node])) 
253   
254              # calculate the error 
255              errors[error_node] = previous_error * corr_factor 
256   
257          return errors 
258   
259   
260 -class OutputLayer(AbstractLayer): 
261 -    def __init__(self, num_nodes, activation=logistic_function): 
262          """Initialize the Output Layer. 
263   
264          Arguments: 
265   
266          o num_nodes -- The number of nodes in this layer. This corresponds 
267          to the number of outputs in the neural network. 
268   
269          o activation -- The transformation function used to transform 
270          predicted values. 
271          """ 
272          AbstractLayer.__init__(self, num_nodes, 0) 
273   
274          self._activation = activation 
275   
276          self.values = {} 
277          for node in self.nodes: 
278              self.values[node] = 0 
279   
280 -    def update(self, previous_layer): 
281          """Update the value of output nodes from the previous layers. 
282   
283          Arguments: 
284   
285          o previous_layer -- The hidden layer preceding this. 
286          """ 
287          # update all of the nodes in this layer 
288          for update_node in self.nodes: 
289              # sum up the contribution from all of the previous inputs 
290              sum = 0.0 
291              for node in previous_layer.nodes: 
292                  sum += (previous_layer.values[node] * 
293                          previous_layer.weights[(node, update_node)]) 
294   
295              self.values[update_node] = self._activation(sum) 
296   
297 -    def backpropagate(self, outputs, learning_rate, momentum): 
298          """Calculate the backpropagation error at a given node. 
299   
300          This calculates the error term using the formula: 
301   
302          p = (z - t) z (1 - z) 
303   
304          where z is the calculated value for the node, and t is the 
305          real value. 
306   
307          Arguments: 
308   
309          o outputs - The list of output values we use to calculate the 
310          errors in our predictions. 
311          """ 
312          errors = {} 
313          for node in self.nodes: 
314              calculated_value = self.values[node] 
315              real_value = outputs[node - 1] 
316   
317              errors[node] = ((real_value - calculated_value) * 
318                              calculated_value * 
319                              (1 - calculated_value)) 
320   
321          return errors 
322   
323 -    def get_error(self, real_value, node_number): 
324          """Return the error value at a particular node. 
325          """ 
326          predicted_value = self.values[node_number] 
327          return 0.5 * math.pow((real_value - predicted_value), 2) 
328   
329 -    def set_weight(self, this_node, next_node, value): 
330          raise NotImplementedError("Can't set weights for the output layer") 
331