#include <math.h>
#include <time.h>
#include "ind_types.h"
#include "ipc.h"
#include "cmdparam.h"
#include "binarysave.h"
#include "vgen.h"
#include "inputs.h"
#include "file_io.h"
#include "globals.h"
#include "kernel.h"
#include "kernelwrapper.h"
#include "stringutils.h"
#include "lissom.h"
Include dependency graph for kernel.c:

Go to the source code of this file.
Compounds | |
| struct | cmdobj_init_network |
| struct | cmdobj_kill_connections |
| struct | cmdobj_step |
| struct | cmdobj_testing |
| struct | cmdobj_training |
| struct | cmdobj_uninit_network |
Defines | |
| #define | ACTLIST_START(actlist) 1 |
| Wrappers to hide structure of activity lists (actlists). More... | |
| #define | ACTLIST_COUNT(actlist) ((actlist)[0]) |
| #define | BINARY_WEIGHTS_FILE |
| Save binary files instead of ASCII. More... | |
| #define | VERIFY_WEIGHT_SAVES_ERROR_LIMIT 10 |
| Maximum number of errors to report when verifying a weight file. More... | |
Typedefs | |
| typedef int | actlist [NMAX+1] |
Functions | |
| int | binary_search (const int list[], int lower, int upper, int value) |
| Holds copy of weights for comparison. More... | |
| void | clear_weights (void) |
| Intialize all weights to known values. More... | |
| void | crop_actlist (const actlist, int lowval, int highval, int *lowidx, int *highidx) |
| Given an ordered list and a range [lowval,highval], returns the range of list indexes [lowidx,highidx] which contain values in that range. More... | |
| double | dist_lat_wt (double value, double amt_of_randomness) |
| This function returns a value randomly distributed within a range value*amt_of_randomness on either side of value. More... | |
| double | lat_resp (int ui, int uj, int radius, int array_width, l_weight *weights, int weight_idx_bound) |
| Computes lateral excitation or inhibition response with the given weight vectors and radius. More... | |
| double | lat_resp_rad1 (int ui, int uj, int radius, int array_width, int array_width_2, l_weight *weights, int weight_idx_bound) |
| Optimized case when radius is just 0 or 1, mainly for excitation resp of organized map. More... | |
| void | modify_lat_wts (void) |
| Modify lateral weights by normalized Hebbian rule. More... | |
| void | modify_latwt_loop (int i, int j, int radius, int array_width, l_weight *weights, int weight_idx_bound, double alpha) |
| Assumes activity has been collected into the prev_map_activity array. More... | |
| void | modify_latwt_loop_killed (int i, int j, int radius, int array_width, l_weight *weights, int weight_idx_bound, double alpha) |
| void | modify_latwt_loop_rad1 (int ui, int uj, int radius, int array_width, int array_width_2, l_weight *weights, int weight_idx_bound, double alpha) |
| Optimized case when radius is just 1. More... | |
| void | modify_weights (void) |
| Change afferent weights by normalized Hebbian rule (sum of weights normalization). More... | |
| void | normalize_afferent_wts (int lowk, int highk, int lowl, int highl, int row, int j, double length) |
| void | normalize_lat_wts (int i, int j, int radius, int array_width, l_weight *weights) |
| Renormalize a set of lateral weights of the given neuron, such as after changing the radius. More... | |
| double | prune_circular_aff_weights (int ui, int uj, int radius, a_weight weights[MAX_NUM_EYES][WTMAX][WTMAX], double length) |
| Kill off connections that aren't within the specified radius. More... | |
| void | prune_circular_lat_weights (int ui, int uj, int radius_sq, int array_width, l_weight *weights) |
| Kill off connections that aren't within the specified radius. More... | |
| void | receptor_normalize (void) |
| Output normalization for receptors. More... | |
| void | reduce_lateral_radius (int ui, int uj, int old_radius, int new_radius, int array_width, l_weight *weights) |
| Redistribute and renormalize lateral excitation. More... | |
| void | set_markers (int radius) |
| Marks whether neurons are active or not. More... | |
| void | setup_lateral_weights (int ui, int uj, int radius, int array_width, l_weight *weights, int preset, double sigma, double randomness_scale, int* con_killed_flag) |
| Sets up lateral weights for one neuron based on where it is in the map. More... | |
| double | sigmoid (double activity, double sigdelta, double sigbeta) |
| Transform the activity to a sigmoid response between 0 and 1.0. More... | |
| void | verify_actlist (actlist list) |
| Verifies that the given actlist is in increasing order. More... | |
| void | verify_actlist_array (actlist lists[],int num) |
| Call verify_actlist for every element of array. More... | |
| void | verify_saved_lateral_weights ( int i, int j, int radius, int array_width, l_weight *old_weights, l_weight *new_weights, double tolerance, const char *description) |
| void | verify_saved_weights ( double tolerance ) |
| cmdstat | cmd_init_network ( int argc, const char *argv[] ) |
| cmdstat | cmd_kill_connections ( int argc, const char *argv[] ) |
| Dead connections have a small negative value on them. More... | |
| cmdstat | cmd_step ( int argc, const char *argv[] ) |
| cmdstat | cmd_testing ( int argc, const char *argv[] ) |
| cmdstat | cmd_training ( int argc, const char *argv[] ) |
| Main command to train the network until the specified iteration. More... | |
| cmdstat | cmd_uninit_network ( int argc, const char *argv[] ) |
| Initializations that are independent of any parameter values. More... | |
| void | kernel_init_hook ( void ) |
| void | present_inputs (int learn) |
| Main function called at each iteration to present inputs and modify the weights accordingly. More... | |
| void | settle_responses (int learn) |
| Propagate input and let the network settle. More... | |
| void | response_to_input (void) |
| Optimized version of input response calculation. More... | |
| void | compute_responses (int ts) |
| Add up input and lateral activations. More... | |
| void | initialize_actlists ( void ) |
| Initialize the activity list counts to zero. More... | |
| void | initialize_markers ( void ) |
| Initialize activity markers for optimization of lat_exc computations. More... | |
| double | network_size_megabytes ( void ) |
| Memory requirements per PE. More... | |
| int | network_size_connections ( void ) |
| Total number of connections. More... | |
| void | init_weights (void) |
| Initialize active weights to specified values. More... | |
| void | setup_latw (void) |
| Obsolete version; retained only for use only with kurtosis_contrast. More... | |
| int | change_lateral_exc_radius (int old_radius, int new_radius) |
| Change lateral excitatory radius for every row. More... | |
| void | collect_activation_data (int dest_pe) |
| Some data is ordinarily left distributed on each PE because it is not needed on every PE for the core LISSOM algorithm. More... | |
| int | save_current (const char *filename) |
| Appends snapshot to filebase.<t>. More... | |
| int | load_current (int load_snap, const char *filename) |
| Loads ASCII or binary-format snapshot from filebase.<t> from which to continue. More... | |
Variables | |
| double | alpha_exc = 0.002 |
| double | alpha_inh = 0.0002 |
| double | alpha_input = 0.0007 |
| double | beta = 0.7 |
| double | delta = 0.1 |
| int | display = 0 |
| int | dynamics = 0 |
| double | exc_death = 0 |
| double | inh_death = 0.00001 |
| double | lat_exc_randomness = 1.0 |
| double | lat_inh_randomness = 1.0 |
| double | noise = 0.0 |
| int | preset_aff_wts = True |
| double | preset_sigma_aff = Uninitialized |
| double | preset_sigma_lat = Uninitialized |
| double | preset_sigma_exc = Uninitialized |
| double | preset_sigma_inh = Uninitialized |
| double | randomness = 0 |
| int | seed = 87654321 |
| int | smooth_circular_outlines = False |
| double | smooth_circular_radius_trim = -0.25 |
| int | tend = MAXITERATION |
| int | essential_training_hooks_only = False |
| Whether to skip non-catchup hooks in train. More... | |
| double | accumulate_sum [NMAX] |
| To add up weights without roundoff. More... | |
| actlist | activity_list [NROWS] |
| Maintains activity lists. More... | |
| int | activity_marker [NMAX][NMAX] |
| True if neighbors are active. More... | |
| double | aff_norm_sum [MAX_NUM_EYES*RNMAX*RNMAX] |
| To do a parallel fetch of the data. More... | |
| double | afferent_sum [MAX_NUM_EYES*RNMAX*RNMAX] |
| To accumulate wts of each receptor. More... | |
| int | exc_connections_killed = False |
| Flag for use in modify_weights. More... | |
| int | inh_connections_killed = False |
| Flag for use in modify_weights. More... | |
| double | gather_activity [NMAX][NMAX+1] |
| To collect activity in each row. More... | |
| double | map_activity [NMAX][NMAX] |
| Current map_activity over network. More... | |
| actlist | prev_activity_list [NMAX] |
| Maintains activity lists. More... | |
| double | resp_to_inp [NMAX][NMAX] |
| Temporary to compute extern response. More... | |
| int | RN_sq |
| RN*RN precomputed for index calculations. More... | |
| double | temp_sum [MAX_NUM_EYES*RNMAX*RNMAX] |
| To keep the temporary sum. More... | |
| Wts | wts [NROWS][NMAX] |
| One or more row(s) of weights for each processor. More... | |
RETINAL AND CORTICAL ARRAY LAYOUT
The routines in this file (and most others) assume the following organizations:
World Retina CortexRN-1 ------------ 0 ----------- 0 ----------- | | | | | | | | | | | | y | | r | | j | | | | | | | | | | | | | | 0.0 ----------- RN-1 ----------- N-1 ----------- 0.0 x RN-1 0 c RN-1 0 i N-1
Unfortunately, even though the `i' index is called a row and corresponds to a C/C++ array row, it is currently plotted along the vertical dimension instead of the horizontal, as shown above. This will hopefully change soon so that the memory layout is a better match to the plotted data.
The following arrays all share the cortical map structure described above; not all are declared in this file:
map 2D grid of units; weights active only in certain rows.resp_to_inp Activity level of each unit due only to the afferent input, i.e. the response before settling due to the lateral interactions.
map_activity Activity level of each unit in the current iteration during and after settling.
prev_map_activity Settled activity level of each unit in the previous iteration.
activity_marker Each neuron marked true if any neighbor in a given radius is active; the rest are exempt from further calculation.
or_pref Orientation preferences for each neuron as an integer in the range [0,NANGLES].
od_pref Ocular dominance preferences for each neuron as a float in the range [0,1].
DISTRIBUTED CORTICAL MAPS
For those arrays which are distributed across multiple PEs, the `rows' are strided across the cortex such that for each local row i in [0,nrows), the row in the map is pe+i*NPEs. (E.g. if there are 64 PEs handling 3 rows each, PE 32 handles rows 32, 96, and 160). I.e.:
Cortex Local Rows Map Rows0.-----------. 0.---. 0...-....-... | | | | : | | | | : | | | | : | | | | : j | | j | | j : | | | | : | | | | : | | | | : | | | | : | | | | : N-1`-----------' N-1`---' N-1..`-'..`-'.. 0 i N-1 0 i (nrows-1) 0 MAPROW(i) N-1
(where MAPROW(i) = pe+i*NPEs for local row i)
Unfortunately, the terms `i' and `row' are often used interchangeably for either of these two types of indexes, so be careful to distinguish between them when reading the code.
In this file, the only distributed cortical map is the `wts' array, which has the weights to each neuron stored on this PE (nrows worth of data). The weights are also linked in to the global map at appropriate positions, which provides for an easy translation between `local' [0,nrows) and `map' [0,N) rows.
The afferent weights are stored as a 3D array of values, one dimension being the eye number (usually 0 or 1) and the other two being the two spatial dimensions of the retina. The lateral weights are each stored as a 1-D array whose format is described in section LATERAL CONNECTIONS below.
COLLAPSED ROW-ORIENTED ARRAYS
Other arrays contain a single element for each row; these are often used when computing a quantity first for each row on the various PEs, then combining the result from all the PEs.
One element per local row:
activity_listFor each local row, contains a list of the column numbers of units which were active in that row. The first element in each row is the `count' of active units in that row, and the following `count' elements are the numbers of the active columns in that row. At the end of the iteration, this data is propagated to the other processors and stored in the map-oriented prev_activity_list.
One element per map row:
prev_activity_listFor each row in the map, contains a list of the column numbers of units which were active in the preceding iteration. See the description of activity_list below. Only weights to/from units active in the preceding iteration are modified in this one.
gather_activity
Like prev_activity_list, except that each element is an activity level, not a column number. (The first element still represents the `count}.)
LATERAL CONNECTIONS
Instead of being stored in a two-dimensional array like the afferent weights, lateral connections of each type are stored as one-dimensional arrays for which the 2D coordinates are computed by hand. I imagine that this complicated arrangement was chosen by Joseph Sirosh because the size of the lateral connection array varies periodically during training. If a fixed size but partially-populated 2D array was used instead, the active connections would be scattered throughout the array, resulting in poor locality of reference, which would lead to a high cache miss rate, which would lead to poor performance.
However, the current implementation contains many useless connections at the cortex boundaries, where as many as half of the connections are unused. Plus it contains all the dead weights after connections have been killed, so it doesn't get any faster as training progresses. Finally, the current implementation is an unholy mess to program, with index calculations duplicated all over the code in different forms. There must be a better way, probably some sort of optimized sparse array format.
In any case, as it currently stands, the formula for computing the 1-dimensional array index for the connection to unit (ui,uj) from unit (k,l) is:
index(k,l) = (k-(ui-radius))*ar_width + (l-(uj-radius))
where radius is either the current `inh_rad' or the current `exc_rad', depending upon the connection type, and ar_width is similarly either `inh_array_width' or `exc_array_width'.
This equation can be simplified slightly to:
index(k,l) = (k-ui+radius)*ar_width + l - uj + radius
which is how it usually appears in the code unless otherwise optimized.
If computing a number of such indexes for various values of l (various `columns' in a `row', using the terminology used in the code), the index can be partially computed beforehand to save calculation time:
partial_index(k) = (k-ui+radius)*ar_width - uj + radius)
The full index is then
index(k,l) = partial_index(k) + l
Macros have been defined in lissom.h to hide all of these details, but for computational reasons it's important to understand what's going on when they are used.
Definition in file kernel.c.
|
|
|
|
|
Wrappers to hide structure of activity lists (actlists). The activity_list and prev_activity_list are ordered lists stored as arrays whose first element indicates the number of elements in the list, and whose remaining "count" elements (starting at index 1) are the elements in the list (up to one element per neuron in the row, thus NMAX maximum). The gather_activity array is the same type except that the elements are floating-point. Definition at line 245 of file kernel.c. Referenced by compute_responses(), crop_actlist(), present_inputs(), and verify_actlist(). |
|
|
Save binary files instead of ASCII.
|
|
|
Maximum number of errors to report when verifying a weight file.
|
|
|
|
|
|
Holds copy of weights for comparison. If the value is found, its index is returned; otherwise the next higher index is returned. Definition at line 1065 of file kernel.c. Referenced by crop_actlist(). |
|
|
Change lateral excitatory radius for every row.
|
|
|
Intialize all weights to known values.
Definition at line 1890 of file kernel.c. Referenced by init_weights(), load_current(), and save_current(). |
|
|
|
|
|
Dead connections have a small negative value on them. These connections are not modified afterwards. Nor do they make any significant contribution to the response on the map because of their small value. |
|
|
|
|
|
|
|
|
Main command to train the network until the specified iteration.
|
|
|
Initializations that are independent of any parameter values.
|
|
|
Some data is ordinarily left distributed on each PE because it is not needed on every PE for the core LISSOM algorithm. This procedure collects such data onto the specified PE for simpler graphing and/or analysis. If pe is Uninitialized the data is broadcast to all PEs. This does not need to be called when NPES==1, but it won't take any time to do so. |
|
|
Add up input and lateral activations.
|
|
|
Given an ordered list and a range [lowval,highval], returns the range of list indexes [lowidx,highidx] which contain values in that range. If (lowidx>highidx) on return, there is no item in the list in the given range. Definition at line 1090 of file kernel.c. Referenced by lat_resp(), and modify_latwt_loop(). |
|
|
This function returns a value randomly distributed within a range value*amt_of_randomness on either side of value.
Definition at line 2209 of file kernel.c. Referenced by setup_lateral_weights(). |
|
|
Initialize active weights to specified values. Each neuron is assumed to have a receptive field on the retina. The size of the receptive field of a neuron is (RN/N). Each neuron receives input from a square area of radius rf_radius from around the center of its receptive field projected on the retina. With the two retinas, (centerx+RN, centery+RN) gives the corresponding center in the second retina The total sum of weights is normalized to 1.0. |
|
|
Initialize the activity list counts to zero.
Definition at line 1160 of file kernel.c. Referenced by settle_responses(). |
|
|
Initialize activity markers for optimization of lat_exc computations. Initially all markers must be True. Definition at line 1173 of file kernel.c. Referenced by settle_responses(). |
|
|
|
|
|
Computes lateral excitation or inhibition response with the given weight vectors and radius. The response is calculated only from active neurons. Definition at line 1221 of file kernel.c. Referenced by compute_responses(). |
|
|
Optimized case when radius is just 0 or 1, mainly for excitation resp of organized map.
Definition at line 1257 of file kernel.c. Referenced by compute_responses(). |
|
|
Loads ASCII or binary-format snapshot from filebase.<t> from which to continue. The ASCII format is deprecated, and may eventually be phased out. |
|
|
Modify lateral weights by normalized Hebbian rule. The normalization keeps sum of lateral excitatory weights and sum of lateral inhibitory weights constant and equal to 1.0. Definition at line 1488 of file kernel.c. Referenced by present_inputs(), and settle_responses(). |
|
|
Assumes activity has been collected into the prev_map_activity array.
Definition at line 1514 of file kernel.c. Referenced by modify_lat_wts(). |
|
|
Optimized case when radius is just 1. Assumes activity has been collected into the prev_map_activity array. Definition at line 1567 of file kernel.c. Referenced by modify_lat_wts(). |
|
|
Change afferent weights by normalized Hebbian rule (sum of weights normalization).
Definition at line 1338 of file kernel.c. Referenced by present_inputs(), and settle_responses(). |
|
|
Total number of connections.
|
|
|
Memory requirements per PE.
Definition at line 1727 of file kernel.c. Referenced by main(). |
|
|
|
|
|
Renormalize a set of lateral weights of the given neuron, such as after changing the radius.
Definition at line 1669 of file kernel.c. Referenced by reduce_lateral_radius(), and setup_lateral_weights(). |
|
|
Main function called at each iteration to present inputs and modify the weights accordingly. When this routine completes, the following data should be available to plot or analyze (on each PE):
Variable Data Portion Same on each PE? To get all of init_activity for graphing, must call collect_activation_data() if NPES>1. |
|
|
Kill off connections that aren't within the specified radius.
Definition at line 2152 of file kernel.c. Referenced by init_weights(). |
|
|
Kill off connections that aren't within the specified radius.
Definition at line 2110 of file kernel.c. Referenced by reduce_lateral_radius(), and setup_lateral_weights(). |
|
|
Output normalization for receptors.
Definition at line 1408 of file kernel.c. Referenced by modify_weights(). |
|
|
Redistribute and renormalize lateral excitation. Lat_exc wts are indexed based on the original exc_array_width, and the current excitation radius. So the weights from the center of the exc_wts array must be copied to this new index range, for the indexing to be still correct. Works only for reducing, not increasing lateral radius. Definition at line 2302 of file kernel.c. Referenced by change_lateral_exc_radius(). |
|
|
Optimized version of input response calculation.
Definition at line 896 of file kernel.c. Referenced by settle_responses(). |
|
|
Appends snapshot to filebase.<t>. The output is ASCII weight values. The ASCII format is deprecated, and may eventually be phased out. |
|
|
Marks whether neurons are active or not. If there is any activity in the given neighborhood radius, markers are set to True, otherwise they are set to False for the remainder of the settling iteration. Saves a lot of computation. Definition at line 1191 of file kernel.c. Referenced by compute_responses(). |
|
|
Propagate input and let the network settle.
|
|
|
Sets up lateral weights for one neuron based on where it is in the map. Neurons at the border have fewer connections than at the center. Note that inhibition wts are initialized to positive. The lateral wts are distributed around the mean with the given randomness. The lowk,lowl,highk,highl variables keep track of the spread of connections. Connections can at most spread to both borders of the network. Definition at line 2068 of file kernel.c. Referenced by init_weights(), and setup_latw(). |
|
|
Obsolete version; retained only for use only with kurtosis_contrast.
|
|
|
Transform the activity to a sigmoid response between 0 and 1.0. Between the thresholds, response is linear. Definition at line 1708 of file kernel.c. Referenced by compute_responses(). |
|
|
Verifies that the given actlist is in increasing order.
Definition at line 1130 of file kernel.c. Referenced by verify_actlist_array(). |
|
|
Call verify_actlist for every element of array.
|
|
|
RN*RN precomputed for index calculations.
|
|
|
To add up weights without roundoff.
|
|
|
Maintains activity lists.
|
|
|
True if neighbors are active.
|
|
|
To do a parallel fetch of the data.
|
|
|
To accumulate wts of each receptor.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Whether to skip non-catchup hooks in train.
|
|
|
Flag for use in modify_weights.
|
|
|
|
|
|
To collect activity in each row.
|
|
|
Flag for use in modify_weights.
|
|
|
|
|
|
|
|
|
|
|
|
Current map_activity over network.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Maintains activity lists.
|
|
|
|
|
|
Temporary to compute extern response.
|
|
|
|
|
|
|
|
|
|
|
|
To keep the temporary sum.
|
|
|
|
|
|
One or more row(s) of weights for each processor.
|
1.2.1 written by Dimitri van Heesch,
© 1997-2000