Main Page   Namespace List   Class Hierarchy   Compound List   File List   Namespace Members   Compound Members   File Members  

binarysave.c File Reference

Routines for binary weight file writing/reading. More...

#include <stdio.h>
#include <time.h>
#include <sys/types.h>
#include "ipc.h"
#include "cmdparam.h"
#include "ind_types.h"
#include "lissom.h"
#include "globals.h"
#include "kernel.h"
#include "analyze.h"
#include "binarysave.h"

Include dependency graph for binarysave.c:

Include dependency graph

Go to the source code of this file.

Compounds

struct  LFBuffer

Defines

#define LF_BUFFER_VERIFY
 Define if each buffer should be interpreted to verify before writing it. More...

#define NEURONS_PER_BUFFER   1
 Number of neurons to be handled in one buffer for saving and loading. More...

#define PARTS_PER_ROW   (NMAX/NEURONS_PER_BUFFER)
 Alternate way of specifying NEURONS_PER_BUFFER. More...

#define parts_per_row   (N/NEURONS_PER_BUFFER)
#define LF_FLOAT_SIZE   sizeof(LFFloatType)
#define SMALLEST_LFTOK   1
#define LFTOK_NO_TOKEN   0
#define LFTOK_SNAPSHOT_HEADER   1
#define LFTOK_ROW_HEADER   2
#define LFTOK_NEURON_HEADER_OLD   3
#define LFTOK_AFF_WEIGHTS   4
#define LFTOK_LEXC_WEIGHTS_OLD   5
#define LFTOK_LINH_WEIGHT_CHUNK   6
#define LFTOK_NEURON_TRAILER   7
#define LFTOK_SNAPSHOT_TRAILER   8
#define LFTOK_LEXC_WEIGHTS   9
#define LFTOK_NEURON_HEADER   10
#define LARGEST_LFTOK   10
#define LF_UNUSED   0x00
 Use to fill in a token field that is unused. More...

#define LF_ALL_EYES   99
 Indicates that afferent weights are for all eyes. More...

#define LF_BUFFER_SIZE
 Very conservative estimate of number of 32-bit words needed to store binary file representation of one part-row. More...

#define LF_TOKEN_GRAIN_SIZE   4
 Units for token lengths. More...

#define LFBufferDataType_swap_endian   i32_swap_endian
#define LF_FLOAT_TOLERANCE   0.0000001
#define LF_NO_ERROR   0
#define LF_ERROR   -1
#define LF_MAXWARNINGS   5
#define compareOrSet_proto(type)
#define MAKE_SURE_NEURON_HEADER_INFO_AVAILABLE
#define compareOrSet_proc(type)
 Family of routines to set a value of different float types. More...


Typedefs

typedef f32 LFFloatType
 Format for each floating point number stored in the weights file. More...

typedef i32 LFBufferDataType
 The size of this should match LF_TOKEN_GRAIN_SIZE. More...


Functions

LFTokenType makeToken (int name, int length, int infoa, int infob )
 Combines the given parameters and returns as a token. More...

int tokenName ( LFTokenType token )
int tokenLength ( LFTokenType token )
int tokenInfoa ( LFTokenType token )
int tokenInfob ( LFTokenType token )
void printToken ( LFTokenType token, int offset )
const char* tokenNameString (LFTokenType token)
int writeToken (FILE *file, LFTokenType token )
 Writes the given token to a binary file. More...

LFTokenType peekToken (FILE *file )
 Reads and returns a single token. More...

LFTokenType readToken (FILE *file )
 Reads and returns a single token. More...

LFTokenType readTokenExpecting (int tokenNameExpected, FILE *file)
 Read a token from file expecting one of a particular name. More...

int writeItem (FILE *file, void *itemptr)
 Write an item of the same size as LFBufferDataType to a binary buffer. More...

LFBufferDataType readItem (FILE* file)
 Reads an item from a binary buffer. More...

int writeFloat (FILE *file, f32 aFloat )
 Outputs a floating point number to a binary file. More...

f32 readFloat (FILE *file)
 Reads a floating point number from a binary file. More...

int writeInteger (FILE *file, int anInteger )
 Outputs an integer to a binary file. More...

int readInteger (FILE *file)
 Reads an integer from a binary file. More...

void printTokensInBuffer ( LFBuffer *buffer )
int binaryWeightsFileReadInArbitraryOrder (FILE *file, LFBuffer *buffer)
int binaryWeightsFileReadInFixedOrder (FILE *file, LFBuffer *buffer)
 Read file assuming layout of rows and part-rows is known. More...

int binaryWeightsBufferFill ( LFBuffer *buffer, int current_map_row, int current_local_row, int part_of_row )
 Fill binary weight buffer with the tokens for one part-row. More...

int binaryWeightsBufferProcess ( LFBuffer *buffer, int verify)
 Read and process a binary weight buffer. More...

int binaryWeightsBufferWriteLatWeights ( LFBuffer *buffer, int token_name, int i, int j, int radius, int array_width, l_weight *weights)
int binaryWeightsBufferReadLatWeights ( LFBuffer *buffer, LFTokenType token, int i, int j, int radius, int array_width, l_weight *weights, int verify, const char *description)
void LFBufferGet ( LFBuffer *buffer, int pe )
 Copy an entire buffer from the same position on another PE. More...

void LFBufferPut ( LFBuffer *buffer, int pe )
 Copy an entire buffer from the same position on another PE. More...

void LFBufferClear ( LFBuffer *buffer )
 Clear the entire buffer. More...

int LFBufferLength ( LFBuffer *buffer )
 Returns the number of items in the buffer. More...

int LFBufferWrite ( LFBuffer *buffer, FILE *file )
 Write an entire buffer full of data to a binary file. More...

int LFBufferRead ( LFBuffer *buffer, FILE *file, int length)
 Read the given number of items from a binary file. More...

int LFBufferWriteToken (LFBuffer *buffer, LFTokenType token )
 Writes the given token to a binary buffer. More...

LFTokenType LFBufferReadToken (LFBuffer *buffer )
 Reads and returns a single token. More...

int LFBufferWriteItem (LFBuffer *buffer, void *itemptr)
 Write an item of the same size as LFBufferDataType to a binary buffer. More...

LFBufferDataType LFBufferReadItem (LFBuffer *buffer)
 Reads an item from a binary buffer. More...

int LFBufferWriteFloat (LFBuffer *buffer, f32 aFloat )
 Outputs a floating point number to a binary buffer. More...

f32 LFBufferReadFloat (LFBuffer *buffer)
 Reads a floating point number from a binary buffer. More...

int LFBufferWriteInteger (LFBuffer *buffer, int anInteger )
 Outputs an integer to a binary buffer. More...

int LFBufferReadInteger ( LFBuffer *buffer)
 Reads an integer from a binary buffer. More...

int compareParameter (int weight_file_parameter, int param_file_parameter, const char *description)
 Ensure that a weight file is appropriate for the current parameters. More...

int compareOrSetInt (int *destination, int value, int verify, const char *description)
 Ensure that a weight file is appropriate for the current parameters. More...

 compareOrSet_proto (f64)
 compareOrSet_proto (a_weight)
 compareOrSet_proto (l_weight)
void binarysave_init_hook ( void )
int binaryWeightsFileWrite ( FILE *file )
 Write network weights to a binary file. More...

int binaryWeightsFileRead ( FILE *file )
 Read and process a binary weight file. More...

void printTokens ( FILE *file )
 Prints the tokens present in a binary file for debugging. More...


Variables

int save_afferent_weights_only = False
int load_afferent_weights_only = False
int tokensWritten = 0
 Total number of tokens output so far. More...

int tokensRead = 0
 Total number of tokens input so far. More...

LFBuffer binaryBuffer
int status = 0
int errorsEncountered = 0
 Number of errors encountered in this load or save operation. More...

int bigendian = -1
 Initially undefined; define before use. More...


Detailed Description

Routines for binary weight file writing/reading.

Header:
/u/nn/cvsroot/lissom/src/binarysave.c,v 1.38 2000/04/21 20:47:19 jbednar Exp

Definition in file binarysave.c.


Define Documentation

#define LARGEST_LFTOK   10
 

Definition at line 71 of file binarysave.c.

#define LFBufferDataType_swap_endian   i32_swap_endian
 

Definition at line 96 of file binarysave.c.

#define LFTOK_AFF_WEIGHTS   4
 

Definition at line 64 of file binarysave.c.

#define LFTOK_LEXC_WEIGHTS   9
 

Definition at line 69 of file binarysave.c.

#define LFTOK_LEXC_WEIGHTS_OLD   5
 

Definition at line 65 of file binarysave.c.

#define LFTOK_LINH_WEIGHT_CHUNK   6
 

Definition at line 66 of file binarysave.c.

#define LFTOK_NEURON_HEADER   10
 

Definition at line 70 of file binarysave.c.

#define LFTOK_NEURON_HEADER_OLD   3
 

Definition at line 63 of file binarysave.c.

#define LFTOK_NEURON_TRAILER   7
 

Definition at line 67 of file binarysave.c.

#define LFTOK_NO_TOKEN   0
 

Definition at line 60 of file binarysave.c.

#define LFTOK_ROW_HEADER   2
 

Definition at line 62 of file binarysave.c.

#define LFTOK_SNAPSHOT_HEADER   1
 

Definition at line 61 of file binarysave.c.

#define LFTOK_SNAPSHOT_TRAILER   8
 

Definition at line 68 of file binarysave.c.

#define LF_ALL_EYES   99
 

Indicates that afferent weights are for all eyes.

Definition at line 76 of file binarysave.c.

#define LF_BUFFER_SIZE
 

Initializer:

\
  (int)(1.02 * (NMAX/PARTS_PER_ROW) * ( 2*WTMAX*WTMAX + MAX_EXC_DIMENSION + 1.0*MAX_INH_DIMENSION ))
Very conservative estimate of number of 32-bit words needed to store binary file representation of one part-row.

Assumes zero compression, full excitatory radius, and 2% file overhead.

Definition at line 84 of file binarysave.c.

#define LF_BUFFER_VERIFY
 

Define if each buffer should be interpreted to verify before writing it.

Definition at line 35 of file binarysave.c.

#define LF_ERROR   -1
 

Definition at line 110 of file binarysave.c.

#define LF_FLOAT_SIZE   sizeof(LFFloatType)
 

Definition at line 49 of file binarysave.c.

#define LF_FLOAT_TOLERANCE   0.0000001
 

Definition at line 106 of file binarysave.c.

#define LF_MAXWARNINGS   5
 

Definition at line 112 of file binarysave.c.

#define LF_NO_ERROR   0
 

Definition at line 109 of file binarysave.c.

#define LF_TOKEN_GRAIN_SIZE   4
 

Units for token lengths.

Each token must be an even multiple of this number. Token headers must also be this length.

Definition at line 93 of file binarysave.c.

#define LF_UNUSED   0x00
 

Use to fill in a token field that is unused.

Definition at line 74 of file binarysave.c.

#define MAKE_SURE_NEURON_HEADER_INFO_AVAILABLE
 

Initializer:

\
  if (current_map_row == Uninitialized || current_j == Uninitialized) {  \
    ipc_notify(IPC_ALL,IPC_ERROR,"Missing neuron header");                           \
    return LF_ERROR;                                                       \
  }

Definition at line 729 of file binarysave.c.

#define NEURONS_PER_BUFFER   1
 

Number of neurons to be handled in one buffer for saving and loading.

The larger the number, the more efficient, but the more memory is required. Must be smaller than N, and N must be an even multiple of it.

Definition at line 42 of file binarysave.c.

#define PARTS_PER_ROW   (NMAX/NEURONS_PER_BUFFER)
 

Alternate way of specifying NEURONS_PER_BUFFER.

Must divide into N evenly.

Definition at line 44 of file binarysave.c.

#define SMALLEST_LFTOK   1
 

Definition at line 59 of file binarysave.c.

#define compareOrSet_proc( type )
 

Initializer:

\
type compareOrSet_ ## type (type *destination, type value, int verify, const char *description) \
{                                                                                               \
  if (!verify)                                                                                  \
    *destination = value;                                                                       \
  else                                                                                          \
                        \
    if ((*destination == 0 && ((*destination - value)                 > LF_FLOAT_TOLERANCE ))|| \
        (*destination != 0 && ((*destination - value)/(*destination)) > LF_FLOAT_TOLERANCE ) ) {\
      static int warnings=0;                                                                    \
      warnings++;                                                                               \
      if (warnings < LF_MAXWARNINGS)                                                            \
        ipc_notify(IPC_ALL,IPC_ERROR,"compareOrSet_" #type ": Value for %s stored in buffer (%e) differs from value in memory (%e)",    \
                  description, (double)value, (double)(*destination));                             \
      else if (warnings == LF_MAXWARNINGS)                                                      \
        ipc_notify(IPC_ALL,IPC_ERROR,"compareOrSet_" #type ": Maximum number of warnings reached (%d)",warnings);       \
    }                                                                                           \
                                                                                                \
  return *destination;                                                                          \
}
Family of routines to set a value of different float types.

Can verify instead if in verify mode.

Definition at line 1532 of file binarysave.c.

#define compareOrSet_proto( type )
 

Initializer:

\
type compareOrSet_ ## type(type *destination, type value, int verify, const char *description)

Definition at line 195 of file binarysave.c.

#define parts_per_row   (N/NEURONS_PER_BUFFER)
 

Definition at line 45 of file binarysave.c.


Typedef Documentation

typedef i32 LFBufferDataType
 

The size of this should match LF_TOKEN_GRAIN_SIZE.

Definition at line 95 of file binarysave.c.

typedef f32 LFFloatType
 

Format for each floating point number stored in the weights file.

Definition at line 48 of file binarysave.c.


Function Documentation

void LFBufferClear ( LFBuffer * buffer )
 

Clear the entire buffer.

Definition at line 1123 of file binarysave.c.

Referenced by binaryWeightsBufferProcess(), binaryWeightsFileRead(), binaryWeightsFileReadInFixedOrder(), and binaryWeightsFileWrite().

void LFBufferGet ( LFBuffer * buffer,
int pe )
 

Copy an entire buffer from the same position on another PE.

Definition at line 1103 of file binarysave.c.

Referenced by binaryWeightsFileWrite().

int LFBufferLength ( LFBuffer * buffer )
 

Returns the number of items in the buffer.

Definition at line 1132 of file binarysave.c.

Referenced by LFBufferRead(), LFBufferWrite(), binaryWeightsBufferProcess(), and binaryWeightsFileWrite().

void LFBufferPut ( LFBuffer * buffer,
int pe )
 

Copy an entire buffer from the same position on another PE.

Definition at line 1113 of file binarysave.c.

Referenced by binaryWeightsFileReadInFixedOrder().

int LFBufferRead ( LFBuffer * buffer,
FILE * file,
int length )
 

Read the given number of items from a binary file.

If less than that is available, there is no error, but the length is less than the maximum.

Definition at line 1156 of file binarysave.c.

Referenced by binaryWeightsFileReadInFixedOrder().

f32 LFBufferReadFloat ( LFBuffer * buffer )
 

Reads a floating point number from a binary buffer.

Definition at line 1284 of file binarysave.c.

Referenced by binaryWeightsBufferProcess().

int LFBufferReadInteger ( LFBuffer * buffer )
 

Reads an integer from a binary buffer.

Definition at line 1305 of file binarysave.c.

Referenced by binaryWeightsBufferProcess().

LFBufferDataType LFBufferReadItem ( LFBuffer * buffer )
 

Reads an item from a binary buffer.

Definition at line 1252 of file binarysave.c.

Referenced by LFBufferReadFloat(), and LFBufferReadInteger().

LFTokenType LFBufferReadToken ( LFBuffer * buffer )
 

Reads and returns a single token.

Definition at line 1200 of file binarysave.c.

Referenced by binaryWeightsBufferProcess().

int LFBufferWrite ( LFBuffer * buffer,
FILE * file )
 

Write an entire buffer full of data to a binary file.

Definition at line 1138 of file binarysave.c.

Referenced by binaryWeightsFileWrite().

int LFBufferWriteFloat ( LFBuffer * buffer,
f32 aFloat )
 

Outputs a floating point number to a binary buffer.

The number is first converted to an f32.

Definition at line 1275 of file binarysave.c.

Referenced by binaryWeightsBufferFill().

int LFBufferWriteInteger ( LFBuffer * buffer,
int anInteger )
 

Outputs an integer to a binary buffer.

The number is first converted to an i32.

Definition at line 1295 of file binarysave.c.

Referenced by binaryWeightsBufferFill().

int LFBufferWriteItem ( LFBuffer * buffer,
void * itemptr )
 

Write an item of the same size as LFBufferDataType to a binary buffer.

Definition at line 1231 of file binarysave.c.

Referenced by LFBufferWriteFloat(), and LFBufferWriteInteger().

int LFBufferWriteToken ( LFBuffer * buffer,
LFTokenType token )
 

Writes the given token to a binary buffer.

Definition at line 1173 of file binarysave.c.

Referenced by binaryWeightsBufferFill().

int binaryWeightsBufferFill ( LFBuffer * buffer,
int current_map_row,
int current_local_row,
int part_of_row )
 

Fill binary weight buffer with the tokens for one part-row.

Definition at line 320 of file binarysave.c.

Referenced by binaryWeightsFileWrite().

int binaryWeightsBufferProcess ( LFBuffer * buffer,
int verify )
 

Read and process a binary weight buffer.

Definition at line 738 of file binarysave.c.

Referenced by binaryWeightsFileReadInFixedOrder(), and binaryWeightsFileWrite().

int binaryWeightsBufferReadLatWeights ( LFBuffer * buffer,
LFTokenType token,
int i,
int j,
int radius,
int array_width,
l_weight * weights,
int verify,
const char * description )
 

Definition at line 554 of file binarysave.c.

int binaryWeightsBufferWriteLatWeights ( LFBuffer * buffer,
int token_name,
int i,
int j,
int radius,
int array_width,
l_weight * weights )
 

Definition at line 526 of file binarysave.c.

int binaryWeightsFileRead ( FILE * file )
 

Read and process a binary weight file.

Definition at line 464 of file binarysave.c.

int binaryWeightsFileReadInArbitraryOrder ( FILE * file,
LFBuffer * buffer )
 

Definition at line 665 of file binarysave.c.

int binaryWeightsFileReadInFixedOrder ( FILE * file,
LFBuffer * buffer )
 

Read file assuming layout of rows and part-rows is known.

This is slightly more efficient than binaryWeightsFileReadInArbitraryOrder but does not work with files created using different numbers of PEs than there are when they are loaded back in.

Definition at line 598 of file binarysave.c.

Referenced by binaryWeightsFileRead().

int binaryWeightsFileWrite ( FILE * file )
 

Write network weights to a binary file.

Definition at line 230 of file binarysave.c.

void binarysave_init_hook ( void )
 

Definition at line 208 of file binarysave.c.

int compareOrSetInt ( int * destination,
int value,
int verify,
const char * description )
 

Ensure that a weight file is appropriate for the current parameters.

Definition at line 1510 of file binarysave.c.

Referenced by binaryWeightsBufferProcess().

int compareParameter ( int weight_file_parameter,
int param_file_parameter,
const char * description )
 

Ensure that a weight file is appropriate for the current parameters.

Definition at line 1496 of file binarysave.c.

Referenced by binaryWeightsBufferProcess(), and binaryWeightsFileRead().

LFTokenType makeToken ( int name,
int length,
int infoa,
int infob )
 

Combines the given parameters and returns as a token.

Even though the parameters are specified as 64-bit, each must use only the lower 16 bits or truncation will occur.

Definition at line 1042 of file binarysave.c.

Referenced by binaryWeightsBufferFill(), binaryWeightsFileWrite(), and readTokenExpecting().

LFTokenType peekToken ( FILE * file )
 

Reads and returns a single token.

Definition at line 1365 of file binarysave.c.

void printToken ( LFTokenType token,
int offset )
 

Definition at line 1089 of file binarysave.c.

void printTokens ( FILE * file )
 

Prints the tokens present in a binary file for debugging.

Definition at line 962 of file binarysave.c.

void printTokensInBuffer ( LFBuffer * buffer )
 

Definition at line 996 of file binarysave.c.

f32 readFloat ( FILE * file )
 

Reads a floating point number from a binary file.

Definition at line 1464 of file binarysave.c.

Referenced by printTokens().

int readInteger ( FILE * file )
 

Reads an integer from a binary file.

Definition at line 1485 of file binarysave.c.

Referenced by binaryWeightsFileRead(), binaryWeightsFileReadInFixedOrder(), and printTokens().

LFBufferDataType readItem ( FILE * file )
 

Reads an item from a binary buffer.

Definition at line 1434 of file binarysave.c.

Referenced by readFloat(), and readInteger().

LFTokenType readToken ( FILE * file )
 

Reads and returns a single token.

Definition at line 1338 of file binarysave.c.

Referenced by printTokens(), and readTokenExpecting().

LFTokenType readTokenExpecting ( int tokenNameExpected,
FILE * file )
 

Read a token from file expecting one of a particular name.

Definition at line 1393 of file binarysave.c.

Referenced by binaryWeightsFileRead(), and binaryWeightsFileReadInFixedOrder().

int tokenInfoa ( LFTokenType token )
 

Definition at line 1061 of file binarysave.c.

int tokenInfob ( LFTokenType token )
 

Definition at line 1062 of file binarysave.c.

int tokenLength ( LFTokenType token )
 

Definition at line 1060 of file binarysave.c.

int tokenName ( LFTokenType token )
 

Definition at line 1059 of file binarysave.c.

const char * tokenNameString ( LFTokenType token )
 

Definition at line 1066 of file binarysave.c.

int writeFloat ( FILE * file,
f32 aFloat )
 

Outputs a floating point number to a binary file.

The number is first converted to an f32.

Definition at line 1455 of file binarysave.c.

int writeInteger ( FILE * file,
int anInteger )
 

Outputs an integer to a binary file.

The number is first converted to an i32.

Definition at line 1475 of file binarysave.c.

Referenced by binaryWeightsFileWrite().

int writeItem ( FILE * file,
void * itemptr )
 

Write an item of the same size as LFBufferDataType to a binary buffer.

Definition at line 1414 of file binarysave.c.

Referenced by writeFloat(), and writeInteger().

int writeToken ( FILE * file,
LFTokenType token )
 

Writes the given token to a binary file.

Definition at line 1315 of file binarysave.c.

Referenced by binaryWeightsFileWrite().


Variable Documentation

int bigendian = -1
 

Initially undefined; define before use.

Definition at line 137 of file binarysave.c.

LFBuffer binaryBuffer
 

Definition at line 131 of file binarysave.c.

int errorsEncountered = 0
 

Number of errors encountered in this load or save operation.

Definition at line 134 of file binarysave.c.

int load_afferent_weights_only = False
 

Definition at line 121 of file binarysave.c.

int save_afferent_weights_only = False
 

Definition at line 120 of file binarysave.c.

int status = 0
 

Definition at line 132 of file binarysave.c.

int tokensRead = 0
 

Total number of tokens input so far.

Definition at line 129 of file binarysave.c.

int tokensWritten = 0
 

Total number of tokens output so far.

Definition at line 128 of file binarysave.c.


Generated at Mon Aug 21 00:30:56 2000 for RF-LISSOM by doxygen1.2.1 written by Dimitri van Heesch, © 1997-2000