diff options
Diffstat (limited to 'nnpackage/schema/circle_schema.fbs')
-rw-r--r-- | nnpackage/schema/circle_schema.fbs | 349 |
1 files changed, 341 insertions, 8 deletions
diff --git a/nnpackage/schema/circle_schema.fbs b/nnpackage/schema/circle_schema.fbs index dbb0d3526..3972056f9 100644 --- a/nnpackage/schema/circle_schema.fbs +++ b/nnpackage/schema/circle_schema.fbs @@ -1,4 +1,4 @@ -// Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved +// Copyright (c) 2019~2020 Samsung Electronics Co., Ltd. All Rights Reserved // Copyright 2017 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +14,20 @@ // limitations under the License. // Revision History -// Version 0: Initial version. Based on TensorFlow Lite v1.13.1 schema. +// +// Version Major.Minor +// +// Major version is schema version. +// We keep schema version if it is compatible +// Minor version is for human communication +// It will not be stored in circle model. +// +// Version 0.0: Initial version. Based on TensorFlow Lite v1.13.1 schema. +// Version 0.1: Based on TF v2.2-rc2 + more (from TensorFlow `56d281c`) +// `BATCH_MATMUL` operator, `FLOAT64` tensor type, +// `asymmetric_quantize_inputs` for several operator options +// Version 0.2: BCQ_GATHER and BCQ_FULLY_CONNECTED are added. +// Version 0.3: SHUFFLED16x1FLOAT32 is added. namespace circle; @@ -38,6 +51,7 @@ enum TensorType : byte { INT16 = 7, COMPLEX64 = 8, INT8 = 9, + FLOAT64 = 10, } // Custom quantization parameters for experimenting with new quantization @@ -62,9 +76,119 @@ table QuantizationParameters { scale:[float]; // For dequantizing the tensor's values. zero_point:[long]; - // If this is not none, the quantization parameters above are ignored and the - // value of the QuantizationDetails union below should be used. + // If this is not none, the other quantization parameters (i.e. min, max, + // scale, zero_point fields above) are ignored and the value of the + // QuantizationDetails union should be used. details:QuantizationDetails; + + // Specifies the dimension of the Tensor's shape that the scales and + // zero_points correspond to. For example, a tensor t, with dims=[4, 3, 2, 1] + // with quantization params: + // scale=[1.0, 2.0, 3.0], zero_point=[1, 2, 3], quantization_dimension=1 + // will be quantized across the second dimension of t. + // t[:, 0, :, :] will have scale[0]=1.0, zero_point[0]=1 + // t[:, 1, :, :] will have scale[1]=2.0, zero_point[0]=2 + // t[:, 2, :, :] will have scale[2]=3.0, zero_point[0]=3 + quantized_dimension:int; +} + +// Sparse tensors. +// We use a modification of the TACO format. +// Reference: http://tensor-compiler.org/kjolstad-oopsla17-tensor-compiler.pdf +// +// To encode a conceptual n-dimensional dense tensor with dims (d0, ..., dn-1), +// potentially with a k-dimensional block (0 <= k <= n) with dims +// (dn, ..., dn+k-1), the format needs to specify: +// 1. In what order to traverse these dimensions. For example, to store a 2-D +// matrix in row major order, the traversal order would be (d0, d1), +// whereas to store it in column major order, the traversal order would be +// (d1, d0). If the 2-D matrix has a 2-D inner block, the traversal order +// could be (d0, d1, d2, d3). +// 2. How each block dimension in (dn, ..., dn+k-1) maps to the original +// tensor dimension in (d0, ..., dn-1). +// 3. In the traversal order defined above, the format (dense vs. sparse) and +// index metadata for each dimension. For a dense dimension, this is just +// the size of that dimension. For a sparse dimension, it's the same as +// the compressed index defined in the Compressed Sparse Row (CSR) format. +// (http://scipy-lectures.org/advanced/scipy_sparse/csr_matrix.html) + +// The storage type for a dimension. Currently we support: +// 1. DENSE: each coordinate in this dimension is stored implicitly. +// 2. SPARSE_CSR: only the coordinates with non-zero elements are stored. The +// compression technique is the same what CSR uses. +// More types like a sparse dimension with a different compression technique +// could be added to the list in the future. +enum DimensionType : byte { + DENSE = 0, + SPARSE_CSR = 1, +} + +table Int32Vector { + values:[int]; +} + +table Uint16Vector { + values:[ushort] (force_align: 4); +} + +table Uint8Vector { + values:[ubyte] (force_align: 4); +} + +// Variable-typed buffer to store the index metadata for a sparse dimension. +// The widest type is Int32 instead of UInt32 because tensor's shape is a int32 +// vector. We don't want the per-dimensional index to overflow that range. +union SparseIndexVector { + Int32Vector, + Uint16Vector, + Uint8Vector +} + +table DimensionMetadata { + // Whether a dimension is dense or sparse. + format:DimensionType; + // Index metadata used for a dimension. + // - If format is DimensionType.DENSE then we use the dense_size field to + // store the size of that dimension. Each index in that dimension is + // stored implicitly. + // - If format is DimensionType.SPARSE_CSR then we use array_segments and + // array_indices to encode that dimension. array_segments represents how + // to segment the indices array, each segment corresponds to one element + // in the previous dimension. array_indices represents the index of the + // non-zero elements within this dimension (as those in the CSR matrix + // format, where the first array is row pointers and the second array is + // column indices). + dense_size:int; + array_segments:SparseIndexVector; + array_indices:SparseIndexVector; +} + +// Parameters to encode a sparse TfLite tensor. +table SparsityParameters { + // The traversal order of the dimensions defined in the `shape` field of the + // conceptual dense tensor. For a n-dimensional tensors with dims (d0, d1, + // ..., dn-1), + // - if not block sparse, the traversal_order is just a permutation of (d0, + // ..., dn-1). For example, a 2-D matrix stored in row-major order would + // have traversal_order = (d0, d1). + // - if block sparse with a k-dimensional block (0 <= k <= n), the + // traversal_order has n + k elements. The first n elements are still a + // permutation of (d0, ..., dn-1). The lask k elements are a permutation + // of (dn, ..., dn+k-1), defining how to traverse a block internally. For + // example, a 2-D matrix with 2-D blocks, both stored in row-major order + // would have traversal_order = (d0, d1, d2, d3). + traversal_order:[int]; + // For an n-dimensional tensor with a k-dimensional block (0 <= k <= n), + // stores how a block dimension in (dn, ..., dn+k-1) maps to the original + // tensor dimension in (d0, ..., dn). + // It's stored in the order of (dn, ..., dn+k-1). + // If not block-sparse, this field is NULL. + block_map:[int]; + // In the traversal order defined above, the metadata needed for + // each dimension to locate the non-zero values in the original dense tensor. + // The size of the dim_metadata array = the size of the traversal_order array + // = n + k. + dim_metadata:[DimensionMetadata]; } table Tensor { @@ -86,19 +210,28 @@ table Tensor { quantization:QuantizationParameters; // Optional. is_variable:bool = false; + + // Parameters to encode a sparse tensor. See the example in + // tensorflow/lite/testdata/sparse_tensor.json. + sparsity:SparsityParameters; // Optional. + + // Encodes `shape` with unknown dimensions. Unknown dimensions are + // represented with -1. + shape_signature:[int]; // Optional. } // A list of builtin operators. Builtin operators are slightly faster than custom // ones, but not by much. Moreover, while custom operators accept an opaque // object containing configuration parameters, builtins have a predetermined // set of acceptable options. -enum BuiltinOperator : byte { + +enum BuiltinOperator : ubyte { ADD = 0, AVERAGE_POOL_2D = 1, CONCATENATION = 2, CONV_2D = 3, DEPTHWISE_CONV_2D = 4, - // DEPTH_TO_SPACE = 5, + DEPTH_TO_SPACE = 5, DEQUANTIZE = 6, EMBEDDING_LOOKUP = 7, FLOOR = 8, @@ -203,6 +336,33 @@ enum BuiltinOperator : byte { MIRROR_PAD = 100, ABS = 101, SPLIT_V = 102, + UNIQUE = 103, + CEIL = 104, + REVERSE_V2 = 105, + ADD_N = 106, + GATHER_ND = 107, + COS = 108, + WHERE = 109, + RANK = 110, + ELU = 111, + REVERSE_SEQUENCE = 112, + MATRIX_DIAG = 113, + QUANTIZE = 114, + MATRIX_SET_DIAG = 115, + ROUND = 116, + HARD_SWISH = 117, + IF = 118, + WHILE = 119, + NON_MAX_SUPPRESSION_V4 = 120, + NON_MAX_SUPPRESSION_V5 = 121, + SCATTER_ND = 122, + SELECT_V2 = 123, + DENSIFY = 124, + SEGMENT_SUM = 125, + BATCH_MATMUL = 126, + BCQ_GATHER = 252, + BCQ_FULLY_CONNECTED = 253, + INSTANCE_NORM = 254, } // Options for the builtin operators. @@ -286,6 +446,31 @@ union BuiltinOptions { MirrorPadOptions, AbsOptions, SplitVOptions, + UniqueOptions, + ReverseV2Options, + AddNOptions, + GatherNdOptions, + CosOptions, + WhereOptions, + RankOptions, + ReverseSequenceOptions, + MatrixDiagOptions, + QuantizeOptions, + MatrixSetDiagOptions, + HardSwishOptions, + IfOptions, + WhileOptions, + DepthToSpaceOptions, + NonMaxSuppressionV4Options, + NonMaxSuppressionV5Options, + ScatterNdOptions, + SelectV2Options, + DensifyOptions, + SegmentSumOptions, + BatchMatMulOptions, + BCQGatherOptions = 252, + BCQFullyConnectedOptions = 253, + InstanceNormOptions = 254, } enum Padding : byte { SAME, VALID } @@ -322,6 +507,9 @@ table DepthwiseConv2DOptions { padding:Padding; stride_w:int; stride_h:int; + // `depth_multiplier` is redundant. It's used by CPU kernels in + // TensorFlow 2.0 or below, but ignored in versions above. + // See comments in lite/c/builtin_op_data.h for more details. depth_multiplier:int; fused_activation_function:ActivationFunctionType; // Parameters for DepthwiseConv version 2 or above. @@ -348,17 +536,22 @@ table LSHProjectionOptions { table SVDFOptions { rank:int; fused_activation_function:ActivationFunctionType; + // For weights-only quantization, use asymmetric quantization for non + // constant inputs at evaluation time. + asymmetric_quantize_inputs:bool; } // An implementation of TensorFlow RNNCell. table RNNOptions { fused_activation_function:ActivationFunctionType; + asymmetric_quantize_inputs:bool; } // An implementation of TensorFlow dynamic_rnn with RNNCell. table SequenceRNNOptions { time_major:bool; fused_activation_function:ActivationFunctionType; + asymmetric_quantize_inputs:bool; } // An implementation of TensorFlow bidrectional_dynamic_rnn with RNNCell. @@ -366,11 +559,13 @@ table BidirectionalSequenceRNNOptions { time_major:bool; fused_activation_function:ActivationFunctionType; merge_outputs: bool; + asymmetric_quantize_inputs:bool; } enum FullyConnectedOptionsWeightsFormat: byte { DEFAULT = 0, SHUFFLED4x16INT8 = 1, + SHUFFLED16x1FLOAT32 = 127 } // An implementation of TensorFlow fully_connected (a.k.a Dense) layer. @@ -380,6 +575,16 @@ table FullyConnectedOptions { // Parameters for FullyConnected version 2 or above. weights_format:FullyConnectedOptionsWeightsFormat = DEFAULT; + + // Parameters for FullyConnected version 5 or above. + // If set to true, then the number of dimension is preserved. Furthermore, + // all but the last dimension of the input and output shapes will be equal. + keep_num_dims: bool; + + // Parameters for FullyConnected version 7 or above. + // If set to true, then weights-only op will use asymmetric quantization for + // inputs. + asymmetric_quantize_inputs: bool; } table SoftmaxOptions { @@ -428,6 +633,9 @@ table LSTMOptions { // Parameters for LSTM version 2 or above. // Basic kernel is only supported in version 2 or above. kernel_type: LSTMKernelType = FULL; + + // Parameters for LSTM version 4 or above. + asymmetric_quantize_inputs: bool; } // An implementation of TensorFlow dynamic_rnn with LSTMCell. @@ -438,21 +646,35 @@ table UnidirectionalSequenceLSTMOptions { // If true then first dimension is sequence, otherwise batch. time_major:bool; + + // Parameter for Unidirectional Sequence LSTM version 4. + asymmetric_quantize_inputs:bool; } table BidirectionalSequenceLSTMOptions { + // Parameters supported by version 1: fused_activation_function:ActivationFunctionType; cell_clip: float; // Optional, 0.0 means no clipping proj_clip: float; // Optional, 0.0 means no clipping // If true, store the outputs of both directions into the first output. merge_outputs: bool; + + // Parameters supported by version 2: + // If true then first dimension is sequence, otherwise batch. + // Version 1 implementations assumed time_major to be true, so this default + // value should never change. + time_major: bool = true; + + // Parameters for version 3 or above. + asymmetric_quantize_inputs:bool; } table ResizeBilinearOptions { new_height: int (deprecated); new_width: int (deprecated); align_corners: bool; + half_pixel_centers: bool; } table ResizeNearestNeighborOptions { @@ -491,6 +713,10 @@ table SpaceToDepthOptions { block_size: int; } +table DepthToSpaceOptions { + block_size: int; +} + table SubOptions { fused_activation_function:ActivationFunctionType; } @@ -522,6 +748,9 @@ table TransposeOptions { table ExpOptions { } +table CosOptions { +} + table ReducerOptions { keep_dims: bool; } @@ -616,6 +845,9 @@ table ShapeOptions { out_type : TensorType; } +table RankOptions { +} + table PowOptions { } @@ -645,6 +877,9 @@ table AbsOptions { } +table HardSwishOptions { +} + table LogicalAndOptions { } @@ -692,6 +927,84 @@ table MirrorPadOptions { mode:MirrorPadMode; } +table UniqueOptions { + idx_out_type:TensorType = INT32; +} + +table ReverseV2Options { +} + +table AddNOptions { +} + +table GatherNdOptions { +} + +table WhereOptions { +} + +table ReverseSequenceOptions { + seq_dim:int; + batch_dim:int = 0; +} + +table MatrixDiagOptions { +} + +table QuantizeOptions { +} + +table MatrixSetDiagOptions { +} + +table IfOptions { + then_subgraph_index:int; + else_subgraph_index:int; +} + +table WhileOptions { + cond_subgraph_index:int; + body_subgraph_index:int; +} + +table NonMaxSuppressionV4Options { +} + +table NonMaxSuppressionV5Options { +} + +table ScatterNdOptions { +} + +table SelectV2Options { +} + +table DensifyOptions { +} + +table SegmentSumOptions { +} + +table BatchMatMulOptions { + adjoint_lhs:bool; + adjoint_rhs:bool; +} + +table BCQGatherOptions { + input_hidden_size: int; + axis: int; +} + +table BCQFullyConnectedOptions { + weights_hidden_size: int; + fused_activation_function:ActivationFunctionType; +} + +table InstanceNormOptions { + epsilon:float; + fused_activation_function:ActivationFunctionType; +} + // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a // builtin, or a string if the operator is custom. table OperatorCode { @@ -725,7 +1038,7 @@ table Operator { // complicate map lookups. opcode_index:uint; - // Optional input and output tensors are indicated by -1. + // Optional input are indicated by -1. inputs:[int]; outputs:[int]; @@ -742,6 +1055,15 @@ table Operator { // If the list is empty, no variable is mutated in this operator. // The list either has the same length as `inputs`, or is empty. mutating_variable_inputs:[bool]; + + // A list of indices to the subgraph's "tensors" that are internal to an Op. + // Internal tensors are those that do not flow in or out of the operation, + // but instead are part of internal computation. As such, the operation's + // implementation may manage its memory more efficiently. They are needed + // however (i.e. not just an implementation detail) since they are part of the + // computation, which may require relevant metadata such as quantization + // parameters. + intermediates:[int]; } // The root type, defining a subgraph, which typically represents an entire @@ -775,6 +1097,13 @@ table Buffer { data:[ubyte] (force_align: 16); } +table Metadata { + // A human readable string to uniquely identify a Metadata. + name:string; + // An index to the buffers table. + buffer:uint; +} + table Model { // Version of the schema. version:uint; @@ -797,8 +1126,12 @@ table Model { // their buffer. buffers:[Buffer]; - // Metadata about the model. Indirects into the existings buffers list. + // Metadata about the model. Indirects into the existings buffers list. + // Deprecated, prefer to use metadata field. metadata_buffer:[int]; + + // Metadata about the model. + metadata:[Metadata]; } root_type Model; |