Chiebot-Mirror
/
FFmpeg
mirror of https://github.com/FFmpeg/FFmpeg.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

dnn: export operand info in python script and load in c code

Browse Source 
Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
Signed-off-by: Pedro Arthur <bygrandao@gmail.com>
pull/321/head^2
Guo, Yejun 5 years ago committed by Pedro Arthur
parent 2d5e39c13e
commit 83e0b71f66
4 changed files with 142 additions and 22 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 49
      libavfilter/dnn/dnn_backend_native.c
  2. 2
      libavfilter/dnn/dnn_backend_native.h
  3. 2
      libavfilter/dnn_interface.h
  4. 111
      tools/python/convert_from_tensorflow.py

49
libavfilter/dnn/dnn_backend_native.c
Unescape View File

@ -72,7 +72,6 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
ConvolutionalParams *conv_params;
DepthToSpaceParams *depth_to_space_params;
LayerPadParams *pad_params;
int32_t operand_index = 0;
model = av_malloc(sizeof(DNNModel));
if (!model){
@ -93,9 +92,10 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
}
model->model = (void *)network;
avio_seek(model_file_context, file_size - 4, SEEK_SET);
avio_seek(model_file_context, file_size - 8, SEEK_SET);
network->layers_num = (int32_t)avio_rl32(model_file_context);
dnn_size = 4;
network->operands_num = (int32_t)avio_rl32(model_file_context);
dnn_size = 8;
avio_seek(model_file_context, 0, SEEK_SET);
network->layers = av_mallocz(network->layers_num * sizeof(Layer));
@ -105,11 +105,6 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
return NULL;
}
/**
* Operands should be read from model file, the whole change will be huge.
* to make things step by step, we first mock the operands, instead of reading from model file.
*/
network->operands_num = network->layers_num + 1;
network->operands = av_mallocz(network->operands_num * sizeof(DnnOperand));
if (!network->operands){
avio_closep(&model_file_context);
@ -120,8 +115,6 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
for (layer = 0; layer < network->layers_num; ++layer){
layer_type = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
network->layers[layer].input_operand_indexes[0] = operand_index++;
network->layers[layer].output_operand_index = operand_index;
switch (layer_type){
case CONV:
conv_params = av_malloc(sizeof(ConvolutionalParams));
@ -162,6 +155,9 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
for (i = 0; i < conv_params->output_num; ++i){
conv_params->biases[i] = av_int2float(avio_rl32(model_file_context));
}
network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 8;
network->layers[layer].type = CONV;
network->layers[layer].params = conv_params;
break;
@ -174,6 +170,9 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
}
depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 8;
network->layers[layer].type = DEPTH_TO_SPACE;
network->layers[layer].params = depth_to_space_params;
break;
@ -191,6 +190,9 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
pad_params->paddings[i][1] = avio_rl32(model_file_context);
dnn_size += 8;
}
network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 8;
network->layers[layer].type = MIRROR_PAD;
network->layers[layer].params = pad_params;
break;
@ -201,6 +203,33 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
}
}
for (int32_t i = 0; i < network->operands_num; ++i){
DnnOperand *oprd;
int32_t name_len;
int32_t operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
oprd = &network->operands[operand_index];
name_len = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
dnn_size += name_len;
oprd->type = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
oprd->data_type = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
for (int32_t dim = 0; dim < 4; ++dim) {
oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
}
oprd->isNHWC = 1;
}
avio_closep(&model_file_context);
if (dnn_size != file_size){

2
libavfilter/dnn/dnn_backend_native.h
Unescape View File

@ -36,7 +36,7 @@ typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc;
typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam;
typedef enum {DOT_INPUT, DOT_INTERMEDIATE, DOT_OUTPUT} DNNOperandType;
typedef enum {DOT_INPUT = 1, DOT_OUTPUT = 2, DOT_INTERMEDIATE = DOT_INPUT | DOT_INPUT} DNNOperandType;
typedef struct Layer{
DNNLayerType type;

2
libavfilter/dnn_interface.h
Unescape View File

@ -32,7 +32,7 @@ typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType;
typedef enum {DNN_NATIVE, DNN_TF} DNNBackendType;
typedef enum {DNN_FLOAT, DNN_UINT8} DNNDataType;
typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
typedef struct DNNInputData{
void *data;

111
tools/python/convert_from_tensorflow.py
Unescape View File

@ -23,6 +23,37 @@ import sys, struct
__all__ = ['convert_from_tensorflow']
class Operand(object):
IOTYPE_INPUT = 1
IOTYPE_OUTPUT = 2
IOTYPE_INTERMEDIATE = IOTYPE_INPUT | IOTYPE_OUTPUT
DTYPE_FLOAT = 1
DTYPE_UINT8 = 4
index = 0
def __init__(self, name, dtype, dims):
self.name = name
self.dtype = dtype
self.dims = dims
self.iotype = 0
self.used_count = 0
self.index = Operand.index
Operand.index = Operand.index + 1
self.iotype2str = {Operand.IOTYPE_INPUT: 'in', Operand.IOTYPE_OUTPUT: 'out', Operand.IOTYPE_INTERMEDIATE: 'inout'}
self.dtype2str = {Operand.DTYPE_FLOAT: 'DT_FLOAT', Operand.DTYPE_UINT8: 'DT_UINT8'}
def add_iotype(self, iotype):
self.iotype = self.iotype | iotype
if iotype == Operand.IOTYPE_INPUT:
self.used_count = self.used_count + 1
def __str__(self):
return "{}: (name: {}, iotype: {}, dtype: {}, dims: ({},{},{},{}) used_count: {})".format(self.index,
self.name, self.iotype2str[self.iotype], self.dtype2str[self.dtype],
self.dims[0], self.dims[1], self.dims[2], self.dims[3], self.used_count)
def __lt__(self, other):
return self.index < other.index
class TFConverter:
def __init__(self, graph_def, nodes, outfile, dump4tb):
self.graph_def = graph_def
@ -37,8 +68,28 @@ class TFConverter:
self.conv_paddings = {'VALID':0, 'SAME':1}
self.converted_nodes = set()
self.conv2d_scope_names = set()
self.conv2d_scopename_inputname_dict = {}
self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3}
self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2}
self.name_operand_dict = {}
def add_operand(self, name, type):
node = self.name_node_dict[name]
if name not in self.name_operand_dict:
dtype = node.attr['dtype'].type
if dtype == 0:
dtype = node.attr['T'].type
dims = [-1,-1,-1,-1]
if 'shape' in node.attr:
dims[0] = node.attr['shape'].shape.dim[0].size
dims[1] = node.attr['shape'].shape.dim[1].size
dims[2] = node.attr['shape'].shape.dim[2].size
dims[3] = node.attr['shape'].shape.dim[3].size
operand = Operand(name, dtype, dims)
self.name_operand_dict[name] = operand;
self.name_operand_dict[name].add_iotype(type)
return self.name_operand_dict[name].index
def dump_for_tensorboard(self):
@ -60,11 +111,10 @@ class TFConverter:
# the BiasAdd name is possible be changed into the output name,
# if activation is None, and BiasAdd.next is the last op which is Identity
if conv2d_scope_name + '/BiasAdd' in self.edges:
activation = self.edges[conv2d_scope_name + '/BiasAdd'][0]
activation = activation.op
anode = self.edges[conv2d_scope_name + '/BiasAdd'][0]
else:
activation = 'None'
return knode, bnode, dnode, activation
anode = None
return knode, bnode, dnode, anode
def dump_conv2d_to_file(self, node, f):
@ -73,16 +123,21 @@ class TFConverter:
self.converted_nodes.add(node.name)
scope_name = TFConverter.get_scope_name(node.name)
#knode for kernel, bnode for bias, dnode for dilation
knode, bnode, dnode, activation = self.get_conv2d_params(scope_name)
#knode for kernel, bnode for bias, dnode for dilation, anode for activation
knode, bnode, dnode, anode = self.get_conv2d_params(scope_name)
if dnode is not None:
dilation = struct.unpack('i', dnode.attr['value'].tensor.tensor_content[0:4])[0]
else:
dilation = 1
if anode is not None:
activation = anode.op
else:
activation = 'None'
padding = node.attr['padding'].s.decode("utf-8")
# conv2d with dilation > 1 generates tens of nodes, not easy to parse them, so use tricky.
# conv2d with dilation > 1 generates tens of nodes, not easy to parse them, so use this tricky method.
if dilation > 1 and scope_name + '/stack' in self.name_node_dict:
if self.name_node_dict[scope_name + '/stack'].op == "Const":
padding = 'SAME'
@ -107,6 +162,15 @@ class TFConverter:
bias = btensor.tensor_content
f.write(bias)
input_name = self.conv2d_scopename_inputname_dict[scope_name]
input_operand_index = self.add_operand(input_name, Operand.IOTYPE_INPUT)
if anode is not None:
output_operand_index = self.add_operand(anode.name, Operand.IOTYPE_OUTPUT)
else:
output_operand_index = self.add_operand(self.edges[bnode.name][0].name, Operand.IOTYPE_OUTPUT)
np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
def dump_depth2space_to_file(self, node, f):
assert(node.op == 'DepthToSpace')
@ -114,6 +178,9 @@ class TFConverter:
block_size = node.attr['block_size'].i
np.array([self.op2code[node.op], block_size], dtype=np.uint32).tofile(f)
self.converted_nodes.add(node.name)
input_operand_index = self.add_operand(node.input[0], Operand.IOTYPE_INPUT)
output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
def dump_mirrorpad_to_file(self, node, f):
@ -127,6 +194,9 @@ class TFConverter:
paddings = pnode.attr['value'].tensor.tensor_content
f.write(paddings)
self.converted_nodes.add(node.name)
input_operand_index = self.add_operand(node.input[0], Operand.IOTYPE_INPUT)
output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f)
def dump_layers_to_file(self, f):
@ -147,10 +217,21 @@ class TFConverter:
self.dump_mirrorpad_to_file(node, f)
def dump_operands_to_file(self, f):
operands = sorted(self.name_operand_dict.values())
for operand in operands:
#print('{}'.format(operand))
np.array([operand.index, len(operand.name)], dtype=np.uint32).tofile(f)
f.write(operand.name.encode('utf-8'))
np.array([operand.iotype, operand.dtype], dtype=np.uint32).tofile(f)
np.array([operand.dims[0], operand.dims[1], operand.dims[2], operand.dims[3]], dtype=np.uint32).tofile(f)
def dump_to_file(self):
with open(self.outfile, 'wb') as f:
self.dump_layers_to_file(f)
np.array([self.layer_number], dtype=np.uint32).tofile(f)
self.dump_operands_to_file(f)
np.array([self.layer_number, len(self.name_operand_dict)], dtype=np.uint32).tofile(f)
def generate_name_node_dict(self):
@ -212,19 +293,29 @@ class TFConverter:
return name[0:index]
def generate_conv2d_scope_names(self):
def generate_conv2d_scope_info(self):
# conv2d is a sub block in graph, get the scope name
for node in self.nodes:
if node.op == 'Conv2D':
scope = TFConverter.get_scope_name(node.name)
self.conv2d_scope_names.add(scope)
# get the input name to the conv2d sub block
for node in self.nodes:
scope = TFConverter.get_scope_name(node.name)
if scope in self.conv2d_scope_names:
if node.op == 'Conv2D' or node.op == 'Shape':
for inp in node.input:
if TFConverter.get_scope_name(inp) != scope:
self.conv2d_scopename_inputname_dict[scope] = inp
def run(self):
self.generate_name_node_dict()
self.generate_output_names()
self.remove_identity()
self.generate_edges()
self.generate_conv2d_scope_names()
self.generate_conv2d_scope_info()
if self.dump4tb:
self.dump_for_tensorboard()

Write Preview
Loading…
Cancel
Save