|
|
|
@@ -1,504 +0,0 @@ |
|
|
|
# -*- coding: utf-8 -*- |
|
|
|
# MegEngine is Licensed under the Apache License, Version 2.0 (the "License") |
|
|
|
# |
|
|
|
# Copyright (c) 2014-2020 Megvii Inc. All rights reserved. |
|
|
|
# |
|
|
|
# Unless required by applicable law or agreed to in writing, |
|
|
|
# software distributed under the License is distributed on an |
|
|
|
# "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
|
|
import argparse |
|
|
|
import collections |
|
|
|
import textwrap |
|
|
|
import os |
|
|
|
import hashlib |
|
|
|
import struct |
|
|
|
|
|
|
|
class member_defs: |
|
|
|
"""contain classes to define members of an opr param""" |
|
|
|
|
|
|
|
Dtype = collections.namedtuple('Dtype', ['cname', 'pycvt', 'pyfmt', |
|
|
|
'cppjson', 'cname_attr']) |
|
|
|
Dtype.__new__.__defaults__ = ('', ) |
|
|
|
uint32 = Dtype('uint32_t', 'int', 'I', 'NumberInt') |
|
|
|
uint64 = Dtype('uint64_t', 'int', 'Q', 'NumberInt', |
|
|
|
'alignas(sizeof(uint64_t)) ') |
|
|
|
int32 = Dtype('int32_t', 'int', 'i', 'NumberInt') |
|
|
|
float32 = Dtype('float', 'float', 'f', 'Number') |
|
|
|
float64 = Dtype('double', 'float', 'd', 'Number') |
|
|
|
dtype = Dtype('DTypeEnum', '_as_dtype_num', 'I', 'Number') |
|
|
|
bool = Dtype('bool', 'bool', '?', 'Bool') |
|
|
|
|
|
|
|
class Base: |
|
|
|
pass |
|
|
|
|
|
|
|
|
|
|
|
class Doc: |
|
|
|
"""wrap an identifier to associate document |
|
|
|
|
|
|
|
note: if the doc starts with a linebreak, it would not be reforamtted. |
|
|
|
""" |
|
|
|
__slots__ = ['id', 'doc'] |
|
|
|
|
|
|
|
def __init__(self, id_, doc): |
|
|
|
assert isinstance(id_, str) and isinstance(doc, str), (id_, doc) |
|
|
|
self.id = id_ |
|
|
|
self.doc = doc |
|
|
|
|
|
|
|
@property |
|
|
|
def no_reformat(self): |
|
|
|
"""whether reformat is disallowed for this doc string""" |
|
|
|
return self.doc.startswith('\n') |
|
|
|
|
|
|
|
@property |
|
|
|
def raw_lines(self): |
|
|
|
"""the doc lines when ``no_format`` is true""" |
|
|
|
ret = self.doc.split('\n') |
|
|
|
assert not ret[0] |
|
|
|
return ret[1:] |
|
|
|
|
|
|
|
@classmethod |
|
|
|
def make(cls, v): |
|
|
|
"""make doc object from str or doc""" |
|
|
|
if isinstance(v, cls): |
|
|
|
return v |
|
|
|
assert isinstance(v, str) |
|
|
|
return cls(v, '') |
|
|
|
|
|
|
|
def __str__(self): |
|
|
|
return self.id |
|
|
|
|
|
|
|
def __eq__(self, rhs): |
|
|
|
if isinstance(rhs, str): |
|
|
|
return self.id == rhs |
|
|
|
return (isinstance(rhs, Doc) and |
|
|
|
(self.id, self.doc) == (rhs.id, rhs.doc)) |
|
|
|
|
|
|
|
|
|
|
|
class Enum(Base): |
|
|
|
"""define an enum; the result would contain both an enum class def and its |
|
|
|
corresponding data field |
|
|
|
|
|
|
|
:param default: index of default member value |
|
|
|
|
|
|
|
:attr name_field: name of the data field of this enum in the param |
|
|
|
struct |
|
|
|
:attr member_alias: list of (member, alias) pairs |
|
|
|
""" |
|
|
|
__slots__ = ['name', 'name_field', 'members', 'default', |
|
|
|
'member_alias'] |
|
|
|
|
|
|
|
all_enums = {} |
|
|
|
"""(param_name, name) => enum""" |
|
|
|
|
|
|
|
def __init__(self, param_name, name, name_field, members, default, |
|
|
|
member_alias): |
|
|
|
name = member_defs.Doc.make(name) |
|
|
|
assert name.id[0].isupper() |
|
|
|
members = tuple(map(member_defs.Doc.make, members)) |
|
|
|
if isinstance(default, str): |
|
|
|
if default not in name_field: |
|
|
|
raise ValueError( |
|
|
|
"Default value '{}' does not exist.".format(default)) |
|
|
|
default = name_field.index(default) |
|
|
|
assert isinstance(default, int) |
|
|
|
self.name = name |
|
|
|
self.name_field = self.get_name_field(name.id, name_field) |
|
|
|
self.members = members |
|
|
|
self.default = default |
|
|
|
|
|
|
|
self.all_enums[(param_name, name.id)] = self |
|
|
|
|
|
|
|
assert isinstance(member_alias, list) |
|
|
|
self.member_alias = member_alias |
|
|
|
|
|
|
|
@classmethod |
|
|
|
def get_name_field(cls, name, name_field): |
|
|
|
if name_field is None: |
|
|
|
name_field = name[0].lower() + name[1:] |
|
|
|
assert isinstance(name_field, str) |
|
|
|
return name_field |
|
|
|
|
|
|
|
class Field(Base): |
|
|
|
"""define a normal data field""" |
|
|
|
__slots__ = ['name', 'dtype', 'default'] |
|
|
|
|
|
|
|
def __init__(self, name, dtype, default): |
|
|
|
assert isinstance(dtype, member_defs.Dtype) |
|
|
|
self.name = member_defs.Doc.make(name) |
|
|
|
self.dtype = dtype |
|
|
|
self.default = default |
|
|
|
|
|
|
|
class Const(Base): |
|
|
|
"""define a const data field""" |
|
|
|
__slots__ = ['name', 'dtype', 'default'] |
|
|
|
|
|
|
|
def __init__(self, name, dtype, default): |
|
|
|
assert isinstance(dtype, member_defs.Dtype) |
|
|
|
self.name = member_defs.Doc.make(name) |
|
|
|
self.dtype = dtype |
|
|
|
self.default = default |
|
|
|
|
|
|
|
class EnumAlias(Base): |
|
|
|
"""alias of enum type from another param""" |
|
|
|
__slots__ = ['name', 'name_field', 'src_class', 'src_name', 'default'] |
|
|
|
|
|
|
|
def __init__(self, name, name_field, src_class, src_name, default): |
|
|
|
self.name = name |
|
|
|
self.name_field = member_defs.Enum.get_name_field(name, name_field) |
|
|
|
self.src_class = src_class |
|
|
|
if src_name is None: |
|
|
|
src_name = name |
|
|
|
self.src_name = src_name |
|
|
|
self.default = default |
|
|
|
|
|
|
|
@property |
|
|
|
def src_enum(self): |
|
|
|
"""source Enum class""" |
|
|
|
return member_defs.Enum.all_enums[(self.src_class, self.src_name)] |
|
|
|
|
|
|
|
def get_default(self): |
|
|
|
"""get default index; fallback to src index if default is not |
|
|
|
set""" |
|
|
|
if self.default is None: |
|
|
|
return self.src_enum.default |
|
|
|
return self.default |
|
|
|
|
|
|
|
|
|
|
|
class ParamDef: |
|
|
|
"""""" |
|
|
|
__all_tags = set() |
|
|
|
all_param_defs = [] |
|
|
|
|
|
|
|
__slots__ = ['name', 'members', 'tag', 'is_legacy'] |
|
|
|
|
|
|
|
def __init__(self, name, doc='', *, version=0, is_legacy=False): |
|
|
|
self.members = [] |
|
|
|
self.all_param_defs.append(self) |
|
|
|
h = hashlib.sha256(name.encode('utf-8')) |
|
|
|
if version: |
|
|
|
h.update(struct.pack('<I', version)) |
|
|
|
if is_legacy: |
|
|
|
name += 'V{}'.format(version) |
|
|
|
self.name = member_defs.Doc(name, doc) |
|
|
|
self.tag = int(h.hexdigest()[:8], 16) |
|
|
|
self.is_legacy = is_legacy |
|
|
|
if self.tag < 1024: |
|
|
|
self.tag += 1024 |
|
|
|
assert self.tag not in self.__all_tags, ( |
|
|
|
'tag hash confliction: name={} tag={}'.format(name, self.tag)) |
|
|
|
self.__all_tags.add(self.tag) |
|
|
|
|
|
|
|
def add_fields(self, dtype, *names_defaults): |
|
|
|
assert isinstance(dtype, str) |
|
|
|
dtype = getattr(member_defs, dtype) |
|
|
|
assert len(names_defaults) % 2 == 0 |
|
|
|
for i, j in zip(names_defaults[::2], names_defaults[1::2]): |
|
|
|
self.members.append(member_defs.Field(i, dtype, j)) |
|
|
|
return self |
|
|
|
|
|
|
|
def add_enum(self, name, *members, default=0, name_field=None, |
|
|
|
member_alias=[]): |
|
|
|
self.members.append(member_defs.Enum( |
|
|
|
self.name.id, name, name_field, members, default, member_alias)) |
|
|
|
return self |
|
|
|
|
|
|
|
def add_enum_alias(self, name, src_class, src_name=None, name_field=None, |
|
|
|
default=None): |
|
|
|
self.members.append(member_defs.EnumAlias( |
|
|
|
name, name_field, src_class, src_name, default)) |
|
|
|
return self |
|
|
|
|
|
|
|
def add_const(self, dtype, *names_defaults): |
|
|
|
assert isinstance(dtype, str) |
|
|
|
dtype = getattr(member_defs, dtype) |
|
|
|
assert len(names_defaults) % 2 == 0 |
|
|
|
for i, j in zip(names_defaults[::2], names_defaults[1::2]): |
|
|
|
self.members.append(member_defs.Const(i, dtype, j)) |
|
|
|
return self |
|
|
|
|
|
|
|
|
|
|
|
class WriterBase: |
|
|
|
"""base class for output file writer""" |
|
|
|
|
|
|
|
_fout = None |
|
|
|
_input_hash = None |
|
|
|
_cur_class = None |
|
|
|
|
|
|
|
def __call__(self, fout): |
|
|
|
self._fout = fout |
|
|
|
|
|
|
|
def set_input_hash(self, h): |
|
|
|
self._input_hash = h |
|
|
|
return self |
|
|
|
|
|
|
|
def _get_header(self): |
|
|
|
return 'generated by {} for {}'.format( |
|
|
|
os.path.basename(__file__), |
|
|
|
self._input_hash |
|
|
|
) |
|
|
|
|
|
|
|
def _process(self, defs): |
|
|
|
dispatch = { |
|
|
|
member_defs.Enum: self._on_member_enum, |
|
|
|
member_defs.EnumAlias: self._on_member_enum_alias, |
|
|
|
member_defs.Field: self._on_member_field, |
|
|
|
member_defs.Const: self._on_const_field |
|
|
|
} |
|
|
|
for i in defs: |
|
|
|
assert isinstance(i, ParamDef) |
|
|
|
if i.is_legacy: |
|
|
|
continue |
|
|
|
self._cur_class = i.name |
|
|
|
self._on_param_begin(i) |
|
|
|
for j in i.members: |
|
|
|
dispatch[type(j)](j) |
|
|
|
self._on_param_end(i) |
|
|
|
|
|
|
|
def _on_param_begin(self, p): |
|
|
|
""":type p: :class:`.ParamDef`""" |
|
|
|
|
|
|
|
def _on_param_end(self, p): |
|
|
|
""":type p: :class:`.ParamDef`""" |
|
|
|
|
|
|
|
def _on_member_enum(self, e): |
|
|
|
""":type p: :class:`.Enum`""" |
|
|
|
|
|
|
|
def _on_member_enum_alias(self, e): |
|
|
|
""":type p: :class:`.EnumAlias`""" |
|
|
|
|
|
|
|
def _on_member_field(self, f): |
|
|
|
""":type p: :class:`.Field`""" |
|
|
|
|
|
|
|
def _on_const_field(self, f): |
|
|
|
""":type p: :class:`.Const`""" |
|
|
|
|
|
|
|
|
|
|
|
class IndentWriterBase(WriterBase): |
|
|
|
_cur_indent = '' |
|
|
|
|
|
|
|
def _indent(self): |
|
|
|
self._cur_indent += ' ' * 4 |
|
|
|
|
|
|
|
def _unindent(self): |
|
|
|
self._cur_indent = self._cur_indent[:-4] |
|
|
|
|
|
|
|
def _write(self, content, *fmt, indent=0): |
|
|
|
if indent < 0: |
|
|
|
self._unindent() |
|
|
|
|
|
|
|
self._fout.write(self._cur_indent) |
|
|
|
if fmt: |
|
|
|
content = content % fmt |
|
|
|
self._fout.write(content) |
|
|
|
self._fout.write('\n') |
|
|
|
|
|
|
|
if indent > 0: |
|
|
|
self._indent() |
|
|
|
|
|
|
|
|
|
|
|
class PyWriter(IndentWriterBase): |
|
|
|
|
|
|
|
_static_members = None |
|
|
|
_non_static_members = None |
|
|
|
_enums = None |
|
|
|
_enum_map = None |
|
|
|
|
|
|
|
def __call__(self, fout, defs): |
|
|
|
super().__call__(fout) |
|
|
|
self._enum_map = {} |
|
|
|
self._write('// %s', self._get_header()) |
|
|
|
self._write('#include "megbrain/imperative/opdef/all.h"') |
|
|
|
self._write('') |
|
|
|
self._write('using namespace mgb::imperative;') |
|
|
|
self._write('') |
|
|
|
self._process(defs) |
|
|
|
|
|
|
|
def _on_param_begin(self, p): |
|
|
|
self._enums = [] |
|
|
|
self._non_static_members = [] |
|
|
|
self._static_members = [] |
|
|
|
|
|
|
|
def _reg_enum_single(self, cur_def, e): |
|
|
|
alias = None |
|
|
|
if isinstance(e, member_defs.Enum): |
|
|
|
src = e |
|
|
|
else: |
|
|
|
assert isinstance(e, member_defs.EnumAlias) |
|
|
|
src = e.src_enum |
|
|
|
alias = e |
|
|
|
|
|
|
|
src_py_name = self._enum_map.get(src, None) |
|
|
|
if src_py_name is not None: |
|
|
|
py_name = '{}{}Enum'.format(cur_def, src.name if alias is None else alias.name) |
|
|
|
self._write('m.attr("{}") = m.attr("{}");\n'.format(py_name, src_py_name)) |
|
|
|
return |
|
|
|
|
|
|
|
if alias is None: |
|
|
|
enum_name = str(src.name) |
|
|
|
else: |
|
|
|
enum_name = str(alias.name) |
|
|
|
c_name = 'opdef::{}::{}'.format(cur_def, enum_name) |
|
|
|
py_name = '{}{}Enum'.format(cur_def, enum_name) |
|
|
|
self._write('py::enum_<{}>(m, "{}")'.format(c_name, py_name), indent=1) |
|
|
|
for i in src.members: |
|
|
|
self._write('.value("{0}", {1}::{0})'.format(i, c_name)) |
|
|
|
self._write(';\n', indent=-1) |
|
|
|
self._enum_map[src] = py_name |
|
|
|
|
|
|
|
def _on_param_end(self, p): |
|
|
|
cur_def = '{}Def'.format(p.name) |
|
|
|
for e in self._enums: |
|
|
|
self._reg_enum_single(cur_def, e) |
|
|
|
self._write('py::class_<opdef::{0}>(m, "{0}")'.format(cur_def), indent=1) |
|
|
|
# TODO: use ctor with given default value |
|
|
|
self._write('.def(py::init<>())') |
|
|
|
for i in self._static_members: |
|
|
|
assert isinstance(i, member_defs.Const) |
|
|
|
self._write('.def_property_readonly_static("{0}", []() {{ return opdef::{1}::{0}; }})'.format(i.name, cur_def)) |
|
|
|
for i in self._non_static_members: |
|
|
|
fname = None |
|
|
|
if isinstance(i, member_defs.Field): |
|
|
|
fname = i.name |
|
|
|
else: |
|
|
|
assert isinstance(i, (member_defs.Enum, member_defs.EnumAlias)) |
|
|
|
fname = i.name_field |
|
|
|
self._write('.def_readwrite("{0}", &opdef::{1}::{0})'.format(fname, cur_def)) |
|
|
|
self._write(';\n', indent=-1) |
|
|
|
|
|
|
|
|
|
|
|
def _on_member_enum(self, e,): |
|
|
|
self._enums.append(e) |
|
|
|
self._non_static_members.append(e) |
|
|
|
|
|
|
|
def _on_member_enum_alias(self, e): |
|
|
|
self._enums.append(e) |
|
|
|
self._non_static_members.append(e) |
|
|
|
|
|
|
|
def _on_member_field(self, f): |
|
|
|
self._non_static_members.append(f) |
|
|
|
|
|
|
|
def _on_const_field(self, f): |
|
|
|
self._static_members.append(f) |
|
|
|
|
|
|
|
|
|
|
|
class CPPWriter(IndentWriterBase): |
|
|
|
_param_namespace = 'opdef' |
|
|
|
|
|
|
|
_ctor_args = None |
|
|
|
"""list of (text in func param, var name); func param name must be var name |
|
|
|
appended by an underscore""" |
|
|
|
_non_static_members = None |
|
|
|
|
|
|
|
def __call__(self, fout, defs): |
|
|
|
super().__call__(fout) |
|
|
|
self._write('// %s', self._get_header()) |
|
|
|
self._write('#pragma once') |
|
|
|
self._write('#include "megdnn.h"') |
|
|
|
# which defined in megbrain/tools/param_defs/mgb_opr_param_defs.py |
|
|
|
self._write('#include "megbrain/opr/param_defs.h"') |
|
|
|
self._write('#include <stdint.h>') |
|
|
|
self._write('namespace mgb {') |
|
|
|
self._write('namespace imperative {') |
|
|
|
self._write('namespace %s {', self._param_namespace) |
|
|
|
self._write('namespace {') |
|
|
|
self._write('#include "megdnn/dtype.h"') |
|
|
|
self._write('using DTypeEnum = megdnn::DTypeEnum;') |
|
|
|
self._write('} // anonymous namespace') |
|
|
|
self._process(defs) |
|
|
|
self._write('} // namespace %s', self._param_namespace) |
|
|
|
self._write('} // namespace imperative') |
|
|
|
self._write('} // namespace mgb') |
|
|
|
self._write('// vim: syntax=cpp.doxygen') |
|
|
|
|
|
|
|
def _on_param_begin(self, p): |
|
|
|
self._write('struct %sDef {', p.name, indent=1) |
|
|
|
self._ctor_args = [] |
|
|
|
self._non_static_members = [] |
|
|
|
|
|
|
|
def _add_ctor_args(self, typename, default, varname): |
|
|
|
self._ctor_args.append(( |
|
|
|
'{} {}_={}'.format(typename, varname, default), |
|
|
|
varname)) |
|
|
|
|
|
|
|
def _on_param_end(self, p): |
|
|
|
''' |
|
|
|
MegDNN param structures are not packed and we need to initialize the structure |
|
|
|
paddings to zero or it would break MegBrain hash system. We do memset(0) in default |
|
|
|
ctor and use a trick, wrapping non-static members in a anonymous union which would |
|
|
|
copy the object representation in its default copy/move ctor, for copy/move ctor. |
|
|
|
> The implicitly-defined copy/move constructor for a non-union class X performs |
|
|
|
> a memberwise copy/move of its bases and members. [class.copy.ctor 14] |
|
|
|
> The implicitly-defined copy/move constructor for a union X copies the object |
|
|
|
> representation (6.9) of X. [class.copy.ctor 15] |
|
|
|
''' |
|
|
|
if self._non_static_members: |
|
|
|
self._write('union { struct {') |
|
|
|
for i in self._non_static_members: |
|
|
|
if isinstance(i, member_defs.Field): |
|
|
|
self._write('%s%s %s;', i.dtype.cname_attr, i.dtype.cname, i.name) |
|
|
|
else: |
|
|
|
assert isinstance(i, (member_defs.Enum, member_defs.EnumAlias)) |
|
|
|
self._write('%s %s;', i.name, i.name_field) |
|
|
|
self._write('}; };') |
|
|
|
param_list = [] |
|
|
|
if self._ctor_args: |
|
|
|
pdefs, varnames = zip(*self._ctor_args) |
|
|
|
self._write('%sDef(%s) {', p.name, ', '.join(pdefs), indent=1) |
|
|
|
self._write('memset(this, 0, sizeof(*this));') |
|
|
|
for var in varnames: |
|
|
|
self._write('this->%s = %s_;', var, var) |
|
|
|
param_list.append(str(var)) |
|
|
|
self._write('}', indent=-1) |
|
|
|
self._write('megdnn::param::%s param() {', self._cur_class, indent=1) |
|
|
|
self._write('return {%s};', ','.join(param_list)) |
|
|
|
self._write('}', indent=-1) |
|
|
|
self._write('};\n', indent=-1) |
|
|
|
|
|
|
|
|
|
|
|
def __on_member_enum(self, e, default_value): |
|
|
|
self._write('using %s = megdnn::param::%s::%s;', e.name, self._cur_class, e.name) |
|
|
|
self._non_static_members.append(e) |
|
|
|
self._add_ctor_args(e.name, default_value, e.name_field) |
|
|
|
|
|
|
|
def _on_member_enum(self, e,): |
|
|
|
self.__on_member_enum(e, '{}::{}'.format(e.name, e.members[e.default])) |
|
|
|
|
|
|
|
def _on_member_enum_alias(self, e): |
|
|
|
self.__on_member_enum(e, '{}::{}'.format(e.name, e.src_enum.members[e.get_default()])) |
|
|
|
|
|
|
|
def _on_member_field(self, f): |
|
|
|
self._non_static_members.append(f) |
|
|
|
self._add_ctor_args(f.dtype.cname, f.default, f.name) |
|
|
|
|
|
|
|
def _on_const_field(self, f): |
|
|
|
if 'int' in f.dtype.cname: |
|
|
|
self._write('static constexpr %s%s %s = %s;', f.dtype.cname_attr, f.dtype.cname, f.name, f.default) |
|
|
|
else: |
|
|
|
self._write('static const %s%s %s = %s;', f.dtype.cname_attr, f.dtype.cname, f.name, f.default) |
|
|
|
|
|
|
|
def main(): |
|
|
|
parser = argparse.ArgumentParser( |
|
|
|
'generate opr param defs from description file') |
|
|
|
parser.add_argument('-t', '--type', choices=['c++', 'py'], default='c++', |
|
|
|
help='output type') |
|
|
|
parser.add_argument('input') |
|
|
|
parser.add_argument('output') |
|
|
|
args = parser.parse_args() |
|
|
|
|
|
|
|
with open(args.input) as fin: |
|
|
|
inputs = fin.read() |
|
|
|
exec(inputs, {'pdef': ParamDef, 'Doc': member_defs.Doc}) |
|
|
|
input_hash = hashlib.sha256() |
|
|
|
input_hash.update(inputs.encode(encoding='UTF-8')) |
|
|
|
input_hash = input_hash.hexdigest() |
|
|
|
|
|
|
|
if args.type == 'py': |
|
|
|
writer = PyWriter() |
|
|
|
else: |
|
|
|
writer = CPPWriter() |
|
|
|
|
|
|
|
with open(args.output, 'w') as fout: |
|
|
|
writer.set_input_hash(input_hash)(fout, ParamDef.all_param_defs) |
|
|
|
|
|
|
|
if __name__ == '__main__': |
|
|
|
main() |
Megvii Engine Team
4 years ago