blob: ec9e3cf85e1de662f373164c585547179625013f (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
|
#[[ Generate common python virtual enviornment ]]
find_package(PythonInterp 3 QUIET)
find_package(PythonLibs 3 QUIET)
if(NOT ${PYTHONINTERP_FOUND})
message(STATUS "Build common-artifacts: FALSE (Python3 is missing)")
return()
endif()
if(${PYTHON_VERSION_MINOR} LESS 3)
message(STATUS "Build common-artifacts: FALSE (You need to install Python version higher than 3.3)")
return()
endif()
# Create python virtual environment with tensorflow 1.13.2
set(VIRTUALENV_OVERLAY_TF_1_13_2 "${NNCC_OVERLAY_DIR}/venv_1_13_2")
# Create python virtual environment with tensorflow 2.3.0
set(VIRTUALENV_OVERLAY_TF_2_3_0 "${NNCC_OVERLAY_DIR}/venv_2_3_0")
add_custom_command(
OUTPUT ${VIRTUALENV_OVERLAY_TF_1_13_2}
COMMAND ${PYTHON_EXECUTABLE} -m venv ${VIRTUALENV_OVERLAY_TF_1_13_2}
)
add_custom_command(
OUTPUT ${VIRTUALENV_OVERLAY_TF_2_3_0}
COMMAND ${PYTHON_EXECUTABLE} -m venv ${VIRTUALENV_OVERLAY_TF_2_3_0}
)
# Create requirements.txt and install required pip packages
set(REQUIREMENTS_FILE "requirements.txt")
set(REQUIREMENTS_OVERLAY_PATH_TF_1_13_2 "${VIRTUALENV_OVERLAY_TF_1_13_2}/${REQUIREMENTS_FILE}")
set(REQUIREMENTS_OVERLAY_PATH_TF_2_3_0 "${VIRTUALENV_OVERLAY_TF_2_3_0}/${REQUIREMENTS_FILE}")
# TODO remove version number of '--upgrade pip==20.2.1 setuptools==49.3.0'
# NOTE adding version is for temporary hotfix of setuptools 50.x.y version
add_custom_command(
OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_1_13_2}
COMMAND ${CMAKE_COMMAND} -E echo "tensorflow==1.13.2" > ${REQUIREMENTS_OVERLAY_PATH_TF_1_13_2}
COMMAND ${VIRTUALENV_OVERLAY_TF_1_13_2}/bin/python -m pip --default-timeout=1000 install --upgrade pip==20.2.1 setuptools==49.3.0
COMMAND ${VIRTUALENV_OVERLAY_TF_1_13_2}/bin/python -m pip --default-timeout=1000 install -r ${REQUIREMENTS_OVERLAY_PATH_TF_1_13_2} --upgrade
DEPENDS ${VIRTUALENV_OVERLAY_TF_1_13_2}
)
add_custom_command(
OUTPUT ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
COMMAND ${CMAKE_COMMAND} -E remove -f ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
COMMAND ${CMAKE_COMMAND} -E echo "tensorflow-cpu==2.3.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
COMMAND ${CMAKE_COMMAND} -E echo "flatbuffers==1.12" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
COMMAND ${VIRTUALENV_OVERLAY_TF_2_3_0}/bin/python -m pip --default-timeout=1000 install --upgrade pip==20.2.1 setuptools==49.3.0
COMMAND ${VIRTUALENV_OVERLAY_TF_2_3_0}/bin/python -m pip --default-timeout=1000 install -r ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0} --upgrade
DEPENDS ${VIRTUALENV_OVERLAY_TF_2_3_0}
)
add_custom_target(common_artifacts_python_deps ALL
DEPENDS ${VIRTUALENV_OVERLAY_TF_1_13_2} ${VIRTUALENV_OVERLAY_TF_2_3_0} ${REQUIREMENTS_OVERLAY_PATH_TF_1_13_2} ${REQUIREMENTS_OVERLAY_PATH_TF_2_3_0}
)
#[[ Generate common resources ]]
# TODO add pbtxt
nnas_find_package(HDF5 QUIET)
if(NOT HDF5_FOUND)
message(STATUS "Build common-artifacts: FAILED (missing HDF5)")
return()
endif(NOT HDF5_FOUND)
set(SOURCES src/TestDataGenerator.cpp)
add_executable(testDataGenerator ${SOURCES})
target_include_directories(testDataGenerator PRIVATE ${HDF5_INCLUDE_DIRS})
target_link_libraries(testDataGenerator PRIVATE ${HDF5_CXX_LIBRARIES})
target_link_libraries(testDataGenerator PRIVATE arser)
target_link_libraries(testDataGenerator PRIVATE foder)
target_link_libraries(testDataGenerator PRIVATE luci_import)
target_link_libraries(testDataGenerator PRIVATE luci_interpreter)
target_link_libraries(testDataGenerator PRIVATE mio_circle)
target_link_libraries(testDataGenerator PRIVATE safemain)
unset(TEST_DEPS)
# Include recipe repo
nncc_find_resource(TensorFlowLiteRecipes)
nncc_find_resource(CircleRecipes)
set(TFLITE_RECIPE_REPO "${TensorFlowLiteRecipes_DIR}")
set(CIRCLE_RECIPE_REPO "${CircleRecipes_DIR}")
set(TEST_RECIPE_FILENAME "test.recipe")
set(TEST_RULE_FILENAME "test.rule")
set(MODEL2NNPKG "${NNAS_PROJECT_SOURCE_DIR}/tools/nnpackage_tool/model2nnpkg/model2nnpkg.sh")
# Get test case list
unset(RECIPES)
file(GLOB TFLITE_SUBDIR RELATIVE ${TFLITE_RECIPE_REPO} ${TFLITE_RECIPE_REPO}/*)
foreach(DIR IN ITEMS ${TFLITE_SUBDIR})
if(IS_DIRECTORY ${TFLITE_RECIPE_REPO}/${DIR})
list(APPEND RECIPES ${DIR})
endif()
endforeach()
file(GLOB CIRCLE_SUBDIR RELATIVE ${CIRCLE_RECIPE_REPO} ${CIRCLE_RECIPE_REPO}/*)
foreach(DIR IN ITEMS ${CIRCLE_SUBDIR})
if(IS_DIRECTORY ${CIRCLE_RECIPE_REPO}/${DIR})
list(APPEND RECIPES ${DIR})
endif()
endforeach()
macro(circlize NAME)
set(NO_CIRCLIZE_${NAME} TRUE)
set(NO_OPTIMIZE_${NAME} TRUE)
set(NO_TCGEN_${NAME} TRUE)
endmacro()
macro(optimize NAME)
set(NO_OPTIMIZE_${NAME} TRUE)
endmacro()
macro(tcgenerate NAME)
set(NO_TCGEN_${NAME} TRUE)
endmacro()
include("exclude.lst")
foreach(RECIPE IN ITEMS ${RECIPES})
unset(OPT_FORMAT)
unset(MODEL_FORMAT)
set(RECIPE_FILE "${RECIPE}.recipe")
set(RULE_FILE "${RECIPE}.rule")
set(TFLITE_RECIPE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
set(CIRCLE_RECIPE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RECIPE_FILENAME}")
if(NOT EXISTS "${TFLITE_RECIPE_SOURCE_PATH}")
if(NOT EXISTS "${CIRCLE_RECIPE_SOURCE_PATH}")
message(FATAL_ERROR "Missing recipe of '${RECIPE}' test")
else()
# circle recipe
set(MODEL_FORMAT "circle")
set(RECIPE_SOURCE_PATH ${CIRCLE_RECIPE_SOURCE_PATH})
endif()
else()
# tflite recipe
set(MODEL_FORMAT "tflite")
set(RECIPE_SOURCE_PATH ${TFLITE_RECIPE_SOURCE_PATH})
endif()
set(TFLITE_RULE_SOURCE_PATH "${TFLITE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
set(CIRCLE_RULE_SOURCE_PATH "${CIRCLE_RECIPE_REPO}/${RECIPE}/${TEST_RULE_FILENAME}")
unset(RULE_SOURCE_PATH)
if(EXISTS "${TFLITE_RULE_SOURCE_PATH}")
set(RULE_SOURCE_PATH ${TFLITE_RULE_SOURCE_PATH})
endif()
if(EXISTS "${CIRCLE_RULE_SOURCE_PATH}")
set(RULE_SOURCE_PATH ${CIRCLE_RULE_SOURCE_PATH})
endif()
set(RECIPE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RECIPE_FILE}")
set(RULE_BINARY_PATH "${CMAKE_CURRENT_BINARY_DIR}/${RULE_FILE}")
set(TFLITE_FILE "${RECIPE}.tflite")
set(TFLITE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${TFLITE_FILE}")
set(CIRCLE_FILE "${RECIPE}.circle")
set(CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${CIRCLE_FILE}")
# Copy .recipe
add_custom_command(OUTPUT ${RECIPE_BINARY_PATH}
COMMAND ${CMAKE_COMMAND} -E copy "${RECIPE_SOURCE_PATH}" "${RECIPE_BINARY_PATH}"
DEPENDS ${RECIPE_SOURCE_PATH}
COMMENT "Generate ${RECIPE_FILE}"
)
list(APPEND TEST_DEPS ${RECIPE_BINARY_PATH})
if(DEFINED RULE_SOURCE_PATH)
# Copy .rule
add_custom_command(OUTPUT ${RULE_BINARY_PATH}
COMMAND ${CMAKE_COMMAND} -E copy "${RULE_SOURCE_PATH}" "${RULE_BINARY_PATH}"
DEPENDS ${RULE_SOURCE_PATH}
COMMENT "Generate ${RULE_FILE}"
)
list(APPEND TEST_DEPS ${RULE_BINARY_PATH})
endif()
if(${MODEL_FORMAT} STREQUAL "tflite")
# Generate .tflite
add_custom_command(OUTPUT ${TFLITE_OUTPUT_PATH}
COMMAND $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH} ${TFLITE_OUTPUT_PATH}
DEPENDS $<TARGET_FILE:tflchef-file> ${RECIPE_BINARY_PATH}
COMMENT "Generate ${TFLITE_FILE}"
)
list(APPEND TEST_DEPS ${TFLITE_OUTPUT_PATH})
if(NOT DEFINED NO_CIRCLIZE_${RECIPE})
# Generate .circle
add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
COMMAND $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH} ${CIRCLE_OUTPUT_PATH}
DEPENDS $<TARGET_FILE:tflite2circle> ${TFLITE_OUTPUT_PATH}
COMMENT "Generate ${CIRCLE_FILE}"
)
set(MODEL_FORMAT "circle")
list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
endif()
else()
# Generate .circle
add_custom_command(OUTPUT ${CIRCLE_OUTPUT_PATH}
COMMAND $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH} ${CIRCLE_OUTPUT_PATH}
DEPENDS $<TARGET_FILE:circlechef-file> ${RECIPE_BINARY_PATH}
COMMENT "Generate ${CIRCLE_FILE}"
)
list(APPEND TEST_DEPS ${CIRCLE_OUTPUT_PATH})
endif()
set(OPT_CIRCLE_FILE "${RECIPE}.opt.circle")
set(OPT_CIRCLE_OUTPUT_PATH "${CMAKE_CURRENT_BINARY_DIR}/${OPT_CIRCLE_FILE}")
if(NOT DEFINED NO_OPTIMIZE_${RECIPE})
# Generate optimized .circle
add_custom_command(OUTPUT ${OPT_CIRCLE_OUTPUT_PATH}
COMMAND $<TARGET_FILE:circle2circle> --all ${CIRCLE_OUTPUT_PATH} ${OPT_CIRCLE_OUTPUT_PATH}
DEPENDS $<TARGET_FILE:circle2circle> ${CIRCLE_OUTPUT_PATH}
COMMENT "Generate ${OPT_CIRCLE_FILE}"
)
set(OPT_FORMAT ".opt")
list(APPEND TEST_DEPS ${OPT_CIRCLE_OUTPUT_PATH})
endif()
set(MODEL_FILE "${RECIPE}${OPT_FORMAT}.${MODEL_FORMAT}")
set(MODEL_PATH "${CMAKE_CURRENT_BINARY_DIR}/${MODEL_FILE}")
set(NNPKG_FILE "${RECIPE}${OPT_FORMAT}")
set(NNPKG_PATH "${CMAKE_CURRENT_BINARY_DIR}/${NNPKG_FILE}")
add_custom_command(OUTPUT ${NNPKG_PATH}
COMMAND ${MODEL2NNPKG} ${MODEL_PATH}
DEPENDS ${MODEL2NNPKG} ${MODEL_PATH}
COMMENT "Generate ${RECIPE} nnpackage"
)
list(APPEND TEST_DEPS ${NNPKG_PATH})
set(INPUT_HDF5_FILE "${RECIPE}${OPT_FORMAT}.input.h5")
set(INPUT_BIN_PATH "${CMAKE_CURRENT_BINARY_DIR}/${INPUT_HDF5_FILE}")
set(EXPECTED_HDF5_FILE "${RECIPE}${OPT_FORMAT}.expected.h5")
set(EXPECTED_BIN_PATH "${CMAKE_CURRENT_BINARY_DIR}/${EXPECTED_HDF5_FILE}")
if(NOT DEFINED NO_TCGEN_${RECIPE})
# Generate input.h5, expected.h5
add_custom_command(OUTPUT ${INPUT_BIN_PATH} ${EXPECTED_BIN_PATH}
COMMAND $<TARGET_FILE:testDataGenerator> ${MODEL_FILE}
DEPENDS $<TARGET_FILE:testDataGenerator> ${MODEL_FILE}
COMMENT "Generate ${INPUT_BIN_PATH} and ${EXPECTED_BIN_PATH}"
)
# Generate test directory
set(TC_DIRECTORY "${NNPKG_PATH}/metadata/tc")
add_custom_command(OUTPUT ${TC_DIRECTORY}
COMMAND ${CMAKE_COMMAND} -E make_directory ${TC_DIRECTORY}
DEPENDS ${NNPKG_PATH}
COMMENT "Generate ${RECIPE} nnpackage test directory"
)
# Move input hdf5 file to test directory
set(INPUT_NNPKG_PATH "${TC_DIRECTORY}/input.h5")
add_custom_command(OUTPUT ${INPUT_NNPKG_PATH}
COMMAND ${CMAKE_COMMAND} -E rename ${INPUT_BIN_PATH} ${INPUT_NNPKG_PATH}
DEPENDS ${INPUT_BIN_PATH} ${TC_DIRECTORY}
COMMENT "Move ${INPUT_HDF5_FILE} to nnpackage"
)
# Move expected hdf5 file to test directory
set(EXPECTED_NNPKG_PATH "${TC_DIRECTORY}/expected.h5")
add_custom_command(OUTPUT ${EXPECTED_NNPKG_PATH}
COMMAND ${CMAKE_COMMAND} -E rename ${EXPECTED_BIN_PATH} ${EXPECTED_NNPKG_PATH}
DEPENDS ${EXPECTED_BIN_PATH} ${TC_DIRECTORY}
COMMENT "Move ${EXPECTED_HDF5_FILE} to nnpackage"
)
list(APPEND TEST_DEPS ${TC_DIRECTORY} ${INPUT_BIN_PATH} ${EXPECTED_BIN_PATH}
${INPUT_NNPKG_PATH} ${EXPECTED_NNPKG_PATH})
endif()
endforeach()
add_custom_target(common_artifacts_deps ALL DEPENDS ${TEST_DEPS})
|
