Merge master into 7.0 (onnx#494)

* Updating linear resize dimensions check (onnx#362)

* Updates global pooling functions to work correctly with dynamic shapes (onnx#365)

* Support empty initializers for optional inputs (onnx#366)

* Add support for empty initializers for optional inputs

* Alphabetize importPluginFactory

* Support ceiling mode padding for dynamic inputs (onnx#368)

* Register empty Constant node outputs to support empty weights (onnx#369)

* Update myelin library name on Windows (onnx#371)

* Update logic to import ONNX initializers (onnx#375)

* Adding more type checks (onnx#380)

* Add type check for gather and shapedweights attribute imports (onnx#384)

* Throw warning if seed input is provided for randomuniform nodes (onnx#386)

* Update spacetodepth importer to support fulldims and dynamic shapes (onnx#392)

* Add check to avoid console spam of warnings (onnx#402)

* fix some build warnings/errors on Windows VS2019 (onnx#403)

* remove c++11/14 non-compliant constexpr lambdas

* fix build warnings on VS2019

* disable shape input tensor

* Revert "disable shape input tensor"

This reverts commit 9a49e03.

* Support opset11 padding (onnx#408)

* Fix loop importer scan output calculation (onnx#412)

* Fix typo in operators.md supported onnx operators (onnx#399)

There are two overlapping RNN operators, one supporting and the other not. Since onnx supports RNN, the one with supported N should be removed.

Signed-off-by: juhyung <sonju0427@gmail.com>

* Added optimization only mode which runs optimization passes on the model without converting it to tensorrt. (onnx#420)

* New command line options.
* Updated documentation.
* Currently requires linking against onnx project.

* Support opset8 scan (onnx#433)

* Fix deconv importer and remote instancenormalization epsilon clamp value (onnx#434)

* Fix deconv importer and remote instancenormalization epsilon clamp value

* Remove dilations

* Add check for shape tensor outputs (onnx#437)

* Fix slice caclulation for -INT_MAX (onnx#438)

* Support boolean weight conversion to tensors (onnx#439)

* Fix node output accesser for older versions of protobuf (onnx#441)

* Add const qualifier to isNullTensor() (onnx#446)

* Support negative slicing across an entire axis (onnx#453)

* Keep track of Loop tensor mappings (onnx#454)

* Fix fp16 weight import (onnx#484)

* Fix GEMM import assertion (onnx#485)

Co-authored-by: pranavm-nvidia <49246958+pranavm-nvidia@users.noreply.github.com>
Co-authored-by: George Wu <jywu@microsoft.com>
Co-authored-by: JuHyung Son <sonju0427@gmail.com>
Co-authored-by: Dennis Sandler <sandler.denis@gmail.com>

Author: 5 people

CMakeLists.txt

@@ -111,9 +111,15 @@ find_library(TENSORRT_LIBRARY_INFER nvinfer
 find_library(TENSORRT_LIBRARY_INFER_PLUGIN nvinfer_plugin
   HINTS  ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR}
   PATH_SUFFIXES lib lib64 lib/x64)
-find_library(TENSORRT_LIBRARY_MYELIN myelin
-  HINTS  ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR}
-  PATH_SUFFIXES lib lib64 lib/x64)
+if(WIN32)
+  find_library(TENSORRT_LIBRARY_MYELIN myelin64_1
+    HINTS  ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR}
+    PATH_SUFFIXES lib lib64 lib/x64)
+else()
+  find_library(TENSORRT_LIBRARY_MYELIN myelin
+    HINTS  ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR}
+    PATH_SUFFIXES lib lib64 lib/x64)
+endif()
 set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER} ${TENSORRT_LIBRARY_INFER_PLUGIN} ${TENSORRT_LIBRARY_MYELIN})
 MESSAGE(STATUS "Find TensorRT libs at ${TENSORRT_LIBRARY}")
 find_package_handle_standard_args(
@@ -152,7 +158,7 @@ endif()
 # --------------------------------
 add_executable(onnx2trt ${EXECUTABLE_SOURCES})
 target_include_directories(onnx2trt PUBLIC ${ONNX_INCLUDE_DIRS})
-target_link_libraries(onnx2trt PUBLIC ${PROTOBUF_LIB} nvonnxparser_static ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) #${CUDA_LIBRARIES} 
+target_link_libraries(onnx2trt PUBLIC ${PROTOBUF_LIB} onnx nvonnxparser_static ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) #${CUDA_LIBRARIES} 
 
 # --------------------------------
 # API Tests

ImporterContext.hpp

@@ -48,6 +48,8 @@ class ImporterContext final : public IImporterContext
         mTensorNameCounts; // Keep track of how many times a tensor name shows up, to avoid duplicate naming in TRT.
     StringMap<size_t>
         mLayerNameCounts; // Keep track of how many times a tensor name shows up, to avoid duplicate naming in TRT.
+    std::unordered_set<std::string> mUnsupportedShapeTensors; // Container to hold any shape tensors that are the output of layers that do not support shape tensors.
+    StringMap<std::string> mLoopTensors; // Container to map subgraph tensors to their original outer graph names.
 public:
     ImporterContext(nvinfer1::INetworkDefinition* network, nvinfer1::ILogger* logger)
         : _network(network)
@@ -78,6 +80,14 @@ class ImporterContext final : public IImporterContext
     {
         return mLayerPrecisions;
     }
+    virtual std::unordered_set<std::string>& unsupportedShapeTensors() override
+    {
+        return mUnsupportedShapeTensors;
+    }
+    virtual StringMap<std::string>& loopTensors() override
+    {
+        return mLoopTensors;
+    }
 
     // This actually handles weights as well, but is named this way to be consistent with the tensors()
     virtual void registerTensor(TensorOrWeights tensor, const std::string& basename) override

ModelImporter.cpp

@@ -172,7 +172,8 @@ Status parseGraph(
             auto& output = outputs.at(i);
             ssOutputs << "[" << outputName << " -> " << output.shape() << "], ";
             // Note: This condition is to allow ONNX outputs to be ignored
-            if (output && !outputName.empty())
+            // Always register output weights (even empty ones) as it may be mapped to an unused input
+            if ((output || output.is_weights()) && !outputName.empty())
             {
                 ctx->registerTensor(std::move(output), outputName);
             }
@@ -331,11 +332,11 @@ bool ModelImporter::supportsModel(
             }
         }
     }
-
-    auto checkForInput = [&input_node](::ONNX_NAMESPACE::NodeProto const& node) {
+    auto* ctx = &_importer_ctx;
+    auto checkForInput = [&input_node, &ctx](::ONNX_NAMESPACE::NodeProto const& node) {
         for (auto input : node.input())
         {
-            if (input_node == input)
+            if (input_node == input || ctx->loopTensors()[input_node] == input)
             {
                 return true;
             }
@@ -351,17 +352,23 @@ bool ModelImporter::supportsModel(
         cout << "Failed to sort model topologically, exiting ..." << endl;
         return false;
     }
+
     for (int node_idx : topological_order)
     {
         ::ONNX_NAMESPACE::NodeProto const& node = model.graph().node(node_idx);
+
         // Add the node to the subgraph if:
-        //     1. Importer function is regestiered for the operator type
-        //     2. It is not directly connected to an unsupported input
-        //     3. Parsing did not hit an error on the node
+        //     1. Importer function is registered for the operator type
+        //     2. It is NOT directly connected to an unsupported input
+        //     3. Parsing did NOT hit an error on the node
+        //     4. Any shape tensor output is coming from a supported node
         bool registered = supportsOperator(node.op_type().c_str());
         bool containsInput = (input_node.empty()) ? false : checkForInput(node);
         bool containsIndex = node_idx == error_node;
-        if (registered && !containsInput && !containsIndex)
+        auto const tensor = node.output(0);
+        bool supportedShapeTensorOutput = ctx->unsupportedShapeTensors().count(tensor) == 0 ? true : false;
+
+        if (registered && !containsInput && !containsIndex && supportedShapeTensorOutput)
         {
             if (newSubGraph)
             {
@@ -447,6 +454,18 @@ void removeShapeTensorCasts(IImporterContext* ctx)
             {
                 t.setType(SHAPE_TENSOR_TYPE);
             }
+            // Some layers do not support shape tensor outputs. Keep track of these tensor names
+            // for supportsModel().
+            auto type = layer->getType();
+            auto elementwiseOp = layer->getType() == nvinfer1::LayerType::kELEMENTWISE ? (static_cast<nvinfer1::IElementWiseLayer*>(layer))->getOperation() : nvinfer1::ElementWiseOperation::kSUM;
+            auto reduceOp = layer->getType() == nvinfer1::LayerType::kREDUCE ? (static_cast<nvinfer1::IReduceLayer*>(layer))->getOperation() : nvinfer1::ReduceOperation::kSUM;
+
+            if (!supportsShapeTensor(type, elementwiseOp, reduceOp))
+            {
+                auto name = layer->getOutput(0)->getName();
+                ctx->unsupportedShapeTensors().insert(name);
+                LOG_ERROR("Found " << name << " as a shape tensor output from a layer that does not support it!");
+            }
         }
     }
 }

OnnxAttrs.cpp

@@ -109,7 +109,11 @@ onnx2trt::ShapedWeights OnnxAttrs::get<onnx2trt::ShapedWeights>(const std::strin
 {
     ::ONNX_NAMESPACE::TensorProto const& onnx_weights_tensor = this->at(key)->t();
     onnx2trt::ShapedWeights weights;
-    convertOnnxWeights(onnx_weights_tensor, &weights, mCtx);
+    // Return empty weights if conversion failed
+    if (!convertOnnxWeights(onnx_weights_tensor, &weights, mCtx))
+    {
+        return onnx2trt::ShapedWeights::empty(::ONNX_NAMESPACE::TensorProto_DataType_FLOAT);
+    }
     return weights;
 }
 

README.md

@@ -58,6 +58,15 @@ ONNX models can also be converted to human-readable text:
 
     onnx2trt my_model.onnx -t my_model.onnx.txt
 
+ONNX models can also be optimized by ONNX's optimization libraries.
+To optimize an ONNX model and output a new one use `-m` to specify the output model name and `-O` to specify a semicolon-separated list of optimization passes to apply:
+
+    onnx2trt my_model.onnx -O "pass_1;pass_2;pass_3" -m my_model_optimized.onnx
+
+See more all available optimization passes by running:
+
+    onnx2trt -p
+
 See more usage information by running:
 
     onnx2trt -h

TensorOrWeights.hpp

@@ -67,14 +67,18 @@ class TensorOrWeights
     {
         return _variant == NODE_WEIGHTS;
     }
+    bool isNullTensor() const
+    {
+        return is_tensor() && _tensor == nullptr;
+    }
     nvinfer1::ITensor& tensor()
     {
-        assert(is_tensor());
+        assert(!isNullTensor());
         return *_tensor;
     }
     nvinfer1::ITensor const& tensor() const
     {
-        assert(is_tensor());
+        assert(!isNullTensor());
         return *_tensor;
     }
     ShapedWeights& weights()
@@ -99,6 +103,10 @@ class TensorOrWeights
     {
         return is_tensor() ? _tensor->getType() == nvinfer1::DataType::kINT32 : _weights.type == ::ONNX_NAMESPACE::TensorProto_DataType_INT32;
     }
+    bool isBool() const
+    {
+        return is_tensor() ? _tensor->getType() == nvinfer1::DataType::kBOOL : _weights.type == ::ONNX_NAMESPACE::TensorProto_DataType_BOOL;
+    }
 };
 
 } // namespace onnx2trt