From ee27405418c580d8b10638e7b30835f322e78215 Mon Sep 17 00:00:00 2001 From: zjq Date: Tue, 9 Jun 2026 20:33:21 +0800 Subject: [PATCH 1/7] Refactor ViT TensorRT support --- vit/README.md | 29 ++- vit/gen_wts.py | 193 +++++++++++---- vit/vit.cc | 632 +++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 675 insertions(+), 179 deletions(-) diff --git a/vit/README.md b/vit/README.md index d19217df..10e639ae 100644 --- a/vit/README.md +++ b/vit/README.md @@ -19,21 +19,33 @@ This is a handwritten TensorRT implementation of the Vision Transformers[arxiv.o - Support a dummy profiler by default - Support a dummy output allocator by default - Use optimization profile by default +- Support ViT-B/16, ViT-B/32, ViT-L/16, ViT-L/32 and ViT-H/14 +- Use an optimization profile by default (`min=1 / opt=1 / max=2`) ### 2.2 Current limitations - cannot use `IAttenion` with TensorRT SDK 10.14 ~ 10.15 because of the bugs in TensorRT - TensorRT < 8 is not supported because some ops are not inplemented in cuDNN - SM < 86, TensorRT < 10, CUDA < 12 cases are _NOT_ fully tested yet +- ViT-H/14 uses the HuggingFace ImageNet-21k checkpoint + (`google/vit-huge-patch14-224-in21k`, 21843 classes); no public 1k + fine-tuned checkpoint is available. ### 2.3 Usage 1. use `gen_wts.py` to generate `.wts` file. ```bash -python gen_wts.py +# Choose one of: ViT-B/16, ViT-B/32, ViT-L/16, ViT-L/32, ViT-H/14 +# Aliases b16, b32, l16, l32 and h14 are also accepted. +python gen_wts.py ViT-B/16 +# -> writes models/ViT-B-16.wts ``` +`gen_wts.py` sets model download caches before importing `torch` and +`transformers`: `TORCH_HOME=/mnt/data/storage/torch` and +`HF_HOME=/mnt/data/storage/huggingface`. + 2. build C++ code ```bash @@ -45,15 +57,22 @@ cmake --build build 3. serialize `.wts` model to engine file. ```bash -./build/vit -s +./build/vit -s +# Example: +./build/vit -s models/ViT-B-16.wts models/ViT-B-16.engine ViT-B/16 ``` 4. run inference ```bash -./build/vit -d +./build/vit -d +# Example: +./build/vit -d models/ViT-B-16.engine assets/cats.jpg ``` +The engine uses a dynamic batch profile (`min=1 / opt=1 / max=2`). Passing a +directory with two images runs both samples in one batch. + On **RTX 4080, TensorRT 10.15.1 SDK**, the output looks like: ```bash @@ -110,7 +129,7 @@ Where: - (N): batch size (represented by `N` in your code) - (L): sequence length (number of tokens; dynamic in code via `-1`) -- (D): hidden size, fixed at 768 in this implementation +- (D): hidden size, configured by the selected variant The attention head configuration: @@ -141,7 +160,7 @@ For a standard Transformer block: $$ \mathbf{W}_2 \in \mathbb{R}^{4D \times D}, \ \mathbf{b}_2 \in \mathbb{R}^{D} $$ - Here ($4 D = 3072$). + Here the FFN dimension is configured by the selected variant. ### 3.3 High-Level Block Structure diff --git a/vit/gen_wts.py b/vit/gen_wts.py index 75086eb6..838c5b43 100644 --- a/vit/gen_wts.py +++ b/vit/gen_wts.py @@ -1,64 +1,159 @@ +"""Export ViT weights from HuggingFace to a .wts file for TensorRT. + +Usage: + python gen_wts.py [] + +Examples: + python gen_wts.py ViT-B/16 + python gen_wts.py ViT-L/16 models/vit_l16.wts + python gen_wts.py b32 # alias accepted + +Model types supported (must match the table in vit.cc::getVariantConfig): + ViT-B/16 -> google/vit-base-patch16-224 (img=224, classes=1000) + ViT-B/32 -> google/vit-base-patch32-384 (img=384, classes=1000) + ViT-L/16 -> google/vit-large-patch16-224 (img=224, classes=1000) + ViT-L/32 -> google/vit-large-patch32-384 (img=384, classes=1000) + ViT-H/14 -> google/vit-huge-patch14-224-in21k (img=224, classes=21843, + ImageNet-21k pretrain only; + no public 1k fine-tuned ckpt) + +Default output path (when omitted) is models/.wts where slashes +are replaced by hyphens, e.g. ViT-B/16 -> models/ViT-B-16.wts. This matches +the recommended layout used by exp/run_experiment.py. +""" + +from __future__ import annotations + +import argparse +import os import struct +from pathlib import Path -import cv2 -import numpy as np import torch -from transformers import AutoConfig, AutoImageProcessor, AutoModelForImageClassification +from transformers import AutoConfig, AutoModelForImageClassification +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parent +MODELS_DIR = REPO_ROOT / "models" -def read_imagenet_labels() -> dict[int, str]: - """ - read ImageNet 1000 labels +# Canonical name -> (HF hub id, image size, num classes) +VARIANTS: dict[str, tuple[str, int, int]] = { + "ViT-B/16": ("google/vit-base-patch16-224", 224, 1000), + "ViT-B/32": ("google/vit-base-patch32-384", 384, 1000), + "ViT-L/16": ("google/vit-large-patch16-224", 224, 1000), + "ViT-L/32": ("google/vit-large-patch32-384", 384, 1000), + "ViT-H/14": ("google/vit-huge-patch14-224-in21k", 224, 21843), +} - Returns: - dict[int, str]: labels dict - """ - clsid2label = {} - with open("../assets/imagenet1000_clsidx_to_labels.txt", "r") as f: - for i in f.readlines(): - k, v = i.split(": ") - clsid2label.setdefault(int(k), v[1:-3]) - return clsid2label +ALIASES: dict[str, str] = { + "b16": "ViT-B/16", + "B16": "ViT-B/16", + "B/16": "ViT-B/16", + "b32": "ViT-B/32", + "B32": "ViT-B/32", + "B/32": "ViT-B/32", + "l16": "ViT-L/16", + "L16": "ViT-L/16", + "L/16": "ViT-L/16", + "l32": "ViT-L/32", + "L32": "ViT-L/32", + "L/32": "ViT-L/32", + "h14": "ViT-H/14", + "H14": "ViT-H/14", + "H/14": "ViT-H/14", +} -USE_HF_PREPROCESS = False +def normalize_model_type(name: str) -> str: + if name in VARIANTS: + return name + if name in ALIASES: + return ALIASES[name] + upper = name.upper().replace(" ", "") + if upper.startswith("VIT-"): + cand = "ViT-" + upper[4:] + if cand in VARIANTS: + return cand + raise SystemExit( + f"Unknown model_type: {name!r}. Choose from: {', '.join(VARIANTS.keys())}" + ) -if __name__ == "__main__": - hub_model_id = "google/vit-base-patch16-224" - config = AutoConfig.from_pretrained(hub_model_id) + +def safe_filename(model_type: str) -> str: + """ViT-B/16 -> ViT-B-16 (filesystem-safe).""" + return model_type.replace("/", "-") + + +def require_cache_env() -> None: + missing = [name for name in ("TORCH_HOME", "HF_HOME") if not os.environ.get(name)] + if missing: + raise RuntimeError( + f"Please set required cache environment variables: {', '.join(missing)}" + ) + print(f"Using TORCH_HOME={os.environ['TORCH_HOME']}") + print(f"Using HF_HOME={os.environ['HF_HOME']}") + + +def export_wts(model: torch.nn.Module, out_path: Path) -> None: + out_path.parent.mkdir(parents=True, exist_ok=True) + sd = model.state_dict() + with open(out_path, "w") as f: + f.write(f"{len(sd)}\n") + for k, v in sd.items(): + vr = v.detach().reshape(-1).cpu().numpy() + f.write(f"{k} {vr.size}") + for vv in vr: + f.write(" ") + f.write(struct.pack(">f", float(vv)).hex()) + f.write("\n") + size_mib = out_path.stat().st_size / 1024 / 1024 + print(f"[ok] wrote {out_path} ({size_mib:.1f} MiB, {len(sd)} tensors)") + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument("model_type", help="e.g. ViT-B/16, ViT-L/32, b16, h14") + parser.add_argument( + "output", + nargs="?", + default=None, + help="output .wts path (default: models/.wts)", + ) + args = parser.parse_args() + + require_cache_env() + + model_type = normalize_model_type(args.model_type) + hub_id, img_size, num_classes = VARIANTS[model_type] + out_path = ( + Path(args.output) + if args.output + else MODELS_DIR / f"{safe_filename(model_type)}.wts" + ) + + print( + f"[load] model_type={model_type} hub_id={hub_id} img={img_size} classes={num_classes}" + ) + config = AutoConfig.from_pretrained(hub_id) config._attn_implementation = "eager" + # Force the classifier head size: in21k checkpoints (e.g. ViT-H/14) ship + # without a classifier and HF would otherwise default num_labels=2, + # producing a tiny random head incompatible with our engine table. + config.num_labels = num_classes + config.id2label = {i: str(i) for i in range(num_classes)} + config.label2id = {str(i): i for i in range(num_classes)} model = AutoModelForImageClassification.from_pretrained( - hub_model_id, - ignore_mismatched_sizes=False, + hub_id, + ignore_mismatched_sizes=True, config=config, ) - model.eval() - img = cv2.imread("../assets/cats.jpg", cv2.IMREAD_COLOR) - - if USE_HF_PREPROCESS: - image_processor = AutoImageProcessor.from_pretrained(hub_model_id) - img = image_processor(img, return_tensors="pt") - img = img["pixel_values"] - else: - img: np.array = cv2.resize(img, (224, 224), cv2.INTER_LINEAR) - img = (img.astype(np.float32) / 255.0 - np.array([0.5, 0.5, 0.5])) / np.array([0.5, 0.5, 0.5]) - img = torch.from_numpy(np.transpose(img, (2, 0, 1))[None, ...]) - - output = model(img) - labels = read_imagenet_labels() - for i, j in enumerate(torch.topk(output.logits[0], k=3).indices): - print(f"Top: {i} is {labels[int(j)]}") - - f = open("../models/vit.wts", "w") - f.write("{}\n".format(len(model.state_dict().keys()))) - for k, v in model.state_dict().items(): - print("key: ", k) - print("value: ", v.shape) - vr = v.reshape(-1).cpu().numpy() - f.write("{} {}".format(k, len(vr))) - for vv in vr: - f.write(" ") - f.write(struct.pack(">f", float(vv)).hex()) - f.write("\n") + export_wts(model, out_path) + + +if __name__ == "__main__": + main() diff --git a/vit/vit.cc b/vit/vit.cc index 0a95b60f..762fc098 100644 --- a/vit/vit.cc +++ b/vit/vit.cc @@ -1,9 +1,23 @@ #include #include +#include +#include #include +#include +#include +#include +#include +#include #include +#include #include +#include #include +#include +#include +#include +#include +#include #include "cuda_allocator.h" #include "logging.h" #include "macros.h" @@ -16,17 +30,118 @@ using M = nvinfer1::MatrixOperation; using E = nvinfer1::ElementWiseOperation; using NDCF = nvinfer1::NetworkDefinitionCreationFlag; -static constexpr const int64_t N = 1; -static constexpr const int64_t INPUT_H = 224; -static constexpr const int64_t INPUT_W = 224; +// DataType::kHALF -> FP16 +// DataType::kFLOAT -> FP32 +static constexpr DataType BUILD_PRECISION = DataType::kHALF; +static constexpr int64_t BUILD_MIN_BATCH = 1; +static constexpr int64_t BUILD_OPT_BATCH = 1; +static constexpr int64_t BUILD_MAX_BATCH = 2; + +// ViT model variant table. +// All variants share the same architecture; only sizes differ. +// `model_type` is the canonical name (e.g. "ViT-B/16"); aliases without slash +// (e.g. "b16") are accepted via normalizeModelType(). +struct ViTConfig { + int64_t hidden = 768; + int64_t num_layers = 12; + int64_t num_heads = 12; + int64_t ff_dim = 3072; + int64_t num_classes = 1000; + int64_t patch = 16; + int64_t img_size = 224; + // Optimization profile used by the generated engine. + int64_t min_batch = BUILD_MIN_BATCH; + int64_t opt_batch = BUILD_OPT_BATCH; + int64_t max_batch = BUILD_MAX_BATCH; + float lnorm_eps = 1e-12f; + std::string model_type = "ViT-B/16"; + std::string wts_path; + std::string engine_path; + [[nodiscard]] int64_t num_patches() const { return (img_size / patch) * (img_size / patch); } + [[nodiscard]] int64_t seq_len() const { return num_patches() + 1; } +}; + +static std::string normalizeModelType(const std::string& s) { + // Accept "ViT-B/16", "vit-b/16", "b16", "B/16" -> "ViT-B/16". + std::string t; + for (char c : s) { + if (c != '-' && c != '/' && c != ' ') { + t += static_cast(std::toupper(static_cast(c))); + } + } + // Strip leading "VIT" + if (t.starts_with("VIT")) + t = t.substr(3); + if (t == "B16") + return "ViT-B/16"; + if (t == "B32") + return "ViT-B/32"; + if (t == "L16") + return "ViT-L/16"; + if (t == "L32") + return "ViT-L/32"; + if (t == "H14") + return "ViT-H/14"; + return s; // unknown, return as-is for error reporting +} + +static ViTConfig getVariantConfig(const std::string& raw_name) { + const std::string name = normalizeModelType(raw_name); + ViTConfig c; + c.model_type = name; + if (name == "ViT-B/16") { + c.hidden = 768; + c.num_layers = 12; + c.num_heads = 12; + c.ff_dim = 3072; + c.patch = 16; + c.img_size = 224; + c.num_classes = 1000; + } else if (name == "ViT-B/32") { + c.hidden = 768; + c.num_layers = 12; + c.num_heads = 12; + c.ff_dim = 3072; + c.patch = 32; + c.img_size = 384; + c.num_classes = 1000; + } else if (name == "ViT-L/16") { + c.hidden = 1024; + c.num_layers = 24; + c.num_heads = 16; + c.ff_dim = 4096; + c.patch = 16; + c.img_size = 224; + c.num_classes = 1000; + } else if (name == "ViT-L/32") { + c.hidden = 1024; + c.num_layers = 24; + c.num_heads = 16; + c.ff_dim = 4096; + c.patch = 32; + c.img_size = 384; + c.num_classes = 1000; + } else if (name == "ViT-H/14") { + // HF only ships an ImageNet-21k checkpoint for huge: 21843 classes. + c.hidden = 1280; + c.num_layers = 32; + c.num_heads = 16; + c.ff_dim = 5120; + c.patch = 14; + c.img_size = 224; + c.num_classes = 21843; + } else { + std::cerr << "Unknown model_type: " << raw_name + << " (expected ViT-B/16 | ViT-B/32 | ViT-L/16 | ViT-L/32 | ViT-H/14)\n"; + std::abort(); + } + return c; +} -static constexpr const char* WTS_PATH = "../models/vit.wts"; -static constexpr const char* ENGINE_PATH = "../models/vit.engine"; -static constexpr const char* LABELS_PATH = "../assets/imagenet1000_clsidx_to_labels.txt"; static constexpr const std::array NAMES = {"input", "logits"}; -static constexpr const std::array SIZES = {3 * INPUT_H * INPUT_W, 1000}; static constexpr const std::array mean = {0.5f, 0.5f, 0.5f}; static constexpr const std::array stdv = {0.5f, 0.5f, 0.5f}; +static constexpr const char* LABELS_PATH = "assets/imagenet1000_clsidx_to_labels.txt"; static Logger gLogger; @@ -36,6 +151,8 @@ static auto bytesPerElement(DataType t) -> std::size_t { return 4; case DataType::kHALF: return 2; + case DataType::kINT64: + return 8; case DataType::kINT32: return 4; #if TRT_VERSION >= 8000 @@ -76,9 +193,87 @@ static void convertWeightMapToHalf(WeightMap& w) { struct ViTParam { uint32_t index; uint32_t head_num; + int64_t hidden; + int64_t ff_dim; float lnorm_eps = 1e-12f; }; +struct RepeatedWeightsStorage { + std::vector> f32; + std::vector> f16; + std::vector> i64; +}; + +static auto makeRepeatedBatchWeights(const Weights& src, int64_t repeat, RepeatedWeightsStorage& storage) -> Weights { + if (repeat <= 0 || src.values == nullptr || src.count <= 0) { + std::cerr << "invalid repeated weight\n"; + std::abort(); + } + + if (src.type == DataType::kHALF) { + const auto* values = reinterpret_cast(src.values); + storage.f16.emplace_back(static_cast(src.count * repeat)); + auto& dst = storage.f16.back(); + for (int64_t i = 0; i < repeat; ++i) { + std::copy(values, values + src.count, dst.begin() + i * src.count); + } + return Weights{src.type, dst.data(), static_cast(dst.size())}; + } + + if (src.type == DataType::kFLOAT) { + const auto* values = reinterpret_cast(src.values); + storage.f32.emplace_back(static_cast(src.count * repeat)); + auto& dst = storage.f32.back(); + for (int64_t i = 0; i < repeat; ++i) { + std::copy(values, values + src.count, dst.begin() + i * src.count); + } + return Weights{src.type, dst.data(), static_cast(dst.size())}; + } + + std::cerr << "unsupported repeated weight dtype\n"; + std::abort(); +} + +static auto addShapeConstant(INetworkDefinition* net, std::initializer_list values, + RepeatedWeightsStorage& storage) -> ITensor* { + storage.i64.emplace_back(values); + auto& data = storage.i64.back(); + Dims dims{}; + dims.nbDims = 1; + dims.d[0] = static_cast(data.size()); + auto* c = net->addConstant(dims, Weights{DataType::kINT64, data.data(), static_cast(data.size())}); + return c->getOutput(0); +} + +static auto addBatchedConstantSlice(INetworkDefinition* net, ITensor& input, ITensor& full, int64_t dim1, int64_t dim2, + RepeatedWeightsStorage& storage) -> ITensor* { + auto* input_shape = net->addShape(input); + Dims batch_start{}; + batch_start.nbDims = 1; + batch_start.d[0] = 0; + Dims batch_size{}; + batch_size.nbDims = 1; + batch_size.d[0] = 1; + Dims batch_stride{}; + batch_stride.nbDims = 1; + batch_stride.d[0] = 1; + auto* batch = net->addSlice(*input_shape->getOutput(0), batch_start, batch_size, batch_stride); + auto* size_tail = addShapeConstant(net, {dim1, dim2}, storage); + const std::array size_inputs = {batch->getOutput(0), size_tail}; + auto* size = net->addConcatenation(size_inputs.data(), static_cast(size_inputs.size())); + size->setAxis(0); + + auto* start = addShapeConstant(net, {0, 0, 0}, storage); + auto* stride = addShapeConstant(net, {1, 1, 1}, storage); + const auto full_dims = full.getDimensions(); + auto* slice = + net->addSlice(full, Dims3{0, 0, 0}, Dims3{full_dims.d[0], full_dims.d[1], full_dims.d[2]}, Dims3{1, 1, 1}); + slice->setInput(1, *start); + slice->setInput(2, *size->getOutput(0)); + slice->setInput(3, *stride); + return slice->getOutput(0); +} + static auto addGeLU(INetworkDefinition* net, ITensor& input) -> ILayer* { #if TRT_VERSION < 10000 // tanh approximation: 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))) @@ -129,11 +324,13 @@ static auto addLinearNorm(INetworkDefinition* net, ITensor& input, ITensor& scal auto ViTLayer(INetworkDefinition* net, WeightMap& w, ITensor& input, const ViTParam& param) -> ITensor* { std::string name = "vit.encoder.layer." + std::to_string(param.index); auto attn_name = name + ".attention"; - int64_t attn_head_size = 768LL / param.head_num; + int64_t H = param.hidden; + int64_t F = param.ff_dim; + int64_t attn_head_size = H / param.head_num; - auto* qw = net->addConstant(Dims3{1, 768, 768}, w.at(attn_name + ".attention.query.weight")); - auto* kw = net->addConstant(Dims3{1, 768, 768}, w.at(attn_name + ".attention.key.weight")); - auto* vw = net->addConstant(Dims3{1, 768, 768}, w.at(attn_name + ".attention.value.weight")); + auto* qw = net->addConstant(Dims3{1, H, H}, w.at(attn_name + ".attention.query.weight")); + auto* kw = net->addConstant(Dims3{1, H, H}, w.at(attn_name + ".attention.key.weight")); + auto* vw = net->addConstant(Dims3{1, H, H}, w.at(attn_name + ".attention.value.weight")); /* 1. layer norm before attention */ auto pre_ln_name = name + ".layernorm_before"; auto dims = input.getDimensions(); @@ -143,9 +340,9 @@ auto ViTLayer(INetworkDefinition* net, WeightMap& w, ITensor& input, const ViTPa auto* pre_lnorm = addLinearNorm(net, input, *ln_scale->getOutput(0), *ln_bias->getOutput(0), axes); /** 2. multi-head self-attention */ - auto* qb = net->addConstant(Dims3{1, 1, 768}, w.at(attn_name + ".attention.query.bias")); - auto* kb = net->addConstant(Dims3{1, 1, 768}, w.at(attn_name + ".attention.key.bias")); - auto* vb = net->addConstant(Dims3{1, 1, 768}, w.at(attn_name + ".attention.value.bias")); + auto* qb = net->addConstant(Dims3{1, 1, H}, w.at(attn_name + ".attention.query.bias")); + auto* kb = net->addConstant(Dims3{1, 1, H}, w.at(attn_name + ".attention.key.bias")); + auto* vb = net->addConstant(Dims3{1, 1, H}, w.at(attn_name + ".attention.value.bias")); auto* _lno = pre_lnorm->getOutput(0); // 2.1 Q, K attention matmul auto* _q_attn = net->addMatrixMultiply(*_lno, M::kNONE, *qw->getOutput(0), M::kTRANSPOSE); @@ -191,8 +388,10 @@ auto ViTLayer(INetworkDefinition* net, WeightMap& w, ITensor& input, const ViTPa auto* q_scaled = net->addElementWise(*q_s->getOutput(0), *qk_scale_w->getOutput(0), E::kPROD)->getOutput(0); auto* attn = net->addAttention(*q_scaled, *k_s->getOutput(0), *v_s->getOutput(0), ANO::kSOFTMAX, false); assert(attn != nullptr); - auto status = attn->setDecomposable(false); - assert(status); + if (!attn->setDecomposable(false)) { + std::cerr << "setDecomposable failed\n"; + std::abort(); + } auto* attn_out = net->addShuffle(*attn->getOutput(0)); #else auto* qk = net->addMatrixMultiply(*q_s->getOutput(0), M::kNONE, *k_s->getOutput(0), M::kTRANSPOSE); @@ -204,10 +403,10 @@ auto ViTLayer(INetworkDefinition* net, WeightMap& w, ITensor& input, const ViTPa auto* attn_out = net->addShuffle(*attn_qkv->getOutput(0)); #endif attn_out->setFirstTranspose({0, 2, 1, 3}); - attn_out->setReshapeDimensions(Dims3{0, 0, 768}); + attn_out->setReshapeDimensions(Dims3{0, 0, H}); // 2.4 attention output projection - auto* out_proj_w = net->addConstant(Dims3{1, 768, 768}, w.at(attn_name + ".output.dense.weight"))->getOutput(0); - auto* out_proj_b = net->addConstant(Dims3{1, 1, 768}, w.at(attn_name + ".output.dense.bias"))->getOutput(0); + auto* out_proj_w = net->addConstant(Dims3{1, H, H}, w.at(attn_name + ".output.dense.weight"))->getOutput(0); + auto* out_proj_b = net->addConstant(Dims3{1, 1, H}, w.at(attn_name + ".output.dense.bias"))->getOutput(0); auto* attn_fcw = net->addMatrixMultiply(*attn_out->getOutput(0), M::kNONE, *out_proj_w, M::kTRANSPOSE); auto* attn_fcb = net->addElementWise(*attn_fcw->getOutput(0), *out_proj_b, E::kSUM); attn_fcb->setName((attn_name + ".out_proj").c_str()); @@ -227,8 +426,8 @@ auto ViTLayer(INetworkDefinition* net, WeightMap& w, ITensor& input, const ViTPa /** 6. intermediate (feed-forward) layer and activation */ auto intermediate_name = name + ".intermediate.dense"; std::cout << "Building: " << intermediate_name << "\n"; - auto* iw = net->addConstant(Dims3{1, 3072, 768}, w[intermediate_name + ".weight"]); - auto* ib = net->addConstant(Dims3{1, 1, 3072}, w[intermediate_name + ".bias"]); + auto* iw = net->addConstant(Dims3{1, F, H}, w[intermediate_name + ".weight"]); + auto* ib = net->addConstant(Dims3{1, 1, F}, w[intermediate_name + ".bias"]); ib->setName((intermediate_name + ".bias").c_str()); auto* inter0 = net->addMatrixMultiply(*post_lnorm->getOutput(0), M::kNONE, *iw->getOutput(0), M::kTRANSPOSE); auto* inter1 = net->addElementWise(*inter0->getOutput(0), *ib->getOutput(0), E::kSUM); @@ -237,8 +436,8 @@ auto ViTLayer(INetworkDefinition* net, WeightMap& w, ITensor& input, const ViTPa /** 7. output projection */ auto output_name = name + ".output.dense"; std::cout << "Building: " << output_name << "\n"; - auto* ow = net->addConstant(Dims3{1, 768, 3072}, w[output_name + ".weight"]); - auto* ob = net->addConstant(Dims3{1, 1, 768}, w[output_name + ".bias"]); + auto* ow = net->addConstant(Dims3{1, H, F}, w[output_name + ".weight"]); + auto* ob = net->addConstant(Dims3{1, 1, H}, w[output_name + ".bias"]); ob->setName((output_name + ".bias").c_str()); auto* out0 = net->addMatrixMultiply(*inter_act->getOutput(0), M::kNONE, *ow->getOutput(0), M::kTRANSPOSE); auto* out1 = net->addElementWise(*out0->getOutput(0), *ob->getOutput(0), E::kSUM); @@ -250,13 +449,15 @@ auto ViTLayer(INetworkDefinition* net, WeightMap& w, ITensor& input, const ViTPa } // Creat the engine using only the API without any parser. -auto createEngine(int64_t N, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, +auto createEngine(const ViTConfig& cfg, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, DataType dt) -> ICudaEngine* { - WeightMap w = loadWeights(WTS_PATH); + WeightMap w = loadWeights(cfg.wts_path); if (dt == DataType::kHALF) { convertWeightMapToHalf(w); } + RepeatedWeightsStorage repeated_storage; + #if TRT_VERSION >= 10000 auto* net = builder->createNetworkV2(1U << static_cast(NDCF::kSTRONGLY_TYPED)); #else @@ -264,51 +465,83 @@ auto createEngine(int64_t N, IRuntime* runtime, IBuilder* builder, IBuilderConfi #endif // 1. patch embedding - ITensor* data = net->addInput(NAMES[0], dt, Dims4{-1, 3, INPUT_H, INPUT_W}); + Dims input_dims{.nbDims = 4, .d = {-1, 3, cfg.img_size, cfg.img_size}}; + ITensor* data = net->addInput(NAMES[0], dt, input_dims); std::string name = "vit.embeddings.patch_embeddings.projection."; - auto* embed = net->addConvolutionNd(*data, 768, DimsHW{16, 16}, w[name + "weight"], w[name + "bias"]); + auto* embed = net->addConvolutionNd(*data, cfg.hidden, DimsHW{cfg.patch, cfg.patch}, w[name + "weight"], + w[name + "bias"]); embed->setName("patch embedding"); - embed->setStrideNd(DimsHW{16, 16}); + embed->setStrideNd(DimsHW{cfg.patch, cfg.patch}); auto* s = net->addShuffle(*embed->getOutput(0)); - s->setReshapeDimensions(Dims3{0, 768, 14LL * 14}); + s->setReshapeDimensions(Dims3{0, cfg.hidden, cfg.num_patches()}); s->setSecondTranspose({0, 2, 1}); // 2. add cls token and position embedding - auto* cls_token = net->addConstant(Dims3{1, 1, 768}, w["vit.embeddings.cls_token"]); - auto* pos_embed = net->addConstant(Dims3{1, 197, 768}, w["vit.embeddings.position_embeddings"]); - const std::array _cat = {cls_token->getOutput(0), s->getOutput(0)}; + auto cls_weights = makeRepeatedBatchWeights(w["vit.embeddings.cls_token"], cfg.max_batch, repeated_storage); + auto pos_weights = + makeRepeatedBatchWeights(w["vit.embeddings.position_embeddings"], cfg.max_batch, repeated_storage); + auto* cls_full = net->addConstant(Dims3{cfg.max_batch, 1, cfg.hidden}, cls_weights); + auto* pos_full = net->addConstant(Dims3{cfg.max_batch, cfg.seq_len(), cfg.hidden}, pos_weights); + auto* cls_token = addBatchedConstantSlice(net, *data, *cls_full->getOutput(0), 1, cfg.hidden, repeated_storage); + auto* pos_embed = + addBatchedConstantSlice(net, *data, *pos_full->getOutput(0), cfg.seq_len(), cfg.hidden, repeated_storage); + const std::array _cat = {cls_token, s->getOutput(0)}; auto* cat = net->addConcatenation(_cat.data(), 2); cat->setAxis(1); cat->setName("cat_clstoken_embed"); - auto* pos_added = net->addElementWise(*cat->getOutput(0), *pos_embed->getOutput(0), ElementWiseOperation::kSUM); + auto* pos_added = net->addElementWise(*cat->getOutput(0), *pos_embed, ElementWiseOperation::kSUM); pos_added->setName("position_embed"); // 3. transformer encoder layers ITensor* input = pos_added->getOutput(0); - for (auto i = 0u; i < 12; i++) { - auto* vit = ViTLayer(net, w, *input, {.index = i, .head_num = 12, .lnorm_eps = 1e-12f}); + for (int64_t i = 0; i < cfg.num_layers; i++) { + auto* vit = ViTLayer(net, w, *input, + {.index = static_cast(i), + .head_num = static_cast(cfg.num_heads), + .hidden = cfg.hidden, + .ff_dim = cfg.ff_dim, + .lnorm_eps = cfg.lnorm_eps}); input = vit; } // 4. layer norm after transformer encoder - auto* ln_scale = net->addConstant(Dims3{1, 1, 768}, w["vit.layernorm.weight"]); - auto* ln_bias = net->addConstant(Dims3{1, 1, 768}, w["vit.layernorm.bias"]); + auto* ln_scale = net->addConstant(Dims3{1, 1, cfg.hidden}, w["vit.layernorm.weight"]); + auto* ln_bias = net->addConstant(Dims3{1, 1, cfg.hidden}, w["vit.layernorm.bias"]); uint32_t axes = 1U << static_cast(input->getDimensions().nbDims - 1); auto* post_lnorm = addLinearNorm(net, *input, *ln_scale->getOutput(0), *ln_bias->getOutput(0), axes); - // 6. classifier head - auto* slice = net->addSlice(*post_lnorm->getOutput(0), Dims3{0, 0, 0}, Dims3{N, 1, 768}, Dims3{1, 1, 1}); - auto* shuffle = net->addShuffle(*slice->getOutput(0)); - shuffle->setReshapeDimensions(Dims2{N, 768}); - auto* cls_w = net->addConstant(DimsHW{1000, 768}, w["classifier.weight"]); - auto* cls_b = net->addConstant(DimsHW{1, 1000}, w["classifier.bias"]); + // 6. classifier head -- take CLS token (index 0 along seq axis) via Gather + static int32_t cls_idx_data = 0; + Weights cls_idx_w{DataType::kINT32, &cls_idx_data, 1}; + Dims idx_dims; + idx_dims.nbDims = 1; + idx_dims.d[0] = 1; + auto* idx_const = net->addConstant(idx_dims, cls_idx_w)->getOutput(0); + auto* gather = net->addGather(*post_lnorm->getOutput(0), *idx_const, 1); + auto* shuffle = net->addShuffle(*gather->getOutput(0)); + shuffle->setReshapeDimensions(Dims2{-1, cfg.hidden}); + auto* cls_w = net->addConstant(DimsHW{cfg.num_classes, cfg.hidden}, w["classifier.weight"]); + auto* cls_b = net->addConstant(DimsHW{1, cfg.num_classes}, w["classifier.bias"]); auto* cls_0 = net->addMatrixMultiply(*shuffle->getOutput(0), M::kNONE, *cls_w->getOutput(0), M::kTRANSPOSE); auto* cls_1 = net->addElementWise(*cls_b->getOutput(0), *cls_0->getOutput(0), E::kSUM); + cls_1->getOutput(0)->setName(NAMES[1]); net->markOutput(*cls_1->getOutput(0)); - Dims4 _min{1, 3, INPUT_H, INPUT_W}, _opt{N, 3, INPUT_H, INPUT_W}, _max{2 * N, 3, INPUT_H, INPUT_W}; #if TRT_VERSION >= 8000 config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); - config->setHardwareCompatibilityLevel(HardwareCompatibilityLevel::kAMPERE_PLUS); + config->setBuilderOptimizationLevel(5); +#if TRT_VERSION < 10000 + // Strongly-typed networks (TRT 10+) take their precision from the + // tensor types declared on inputs/weights and reject BuilderFlag::kFP16. + // Pre-TRT-10 networks are weakly typed, so explicitly request FP16 + // tactics when BUILD_PRECISION == kHALF; otherwise TRT may fall back + // to FP32 kernels even though the inputs/weights are half. + if (BUILD_PRECISION == DataType::kHALF) { + config->setFlag(BuilderFlag::kFP16); + } +#endif + Dims _min{.nbDims = 4, .d = {cfg.min_batch, 3, cfg.img_size, cfg.img_size}}; + Dims _opt{.nbDims = 4, .d = {cfg.opt_batch, 3, cfg.img_size, cfg.img_size}}; + Dims _max{.nbDims = 4, .d = {cfg.max_batch, 3, cfg.img_size, cfg.img_size}}; auto* profile = builder->createOptimizationProfile(); profile->setDimensions(NAMES[0], OptProfileSelector::kMIN, _min); profile->setDimensions(NAMES[0], OptProfileSelector::kOPT, _opt); @@ -318,7 +551,7 @@ auto createEngine(int64_t N, IRuntime* runtime, IBuilder* builder, IBuilderConfi ICudaEngine* engine = runtime->deserializeCudaEngine(mem->data(), mem->size()); delete net; #else - builder->setMaxBatchSize(N); + builder->setMaxBatchSize(cfg.max_batch); config->setMaxWorkspaceSize(WORKSPACE_SIZE); ICudaEngine* engine = builder->buildEngineWithConfig(*net, *config); net->destroy(); @@ -341,7 +574,8 @@ auto createEngine(int64_t N, IRuntime* runtime, IBuilder* builder, IBuilderConfi return engine; } -std::vector> doInference(IExecutionContext& context, __half* input, std::size_t batchSize) { +std::vector> doInference(IExecutionContext& context, const void* input, std::size_t batchSize, + const ViTConfig& cfg) { const ICudaEngine& engine = context.getEngine(); cudaStream_t stream; CHECK(cudaStreamCreate(&stream)); @@ -356,34 +590,55 @@ std::vector> doInference(IExecutionContext& context, __half* const int32_t nIO = engine.getNbBindings(); #endif + // SIZES per IO: input is C*H*W per sample; output is num_classes per sample. + const int64_t in_per_sample = 3LL * cfg.img_size * cfg.img_size; + const int64_t out_per_sample = cfg.num_classes; + auto sizeOf = [&](int i) -> int64_t { + return i == 0 ? in_per_sample : out_per_sample; + }; + +#if TRT_VERSION >= 8000 + if (engine.getTensorShape(NAMES[0]).d[0] == -1) { + Dims in_dims{.nbDims = 4, .d = {static_cast(batchSize), 3, cfg.img_size, cfg.img_size}}; + if (!context.setInputShape(NAMES[0], in_dims)) { + std::cerr << "setInputShape failed batch=" << batchSize << "\n"; + std::abort(); + } + } +#endif + buffers.resize(nIO, nullptr); for (auto i = 0; i < nIO; ++i) { #if TRT_VERSION >= 8000 - // TensorRT 8+ use name based SDK auto* tensor_name = engine.getIOTensorName(i); const auto dtype = engine.getTensorDataType(tensor_name); - std::size_t size = batchSize * SIZES[i] * bytesPerElement(dtype); + std::size_t size = batchSize * sizeOf(i) * bytesPerElement(dtype); #if TRT_VERSION >= 10000 - // TensorRT 10+ use outuput allocator - if (i == 0) { + if (engine.getTensorIOMode(tensor_name) == TensorIOMode::kINPUT) { CHECK(cudaMalloc(&buffers[i], size)); CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); - context.setTensorAddress(tensor_name, buffers[i]); + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } } else { context.setOutputAllocator(tensor_name, allocator.get()); } #else - if (i != 0) { + if (engine.getTensorIOMode(tensor_name) == TensorIOMode::kINPUT) { CHECK(cudaMalloc(&buffers[i], size)); + CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); } else { CHECK(cudaMalloc(&buffers[i], size)); - CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); } - context.setTensorAddress(tensor_name, buffers[i]); + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } #endif #else - std::size_t size = batchSize * SIZES[i] * sizeof(float); + std::size_t size = batchSize * sizeOf(i) * sizeof(float); const int32_t idx = engine.getBindingIndex(NAMES[i]); assert(idx == i); CHECK(cudaMalloc(&buffers[i], size)); @@ -394,36 +649,49 @@ std::vector> doInference(IExecutionContext& context, __half* } #if TRT_VERSION >= 8000 - assert(context.enqueueV3(stream)); + if (!context.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); + } #else - assert(context.enqueueV2(buffers.data(), stream, nullptr)); + if (!context.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } #endif std::vector> prob; - for (int i = 1; i < nIO; ++i) { -#if TRT_VERSION >= 10000 + for (int i = 0; i < nIO; ++i) { +#if TRT_VERSION >= 8000 auto* tensor_name = engine.getIOTensorName(i); + if (engine.getTensorIOMode(tensor_name) != TensorIOMode::kOUTPUT) + continue; const auto dtype = engine.getTensorDataType(tensor_name); - std::size_t size = batchSize * SIZES[i] * bytesPerElement(dtype); + std::size_t count = batchSize * out_per_sample; + std::size_t size = count * bytesPerElement(dtype); +#if TRT_VERSION >= 10000 void* out_ptr = allocator->getBuffer(tensor_name); - // D2H data transfer +#else + void* out_ptr = buffers[i]; +#endif if (dtype == DataType::kHALF) { - std::vector<__half> tmp_h(batchSize * SIZES[i]); + std::vector<__half> tmp_h(count); CHECK(cudaMemcpyAsync(tmp_h.data(), out_ptr, size, cudaMemcpyDeviceToHost, stream)); CHECK(cudaStreamSynchronize(stream)); - std::vector tmp(batchSize * SIZES[i]); - for (std::size_t j = 0; j < tmp.size(); ++j) { + std::vector tmp(count); + for (std::size_t j = 0; j < tmp.size(); ++j) tmp[j] = __half2float(tmp_h[j]); - } prob.emplace_back(std::move(tmp)); } else { - std::vector tmp(batchSize * SIZES[i], std::nanf("")); + std::vector tmp(count, std::nanf("")); CHECK(cudaMemcpyAsync(tmp.data(), out_ptr, size, cudaMemcpyDeviceToHost, stream)); prob.emplace_back(std::move(tmp)); } #else - std::vector tmp(batchSize * SIZES[i], std::nanf("")); - std::size_t size = batchSize * SIZES[i] * sizeof(float); + if (i == 0) + continue; + std::vector tmp(batchSize * sizeOf(i), std::nanf("")); + std::size_t size = batchSize * sizeOf(i) * sizeof(float); CHECK(cudaMemcpyAsync(tmp.data(), buffers[i], size, cudaMemcpyDeviceToHost, stream)); prob.emplace_back(std::move(tmp)); #endif @@ -442,11 +710,11 @@ std::vector> doInference(IExecutionContext& context, __half* return prob; } -void APIToModel(int32_t N, IRuntime* runtime, IHostMemory** modelStream) { +void APIToModel(const ViTConfig& cfg, IRuntime* runtime, IHostMemory** modelStream) { IBuilder* builder = createInferBuilder(gLogger); IBuilderConfig* config = builder->createBuilderConfig(); - ICudaEngine* engine = createEngine(N, runtime, builder, config, DataType::kHALF); + ICudaEngine* engine = createEngine(cfg, runtime, builder, config, BUILD_PRECISION); assert(engine != nullptr); (*modelStream) = engine->serialize(); @@ -462,34 +730,111 @@ void APIToModel(int32_t N, IRuntime* runtime, IHostMemory** modelStream) { #endif } +// Recover ViTConfig fields needed by inference (img_size, num_classes) from the +// deserialized engine. The other fields (hidden/layers/heads/ff/patch) are baked +// into the engine and not needed at runtime. +static ViTConfig configFromEngine(const ICudaEngine& engine) { + ViTConfig cfg; + auto in_dims = engine.getTensorShape(NAMES[0]); // [N or -1, 3, H, W] + auto out_dims = engine.getTensorShape(NAMES[1]); // [N or -1, num_classes] + if (in_dims.nbDims != 4 || out_dims.nbDims != 2) { + std::cerr << "Unexpected engine IO shapes\n"; + std::abort(); + } + cfg.img_size = in_dims.d[2]; + cfg.num_classes = out_dims.d[1]; + if (in_dims.d[0] == -1) { + auto pmin = engine.getProfileShape(NAMES[0], 0, OptProfileSelector::kMIN); + auto popt = engine.getProfileShape(NAMES[0], 0, OptProfileSelector::kOPT); + auto pmax = engine.getProfileShape(NAMES[0], 0, OptProfileSelector::kMAX); + cfg.min_batch = pmin.d[0]; + cfg.opt_batch = popt.d[0]; + cfg.max_batch = pmax.d[0]; + } else { + cfg.min_batch = in_dims.d[0]; + cfg.opt_batch = in_dims.d[0]; + cfg.max_batch = in_dims.d[0]; + } + return cfg; +} + +static auto loadLabels() -> std::map { + auto labels = loadImagenetLabelMap(LABELS_PATH); + if (labels.empty()) { + std::cerr << "failed to load labels: " << LABELS_PATH << "\n"; + std::abort(); + } + return labels; +} + +// Collect image files from a path: if a directory, return all *.jpg/*.jpeg/*.png +// files; if a single file, return just that file. +static std::vector collectImages(const std::string& path) { + namespace fs = std::filesystem; + std::vector out; + fs::path p(path); + if (!fs::exists(p)) { + std::cerr << "image path does not exist: " << path << "\n"; + std::abort(); + } + if (fs::is_regular_file(p)) { + out.push_back(path); + } else if (fs::is_directory(p)) { + for (auto& e : fs::directory_iterator(p)) { + if (!e.is_regular_file()) + continue; + auto ext = e.path().extension().string(); + for (auto& c : ext) { + c = static_cast(std::tolower(static_cast(c))); + } + if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".bmp") { + out.push_back(e.path().string()); + } + } + std::sort(out.begin(), out.end()); + } + return out; +} + auto main(int argc, char** argv) -> int { std::cout << "TensorRT version: " << TRT_VERSION << "\n"; - if (argc != 2) { - std::cerr << "arguments not right!\n"; - std::cerr << "./vit -s // serialize model to plan file\n"; - std::cerr << "./vit -d // deserialize plan file and run inference\n"; - + if (argc < 2 || (std::string(argv[1]) == "-s" && argc != 5) || (std::string(argv[1]) == "-d" && argc != 4)) { + std::cerr << "usage:\n" + << " ./vit -s \n" + << " ./vit -d \n" + << " model_type: ViT-B/16 | ViT-B/32 | ViT-L/16 | ViT-L/32 | ViT-H/14\n" + << " (aliases: b16, b32, l16, l32, h14 also accepted)\n" + << " build precision is configured in code via BUILD_PRECISION (see top of vit.cc).\n"; return 1; } + std::string mode = argv[1]; + #ifndef NDEBUG - gLogger.setReportableSeverity(nvinfer1::ILogger::Severity::kVERBOSE); + gLogger.setReportableSeverity(nvinfer1::ILogger::Severity::kINFO); #endif IRuntime* runtime = createInferRuntime(gLogger); assert(runtime != nullptr); - char* trtModelStream{nullptr}; - std::streamsize size{0}; - if (std::string(argv[1]) == "-s") { + if (mode == "-s") { + std::string wts_path = argv[2]; + std::string engine_path = argv[3]; + std::string model_type = argv[4]; + ViTConfig cfg = getVariantConfig(model_type); + cfg.wts_path = wts_path; + cfg.engine_path = engine_path; + const char* prec_str = (BUILD_PRECISION == DataType::kHALF) ? "fp16" : "fp32"; + std::cout << "[cfg] " << cfg.model_type << " hidden=" << cfg.hidden << " layers=" << cfg.num_layers + << " heads=" << cfg.num_heads << " ff=" << cfg.ff_dim << " patch=" << cfg.patch + << " img=" << cfg.img_size << " classes=" << cfg.num_classes + << " batch[min/opt/max]=" << cfg.min_batch << "/" << cfg.opt_batch << "/" << cfg.max_batch + << " precision=" << prec_str << "\n[wts] " << wts_path << "\n[engine] " << engine_path << "\n"; + IHostMemory* modelStream{nullptr}; - APIToModel(N, runtime, &modelStream); + APIToModel(cfg, runtime, &modelStream); assert(modelStream != nullptr); - std::ofstream p(ENGINE_PATH, std::ios::binary | std::ios::trunc); + std::ofstream p(engine_path, std::ios::binary | std::ios::trunc); if (!p) { - std::cerr << "could not open plan output file\n"; - return -1; - } - if (modelStream->size() > static_cast(std::numeric_limits::max())) { - std::cerr << "this model is too large to serialize\n"; + std::cerr << "could not open plan output file: " << engine_path << "\n"; return -1; } const auto* data_ptr = reinterpret_cast(modelStream->data()); @@ -501,68 +846,105 @@ auto main(int argc, char** argv) -> int { modelStream->destroy(); #endif return 0; - } else if (std::string(argv[1]) == "-d") { - std::ifstream file(ENGINE_PATH, std::ios::binary); - if (file.good()) { - file.seekg(0, file.end); - size = file.tellg(); - file.seekg(0, file.beg); - trtModelStream = new char[size]; - assert(trtModelStream); - file.read(trtModelStream, size); - file.close(); - } else { - std::cerr << "read engine file error!\n"; + } else if (mode == "-d") { + std::string engine_path = argv[2]; + std::string image_dir = argv[3]; + + std::ifstream file(engine_path, std::ios::binary); + if (!file.good()) { + std::cerr << "read engine file error: " << engine_path << "\n"; + return -1; + } + file.seekg(0, file.end); + std::streamsize size = file.tellg(); + file.seekg(0, file.beg); + if (size <= 0) { + std::cerr << "empty engine file: " << engine_path << "\n"; return -1; } + std::vector trtModelStream(static_cast(size)); + file.read(trtModelStream.data(), size); + file.close(); #if TRT_VERSION >= 8000 - ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size); + ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream.data(), size); #else - ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size, nullptr); + ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream.data(), size, nullptr); #endif assert(engine != nullptr); auto* context = engine->createExecutionContext(); assert(context != nullptr); - // VIT use default BGR order - auto img = cv::imread("../assets/cats.jpg", cv::IMREAD_COLOR); - auto input = preprocess_img(img, false, mean, stdv, N, INPUT_H, INPUT_W); + ViTConfig cfg = configFromEngine(*engine); + std::cout << "[engine] img=" << cfg.img_size << " classes=" << cfg.num_classes + << " batch[min/opt/max]=" << cfg.min_batch << "/" << cfg.opt_batch << "/" << cfg.max_batch << "\n"; + + auto images = collectImages(image_dir); + if (images.empty()) { + std::cerr << "no images found under: " << image_dir << "\n"; + return -1; + } + // Cap batch to engine's max profile. + int64_t infer_batch = std::min(static_cast(images.size()), cfg.max_batch); + std::cout << "[infer] found " << images.size() << " image(s), using batch=" << infer_batch << "\n"; + + std::vector<__half> input_buf; + input_buf.reserve(infer_batch * 3 * cfg.img_size * cfg.img_size); + for (int64_t i = 0; i < infer_batch; ++i) { + auto img = cv::imread(images[i], cv::IMREAD_COLOR); + if (img.empty()) { + std::cerr << "cannot read image: " << images[i] << "\n"; + return -1; + } + auto one = preprocess_img(img, false, mean, stdv, 1, static_cast(cfg.img_size), + static_cast(cfg.img_size)); + input_buf.insert(input_buf.end(), one.begin(), one.end()); + } + + // Match the engine's declared input dtype. The engine input is FP16 when + // built with `-s ... fp16` (default) and FP32 when built with `... fp32`. + const auto in_dtype = engine->getTensorDataType(NAMES[0]); + std::vector input_buf_f32; + const void* input_ptr = nullptr; + if (in_dtype == DataType::kHALF) { + input_ptr = input_buf.data(); + } else if (in_dtype == DataType::kFLOAT) { + input_buf_f32.resize(input_buf.size()); + for (std::size_t k = 0; k < input_buf.size(); ++k) + input_buf_f32[k] = __half2float(input_buf[k]); + input_ptr = input_buf_f32.data(); + } else { + std::cerr << "unsupported engine input dtype\n"; + return -1; + } Profiler profiler("VisionTransformerProfiler"); - // Warmup: run a few iterations without profiling. for (int i = 0; i < 5; ++i) { - (void)doInference(*context, input.data(), N); + (void)doInference(*context, input_ptr, infer_batch, cfg); } - // Profiled runs context->setProfiler(&profiler); for (int i = 0; i < 20; ++i) { auto start = std::chrono::system_clock::now(); - auto prob = doInference(*context, input.data(), N); + auto prob = doInference(*context, input_ptr, infer_batch, cfg); auto end = std::chrono::system_clock::now(); auto period = std::chrono::duration_cast(end - start); std::cout << period.count() << "us\n"; - for (const auto& vector : prob) { - int idx = 0; - for (auto v : vector) { - std::cout << std::setprecision(4) << v << ", " << std::flush; - if (++idx > 20) { - std::cout << "\n====\n"; - break; - } - } - } - if (i == 19) { - std::cout << "prediction result: \n"; - auto labels = loadImagenetLabelMap(LABELS_PATH); - int _top = 0; - for (auto& [idx, logits] : topk(prob[0], 3)) { - std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits - << ", label: " << labels[idx] << "\n"; + auto labels = (cfg.num_classes == 1000) ? loadLabels() : std::map{}; + for (int64_t b = 0; b < infer_batch; ++b) { + std::cout << "[sample " << b << "] " << images[b] << "\n"; + int _top = 0; + std::vector sample(prob[0].begin() + b * cfg.num_classes, + prob[0].begin() + (b + 1) * cfg.num_classes); + for (auto& [idx, logits] : topk(sample, 3)) { + std::cout << " Top: " << _top++ << " idx: " << idx << ", logits: " << logits; + if (!labels.empty()) + std::cout << ", label: " << labels[idx]; + std::cout << "\n"; + } } std::cout << profiler << "\n"; } From 2e1e51979602038aee6b028c2ca84da4ad6b780d Mon Sep 17 00:00:00 2001 From: zjq Date: Tue, 9 Jun 2026 20:39:56 +0800 Subject: [PATCH 2/7] Fix Release inference for basic classifiers --- .gitignore | 228 ++++++++++++++++++++++++++++++++++++++-- alexnet/alexnet.cc | 17 ++- googlenet/googlenet.cpp | 17 ++- lenet/lenet.cpp | 17 ++- mlp/mlp.cpp | 16 ++- 5 files changed, 275 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 3323bab5..a8823a88 100644 --- a/.gitignore +++ b/.gitignore @@ -19,13 +19,6 @@ build # Built Visual Studio Code Extensions *.vsix -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -!.vscode/*.code-snippets - # Local History for Visual Studio Code .history/ @@ -92,7 +85,6 @@ ppocrv5/*.bin ppocrv5/*.log ppocrv5/*.graph.json ppocrv5/inference.json -ppocrv5/**/__pycache__/ ppocrv5/**/*.pyc ppocrv5/.venv*/ ppocrv5/**/*_layers.txt @@ -105,3 +97,223 @@ ppocrv5/samples/validation/**/*.jpg ppocrv5/samples/validation/**/*.jpeg ppocrv5/samples/validation/**/*.png !ppocrv5/samples/validation/README.md +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +*.lcov +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi/* +!.pixi/config.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule* +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ +# Temporary file for partial code execution +tempCodeRunnerFile.py + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml diff --git a/alexnet/alexnet.cc b/alexnet/alexnet.cc index 6fdfc409..887edbd1 100644 --- a/alexnet/alexnet.cc +++ b/alexnet/alexnet.cc @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include #include "logging.h" @@ -219,7 +221,10 @@ std::vector> doInference(IExecutionContext& context, const st auto s = getSize(engine.getTensorDataType(tensor_name)); std::size_t size = s * batchSize * SIZES[i]; CHECK(cudaMalloc(&buffers[i], size)); - context.setTensorAddress(tensor_name, buffers[i]); + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } #else const int32_t idx = engine.getBindingIndex(NAMES[i]); auto s = getSize(engine.getBindingDataType(idx)); @@ -233,9 +238,15 @@ std::vector> doInference(IExecutionContext& context, const st } #if TRT_VERSION >= 8000 - assert(context.enqueueV3(stream)); + if (!context.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); + } #else - assert(context.enqueueV2(buffers.data(), stream, nullptr)); + if (!context.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } #endif std::vector> prob; diff --git a/googlenet/googlenet.cpp b/googlenet/googlenet.cpp index da2494c8..fd4e590b 100644 --- a/googlenet/googlenet.cpp +++ b/googlenet/googlenet.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include #include "logging.h" @@ -268,7 +270,10 @@ std::vector> doInference(IExecutionContext& context, void* in if (i == 0) { CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); } - context.setTensorAddress(tensor_name, buffers[i]); + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } #else const int32_t idx = engine.getBindingIndex(NAMES[i]); auto s = getSize(engine.getBindingDataType(idx)); @@ -282,9 +287,15 @@ std::vector> doInference(IExecutionContext& context, void* in } #if TRT_VERSION >= 8000 - assert(context.enqueueV3(stream)); + if (!context.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); + } #else - assert(context.enqueueV2(buffers.data(), stream, nullptr)); + if (!context.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } #endif std::vector> prob; diff --git a/lenet/lenet.cpp b/lenet/lenet.cpp index 379edbbd..09aeba7d 100644 --- a/lenet/lenet.cpp +++ b/lenet/lenet.cpp @@ -2,8 +2,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -207,7 +209,10 @@ std::vector> doInference(IExecutionContext& context, void* in if (i == 0) { CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); } - context.setTensorAddress(tensor_name, buffers[i]); + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } #else const int32_t idx = engine.getBindingIndex(NAMES[i]); auto s = getSize(engine.getBindingDataType(idx)); @@ -221,9 +226,15 @@ std::vector> doInference(IExecutionContext& context, void* in } #if TRT_VERSION >= 8000 - assert(context.enqueueV3(stream)); + if (!context.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); + } #else - assert(context.enqueueV2(buffers.data(), stream, nullptr)); + if (!context.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } #endif std::vector> prob; diff --git a/mlp/mlp.cpp b/mlp/mlp.cpp index 3a2fe9ae..562bb257 100644 --- a/mlp/mlp.cpp +++ b/mlp/mlp.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -159,11 +160,20 @@ void doInference(IExecutionContext& ctx, void* input, float* output, int64_t bat auto dims = ctx.getTensorShape(name); auto total = std::accumulate(dims.d, dims.d + dims.nbDims, 1ll, std::multiplies<>()); std::cout << name << "\t" << total << "\n"; - ctx.setTensorAddress(name, buffers[i]); + if (!ctx.setTensorAddress(name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } + } + if (!ctx.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); } - assert(ctx.enqueueV3(stream)); #else - assert(ctx.enqueueV2(buffers.data(), stream, nullptr)); + if (!ctx.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } #endif CHECK(cudaMemcpyAsync(output, buffers[outputIndex], outputSize, cudaMemcpyDeviceToHost, stream)); From 22224da9bd81b9b09dfb6f96c641df8578a910de Mon Sep 17 00:00:00 2001 From: zjq Date: Tue, 9 Jun 2026 20:40:11 +0800 Subject: [PATCH 3/7] Fix Release inference and extend VGG variants --- lprnet/lprnet.cpp | 16 +- mnasnet/mnasnet.cpp | 16 +- squeezenet/FindTensorRT.cmake | 129 ++++++---- squeezenet/gen_wts.py | 86 +++---- squeezenet/macros.h | 5 + squeezenet/squeezenet.cpp | 192 +++++++++------ squeezenet/utils.h | 84 +++---- vgg/CMakeLists.txt | 48 ++-- vgg/FindTensorRT.cmake | 145 +++++++++++ vgg/README.md | 100 ++++++-- vgg/gen_wts.py | 80 ++++++ vgg/logging.h | 298 ++++++++++------------ vgg/macros.h | 32 +++ vgg/utils.h | 268 ++++++++++++++++++++ vgg/vgg.cc | 450 ++++++++++++++++++++++++++++++++++ vgg/vgg11.cpp | 289 ---------------------- 16 files changed, 1521 insertions(+), 717 deletions(-) create mode 100644 vgg/FindTensorRT.cmake create mode 100644 vgg/gen_wts.py create mode 100644 vgg/macros.h create mode 100644 vgg/utils.h create mode 100644 vgg/vgg.cc delete mode 100644 vgg/vgg11.cpp diff --git a/lprnet/lprnet.cpp b/lprnet/lprnet.cpp index 2295a3a2..44de873b 100644 --- a/lprnet/lprnet.cpp +++ b/lprnet/lprnet.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -314,7 +315,10 @@ auto doInference(IExecutionContext& context, void* input, int64_t batchSize) -> if (i == 0) { CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); } - context.setTensorAddress(tensor_name, buffers[i]); + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } #else const int32_t idx = engine.getBindingIndex(NAMES[i]); auto s = getSize(engine.getBindingDataType(idx)); @@ -328,9 +332,15 @@ auto doInference(IExecutionContext& context, void* input, int64_t batchSize) -> } #if TRT_VERSION >= 8000 - assert(context.enqueueV3(stream)); + if (!context.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); + } #else - assert(context.enqueueV2(buffers.data(), stream, nullptr)); + if (!context.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } #endif std::vector> prob; diff --git a/mnasnet/mnasnet.cpp b/mnasnet/mnasnet.cpp index fcbcb3ce..61babc2d 100644 --- a/mnasnet/mnasnet.cpp +++ b/mnasnet/mnasnet.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -284,7 +285,10 @@ std::vector> do_inference(IExecutionContext& context, void* i if (i == 0) { CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); } - context.setTensorAddress(tensor_name, buffers[i]); + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } #else const int32_t idx = engine.getBindingIndex(NAMES[i]); auto s = getSize(engine.getBindingDataType(idx)); @@ -298,9 +302,15 @@ std::vector> do_inference(IExecutionContext& context, void* i } #if TRT_VERSION >= 8000 - assert(context.enqueueV3(stream)); + if (!context.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); + } #else - assert(context.enqueueV2(buffers.data(), stream, nullptr)); + if (!context.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } #endif std::vector> prob; diff --git a/squeezenet/FindTensorRT.cmake b/squeezenet/FindTensorRT.cmake index 6d542776..338f592b 100644 --- a/squeezenet/FindTensorRT.cmake +++ b/squeezenet/FindTensorRT.cmake @@ -1,11 +1,5 @@ cmake_minimum_required(VERSION 3.17.0) -set(TRT_VERSION - $ENV{TRT_VERSION} - CACHE - STRING - "TensorRT version, e.g. \"8.6.1.6\" or \"8.6.1.6+cuda12.0.1.011\", etc") - function(_guess_path var_name required_files) set(_result "") @@ -44,56 +38,95 @@ function(_guess_path var_name required_files) PARENT_SCOPE) endfunction() -# find TensorRT include folder -if(NOT DEFINED TensorRT_INCLUDE_DIR) - if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") - _guess_path( - TensorRT_INCLUDE_DIR "NvInfer.h" "/usr/include/aarch64-linux-gnu" - "/usr/include" "/usr/local/cuda/targets/aarch64-linux/include") - else() - _guess_path( - TensorRT_INCLUDE_DIR "NvInfer.h" - "/usr/local/tensorrt/targets/x86_64-linux-gnu/include" - "/usr/include/x86_64-linux-gnu" "/usr/include") - endif() - message(STATUS "TensorRT includes: ${TensorRT_INCLUDE_DIR}") +# add library +add_library(TensorRT IMPORTED INTERFACE) +add_library(TensorRT::TensorRT ALIAS TensorRT) + +set(TRT_VERSION + "$ENV{TRT_VERSION}" + CACHE + STRING + "TensorRT version, e.g. \"8.6.1.6\" or \"8.6.1.6+cuda12.0.1.011\", \"8.6.1.6.Windows10.x86_64.cuda-12.0\" etc" +) + +if(NOT "${TRT_VERSION}" STREQUAL "" AND NOT "$ENV{TRT_VERSION}" STREQUAL "" + AND NOT "${TRT_VERSION}" STREQUAL "$ENV{TRT_VERSION}") + message( + WARNING + "TRT_VERSION defined by cmake and environment variable both, using the environment variable" + ) endif() -# find TensorRT library folder -if(NOT TensorRT_LIBRARY_DIR) - if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") - _guess_path( - TensorRT_LIBRARY_DIR "libnvinfer.so;libnvinfer_plugin.so" - "/usr/lib/aarch64-linux-gnu;/usr/lib/aarch64-linux-gnu/tegra" "/usr/lib") - else() - _guess_path( - TensorRT_LIBRARY_DIR - "libnvinfer.so;libnvinfer_plugin.so" - "/usr/lib/x86_64-linux-gnu;/usr/local/tensorrt/targets/x86_64-linux-gnu/lib;/usr/lib" +if(NOT "$ENV{TRT_VERSION}" STREQUAL "") + set(TRT_VERSION + "$ENV{TRT_VERSION}" + CACHE STRING "TensorRT version" FORCE) +endif() + +if("${TRT_VERSION}" STREQUAL "") + message(FATAL_ERROR "Please set the TRT_VERSION environment variable") +endif() + +string(REGEX MATCH "^[0-9]+" _match "${TRT_VERSION}") +set(TRT_MAJOR_VERSION "${_match}") +unset(_match) +if("${TRT_MAJOR_VERSION}" STREQUAL "") + message(FATAL_ERROR "Failed to parse TensorRT major version from ${TRT_VERSION}") +endif() + +if(WIN32) + set(TensorRT_DIR "C:/Program Files/TensorRT-${TRT_VERSION}") + if(NOT EXISTS "${TensorRT_DIR}") + message( + FATAL_ERROR + "TensorRT_DIR=${TensorRT_DIR} does not exist!" ) endif() - message(STATUS "TensorRT libraries: ${TensorRT_LIBRARY_DIR}") -endif() -set(TensorRT_LIBRARIES) + if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) + set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 + nvinfer_dispatch_10 nvinfer_lean_10) + message(DEBUG "Using ${_modules}") + else() + set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch + nvinfer_lean) + endif() -message(STATUS "Found TensorRT lib: ${TensorRT_LIBRARIES}") + set(TensorRT_LIBRARY_DIR "${TensorRT_DIR}/lib") + set(TensorRT_INCLUDE_DIR "${TensorRT_DIR}/include") +elseif(UNIX) + string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _trt_arch) + set(_trt_include_candidates) + if(_trt_arch MATCHES "^(aarch64|arm64|arch64)$") + set(_trt_include_candidates "/usr/include/aarch64-linux-gnu" "/usr/include" + "/usr/local/cuda/targets/aarch64-linux/include") + set(_trt_library_candidates + "/usr/local/tensorrt/targets/aarch64-linux-gnu/lib" + "/usr/lib/aarch64-linux-gnu" "/usr/lib/aarch64-linux-gnu/tegra" + "/usr/lib") + elseif(_trt_arch MATCHES "^(x86_64|amd64)$") + set(_trt_include_candidates + "/usr/local/tensorrt/targets/x86_64-linux-gnu/include" + "/usr/include/x86_64-linux-gnu" "/usr/include") + set(_trt_library_candidates + "/usr/local/tensorrt/targets/x86_64-linux-gnu/lib" + "/usr/lib/x86_64-linux-gnu" "/usr/lib") + else() + message(FATAL_ERROR "Unknown architecture") + endif() -# process for different TensorRT version -if(DEFINED TRT_VERSION AND NOT TRT_VERSION STREQUAL "") - string(REGEX MATCH "([0-9]+)" _match ${TRT_VERSION}) - set(TRT_MAJOR_VERSION "${_match}") set(_modules nvinfer nvinfer_plugin) - unset(_match) - - if(TRT_MAJOR_VERSION GREATER_EQUAL 8) + if(${TRT_MAJOR_VERSION} GREATER_EQUAL 8) list(APPEND _modules nvinfer_vc_plugin nvinfer_dispatch nvinfer_lean) endif() -else() - message(FATAL_ERROR "Please set a environment variable \"TRT_VERSION\"") + + _guess_path(TensorRT_LIBRARY_DIR "libnvinfer.so;libnvinfer_plugin.so" + ${_trt_library_candidates}) + message(STATUS "TensorRT libraries: ${TensorRT_LIBRARY_DIR}") + _guess_path(TensorRT_INCLUDE_DIR "NvInfer.h" ${_trt_include_candidates}) + message(STATUS "TensorRT includes: ${TensorRT_INCLUDE_DIR}") endif() -# find and add all modules of TensorRT into list foreach(lib IN LISTS _modules) find_library( TensorRT_${lib}_LIBRARY @@ -102,11 +135,10 @@ foreach(lib IN LISTS _modules) list(APPEND TensorRT_LIBRARIES ${TensorRT_${lib}_LIBRARY}) endforeach() -# make the "TensorRT target" -add_library(TensorRT IMPORTED INTERFACE) -add_library(TensorRT::TensorRT ALIAS TensorRT) target_link_libraries(TensorRT INTERFACE ${TensorRT_LIBRARIES}) +message(STATUS "Found TensorRT libs: ${TensorRT_LIBRARIES}") + set_target_properties( TensorRT PROPERTIES C_STANDARD 17 @@ -119,3 +151,6 @@ set_target_properties( unset(TRT_MAJOR_VERSION) unset(_modules) +unset(_trt_include_candidates) +unset(_trt_library_candidates) +unset(_trt_arch) diff --git a/squeezenet/gen_wts.py b/squeezenet/gen_wts.py index 8ed0ec44..816cd54a 100644 --- a/squeezenet/gen_wts.py +++ b/squeezenet/gen_wts.py @@ -1,72 +1,44 @@ +import os import struct +from pathlib import Path -import cv2 -import numpy as np import torch -import torchvision +from torchvision.models import SqueezeNet1_1_Weights, squeezenet1_1 -def read_imagenet_labels() -> dict[int, str]: - """ - read ImageNet 1000 labels +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parent +MODELS_DIR = REPO_ROOT / "models" +WTS_PATH = MODELS_DIR / "squeezenet.wts" - Returns: - dict[int, str]: labels dict - """ - clsid2label = {} - with open("../assets/imagenet1000_clsidx_to_labels.txt", "r") as f: - for i in f.readlines(): - k, v = i.split(": ") - clsid2label.setdefault(int(k), v[1:-3]) - return clsid2label +def print_cache_env() -> None: + for key in ("TORCH_HOME", "HF_HOME"): + value = os.environ.get(key) + if value: + print(f"{key}={value}") -def preprocess(img: np.array) -> torch.Tensor: - """ - a preprocess method align with ImageNet dataset - - Args: - img (np.array): input image - - Returns: - torch.Tensor: preprocessed image in `NCHW` layout - """ - img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0 - img = cv2.resize(img, (224, 224), interpolation=cv2.INTER_LINEAR) - mean = np.array([0.485, 0.456, 0.406], dtype=np.float32) - std = np.array([0.229, 0.224, 0.225], dtype=np.float32) - img = (img - mean) / std - img = img.transpose(2, 0, 1)[None, ...] - return torch.from_numpy(img) +def export_wts(model: torch.nn.Module, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + state_dict = model.state_dict() + with open(path, "w") as f: + f.write(f"{len(state_dict)}\n") + for key, value in state_dict.items(): + values = value.reshape(-1).cpu().numpy() + f.write(f"{key} {len(values)}") + print(key, value.shape) + for item in values: + f.write(" ") + f.write(struct.pack(">f", float(item)).hex()) + f.write("\n") -def main(): - labels = read_imagenet_labels() - model = torchvision.models.squeezenet1_1(pretrained=True) +def main() -> None: + print_cache_env() + model = squeezenet1_1(weights=SqueezeNet1_1_Weights.DEFAULT) model = model.eval() - - img = cv2.imread("../assets/cats.jpg", cv2.IMREAD_COLOR) - img = preprocess(img) - - with torch.inference_mode(): - output = model(img) - for i, batch in enumerate(torch.topk(output, k=3).indices): - for j, idx in enumerate(batch): - print(f"\tBatch: {i}, Top: {j}, logits: {output[i][idx]:.4f}, label: {labels[int(idx)]}") - print(f"{'=' * 32}") - - with open("../models/squeezenet.wts", "w") as f: - f.write("{}\n".format(len(model.state_dict().keys()))) - for k, v in model.state_dict().items(): - vr = v.reshape(-1).cpu().numpy() - f.write("{} {} ".format(k, len(vr))) - print(k, v.shape) - for vv in vr: - f.write(" ") - f.write(struct.pack(">f", float(vv)).hex()) - f.write("\n") - f.close() + export_wts(model, WTS_PATH) if __name__ == "__main__": diff --git a/squeezenet/macros.h b/squeezenet/macros.h index dc9a4d57..daae7bfb 100644 --- a/squeezenet/macros.h +++ b/squeezenet/macros.h @@ -1,4 +1,5 @@ #pragma once +#include #ifdef API_EXPORTS #if defined(_MSC_VER) @@ -18,6 +19,10 @@ #define TRT_VERSION \ ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) +#if TRT_VERSION < 7220 +#error "TensorRT >= 7.2.2 is required for this demo." +#endif + #if TRT_VERSION >= 8000 #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const diff --git a/squeezenet/squeezenet.cpp b/squeezenet/squeezenet.cpp index 381718c0..1ae46501 100644 --- a/squeezenet/squeezenet.cpp +++ b/squeezenet/squeezenet.cpp @@ -1,85 +1,98 @@ #include +#include #include #include +#include +#include +#include #include +#include #include +#include #include #include +#include #include #include "logging.h" #include "utils.h" -// stuff we know about squeezenet -static constexpr const int N = 1; -static constexpr const int INPUT_H = 224; -static constexpr const int INPUT_W = 224; -static constexpr const int SIZES[] = {3 * INPUT_H * INPUT_W, N * 1000}; -static constexpr const char* NAMES[] = {"data", "prob"}; +static constexpr const std::size_t WORKSPACE_SIZE = 16 << 20; + +static constexpr const int64_t N = 1; +static constexpr const int32_t INPUT_H = 224; +static constexpr const int32_t INPUT_W = 224; +static constexpr const std::array SIZES = {3 * INPUT_H * INPUT_W, N * 1000}; +static constexpr const std::array NAMES = {"data", "prob"}; static constexpr const bool TRT_PREPROCESS = TRT_VERSION >= 8510 ? true : false; -static constexpr const float mean[3] = {0.485f, 0.456f, 0.406f}; -static constexpr const float stdv[3] = {0.229f, 0.224f, 0.225f}; +static constexpr const std::array mean = {0.485f, 0.456f, 0.406f}; +static constexpr const std::array stdv = {0.229f, 0.224f, 0.225f}; -static constexpr const char* WTS_PATH = "../models/squeezenet.wts"; -static constexpr const char* ENGINE_PATH = "../models/squeezenet.engine"; -static constexpr const char* LABELS_PATH = "../assets/imagenet1000_clsidx_to_labels.txt"; +static constexpr const char* WTS_PATH = "models/squeezenet.wts"; +static constexpr const char* ENGINE_PATH = "models/squeezenet.engine"; +static constexpr const char* LABELS_PATH = "assets/imagenet1000_clsidx_to_labels.txt"; using namespace nvinfer1; using WeightMap = std::map; +using NDCF = nvinfer1::NetworkDefinitionCreationFlag; static Logger gLogger; -ILayer* fire(INetworkDefinition* network, WeightMap& m, ITensor& input, const std::string& lname, - int32_t squeeze_planes, int32_t e1x1_planes, int32_t e3x3_planes) { - auto* conv1 = network->addConvolutionNd(input, squeeze_planes, DimsHW{1, 1}, m[lname + "squeeze.weight"], - m[lname + "squeeze.bias"]); +static auto fire(INetworkDefinition* network, WeightMap& weights, ITensor& input, const std::string& lname, + int32_t squeeze_planes, int32_t e1x1_planes, int32_t e3x3_planes) -> ILayer* { + auto* conv1 = network->addConvolutionNd(input, squeeze_planes, DimsHW{1, 1}, weights.at(lname + "squeeze.weight"), + weights.at(lname + "squeeze.bias")); assert(conv1); - auto* relu1 = network->addActivation(*conv1->getOutput(0), ActivationType::kRELU)->getOutput(0); + auto* relu1 = network->addActivation(*conv1->getOutput(0), ActivationType::kRELU); + assert(relu1); std::string _c = lname + "expand1x1"; - auto* conv2 = network->addConvolutionNd(*relu1, e1x1_planes, DimsHW{1, 1}, m[_c + ".weight"], m[_c + ".bias"]); + auto* conv2 = network->addConvolutionNd(*relu1->getOutput(0), e1x1_planes, DimsHW{1, 1}, weights.at(_c + ".weight"), + weights.at(_c + ".bias")); assert(conv2); auto* relu2 = network->addActivation(*conv2->getOutput(0), ActivationType::kRELU); assert(relu2); _c = lname + "expand3x3"; - auto* conv3 = network->addConvolutionNd(*relu1, e3x3_planes, DimsHW{3, 3}, m[_c + ".weight"], m[_c + ".bias"]); + auto* conv3 = network->addConvolutionNd(*relu1->getOutput(0), e3x3_planes, DimsHW{3, 3}, weights.at(_c + ".weight"), + weights.at(_c + ".bias")); assert(conv3); conv3->setPaddingNd(DimsHW{1, 1}); auto* relu3 = network->addActivation(*conv3->getOutput(0), ActivationType::kRELU); assert(relu3); - ITensor* inputTensors[] = {relu2->getOutput(0), relu3->getOutput(0)}; - auto* concat = network->addConcatenation(inputTensors, 2); + std::array inputTensors = {relu2->getOutput(0), relu3->getOutput(0)}; + auto* concat = network->addConcatenation(inputTensors.data(), 2); assert(concat); return concat; } -// Creat the engine using only the API and not any parser. -ICudaEngine* createEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, DataType dt) { +// Create the engine using only the API and not any parser. +static auto createEngine(int32_t batch_size, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, + DataType dt) -> ICudaEngine* { auto weightMap = loadWeights(WTS_PATH); -#if TRT_VERSION >= 10000 - auto* network = builder->createNetworkV2(0); + +#if TRT_VERSION >= 11200 + auto flag = 1U << static_cast(NDCF::kSTRONGLY_TYPED); +#elif TRT_VERSION >= 10000 + auto flag = 0U; #else - auto* network = builder->createNetworkV2(1u << static_cast(NetworkDefinitionCreationFlag::kEXPLICIT_BATCH)); + auto flag = 1U << static_cast(NDCF::kEXPLICIT_BATCH); #endif + auto* network = builder->createNetworkV2(flag); ITensor* data{nullptr}; if constexpr (TRT_PREPROCESS) { -#if TRT_VERSION > 8510 dt = DataType::kUINT8; -#else - dt = DataType::kINT8; -#endif - data = network->addInput(NAMES[0], dt, Dims4{N, INPUT_H, INPUT_W, 3}); + data = network->addInput(NAMES[0], dt, Dims4{batch_size, INPUT_H, INPUT_W, 3}); auto* trans = addTransformLayer(network, *data, true, mean, stdv); data = trans->getOutput(0); } else { - data = network->addInput(NAMES[0], dt, Dims4{N, 3, INPUT_H, INPUT_W}); + data = network->addInput(NAMES[0], dt, Dims4{batch_size, 3, INPUT_H, INPUT_W}); } assert(data); - auto* conv1 = network->addConvolutionNd(*data, 64, DimsHW{3, 3}, weightMap["features.0.weight"], - weightMap["features.0.bias"]); + auto* conv1 = network->addConvolutionNd(*data, 64, DimsHW{3, 3}, weightMap.at("features.0.weight"), + weightMap.at("features.0.bias")); assert(conv1); conv1->setStrideNd(DimsHW{2, 2}); auto* relu1 = network->addActivation(*conv1->getOutput(0), ActivationType::kRELU); @@ -113,8 +126,8 @@ ICudaEngine* createEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IBuil cat1 = fire(network, weightMap, *cat1->getOutput(0), "features.12.", 64, 256, 256); // classifier - auto* conv2 = network->addConvolutionNd(*cat1->getOutput(0), 1000, DimsHW{1, 1}, weightMap["classifier.1.weight"], - weightMap["classifier.1.bias"]); + auto* conv2 = network->addConvolutionNd(*cat1->getOutput(0), 1000, DimsHW{1, 1}, + weightMap.at("classifier.1.weight"), weightMap.at("classifier.1.bias")); assert(conv2); auto* relu2 = network->addActivation(*conv2->getOutput(0), ActivationType::kRELU); assert(relu2); @@ -129,33 +142,47 @@ ICudaEngine* createEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IBuil config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); IHostMemory* mem = builder->buildSerializedNetwork(*network, *config); auto* engine = runtime->deserializeCudaEngine(mem->data(), mem->size()); + delete mem; delete network; #else - builder->setMaxBatchSize(N); + builder->setMaxBatchSize(batch_size); config->setMaxWorkspaceSize(WORKSPACE_SIZE); auto* engine = builder->buildEngineWithConfig(*network, *config); network->destroy(); #endif - std::cout << "build out" << std::endl; + std::cout << "build finished\n"; // Release host memory for (auto& mem : weightMap) { - free((void*)(mem.second.values)); + delete[] static_cast(mem.second.values); } return engine; } -void APIToModel(int32_t N, IRuntime* runtime, IHostMemory** modelStream) { - // Create builder +static void destroyRuntime(IRuntime* runtime) { +#if TRT_VERSION >= 8000 + delete runtime; +#else + runtime->destroy(); +#endif +} + +static void destroyHostMemory(IHostMemory* memory) { +#if TRT_VERSION >= 8000 + delete memory; +#else + memory->destroy(); +#endif +} + +static void APIToModel(int32_t batch_size, IRuntime* runtime, IHostMemory** modelStream) { auto* builder = createInferBuilder(gLogger); auto* config = builder->createBuilderConfig(); - // Create model to populate the network, then set the outputs and create an engine - auto* engine = createEngine(N, runtime, builder, config, DataType::kFLOAT); + auto* engine = createEngine(batch_size, runtime, builder, config, DataType::kFLOAT); assert(engine != nullptr); - // Serialize the engine (*modelStream) = engine->serialize(); #if TRT_VERSION >= 8000 @@ -169,7 +196,8 @@ void APIToModel(int32_t N, IRuntime* runtime, IHostMemory** modelStream) { #endif } -std::vector> doInference(IExecutionContext& context, void* input, int32_t batch_size) { +static auto doInference(IExecutionContext& context, void* input, + int64_t batch_size) -> std::vector> { const auto& engine = context.getEngine(); cudaStream_t stream; CHECK(cudaStreamCreate(&stream)); @@ -192,7 +220,10 @@ std::vector> doInference(IExecutionContext& context, void* in if (i == 0) { CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); } - context.setTensorAddress(tensor_name, buffers[i]); + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } #else const int32_t idx = engine.getBindingIndex(NAMES[i]); auto s = getSize(engine.getBindingDataType(idx)); @@ -206,21 +237,27 @@ std::vector> doInference(IExecutionContext& context, void* in } #if TRT_VERSION >= 8000 - assert(context.enqueueV3(stream)); + if (!context.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); + } #else - assert(context.enqueueV2(buffers.data(), stream, nullptr)); + if (!context.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } #endif std::vector> prob; for (int i = 1; i < nIO; ++i) { - std::vector tmp(batch_size * SIZES[i], std::nan("")); + std::vector tmp(batch_size * SIZES[i], std::nanf("")); std::size_t size = batch_size * SIZES[i] * sizeof(float); CHECK(cudaMemcpyAsync(tmp.data(), buffers[i], size, cudaMemcpyDeviceToHost, stream)); prob.emplace_back(tmp); } CHECK(cudaStreamSynchronize(stream)); - cudaStreamDestroy(stream); + CHECK(cudaStreamDestroy(stream)); for (auto i = 0; i < nIO; ++i) { CHECK(cudaFree(buffers[i])); } @@ -230,9 +267,9 @@ std::vector> doInference(IExecutionContext& context, void* in int main(int argc, char** argv) { checkTrtEnv(); if (argc != 2) { - std::cerr << "arguments not right!" << std::endl; - std::cerr << "./squeezenet -s // serialize model to plan file" << std::endl; - std::cerr << "./squeezenet -d // deserialize plan file and run inference" << std::endl; + std::cerr << "arguments not right!\n"; + std::cerr << "./squeezenet -s // serialize model to plan file\n"; + std::cerr << "./squeezenet -d // deserialize plan file and run inference\n"; return -1; } @@ -240,7 +277,7 @@ int main(int argc, char** argv) { auto* runtime = createInferRuntime(gLogger); assert(runtime != nullptr); char* trtModelStream{nullptr}; - size_t size{0}; + std::streamsize size{0}; if (std::string(argv[1]) == "-s") { IHostMemory* modelStream{nullptr}; @@ -249,15 +286,22 @@ int main(int argc, char** argv) { std::ofstream p(ENGINE_PATH, std::ios::binary | std::ios::trunc); if (!p) { - std::cerr << "could not open plan output file" << std::endl; + std::cerr << "could not open plan output file\n"; + destroyHostMemory(modelStream); + destroyRuntime(runtime); return -1; } - p.write(reinterpret_cast(modelStream->data()), modelStream->size()); -#if TRT_VERSION >= 8000 - delete modelStream; -#else - modelStream->destroy(); -#endif + if (modelStream->size() > static_cast(std::numeric_limits::max())) { + std::cerr << "this model is too large to serialize\n"; + destroyHostMemory(modelStream); + destroyRuntime(runtime); + return -1; + } + const auto* data_ptr = reinterpret_cast(modelStream->data()); + auto data_size = static_cast(modelStream->size()); + p.write(data_ptr, data_size); + destroyHostMemory(modelStream); + destroyRuntime(runtime); return 0; } else if (std::string(argv[1]) == "-d") { std::ifstream file(ENGINE_PATH, std::ios::binary); @@ -269,8 +313,13 @@ int main(int argc, char** argv) { assert(trtModelStream); file.read(trtModelStream, size); file.close(); + } else { + std::cerr << "could not open engine file\n"; + destroyRuntime(runtime); + return -1; } } else { + destroyRuntime(runtime); return -1; } @@ -288,54 +337,55 @@ int main(int argc, char** argv) { cv::Mat img; if constexpr (TRT_PREPROCESS) { // for simplicity, resize image on cpu side - img = cv::imread("../assets/cats.jpg", cv::IMREAD_COLOR); + img = cv::imread("assets/cats.jpg", cv::IMREAD_COLOR); + assert(!img.empty()); cv::resize(img, img, cv::Size(INPUT_W, INPUT_H), 0, 0, cv::INTER_LINEAR); input = static_cast(img.data); } else { - img = cv::imread("../assets/cats.jpg", cv::IMREAD_COLOR); + img = cv::imread("assets/cats.jpg", cv::IMREAD_COLOR); + assert(!img.empty()); flat_img = preprocess_img(img, true, mean, stdv, N, INPUT_H, INPUT_W); input = flat_img.data(); } + assert(input); for (int32_t i = 0; i < 100; ++i) { auto _start = std::chrono::system_clock::now(); auto prob = doInference(*context, input, N); auto _end = std::chrono::system_clock::now(); auto _time = std::chrono::duration_cast(_end - _start).count(); - std::cout << "Execution time: " << _time << "us" << std::endl; + std::cout << "Execution time: " << _time << "us\n"; - for (auto vector : prob) { + for (const auto& vector : prob) { int idx = 0; for (auto v : vector) { std::cout << std::setprecision(4) << v << ", " << std::flush; if (++idx > 20) { - std::cout << "\n====" << std::endl; + std::cout << "\n====\n"; break; } } } if (i == 99) { - std::cout << "prediction result: " << std::endl; + std::cout << "prediction result:\n"; auto labels = loadImagenetLabelMap(LABELS_PATH); int _top = 0; - for (auto& [idx, logits] : topk(prob[0], 3)) { + for (const auto& [idx, logits] : topk(prob[0], 3)) { std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits - << ", label: " << labels[idx] << std::endl; + << ", label: " << labels[idx] << "\n"; } } } delete[] trtModelStream; - // Destroy the engine #if TRT_VERSION >= 8000 delete context; delete engine; - delete runtime; #else context->destroy(); engine->destroy(); - runtime->destroy(); #endif + destroyRuntime(runtime); return 0; } diff --git a/squeezenet/utils.h b/squeezenet/utils.h index 61681b48..d44db18e 100644 --- a/squeezenet/utils.h +++ b/squeezenet/utils.h @@ -1,41 +1,41 @@ #pragma once -#include #include #include +#include #include +#include +#include +#include #include #include #include +#include #include #include -#include #include #include +#include "macros.h" using namespace nvinfer1; -#define WORKSPACE_SIZE (16 << 20) - -#define CHECK(status) \ - do { \ - auto ret = (status); \ - if (ret != cudaSuccess) { \ - std::cerr << "Cuda failure: " << ret << std::endl; \ - abort(); \ - } \ +#define CHECK(status) \ + do { \ + auto ret = (status); \ + if (ret != cudaSuccess) { \ + std::cerr << "Cuda failure: " << ret << "\n"; \ + std::abort(); \ + } \ } while (0) static void checkTrtEnv(int device = 0) { -#if TRT_VERSION < 7220 -#error "TensorRT >= 7.2.2 is required for this demo." -#endif #if TRT_VERSION < 8000 CHECK(cudaGetDevice(&device)); cudaDeviceProp prop{}; CHECK(cudaGetDeviceProperties(&prop, device)); const int sm = prop.major * 10 + prop.minor; if (sm > 86) { - throw std::runtime_error("TensorRT < 8 does not support SM > 86 on this GPU."); + std::cerr << "TensorRT < 8 does not support SM > 86 on this GPU."; + std::abort(); } #endif } @@ -48,7 +48,7 @@ static void checkTrtEnv(int device = 0) { * @return std::map */ static std::map loadWeights(const std::string& file) { - std::cout << "Loading weights: " << file << std::endl; + std::cout << "Loading weights: " << file << "\n"; std::map weightMap; // Open weights file @@ -68,9 +68,10 @@ static std::map loadWeights(const std::string& f input >> name >> std::dec >> wt.count; // Load blob - uint32_t* val = reinterpret_cast(malloc(sizeof(val) * wt.count)); - for (uint32_t x = 0; x < wt.count; ++x) { - input >> std::hex >> val[x]; + auto* val = new uint32_t[static_cast(wt.count)]; + input >> std::hex; + for (auto x = 0ll; x < wt.count; ++x) { + input >> val[x]; } wt.values = val; weightMap[name] = wt; @@ -91,12 +92,13 @@ static std::map loadWeights(const std::string& f * @param w resize width * @return std::vector contiguous flatten image data in float32 type */ -static std::vector preprocess_img(cv::Mat& img, bool bgr2rgb, const float mean[3], const float std[3], int n, - int h, int w) { - const int c = img.channels(); - const std::size_t size = c * h * w; +static std::vector preprocess_img(cv::Mat& img, bool bgr2rgb, const std::array& mean, + const std::array& std, int n, int h, int w) { + const auto c = img.channels(); + const auto size = c * h * w; if (c != 3) { - throw std::runtime_error("this demo only supports 3 channel input image."); + std::cerr << "this demo only supports 3 channel input image.\n"; + std::abort(); } if (bgr2rgb) { cv::cvtColor(img, img, cv::COLOR_BGR2RGB); @@ -104,7 +106,7 @@ static std::vector preprocess_img(cv::Mat& img, bool bgr2rgb, const float cv::resize(img, img, cv::Size(w, h), 0, 0, cv::INTER_LINEAR); img.convertTo(img, CV_32FC3, 1.f / 255); img = (img - cv::Scalar(mean[0], mean[1], mean[2])) / cv::Scalar(std[0], std[1], std[2]); - std::vector chw(n * c * h * w, 0.f); + std::vector chw(static_cast(n) * c * h * w, 0.f); // fill all batch with the same input image for (int i = 0; i < n; ++i) { @@ -112,27 +114,27 @@ static std::vector preprocess_img(cv::Mat& img, bool bgr2rgb, const float for (int x = 0; x < w; ++x) { const cv::Vec3f v = img.at(y, x); chw[i * size + 0 * h * w + y * w + x] = v[0]; - chw[i * size + 1 * h * w + y * h + x] = v[1]; - chw[i * size + 2 * h * w + y * h + x] = v[2]; + chw[i * size + 1 * h * w + y * w + x] = v[1]; + chw[i * size + 2 * h * w + y * w + x] = v[2]; } } } return chw; } -static std::vector> topk(const std::vector& v, int k) { +static auto topk(const std::vector& v, int k) -> std::vector> { if (k <= 0) return {}; - k = std::min(k, v.size()); + auto stride = std::min(k, static_cast(v.size())); std::vector idx(v.size()); std::iota(idx.begin(), idx.end(), 0); - std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) { return v[a] > v[b]; }); + std::partial_sort(idx.begin(), idx.begin() + stride, idx.end(), [&](int a, int b) { return v[a] > v[b]; }); std::vector> out; - out.reserve(k); - for (int i = 0; i < k; ++i) + out.reserve(static_cast(stride)); + for (auto i = 0; i < stride; ++i) out.emplace_back(idx[i], v[idx[i]]); return out; } @@ -163,8 +165,8 @@ static std::map loadImagenetLabelMap(const std::string& path) return labels; } -static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bool bgr2rgb, const float mean[3], - const float std[3]) { +static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bool bgr2rgb, + const std::array& mean, const std::array& std) { struct ScaleParams { std::array shift; std::array scale; @@ -195,7 +197,6 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo in = identity->getOutput(0); #endif } - // Convert from NHWC to NCHW auto* perm = network->addShuffle(*in); assert(perm); @@ -212,11 +213,10 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo auto* _slice = network->addSlice(*perm->getOutput(0), start, size, stride); _slice->setName(name); assert(_slice && _slice->getNbOutputs() == 1); - auto d = _slice->getOutput(0)->getDimensions(); return _slice->getOutput(0); }; - ITensor* channels[] = {add_slice(2, "R"), add_slice(1, "G"), add_slice(0, "B")}; - auto* cat = network->addConcatenation(channels, 3); + std::array channels = {add_slice(2, "R"), add_slice(1, "G"), add_slice(0, "B")}; + auto* cat = network->addConcatenation(channels.data(), 3); assert(cat); cat->setName("RGB"); cat->setAxis(1); @@ -235,7 +235,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo return trans; } -static size_t getSize(DataType dt) { +static std::size_t getSize(DataType dt) { switch (dt) { #if TRT_VERSION >= 8510 case DataType::kUINT8: @@ -248,7 +248,9 @@ static size_t getSize(DataType dt) { return sizeof(int16_t); case DataType::kINT32: return sizeof(int32_t); - default: - throw std::runtime_error("Unsupported data type"); + default: { + std::cerr << "Unsupported data type\n"; + std::abort(); + } } } diff --git a/vgg/CMakeLists.txt b/vgg/CMakeLists.txt index 4c336c72..28ad39af 100644 --- a/vgg/CMakeLists.txt +++ b/vgg/CMakeLists.txt @@ -1,25 +1,37 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 3.14) -project(vgg) +project( + vgg + VERSION 0.1 + LANGUAGES C CXX CUDA) -add_definitions(-std=c++11) +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES 80 86 89 90 100 120) +endif() -option(CUDA_USE_STATIC_CUDA_RUNTIME OFF) -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_BUILD_TYPE Debug) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CUDA_STANDARD 17) +set(CMAKE_CUDA_STANDARD_REQUIRED ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_INCLUDE_CURRENT_DIR TRUE) -include_directories(${PROJECT_SOURCE_DIR}/include) -# include and link dirs of cuda and tensorrt, you need adapt them if yours are different -# cuda -include_directories(/usr/local/cuda/include) -link_directories(/usr/local/cuda/lib64) -# tensorrt -include_directories(/usr/include/x86_64-linux-gnu/) -link_directories(/usr/lib/x86_64-linux-gnu/) +option(CUDA_USE_STATIC_CUDA_RUNTIME "Use static cudaruntime library" OFF) -add_executable(vgg ${PROJECT_SOURCE_DIR}/vgg11.cpp) -target_link_libraries(vgg nvinfer) -target_link_libraries(vgg cudart) +find_package(Threads REQUIRED) +find_package(CUDAToolkit REQUIRED) +find_package(OpenCV REQUIRED) -add_definitions(-O2 -pthread) +if(NOT TARGET TensorRT::TensorRT) + include(FindTensorRT.cmake) +else() + message("TensorRT has been found, skipping for ${PROJECT_NAME}") +endif() +add_executable(${PROJECT_NAME} vgg.cc) + +target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_LIST_DIR} + ${OpenCV_INCLUDE_DIRS}) + +target_link_libraries(${PROJECT_NAME} PUBLIC Threads::Threads CUDA::cudart + TensorRT::TensorRT ${OpenCV_LIBS}) diff --git a/vgg/FindTensorRT.cmake b/vgg/FindTensorRT.cmake new file mode 100644 index 00000000..af75bfe2 --- /dev/null +++ b/vgg/FindTensorRT.cmake @@ -0,0 +1,145 @@ +cmake_minimum_required(VERSION 3.17.0) + +function(_guess_path var_name required_files) + set(_result "") + + foreach(path_entry IN LISTS ARGN) + if(NOT EXISTS "${path_entry}") + message(DEBUG "skip non-existing path '${path_entry}'") + continue() + endif() + + set(_ok TRUE) + foreach(required_file IN LISTS required_files) + if(NOT EXISTS "${path_entry}/${required_file}") + set(_ok FALSE) + message(DEBUG "'${path_entry}' missing '${required_file}'") + break() + endif() + endforeach() + + if(_ok) + list(APPEND _result "${path_entry}") + message(DEBUG "accept '${path_entry}'") + else() + message(DEBUG "reject '${path_entry}'") + endif() + endforeach() + + if(_result STREQUAL "") + message( + FATAL_ERROR + "_guess_path(${var_name}) failed: no valid path found. required_files='${required_files}' candidates='${ARGN}'" + ) + endif() + + set(${var_name} + "${_result}" + PARENT_SCOPE) +endfunction() + +# add library +add_library(TensorRT IMPORTED INTERFACE) +add_library(TensorRT::TensorRT ALIAS TensorRT) + +set(TRT_VERSION + CACHE + STRING + "TensorRT version, e.g. \"8.6.1.6\" or \"8.6.1.6+cuda12.0.1.011\", \"8.6.1.6.Windows10.x86_64.cuda-12.0\" etc" +) + +if(NOT TRT_VERSION STREQUAL "" AND NOT $ENV{TRT_VERSION} STREQUAL "") + message( + WARNING + "TRT_VERSION defined by cmake and environment variable both, using the later one" + ) +endif() + +if(NOT $ENV{TRT_VERSION} STREQUAL "") + set(TRT_VERSION $ENV{TRT_VERSION}) +endif() + +string(REGEX MATCH "([0-9]+)" _match ${TRT_VERSION}) +set(TRT_MAJOR_VERSION "${_match}") +unset(_match) + +if(WIN32) + set(TensorRT_DIR "C:/Program Files/TensorRT-${TRT_VERSION}") + if(NOT EXISTS "${TensorRT_DIR}") + message( + FATAL_ERROR + "TensorRT_DIR=${TensorRT_DIR} does not exist!" + ) + endif() + + if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) + set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 + nvinfer_dispatch_10 nvinfer_lean_10) + message(DEBUG "Using ${_modules}") + else() + set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch + nvinfer_lean) + endif() + + set(TensorRT_LIBRARY_DIR "${TensorRT_DIR}/lib") + set(TensorRT_INCLUDE_DIR "${TensorRT_DIR}/include") +elseif(UNIX) + string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _trt_arch) + set(_trt_include_candidates) + if(_trt_arch MATCHES "^(aarch64|arm64|arch64)$") + set(_trt_include_candidates "/usr/include/aarch64-linux-gnu" "/usr/include" + "/usr/local/cuda/targets/aarch64-linux/include") + set(_trt_library_candidates + "/usr/local/tensorrt/targets/aarch64-linux-gnu/lib" + "/usr/lib/aarch64-linux-gnu" "/usr/lib/aarch64-linux-gnu/tegra" + "/usr/lib") + elseif(_trt_arch MATCHES "^(x86_64|amd64)$") + set(_trt_include_candidates + "/usr/local/tensorrt/targets/x86_64-linux-gnu/include" + "/usr/include/x86_64-linux-gnu" "/usr/include") + set(_trt_library_candidates + "/usr/local/tensorrt/targets/x86_64-linux-gnu/lib" + "/usr/lib/x86_64-linux-gnu" "/usr/lib") + else() + message(FATAL_ERROR "Unknown architecture") + endif() + + set(_modules nvinfer nvinfer_plugin) + if(${TRT_MAJOR_VERSION} GREATER_EQUAL 8) + list(APPEND _modules nvinfer_vc_plugin nvinfer_dispatch nvinfer_lean) + endif() + + _guess_path(TensorRT_LIBRARY_DIR "libnvinfer.so;libnvinfer_plugin.so" + ${_trt_library_candidates}) + message(STATUS "TensorRT libraries: ${TensorRT_LIBRARY_DIR}") + _guess_path(TensorRT_INCLUDE_DIR "NvInfer.h" ${_trt_include_candidates}) + message(STATUS "TensorRT includes: ${TensorRT_INCLUDE_DIR}") +endif() + +foreach(lib IN LISTS _modules) + find_library( + TensorRT_${lib}_LIBRARY + NAMES ${lib} + HINTS ${TensorRT_LIBRARY_DIR}) + list(APPEND TensorRT_LIBRARIES ${TensorRT_${lib}_LIBRARY}) +endforeach() + +target_link_libraries(TensorRT INTERFACE ${TensorRT_LIBRARIES}) + +message(STATUS "Found TensorRT libs: ${TensorRT_LIBRARIES}") + +set_target_properties( + TensorRT + PROPERTIES C_STANDARD 17 + CXX_STANDARD 17 + POSITION_INDEPENDENT_CODE ON + SKIP_BUILD_RPATH TRUE + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "$ORIGIN" + INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") + +unset(TRT_MAJOR_VERSION) +unset(_modules) +unset(_trt_include_candidates) +unset(_trt_library_candidates) +unset(_trt_arch) diff --git a/vgg/README.md b/vgg/README.md index f29afd09..1debe217 100644 --- a/vgg/README.md +++ b/vgg/README.md @@ -1,33 +1,101 @@ -# vgg +# VGG -VGG 11-layer model (configuration "A") from - "Very Deep Convolutional Networks For Large-Scale Image Recognition" +## Introduction -For the Pytorch implementation, you can refer to [pytorchx/vgg](https://github.com/wang-xinyu/pytorchx/tree/master/vgg) +This is a TensorRT implementation of torchvision VGG variants from [Very Deep Convolutional Networks For Large-Scale Image Recognition](https://arxiv.org/pdf/1409.1556.pdf). It supports the common torchvision VGG11/VGG13/VGG16/VGG19 models and their batch-normalized variants. -VGG's architecture is simple, just some conv, relu, maxpool, and fc layers. +VGG's architecture is simple, just some conv, relu, maxpool, and fc layers, e.g., for VGG11: +```bash +VGG( + (features): Sequential( + (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (1): ReLU(inplace=True) + (2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) + (3): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (4): ReLU(inplace=True) + (5): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) + (6): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (7): ReLU(inplace=True) + (8): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (9): ReLU(inplace=True) + (10): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) + (11): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (12): ReLU(inplace=True) + (13): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (14): ReLU(inplace=True) + (15): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) + (16): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (17): ReLU(inplace=True) + (18): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + (19): ReLU(inplace=True) + (20): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) + ) + (avgpool): AdaptiveAvgPool2d(output_size=(7, 7)) + (classifier): Sequential( + (0): Linear(in_features=25088, out_features=4096, bias=True) + (1): ReLU(inplace=True) + (2): Dropout(p=0.5, inplace=False) + (3): Linear(in_features=4096, out_features=4096, bias=True) + (4): ReLU(inplace=True) + (5): Dropout(p=0.5, inplace=False) + (6): Linear(in_features=4096, out_features=1000, bias=True) + ) +) ``` -// 1. generate vgg.wts from [pytorchx/vgg](https://github.com/wang-xinyu/pytorchx/tree/master/vgg) -// 2. put vgg.wts into tensorrtx/vgg +## Usage -// 3. build and run +1. Set the cache locations used by PyTorch/Hugging Face. The export script only reads these paths from environment variables. -cd tensorrtx/vgg +```bash +export TORCH_HOME=/path/to/torch-cache +export HF_HOME=/path/to/hf-cache +``` + +2. Run `gen_wts.py` to generate `.wts` files. No inference is run during export. + +```bash +python gen_wts.py --model vgg11 +``` + +Omit `--model` to export all supported variants, or repeat it for multiple variants. -mkdir build +3. Supported model names: -cd build +| model name | torchvision config | batch norm | +| ---------- | ------------------ | ---------- | +| vgg11 | A | no | +| vgg11_bn | A | yes | +| vgg13 | B | no | +| vgg13_bn | B | yes | +| vgg16 | D | no | +| vgg16_bn | D | yes | +| vgg19 | E | no | +| vgg19_bn | E | yes | -cmake .. +4. build C++ code -make +```bash +pushd tensorrtx/vgg +cmake -S . -B build -G Ninja --fresh +cmake --build build +``` -sudo ./vgg -s // serialize model to plan file i.e. 'vgg.engine' -sudo ./vgg -d // deserialize plan file and run inference +5. serialize wts model to engine file -// 4. see if the output is same as pytorchx/vgg +```bash +./build/vgg -s ``` +Pass a model name to serialize a non-default variant: + +```bash +./build/vgg -s vgg16_bn +``` +6. run inference + +```bash +./build/vgg -d vgg11 +``` diff --git a/vgg/gen_wts.py b/vgg/gen_wts.py new file mode 100644 index 00000000..934fd7ba --- /dev/null +++ b/vgg/gen_wts.py @@ -0,0 +1,80 @@ +import argparse +import os +import struct +from pathlib import Path + +import torch +from torchvision import models + + +MODEL_NAMES = ( + "vgg11", + "vgg11_bn", + "vgg13", + "vgg13_bn", + "vgg16", + "vgg16_bn", + "vgg19", + "vgg19_bn", +) + + +def require_cache_env() -> None: + missing = [name for name in ("TORCH_HOME", "HF_HOME") if not os.environ.get(name)] + if missing: + raise RuntimeError( + f"Please set required cache environment variables: {', '.join(missing)}" + ) + print(f"Using TORCH_HOME={os.environ['TORCH_HOME']}") + print(f"Using HF_HOME={os.environ['HF_HOME']}") + + +def build_model(name: str) -> torch.nn.Module: + weights_cls = getattr(models, f"{name.upper()}_Weights") + model_fn = getattr(models, name) + model = model_fn(weights=weights_cls.DEFAULT) + model.eval() + return model + + +def write_wts(model: torch.nn.Module, output_path: Path) -> None: + state_dict = model.state_dict() + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w") as f: + f.write(f"{len(state_dict.keys())}\n") + for key, value in state_dict.items(): + print(f"key: {key}\tvalue: {value.shape}") + values = value.reshape(-1).cpu().numpy() + f.write(f"{key} {len(values)}") + for item in values: + f.write(" ") + f.write(struct.pack(">f", float(item)).hex()) + f.write("\n") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Export torchvision VGG weights to tensorrtx .wts files." + ) + parser.add_argument( + "--model", + choices=MODEL_NAMES, + action="append", + help="VGG variant to export. Repeat the option for multiple variants. Defaults to all variants.", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("../models"), + help="Directory for generated .wts files.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + require_cache_env() + model_names = args.model or MODEL_NAMES + for model_name in model_names: + print(f"writing {model_name}.wts") + write_wts(build_model(model_name), args.output_dir / f"{model_name}.wts") diff --git a/vgg/logging.h b/vgg/logging.h index 602b69fb..a303c7de 100644 --- a/vgg/logging.h +++ b/vgg/logging.h @@ -17,7 +17,6 @@ #ifndef TENSORRT_LOGGING_H #define TENSORRT_LOGGING_H -#include "NvInferRuntimeCommon.h" #include #include #include @@ -25,32 +24,25 @@ #include #include #include +#include +#include "NvInferRuntime.h" +#include "macros.h" using Severity = nvinfer1::ILogger::Severity; -class LogStreamConsumerBuffer : public std::stringbuf -{ -public: - LogStreamConsumerBuffer(std::ostream& stream, const std::string& prefix, bool shouldLog) - : mOutput(stream) - , mPrefix(prefix) - , mShouldLog(shouldLog) - { - } +class LogStreamConsumerBuffer : public std::stringbuf { + public: + LogStreamConsumerBuffer(std::ostream& stream, std::string prefix, bool shouldLog) + : mOutput(stream), mPrefix(std::move(prefix)), mShouldLog(shouldLog) {} - LogStreamConsumerBuffer(LogStreamConsumerBuffer&& other) - : mOutput(other.mOutput) - { - } + LogStreamConsumerBuffer(LogStreamConsumerBuffer&& other) noexcept : mOutput(other.mOutput) {} - ~LogStreamConsumerBuffer() - { + ~LogStreamConsumerBuffer() override { // std::streambuf::pbase() gives a pointer to the beginning of the buffered part of the output sequence // std::streambuf::pptr() gives a pointer to the current position of the output sequence // if the pointer to the beginning is not equal to the pointer to the current position, // call putOutput() to log the output to the stream - if (pbase() != pptr()) - { + if (pbase() != pptr()) { putOutput(); } } @@ -58,16 +50,13 @@ class LogStreamConsumerBuffer : public std::stringbuf // synchronizes the stream buffer and returns 0 on success // synchronizing the stream buffer consists of inserting the buffer contents into the stream, // resetting the buffer and flushing the stream - virtual int sync() - { + auto sync() -> int override { putOutput(); return 0; } - void putOutput() - { - if (mShouldLog) - { + void putOutput() { + if (mShouldLog) { // prepend timestamp std::time_t timestamp = std::time(nullptr); tm* tm_local = std::localtime(×tamp); @@ -88,12 +77,9 @@ class LogStreamConsumerBuffer : public std::stringbuf } } - void setShouldLog(bool shouldLog) - { - mShouldLog = shouldLog; - } + void setShouldLog(bool shouldLog) { mShouldLog = shouldLog; } -private: + private: std::ostream& mOutput; std::string mPrefix; bool mShouldLog; @@ -103,15 +89,12 @@ class LogStreamConsumerBuffer : public std::stringbuf //! \class LogStreamConsumerBase //! \brief Convenience object used to initialize LogStreamConsumerBuffer before std::ostream in LogStreamConsumer //! -class LogStreamConsumerBase -{ -public: +class LogStreamConsumerBase { + public: LogStreamConsumerBase(std::ostream& stream, const std::string& prefix, bool shouldLog) - : mBuffer(stream, prefix, shouldLog) - { - } + : mBuffer(stream, prefix, shouldLog) {} -protected: + protected: LogStreamConsumerBuffer mBuffer; }; @@ -124,49 +107,49 @@ class LogStreamConsumerBase //! This is necessary to prevent the address of an uninitialized buffer from being passed to std::ostream. //! Please do not change the order of the parent classes. //! -class LogStreamConsumer : protected LogStreamConsumerBase, public std::ostream -{ -public: +class LogStreamConsumer : protected LogStreamConsumerBase, public std::ostream { + public: //! \brief Creates a LogStreamConsumer which logs messages with level severity. //! Reportable severity determines if the messages are severe enough to be logged. LogStreamConsumer(Severity reportableSeverity, Severity severity) - : LogStreamConsumerBase(severityOstream(severity), severityPrefix(severity), severity <= reportableSeverity) - , std::ostream(&mBuffer) // links the stream buffer with the stream - , mShouldLog(severity <= reportableSeverity) - , mSeverity(severity) - { - } - - LogStreamConsumer(LogStreamConsumer&& other) - : LogStreamConsumerBase(severityOstream(other.mSeverity), severityPrefix(other.mSeverity), other.mShouldLog) - , std::ostream(&mBuffer) // links the stream buffer with the stream - , mShouldLog(other.mShouldLog) - , mSeverity(other.mSeverity) - { - } - - void setReportableSeverity(Severity reportableSeverity) - { + : LogStreamConsumerBase(severityOstream(severity), severityPrefix(severity), severity <= reportableSeverity), + std::ostream(&mBuffer) // links the stream buffer with the stream + , + mShouldLog(severity <= reportableSeverity), + mSeverity(severity) {} + + LogStreamConsumer(LogStreamConsumer&& other) noexcept + : LogStreamConsumerBase(severityOstream(other.mSeverity), severityPrefix(other.mSeverity), other.mShouldLog), + std::ostream(&mBuffer) // links the stream buffer with the stream + , + mShouldLog(other.mShouldLog), + mSeverity(other.mSeverity) {} + + void setReportableSeverity(Severity reportableSeverity) { mShouldLog = mSeverity <= reportableSeverity; mBuffer.setShouldLog(mShouldLog); } -private: - static std::ostream& severityOstream(Severity severity) - { + private: + static auto severityOstream(Severity severity) -> std::ostream& { return severity >= Severity::kINFO ? std::cout : std::cerr; } - static std::string severityPrefix(Severity severity) - { - switch (severity) - { - case Severity::kINTERNAL_ERROR: return "[F] "; - case Severity::kERROR: return "[E] "; - case Severity::kWARNING: return "[W] "; - case Severity::kINFO: return "[I] "; - case Severity::kVERBOSE: return "[V] "; - default: assert(0); return ""; + static auto severityPrefix(Severity severity) -> std::string { + switch (severity) { + case Severity::kINTERNAL_ERROR: + return "[F] "; + case Severity::kERROR: + return "[E] "; + case Severity::kWARNING: + return "[W] "; + case Severity::kINFO: + return "[I] "; + case Severity::kVERBOSE: + return "[V] "; + default: + assert(0); + return ""; } } @@ -198,24 +181,19 @@ class LogStreamConsumer : protected LogStreamConsumerBase, public std::ostream //! class to eliminate the inheritance and instead make the nvinfer1::ILogger implementation a member of the Logger //! object. -class Logger : public nvinfer1::ILogger -{ -public: - Logger(Severity severity = Severity::kWARNING) - : mReportableSeverity(severity) - { - } +class Logger : public nvinfer1::ILogger { + public: + Logger(Severity severity = Severity::kWARNING) : mReportableSeverity(severity) {} //! //! \enum TestResult //! \brief Represents the state of a given test //! - enum class TestResult - { - kRUNNING, //!< The test is running - kPASSED, //!< The test passed - kFAILED, //!< The test failed - kWAIVED //!< The test was waived + enum class TestResult { + kRUNNING, //!< The test is running + kPASSED, //!< The test passed + kFAILED, //!< The test failed + kWAIVED //!< The test was waived }; //! @@ -225,10 +203,7 @@ class Logger : public nvinfer1::ILogger //! TODO Once all samples are updated to use this method to register the logger with TensorRT, //! we can eliminate the inheritance of Logger from ILogger //! - nvinfer1::ILogger& getTRTLogger() - { - return *this; - } + auto getTRTLogger() -> nvinfer1::ILogger& { return *this; } //! //! \brief Implementation of the nvinfer1::ILogger::log() virtual method @@ -236,9 +211,8 @@ class Logger : public nvinfer1::ILogger //! Note samples should not be calling this function directly; it will eventually go away once we eliminate the //! inheritance from nvinfer1::ILogger //! - void log(Severity severity, const char* msg) override - { - LogStreamConsumer(mReportableSeverity, severity) << "[TRT] " << std::string(msg) << std::endl; + void log(Severity severity, const char* msg) TRT_NOEXCEPT override { + LogStreamConsumer(mReportableSeverity, severity) << "[TRT] " << std::string(msg) << '\n'; } //! @@ -246,10 +220,7 @@ class Logger : public nvinfer1::ILogger //! //! \param severity The logger will only emit messages that have severity of this level or higher. //! - void setReportableSeverity(Severity severity) - { - mReportableSeverity = severity; - } + void setReportableSeverity(Severity severity) { mReportableSeverity = severity; } //! //! \brief Opaque handle that holds logging information for a particular test @@ -258,20 +229,15 @@ class Logger : public nvinfer1::ILogger //! The sample must call Logger::defineTest() in order to obtain a TestAtom that can be used //! with Logger::reportTest{Start,End}(). //! - class TestAtom - { - public: + class TestAtom { + public: TestAtom(TestAtom&&) = default; - private: + private: friend class Logger; - TestAtom(bool started, const std::string& name, const std::string& cmdline) - : mStarted(started) - , mName(name) - , mCmdline(cmdline) - { - } + TestAtom(bool started, std::string name, std::string cmdline) + : mStarted(started), mName(std::move(name)), mCmdline(std::move(cmdline)) {} bool mStarted; std::string mName; @@ -289,9 +255,8 @@ class Logger : public nvinfer1::ILogger // //! \return a TestAtom that can be used in Logger::reportTest{Start,End}(). //! - static TestAtom defineTest(const std::string& name, const std::string& cmdline) - { - return TestAtom(false, name, cmdline); + static auto defineTest(const std::string& name, const std::string& cmdline) -> TestAtom { + return {false, name, cmdline}; } //! @@ -303,8 +268,7 @@ class Logger : public nvinfer1::ILogger //! \param[in] argv The array of command-line arguments (given as C strings) //! //! \return a TestAtom that can be used in Logger::reportTest{Start,End}(). - static TestAtom defineTest(const std::string& name, int argc, char const* const* argv) - { + static auto defineTest(const std::string& name, int argc, char const* const* argv) -> TestAtom { auto cmdline = genCmdlineString(argc, argv); return defineTest(name, cmdline); } @@ -316,8 +280,7 @@ class Logger : public nvinfer1::ILogger //! //! \param[in] testAtom The handle to the test that has started //! - static void reportTestStart(TestAtom& testAtom) - { + static void reportTestStart(TestAtom& testAtom) { reportTestResult(testAtom, TestResult::kRUNNING); assert(!testAtom.mStarted); testAtom.mStarted = true; @@ -332,98 +295,95 @@ class Logger : public nvinfer1::ILogger //! \param[in] result The result of the test. Should be one of TestResult::kPASSED, //! TestResult::kFAILED, TestResult::kWAIVED //! - static void reportTestEnd(const TestAtom& testAtom, TestResult result) - { + static void reportTestEnd(const TestAtom& testAtom, TestResult result) { assert(result != TestResult::kRUNNING); assert(testAtom.mStarted); reportTestResult(testAtom, result); } - static int reportPass(const TestAtom& testAtom) - { + static auto reportPass(const TestAtom& testAtom) -> int { reportTestEnd(testAtom, TestResult::kPASSED); return EXIT_SUCCESS; } - static int reportFail(const TestAtom& testAtom) - { + static auto reportFail(const TestAtom& testAtom) -> int { reportTestEnd(testAtom, TestResult::kFAILED); return EXIT_FAILURE; } - static int reportWaive(const TestAtom& testAtom) - { + static auto reportWaive(const TestAtom& testAtom) -> int { reportTestEnd(testAtom, TestResult::kWAIVED); return EXIT_SUCCESS; } - static int reportTest(const TestAtom& testAtom, bool pass) - { + static auto reportTest(const TestAtom& testAtom, bool pass) -> int { return pass ? reportPass(testAtom) : reportFail(testAtom); } - Severity getReportableSeverity() const - { - return mReportableSeverity; - } + [[nodiscard]] auto getReportableSeverity() const -> Severity { return mReportableSeverity; } -private: + private: //! //! \brief returns an appropriate string for prefixing a log message with the given severity //! - static const char* severityPrefix(Severity severity) - { - switch (severity) - { - case Severity::kINTERNAL_ERROR: return "[F] "; - case Severity::kERROR: return "[E] "; - case Severity::kWARNING: return "[W] "; - case Severity::kINFO: return "[I] "; - case Severity::kVERBOSE: return "[V] "; - default: assert(0); return ""; + static auto severityPrefix(Severity severity) -> const char* { + switch (severity) { + case Severity::kINTERNAL_ERROR: + return "[F] "; + case Severity::kERROR: + return "[E] "; + case Severity::kWARNING: + return "[W] "; + case Severity::kINFO: + return "[I] "; + case Severity::kVERBOSE: + return "[V] "; + default: + assert(0); + return ""; } } //! //! \brief returns an appropriate string for prefixing a test result message with the given result //! - static const char* testResultString(TestResult result) - { - switch (result) - { - case TestResult::kRUNNING: return "RUNNING"; - case TestResult::kPASSED: return "PASSED"; - case TestResult::kFAILED: return "FAILED"; - case TestResult::kWAIVED: return "WAIVED"; - default: assert(0); return ""; + static auto testResultString(TestResult result) -> const char* { + switch (result) { + case TestResult::kRUNNING: + return "RUNNING"; + case TestResult::kPASSED: + return "PASSED"; + case TestResult::kFAILED: + return "FAILED"; + case TestResult::kWAIVED: + return "WAIVED"; + default: + assert(0); + return ""; } } //! //! \brief returns an appropriate output stream (cout or cerr) to use with the given severity //! - static std::ostream& severityOstream(Severity severity) - { + static auto severityOstream(Severity severity) -> std::ostream& { return severity >= Severity::kINFO ? std::cout : std::cerr; } //! //! \brief method that implements logging test results //! - static void reportTestResult(const TestAtom& testAtom, TestResult result) - { + static void reportTestResult(const TestAtom& testAtom, TestResult result) { severityOstream(Severity::kINFO) << "&&&& " << testResultString(result) << " " << testAtom.mName << " # " - << testAtom.mCmdline << std::endl; + << testAtom.mCmdline << '\n'; } //! //! \brief generate a command line string from the given (argc, argv) values //! - static std::string genCmdlineString(int argc, char const* const* argv) - { + static auto genCmdlineString(int argc, char const* const* argv) -> std::string { std::stringstream ss; - for (int i = 0; i < argc; i++) - { + for (int i = 0; i < argc; i++) { if (i > 0) ss << " "; ss << argv[i]; @@ -434,8 +394,7 @@ class Logger : public nvinfer1::ILogger Severity mReportableSeverity; }; -namespace -{ +namespace { //! //! \brief produces a LogStreamConsumer object that can be used to log messages of severity kVERBOSE @@ -444,9 +403,8 @@ namespace //! //! LOG_VERBOSE(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_VERBOSE(const Logger& logger) -{ - return LogStreamConsumer(logger.getReportableSeverity(), Severity::kVERBOSE); +inline auto LOG_VERBOSE(const Logger& logger) -> LogStreamConsumer { + return {logger.getReportableSeverity(), Severity::kVERBOSE}; } //! @@ -456,9 +414,8 @@ inline LogStreamConsumer LOG_VERBOSE(const Logger& logger) //! //! LOG_INFO(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_INFO(const Logger& logger) -{ - return LogStreamConsumer(logger.getReportableSeverity(), Severity::kINFO); +inline auto LOG_INFO(const Logger& logger) -> LogStreamConsumer { + return {logger.getReportableSeverity(), Severity::kINFO}; } //! @@ -468,9 +425,8 @@ inline LogStreamConsumer LOG_INFO(const Logger& logger) //! //! LOG_WARN(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_WARN(const Logger& logger) -{ - return LogStreamConsumer(logger.getReportableSeverity(), Severity::kWARNING); +inline auto LOG_WARN(const Logger& logger) -> LogStreamConsumer { + return {logger.getReportableSeverity(), Severity::kWARNING}; } //! @@ -480,9 +436,8 @@ inline LogStreamConsumer LOG_WARN(const Logger& logger) //! //! LOG_ERROR(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_ERROR(const Logger& logger) -{ - return LogStreamConsumer(logger.getReportableSeverity(), Severity::kERROR); +inline auto LOG_ERROR(const Logger& logger) -> LogStreamConsumer { + return {logger.getReportableSeverity(), Severity::kERROR}; } //! @@ -493,11 +448,10 @@ inline LogStreamConsumer LOG_ERROR(const Logger& logger) //! //! LOG_FATAL(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_FATAL(const Logger& logger) -{ - return LogStreamConsumer(logger.getReportableSeverity(), Severity::kINTERNAL_ERROR); +inline auto LOG_FATAL(const Logger& logger) -> LogStreamConsumer { + return {logger.getReportableSeverity(), Severity::kINTERNAL_ERROR}; } -} // anonymous namespace +} // anonymous namespace -#endif // TENSORRT_LOGGING_H +#endif // TENSORRT_LOGGING_H diff --git a/vgg/macros.h b/vgg/macros.h new file mode 100644 index 00000000..daae7bfb --- /dev/null +++ b/vgg/macros.h @@ -0,0 +1,32 @@ +#pragma once +#include + +#ifdef API_EXPORTS +#if defined(_MSC_VER) +#define API __declspec(dllexport) +#else +#define API __attribute__((visibility("default"))) +#endif +#else + +#if defined(_MSC_VER) +#define API __declspec(dllimport) +#else +#define API +#endif +#endif // API_EXPORTS + +#define TRT_VERSION \ + ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) + +#if TRT_VERSION < 7220 +#error "TensorRT >= 7.2.2 is required for this demo." +#endif + +#if TRT_VERSION >= 8000 +#define TRT_NOEXCEPT noexcept +#define TRT_CONST_ENQUEUE const +#else +#define TRT_NOEXCEPT +#define TRT_CONST_ENQUEUE +#endif diff --git a/vgg/utils.h b/vgg/utils.h new file mode 100644 index 00000000..b6bff804 --- /dev/null +++ b/vgg/utils.h @@ -0,0 +1,268 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "macros.h" + +using namespace nvinfer1; + +constexpr const std::size_t WORKSPACE_SIZE = 16 << 20; + +#define CHECK(status) \ + do { \ + auto ret = (status); \ + if (ret != cudaSuccess) { \ + std::cerr << "Cuda failure: " << ret << "\n"; \ + std::abort(); \ + } \ + } while (0) + +static void checkTrtEnv(int device = 0) { +#if TRT_VERSION < 8000 + CHECK(cudaGetDevice(&device)); + cudaDeviceProp prop{}; + CHECK(cudaGetDeviceProperties(&prop, device)); + const int sm = prop.major * 10 + prop.minor; + if (sm > 86) { + std::cerr << "TensorRT < 8 does not support SM > 86 on this GPU."; + std::abort(); + } +#endif +} + +/** + * @brief TensorRT weight files have a simple space delimited format: + * [type] [size] + * + * @param file input weight file path + * @return std::map + */ +static std::map loadWeights(const std::string& file) { + std::cout << "Loading weights: " << file << "\n"; + std::map weightMap; + + // Open weights file + std::ifstream input(file); + assert(input.is_open() && "Unable to load weight file."); + + // Read number of weight blobs + int32_t count; + input >> count; + assert(count > 0 && "Invalid weight map file."); + + while (count--) { + nvinfer1::Weights wt{nvinfer1::DataType::kFLOAT, nullptr, 0}; + + // Read name and type of blob + std::string name; + input >> name >> std::dec >> wt.count; + + // Load blob + auto* val = new float[wt.count]; + input >> std::hex; + for (auto x = 0ll; x < wt.count; ++x) { + uint32_t bits; + input >> bits; + std::memcpy(&val[x], &bits, sizeof(bits)); + } + wt.values = val; + weightMap[name] = wt; + } + + return weightMap; +} + +static void releaseWeights(std::map& weightMap) { + for (auto& weight : weightMap) { + delete[] static_cast(weight.second.values); + weight.second.values = nullptr; + } +} + +/** + * @brief a preprocess function aligning with ImageNet preprocess in torchvision, only support 3-channel image + * + * @param img opencv image with BGR layout + * @param bgr2rgb whether to convert BGR to RGB + * @param mean subtract mean + * @param std divide std + * @param n batch size + * @param h resize height + * @param w resize width + * @return std::vector contiguous flatten image data in float32 type + */ +static std::vector preprocess_img(cv::Mat& img, bool bgr2rgb, const std::array& mean, + const std::array& std, int n, int h, int w) { + const auto c = img.channels(); + const auto size = c * h * w; + if (c != 3) { + std::cerr << "this demo only supports 3 channel input image.\n"; + std::abort(); + } + if (bgr2rgb) { + cv::cvtColor(img, img, cv::COLOR_BGR2RGB); + } + cv::resize(img, img, cv::Size(w, h), 0, 0, cv::INTER_LINEAR); + img.convertTo(img, CV_32FC3, 1.f / 255); + img = (img - cv::Scalar(mean[0], mean[1], mean[2])) / cv::Scalar(std[0], std[1], std[2]); + std::vector chw(static_cast(n) * c * h * w, 0.f); + + // fill all batch with the same input image + for (int i = 0; i < n; ++i) { + for (int y = 0; y < h; ++y) { + for (int x = 0; x < w; ++x) { + const cv::Vec3f v = img.at(y, x); + chw[i * size + 0 * h * w + y * w + x] = v[0]; + chw[i * size + 1 * h * w + y * w + x] = v[1]; + chw[i * size + 2 * h * w + y * w + x] = v[2]; + } + } + } + return chw; +} + +static std::vector> topk(const std::vector& v, int k) { + if (k <= 0) + return {}; + auto stride = std::min(k, static_cast(v.size())); + + std::vector idx(v.size()); + std::iota(idx.begin(), idx.end(), 0); + + std::partial_sort(idx.begin(), idx.begin() + stride, idx.end(), [&](int a, int b) { return v[a] > v[b]; }); + + std::vector> out; + out.reserve(stride); + for (auto i = 0; i < stride; ++i) + out.emplace_back(idx[i], v[idx[i]]); + return out; +} + +static std::map loadImagenetLabelMap(const std::string& path) { + std::map labels; + std::ifstream in(path); + if (!in.is_open()) { + return labels; + } + std::string line; + while (std::getline(in, line)) { + auto colon = line.find(':'); + if (colon == std::string::npos) { + continue; + } + auto first_quote = line.find('\'', colon); + if (first_quote == std::string::npos) { + continue; + } + auto second_quote = line.find('\'', first_quote + 1); + if (second_quote == std::string::npos) { + continue; + } + int idx = std::stoi(line.substr(0, colon)); + labels[idx] = line.substr(first_quote + 1, second_quote - first_quote - 1); + } + return labels; +} + +static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bool bgr2rgb, + const std::array& mean, const std::array& std) { + struct ScaleParams { + std::array shift; + std::array scale; + }; + static std::vector> gScaleParams; + auto params = std::make_unique(); + params->shift = {-mean[0] / std[0], -mean[1] / std[1], -mean[2] / std[2]}; + params->scale = {1.f / (std[0] * 255.f), 1.f / (std[1] * 255.f), 1.f / (std[2] * 255.f)}; + + static const Weights empty{DataType::kFLOAT, nullptr, 0ll}; + const Weights shift{DataType::kFLOAT, params->shift.data(), 3ll}; + const Weights scale{DataType::kFLOAT, params->scale.data(), 3ll}; + + gScaleParams.emplace_back(std::move(params)); + + ITensor* in = &input; + if (input.getType() != DataType::kFLOAT) { +#if TRT_VERSION >= 8000 + auto* cast = network->addCast(input, DataType::kFLOAT); + assert(cast); + cast->setName("Cast to FP32"); + in = cast->getOutput(0); +#else + auto* identity = network->addIdentity(input); + assert(identity); + identity->setName("Convert to FP32"); + identity->setOutputType(0, DataType::kFLOAT); + in = identity->getOutput(0); +#endif + } + // Convert from NHWC to NCHW + auto* perm = network->addShuffle(*in); + assert(perm); + perm->setName("NHWC -> NCHW"); + perm->setFirstTranspose(Permutation{0, 3, 1, 2}); + + // Convert from BGR to RGB (optional) + ITensor* data{nullptr}; + if (bgr2rgb) { + auto add_slice = [&](int c, const char* name) -> ITensor* { + auto dims = perm->getOutput(0)->getDimensions(); + Dims4 start = {0, c, 0, 0}, stride = {1, 1, 1, 1}; + Dims4 size = {dims.d[0], 1, dims.d[2], dims.d[3]}; + auto* _slice = network->addSlice(*perm->getOutput(0), start, size, stride); + _slice->setName(name); + assert(_slice && _slice->getNbOutputs() == 1); + return _slice->getOutput(0); + }; + std::array channels = {add_slice(2, "R"), add_slice(1, "G"), add_slice(0, "B")}; + auto* cat = network->addConcatenation(channels.data(), 3); + assert(cat); + cat->setName("RGB"); + cat->setAxis(1); + data = cat->getOutput(0); + } else { + data = perm->getOutput(0); + } + + // Normalize + auto* trans = network->addScale(*data, ScaleMode::kCHANNEL, shift, scale, empty); + assert(trans); + trans->setName("mean & std"); +#if TRT_VERSION >= 8000 + trans->setChannelAxis(1); +#endif + return trans; +} + +static size_t getSize(DataType dt) { + switch (dt) { +#if TRT_VERSION >= 8510 + case DataType::kUINT8: +#endif + case DataType::kINT8: + return sizeof(int8_t); + case DataType::kFLOAT: + return sizeof(float); + case DataType::kHALF: + return sizeof(int16_t); + case DataType::kINT32: + return sizeof(int32_t); + default: { + std::cerr << "Unsupported data type\n"; + std::abort(); + } + } +} diff --git a/vgg/vgg.cc b/vgg/vgg.cc new file mode 100644 index 00000000..fa3b9386 --- /dev/null +++ b/vgg/vgg.cc @@ -0,0 +1,450 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "logging.h" +#include "macros.h" +#include "utils.h" + +using WeightMap = std::map; +using M = nvinfer1::MatrixOperation; +using E = nvinfer1::ElementWiseOperation; +using LayerConfig = std::vector; +using LayerConfigMap = std::map; + +static Logger g_logger; + +struct VggVariant { + std::string name; + std::string cfg_name; + bool batch_norm; + std::string wts_path; + std::string engine_path; +}; + +static constexpr int N = 1; +static constexpr const int32_t INPUT_H = 224; +static constexpr const int32_t INPUT_W = 224; +static constexpr const std::array SIZES = {3 * INPUT_H * INPUT_W, 1000}; +static constexpr const std::array NAMES = {"data", "prob"}; +static constexpr const char* LABELS_PATH = "assets/imagenet1000_clsidx_to_labels.txt"; +static constexpr const bool TRT_PREPROCESS = TRT_VERSION >= 8510; +static constexpr const std::array mean = {0.485f, 0.456f, 0.406f}; +static constexpr const std::array stdv = {0.229f, 0.224f, 0.225f}; + +const LayerConfigMap CFGS = { + {"A", {"64", "M", "128", "M", "256", "256", "M", "512", "512", "M", "512", "512", "M"}}, + {"B", {"64", "64", "M", "128", "128", "M", "256", "256", "M", "512", "512", "M", "512", "512", "M"}}, + {"D", + {"64", "64", "M", "128", "128", "M", "256", "256", "256", "M", "512", "512", "512", "M", "512", "512", "512", + "M"}}, + {"E", {"64", "64", "M", "128", "128", "M", "256", "256", "256", "256", "M", + "512", "512", "512", "512", "M", "512", "512", "512", "512", "M"}}}; + +static auto normalizeModelName(const std::string& raw_name) -> std::string { + std::string name; + name.reserve(raw_name.size()); + for (char ch : raw_name) { + const auto c = static_cast(ch); + name.push_back(ch == '-' ? '_' : static_cast(std::tolower(c))); + } + return name; +} + +static auto getVariantConfig(const std::string& raw_name) -> VggVariant { + const auto name = normalizeModelName(raw_name); + static const std::map> variants = { + {"vgg11", {"A", false}}, {"vgg11_bn", {"A", true}}, {"vgg13", {"B", false}}, {"vgg13_bn", {"B", true}}, + {"vgg16", {"D", false}}, {"vgg16_bn", {"D", true}}, {"vgg19", {"E", false}}, {"vgg19_bn", {"E", true}}, + }; + const auto iter = variants.find(name); + if (iter == variants.end()) { + std::cerr << "Unknown VGG variant: " << raw_name + << " (expected vgg11 | vgg11_bn | vgg13 | vgg13_bn | vgg16 | vgg16_bn | vgg19 | vgg19_bn)\n"; + std::abort(); + } + + return VggVariant{name, iter->second.first, iter->second.second, "models/" + name + ".wts", + "models/" + name + ".engine"}; +} + +static auto addBatchNorm2d(INetworkDefinition* network, WeightMap& w, ITensor& input, const std::string& lname, + float eps = 1e-5) -> IScaleLayer* { + const float* gamma = static_cast(w[lname + ".weight"].values); + const float* beta = static_cast(w[lname + ".bias"].values); + const float* mean = static_cast(w[lname + ".running_mean"].values); + const float* var = static_cast(w[lname + ".running_var"].values); + int64_t len = w[lname + ".running_var"].count; + + auto* scval = new float[len]; + for (int i = 0; i < len; i++) { + scval[i] = gamma[i] / sqrt(var[i] + eps); + } + Weights scale{DataType::kFLOAT, scval, len}; + + auto* shval = new float[len]; + for (int i = 0; i < len; i++) { + shval[i] = beta[i] - mean[i] * gamma[i] / sqrt(var[i] + eps); + } + Weights shift{DataType::kFLOAT, shval, len}; + + auto* pval = new float[len]; + for (int i = 0; i < len; i++) { + pval[i] = 1.0f; + } + Weights power{DataType::kFLOAT, pval, len}; + + w[lname + ".scale"] = scale; + w[lname + ".shift"] = shift; + w[lname + ".power"] = power; + IScaleLayer* scale_1 = network->addScale(input, ScaleMode::kCHANNEL, shift, scale, power); + assert(scale_1); + scale_1->setName(lname.c_str()); + return scale_1; +} + +static auto make_layers(INetworkDefinition* net, WeightMap& w, ITensor& input, const LayerConfig& cfg, + bool use_bn) -> ITensor* { + auto* tensor = &input; + int32_t idx = 0; + std::string name = "features."; + + for (const auto& _v : cfg) { + if (_v == "M") { + auto* pool = net->addPoolingNd(*tensor, PoolingType::kMAX, DimsHW{2, 2}); + pool->setStrideNd(DimsHW{2, 2}); + pool->setName((name + std::to_string(idx)).c_str()); + tensor = pool->getOutput(0); + ++idx; + continue; + } + const int32_t v = std::stoi(_v); + auto _name = "features." + std::to_string(idx); + auto* _conv = net->addConvolutionNd(*tensor, v, DimsHW{3, 3}, w.at(_name + ".weight"), w.at(_name + ".bias")); + _conv->setPaddingNd(DimsHW{1, 1}); + _conv->setName(_name.c_str()); + tensor = _conv->getOutput(0); + ++idx; + if (use_bn) { + auto _bn_name = "features." + std::to_string(idx); + auto* bn = addBatchNorm2d(net, w, *tensor, _bn_name); + tensor = bn->getOutput(0); + ++idx; + } + auto* relu = net->addActivation(*tensor, ActivationType::kRELU); + relu->setName(("features." + std::to_string(idx)).c_str()); + tensor = relu->getOutput(0); + ++idx; + } + + return tensor; +} + +auto create_engine(const VggVariant& variant, int32_t batch_size, IRuntime* runtime, IBuilder* builder, + IBuilderConfig* config, DataType dt) -> ICudaEngine* { + auto w = loadWeights(variant.wts_path); +#if TRT_VERSION >= 10000 + auto* net = builder->createNetworkV2(1U << static_cast(NetworkDefinitionCreationFlag::kSTRONGLY_TYPED)); +#else + auto* net = builder->createNetworkV2(1u << static_cast(NetworkDefinitionCreationFlag::kEXPLICIT_BATCH)); +#endif + + ITensor* input{nullptr}; + if constexpr (TRT_PREPROCESS) { + // for simplicity, resize image on cpu side + dt = DataType::kUINT8; + input = net->addInput(NAMES[0], dt, Dims4{batch_size, INPUT_H, INPUT_W, 3}); + auto* trans = addTransformLayer(net, *input, true, mean, stdv); + input = trans->getOutput(0); + } else { + input = net->addInput(NAMES[0], dt, Dims4{batch_size, 3, INPUT_H, INPUT_W}); + } + assert(input); + + auto* features = make_layers(net, w, *input, CFGS.at(variant.cfg_name), variant.batch_norm); + + auto* _avg_pool = net->addPoolingNd(*features, PoolingType::kAVERAGE, Dims2{1, 1}); + auto* _flatten = net->addShuffle(*_avg_pool->getOutput(0)); + assert(_avg_pool && _flatten); + _flatten->setReshapeDimensions(Dims2{batch_size, -1}); + + auto* _fc1w = + net->addConstant(Dims2{4096, static_cast(512 * 7 * 7)}, w["classifier.0.weight"])->getOutput(0); + auto* _fc1b = net->addConstant(Dims2{1, 4096}, w["classifier.0.bias"])->getOutput(0); + auto* _fc2w = net->addConstant(Dims2{4096, 4096}, w["classifier.3.weight"])->getOutput(0); + auto* _fc2b = net->addConstant(Dims2{1, 4096}, w["classifier.3.bias"])->getOutput(0); + auto* _fc3w = net->addConstant(Dims2{1000, 4096}, w["classifier.6.weight"])->getOutput(0); + auto* _fc3b = net->addConstant(Dims2{1, 1000}, w["classifier.6.bias"])->getOutput(0); + assert(_fc1w && _fc1b && _fc2w && _fc2b && _fc3w && _fc3b); + + auto* _fc1_0 = net->addMatrixMultiply(*_flatten->getOutput(0), M::kNONE, *_fc1w, M::kTRANSPOSE); + auto* _fc1_1 = net->addElementWise(*_fc1_0->getOutput(0), *_fc1b, E::kSUM); + auto* _relu1 = net->addActivation(*_fc1_1->getOutput(0), ActivationType::kRELU); + + auto* _fc2_0 = net->addMatrixMultiply(*_relu1->getOutput(0), M::kNONE, *_fc2w, M::kTRANSPOSE); + auto* _fc2_1 = net->addElementWise(*_fc2_0->getOutput(0), *_fc2b, E::kSUM); + auto* _relu2 = net->addActivation(*_fc2_1->getOutput(0), ActivationType::kRELU); + + auto* _fc3_0 = net->addMatrixMultiply(*_relu2->getOutput(0), M::kNONE, *_fc3w, M::kTRANSPOSE); + auto* _fc3_1 = net->addElementWise(*_fc3_0->getOutput(0), *_fc3b, E::kSUM); + + _fc3_1->getOutput(0)->setName(NAMES[1]); + net->markOutput(*_fc3_1->getOutput(0)); + +#if TRT_VERSION >= 8000 + config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); + auto* _serialized = builder->buildSerializedNetwork(*net, *config); + auto* _engine = runtime->deserializeCudaEngine(_serialized->data(), _serialized->size()); + delete _serialized; + delete net; +#else + builder->setMaxBatchSize(N); + config->setMaxWorkspaceSize(WORKSPACE_SIZE); + auto* _engine = builder->buildEngineWithConfig(*net, *config); + net->destroy(); +#endif + std::cout << "build out" << '\n'; + + releaseWeights(w); + + return _engine; +} + +void APIToModel(const VggVariant& variant, int32_t batch_size, IRuntime* runtime, IHostMemory** model_stream) { + auto* builder = createInferBuilder(g_logger); + auto* config = builder->createBuilderConfig(); + + auto* engine = create_engine(variant, batch_size, runtime, builder, config, DataType::kFLOAT); + assert(engine != nullptr); + + (*model_stream) = engine->serialize(); + +#if TRT_VERSION >= 8000 + delete engine; + delete config; + delete builder; +#else + engine->destroy(); + config->destroy(); + builder->destroy(); +#endif +} + +auto doInference(IExecutionContext& context, void* input, std::size_t batch_size) -> std::vector> { + const ICudaEngine& engine = context.getEngine(); + cudaStream_t stream; + CHECK(cudaStreamCreate(&stream)); + std::vector buffers; + +#if TRT_VERSION >= 8000 + const int32_t nIO = engine.getNbIOTensors(); +#else + const int32_t nIO = engine.getNbBindings(); +#endif + + buffers.resize(nIO); + for (auto i = 0; i < nIO; ++i) { + std::size_t size = 0; +#if TRT_VERSION >= 8000 + auto* tensor_name = engine.getIOTensorName(i); + const std::string name = tensor_name; + auto s = getSize(engine.getTensorDataType(tensor_name)); + size = s * batch_size * (name == NAMES[0] ? SIZES[0] : SIZES[1]); + CHECK(cudaMalloc(&buffers[i], size)); + if (name == NAMES[0]) { + CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); + } + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } +#else + const int32_t idx = engine.getBindingIndex(NAMES[i]); + auto s = getSize(engine.getBindingDataType(idx)); + assert(idx == i); + size = s * batch_size * SIZES[i]; + CHECK(cudaMalloc(&buffers[i], size)); + if (i == 0) { + CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); + } +#endif + } + +#if TRT_VERSION >= 8000 + if (!context.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); + } +#else + if (!context.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } +#endif + + std::vector> prob; + for (int i = 0; i < nIO; ++i) { +#if TRT_VERSION >= 8000 + const std::string name = engine.getIOTensorName(i); + if (name == NAMES[0]) { + continue; + } + constexpr auto output_size = SIZES[1]; +#else + if (i == 0) { + continue; + } + const auto output_size = SIZES[i]; +#endif + std::vector tmp(batch_size * output_size, std::nanf("")); + std::size_t size = batch_size * output_size * sizeof(float); + CHECK(cudaMemcpyAsync(tmp.data(), buffers[i], size, cudaMemcpyDeviceToHost, stream)); + prob.emplace_back(tmp); + } + CHECK(cudaStreamSynchronize(stream)); + + for (auto i = 0; i < nIO; ++i) { + CHECK(cudaFree(buffers[i])); + } + CHECK(cudaStreamDestroy(stream)); + return prob; +} + +auto main(int argc, char** argv) -> int { + checkTrtEnv(); + if (argc < 2 || argc > 3) { + std::cerr << "arguments not right!" << '\n'; + std::cerr << "./vgg -s [model] // serialize model to plan file" << '\n'; + std::cerr << "./vgg -d [model] // deserialize plan file and run inference" << '\n'; + std::cerr << "model choices: vgg11, vgg11_bn, vgg13, vgg13_bn, vgg16, vgg16_bn, vgg19, vgg19_bn" << '\n'; + return -1; + } + const auto variant = getVariantConfig(argc == 3 ? argv[2] : "vgg11"); + std::cout << "Using VGG variant: " << variant.name << '\n'; + + auto* runtime = createInferRuntime(g_logger); + assert(runtime != nullptr); + char* trt_model_stream{nullptr}; + std::streamsize size{0}; + + if (std::string(argv[1]) == "-s") { + IHostMemory* mem{nullptr}; + APIToModel(variant, 1, runtime, &mem); + assert(mem != nullptr); + + std::ofstream _plan(variant.engine_path, std::ios::binary | std::ios::trunc); + if (!_plan) { + std::cerr << "could not open plan output file" << '\n'; + return -1; + } + if (mem->size() > static_cast(std::numeric_limits::max())) { + std::cerr << "this model is too large to serialize\n"; + return -1; + } + const auto* data_ptr = reinterpret_cast(mem->data()); + auto data_size = static_cast(mem->size()); + _plan.write(data_ptr, data_size); +#if TRT_VERSION >= 8000 + delete mem; + delete runtime; +#else + mem->destroy(); + runtime->destroy(); +#endif + return 0; + } else if (std::string(argv[1]) == "-d") { + std::ifstream _file(variant.engine_path, std::ios::binary); + if (_file.good()) { + _file.seekg(0, _file.end); + size = _file.tellg(); + _file.seekg(0, _file.beg); + trt_model_stream = new char[size]; + assert(trt_model_stream); + _file.read(trt_model_stream, size); + _file.close(); + } else { + std::cerr << "could not open engine file" << '\n'; + return -1; + } + } else { + return 1; + } + +#if TRT_VERSION >= 8000 + auto* engine = runtime->deserializeCudaEngine(trt_model_stream, size); +#else + auto* engine = runtime->deserializeCudaEngine(trt_model_stream, size, nullptr); +#endif + assert(engine != nullptr); + auto* context = engine->createExecutionContext(); + assert(context != nullptr); + + const std::string img_path = "assets/cats.jpg"; + void* input = nullptr; + std::vector flat_img; + cv::Mat img; + if constexpr (TRT_PREPROCESS) { + // for simplicity, resize image on cpu side + img = cv::imread(img_path, cv::IMREAD_COLOR); + cv::resize(img, img, cv::Size(INPUT_W, INPUT_H), 0, 0, cv::INTER_LINEAR); + input = static_cast(img.data); + } else { + img = cv::imread(img_path, cv::IMREAD_COLOR); + flat_img = preprocess_img(img, true, mean, stdv, N, INPUT_H, INPUT_W); + input = flat_img.data(); + } + assert(input); + + for (int32_t i = 0; i < 100; ++i) { + auto _start = std::chrono::system_clock::now(); + auto prob = doInference(*context, input, 1); + auto _end = std::chrono::system_clock::now(); + auto _time = std::chrono::duration_cast(_end - _start).count(); + std::cout << "Execution time: " << _time << "ms" << '\n'; + + for (const auto& vector : prob) { + int idx = 0; + for (auto v : vector) { + std::cout << std::setprecision(4) << v << ", " << std::flush; + if (++idx > 20) { + std::cout << "\n====" << '\n'; + break; + } + } + } + + if (i == 99) { + std::cout << "prediction result:\n"; + auto labels = loadImagenetLabelMap(LABELS_PATH); + int _top = 0; + for (auto& [idx, logits] : topk(prob[0], 3)) { + std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits + << ", label: " << labels[idx] << '\n'; + } + } + } + + delete[] trt_model_stream; +#if TRT_VERSION >= 8000 + delete context; + delete engine; + delete runtime; +#else + context->destroy(); + engine->destroy(); + runtime->destroy(); +#endif + return 0; +} diff --git a/vgg/vgg11.cpp b/vgg/vgg11.cpp deleted file mode 100644 index 45ac7514..00000000 --- a/vgg/vgg11.cpp +++ /dev/null @@ -1,289 +0,0 @@ -#include "NvInfer.h" -#include "cuda_runtime_api.h" -#include -#include -#include -#include -#include -#include -#include "logging.h" - -#define CHECK(status) \ - do\ - {\ - auto ret = (status);\ - if (ret != 0)\ - {\ - std::cerr << "Cuda failure: " << ret << std::endl;\ - abort();\ - }\ - } while (0) - -// stuff we know about the network and the input/output blobs -static const int INPUT_H = 224; -static const int INPUT_W = 224; -static const int OUTPUT_SIZE = 1000; - -const char* INPUT_BLOB_NAME = "data"; -const char* OUTPUT_BLOB_NAME = "prob"; - -using namespace nvinfer1; - -static Logger gLogger; - -// Load weights from files shared with TensorRT samples. -// TensorRT weight files have a simple space delimited format: -// [type] [size] -std::map loadWeights(const std::string file) -{ - std::cout << "Loading weights: " << file << std::endl; - std::map weightMap; - - // Open weights file - std::ifstream input(file); - assert(input.is_open() && "Unable to load weight file."); - - // Read number of weight blobs - int32_t count; - input >> count; - assert(count > 0 && "Invalid weight map file."); - - while (count--) - { - Weights wt{DataType::kFLOAT, nullptr, 0}; - uint32_t size; - - // Read name and type of blob - std::string name; - input >> name >> std::dec >> size; - wt.type = DataType::kFLOAT; - - // Load blob - uint32_t* val = reinterpret_cast(malloc(sizeof(val) * size)); - for (uint32_t x = 0, y = size; x < y; ++x) - { - input >> std::hex >> val[x]; - } - wt.values = val; - - wt.count = size; - weightMap[name] = wt; - } - - return weightMap; -} - -// Creat the engine using only the API and not any parser. -ICudaEngine* createEngine(unsigned int maxBatchSize, IBuilder* builder, IBuilderConfig* config, DataType dt) -{ - INetworkDefinition* network = builder->createNetworkV2(0U); - - // Create input tensor of shape { 3, INPUT_H, INPUT_W } with name INPUT_BLOB_NAME - ITensor* data = network->addInput(INPUT_BLOB_NAME, dt, Dims3{3, INPUT_H, INPUT_W}); - assert(data); - - std::map weightMap = loadWeights("../vgg.wts"); - Weights emptywts{DataType::kFLOAT, nullptr, 0}; - - IConvolutionLayer* conv1 = network->addConvolutionNd(*data, 64, DimsHW{3, 3}, weightMap["features.0.weight"], weightMap["features.0.bias"]); - assert(conv1); - conv1->setPaddingNd(DimsHW{1, 1}); - IActivationLayer* relu1 = network->addActivation(*conv1->getOutput(0), ActivationType::kRELU); - assert(relu1); - IPoolingLayer* pool1 = network->addPoolingNd(*relu1->getOutput(0), PoolingType::kMAX, DimsHW{2, 2}); - assert(pool1); - pool1->setStrideNd(DimsHW{2, 2}); - - conv1 = network->addConvolutionNd(*pool1->getOutput(0), 128, DimsHW{3, 3}, weightMap["features.3.weight"], weightMap["features.3.bias"]); - conv1->setPaddingNd(DimsHW{1, 1}); - relu1 = network->addActivation(*conv1->getOutput(0), ActivationType::kRELU); - pool1 = network->addPoolingNd(*relu1->getOutput(0), PoolingType::kMAX, DimsHW{2, 2}); - pool1->setStrideNd(DimsHW{2, 2}); - - conv1 = network->addConvolutionNd(*pool1->getOutput(0), 256, DimsHW{3, 3}, weightMap["features.6.weight"], weightMap["features.6.bias"]); - conv1->setPaddingNd(DimsHW{1, 1}); - relu1 = network->addActivation(*conv1->getOutput(0), ActivationType::kRELU); - conv1 = network->addConvolutionNd(*relu1->getOutput(0), 256, DimsHW{3, 3}, weightMap["features.8.weight"], weightMap["features.8.bias"]); - conv1->setPaddingNd(DimsHW{1, 1}); - relu1 = network->addActivation(*conv1->getOutput(0), ActivationType::kRELU); - pool1 = network->addPoolingNd(*relu1->getOutput(0), PoolingType::kMAX, DimsHW{2, 2}); - pool1->setStrideNd(DimsHW{2, 2}); - - conv1 = network->addConvolutionNd(*pool1->getOutput(0), 512, DimsHW{3, 3}, weightMap["features.11.weight"], weightMap["features.11.bias"]); - conv1->setPaddingNd(DimsHW{1, 1}); - relu1 = network->addActivation(*conv1->getOutput(0), ActivationType::kRELU); - conv1 = network->addConvolutionNd(*relu1->getOutput(0), 512, DimsHW{3, 3}, weightMap["features.13.weight"], weightMap["features.13.bias"]); - conv1->setPaddingNd(DimsHW{1, 1}); - relu1 = network->addActivation(*conv1->getOutput(0), ActivationType::kRELU); - pool1 = network->addPoolingNd(*relu1->getOutput(0), PoolingType::kMAX, DimsHW{2, 2}); - pool1->setStrideNd(DimsHW{2, 2}); - - conv1 = network->addConvolutionNd(*pool1->getOutput(0), 512, DimsHW{3, 3}, weightMap["features.16.weight"], weightMap["features.16.bias"]); - conv1->setPaddingNd(DimsHW{1, 1}); - relu1 = network->addActivation(*conv1->getOutput(0), ActivationType::kRELU); - conv1 = network->addConvolutionNd(*relu1->getOutput(0), 512, DimsHW{3, 3}, weightMap["features.18.weight"], weightMap["features.18.bias"]); - conv1->setPaddingNd(DimsHW{1, 1}); - relu1 = network->addActivation(*conv1->getOutput(0), ActivationType::kRELU); - pool1 = network->addPoolingNd(*relu1->getOutput(0), PoolingType::kMAX, DimsHW{2, 2}); - pool1->setStrideNd(DimsHW{2, 2}); - - IFullyConnectedLayer* fc1 = network->addFullyConnected(*pool1->getOutput(0), 4096, weightMap["classifier.0.weight"], weightMap["classifier.0.bias"]); - assert(fc1); - relu1 = network->addActivation(*fc1->getOutput(0), ActivationType::kRELU); - fc1 = network->addFullyConnected(*relu1->getOutput(0), 4096, weightMap["classifier.3.weight"], weightMap["classifier.3.bias"]); - relu1 = network->addActivation(*fc1->getOutput(0), ActivationType::kRELU); - fc1 = network->addFullyConnected(*relu1->getOutput(0), 1000, weightMap["classifier.6.weight"], weightMap["classifier.6.bias"]); - - fc1->getOutput(0)->setName(OUTPUT_BLOB_NAME); - std::cout << "set name out" << std::endl; - network->markOutput(*fc1->getOutput(0)); - - // Build engine - builder->setMaxBatchSize(maxBatchSize); - config->setMaxWorkspaceSize(1 << 20); - ICudaEngine* engine = builder->buildEngineWithConfig(*network, *config); - std::cout << "build out" << std::endl; - - // Don't need the network any more - network->destroy(); - - // Release host memory - for (auto& mem : weightMap) - { - free((void*) (mem.second.values)); - } - - return engine; -} - -void APIToModel(unsigned int maxBatchSize, IHostMemory** modelStream) -{ - // Create builder - IBuilder* builder = createInferBuilder(gLogger); - IBuilderConfig* config = builder->createBuilderConfig(); - - // Create model to populate the network, then set the outputs and create an engine - ICudaEngine* engine = createEngine(maxBatchSize, builder, config, DataType::kFLOAT); - assert(engine != nullptr); - - // Serialize the engine - (*modelStream) = engine->serialize(); - - // Close everything down - engine->destroy(); - builder->destroy(); - config->destroy(); -} - -void doInference(IExecutionContext& context, float* input, float* output, int batchSize) -{ - const ICudaEngine& engine = context.getEngine(); - - // Pointers to input and output device buffers to pass to engine. - // Engine requires exactly IEngine::getNbBindings() number of buffers. - assert(engine.getNbBindings() == 2); - void* buffers[2]; - - // In order to bind the buffers, we need to know the names of the input and output tensors. - // Note that indices are guaranteed to be less than IEngine::getNbBindings() - const int inputIndex = engine.getBindingIndex(INPUT_BLOB_NAME); - const int outputIndex = engine.getBindingIndex(OUTPUT_BLOB_NAME); - - // Create GPU buffers on device - CHECK(cudaMalloc(&buffers[inputIndex], batchSize * 3 * INPUT_H * INPUT_W * sizeof(float))); - CHECK(cudaMalloc(&buffers[outputIndex], batchSize * OUTPUT_SIZE * sizeof(float))); - - // Create stream - cudaStream_t stream; - CHECK(cudaStreamCreate(&stream)); - - // DMA input batch data to device, infer on the batch asynchronously, and DMA output back to host - CHECK(cudaMemcpyAsync(buffers[inputIndex], input, batchSize * 3 * INPUT_H * INPUT_W * sizeof(float), cudaMemcpyHostToDevice, stream)); - context.enqueue(batchSize, buffers, stream, nullptr); - CHECK(cudaMemcpyAsync(output, buffers[outputIndex], batchSize * OUTPUT_SIZE * sizeof(float), cudaMemcpyDeviceToHost, stream)); - cudaStreamSynchronize(stream); - - // Release stream and buffers - cudaStreamDestroy(stream); - CHECK(cudaFree(buffers[inputIndex])); - CHECK(cudaFree(buffers[outputIndex])); -} - -int main(int argc, char** argv) -{ - if (argc != 2) { - std::cerr << "arguments not right!" << std::endl; - std::cerr << "./vgg -s // serialize model to plan file" << std::endl; - std::cerr << "./vgg -d // deserialize plan file and run inference" << std::endl; - return -1; - } - - // create a model using the API directly and serialize it to a stream - char *trtModelStream{nullptr}; - size_t size{0}; - - if (std::string(argv[1]) == "-s") { - IHostMemory* modelStream{nullptr}; - APIToModel(1, &modelStream); - assert(modelStream != nullptr); - - std::ofstream p("vgg.engine", std::ios::binary); - if (!p) { - std::cerr << "could not open plan output file" << std::endl; - return -1; - } - p.write(reinterpret_cast(modelStream->data()), modelStream->size()); - modelStream->destroy(); - return 1; - } else if (std::string(argv[1]) == "-d") { - std::ifstream file("vgg.engine", std::ios::binary); - if (file.good()) { - file.seekg(0, file.end); - size = file.tellg(); - file.seekg(0, file.beg); - trtModelStream = new char[size]; - assert(trtModelStream); - file.read(trtModelStream, size); - file.close(); - } - } else { - return -1; - } - - static float data[3 * INPUT_H * INPUT_W]; - for (int i = 0; i < 3 * INPUT_H * INPUT_W; i++) - data[i] = 1; - - IRuntime* runtime = createInferRuntime(gLogger); - assert(runtime != nullptr); - ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size); - assert(engine != nullptr); - IExecutionContext* context = engine->createExecutionContext(); - assert(context != nullptr); - delete[] trtModelStream; - - // Run inference - static float prob[OUTPUT_SIZE]; - for (int i = 0; i < 10; i++) { - auto start = std::chrono::system_clock::now(); - doInference(*context, data, prob, 1); - auto end = std::chrono::system_clock::now(); - std::cout << std::chrono::duration_cast(end - start).count() << "ms" << std::endl; - } - - // Destroy the engine - context->destroy(); - engine->destroy(); - runtime->destroy(); - - // Print histogram of the output distribution - std::cout << "\nOutput:\n\n"; - for (unsigned int i = 0; i < OUTPUT_SIZE; i++) - { - std::cout << prob[i] << ", "; - if (i % 10 == 0) std::cout << i / 10 << std::endl; - } - std::cout << std::endl; - - return 0; -} From 8d567cbec1e380882aebca720e0804821d10360c Mon Sep 17 00:00:00 2001 From: zjq Date: Tue, 9 Jun 2026 20:42:02 +0800 Subject: [PATCH 4/7] Fix CSRNet assets and add ShuffleNetV2 variants --- assets/IMG_1.jpg | Bin 0 -> 143045 bytes csrnet/CMakeLists.txt | 49 +- csrnet/FindTensorRT.cmake | 145 ++++++ csrnet/README.md | 91 ++-- csrnet/config.h | 16 - csrnet/csrnet.cpp | 894 +++++++++++++++------------------- csrnet/gen_wts.py | 202 +++++++- csrnet/logging.h | 736 +++++++++++++--------------- csrnet/macros.h | 26 +- csrnet/utils.h | 157 ++++++ shufflenetv2/gen_wts.py | 136 +++--- shufflenetv2/shufflenetv2.cpp | 524 ++++++++++---------- shufflenetv2/utils.h | 7 +- 13 files changed, 1637 insertions(+), 1346 deletions(-) create mode 100644 assets/IMG_1.jpg create mode 100644 csrnet/FindTensorRT.cmake delete mode 100644 csrnet/config.h create mode 100644 csrnet/utils.h diff --git a/assets/IMG_1.jpg b/assets/IMG_1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5d43d292015180fbf5dbe01d290bf35fa61b33dd GIT binary patch literal 143045 zcmbT7Wl$Z_^QJHE5Zv88xI=JvcXxM(00F`cT-@FD;#@qz-Q9x|Jh%i|e*d+q{j|G1 zHD6}VO!ZS!=S-isyWdydw*j9NWaMQ4P*4B>)W-pMUk6A65aHnw;NcJv5D<`%5Rp-F z&`?oOPzkZHF>pwU$jL~FNJuDX*yt&!Sg1)z82A`jK67w$bCc5xi12d?vvF~A{^uo7 zNJvPiD5wNzXat;;B$S;0&-UH}z(j-rzye^Pr~uHIP%xNK?*jnx4?p3c{>uRW!=RvH zVBz2q5Rs5kJ_fXW0zgB-z(B*oz`?=7evA(OxDSBEgu|lb5QoQBH%Fjy$Kec1EJUQ1 zXz0b&nEgw`W#JKygp7wzKuAPON6)~>#LdIY$1fl#DJ3ltEX>ZX=QC= zYiIA^>E-R?>*pU3@hvhc`g;s8DLExIEj=SMtEjl7w5+_MvZ}GExdqbN*51+AKQK5n zJTf{qH@~pBw7jyqwzIppe{gv8=lJC6`sViT{^9ZI`9EAxADsWw{yW(J!G-z31q}-e z1B>tgjWgGS$3KYIj?Xph`voh^^d(uz5w+AZz)L=`DPn&;~ zF=xGnzR9XxLuK5z{~M3pj+Zo{F4d5yKX~P;K2^@(#$X zG232v2kiVRv-8W}f#jlxZ&A+F3P0{iEMvkEz>qZV&9~K2bmUC37r@{y+lx{!z3`PA z9sk|Hov*`*<%EaD1aZv+_kh=axPR0%R5i1T791=Q7PQ-u{E8B;-JA1Eb$>vs5%W7V z;SK339f*(Arb)2Lvi+HjFBKNt6LF{!ckg}HYNPPtubE1jbVd994zTe{R_&Y~Yl8E9 z8~xxAe1>0*vWg{x+V~}D${ibaZ2HB|Kk6^9)tL0TwX47TE*e-A!@OH#s zSB%;{k9+n`91g>5J)OZaq`uIIaE@$tce0(lr-{wiSf4Y%!E8hKV{ac^{!fc+ej zz-1DxqV6NlC9n?H2mJX;P_(d)Nc|L;S4A~_`3g)e#M5T;Z@6+$tpTou@oIX6>uFv# zmA25q?m?=5tvw!p{k6ECj|GQI3bugUa{AafYDU`fw>a)1crY0=T}GndCPJ4`zs!1U zKQ|sK%ek?BMSMM=((Hz_Z<*ZkkF2I-3GEgZdWBQJZxKCpxsYQvxb5s^(TS^ z28Fp9^VxiCNfDJWrg=^c8%DaOn=zN9bTw1W6J*_uYzGeB0YC@BFNZ>@uj06lJpmgA z=@Ga8Vr-Sq2}bA>0x+NELul4VJ-2_fM88^eIsN?onS7~AQ?ul^29C23NmK;Rd3j$E zeijXFj|_}Uz^h}{RR~oQ!_hjPB~wA3`l+cGTCcBmW|VmUEsKi>rAn zwgWu7%Xrx&IuAY06iesX`-!{A&Mh=Kv@wqINlu3l_xH(9u7+S#_9Nu0OsthQZx`;fqI`%dv9!G5PYEx~ka_C^12BPZNpVie@m zxh|NKXM`m@fhWM^v@TJWcd+M7r)hme-+EgWTmR@SmPF-a4@kaq_qN1P#my%xe# zSZVAPvC+$vA~HtW@@>z_htqOU-Qc=AhelA-y9dtR;CGr{J=q`7W4fS)noNd4$m1 zIreIZug)}IL={(aFpp*Kw~`|%+fJ$e{KJ%1!Ia;L01w$(b>&Eb#hfgPuQwOucsiTe zEX}1Hq^zE1(eDCX*J`ff7?`PU4Pqo?@1W79mB9dGtr!~`;pT~Dcu3@~+>+=Pfg2`A zY1+Ys)d*$>!vOuwIcG0amS0{Q?+lt{;qom7<6=^_|2?sm@CSa zeeOA;30(4~w)Hg7#x9$;7P063eS1={a{Haa5a<85hu)ab5WLY0D(i@sf+ z!Q#s+14=6D_o~4hsAKI_6S)`=BXO)-G;TdBoy6y~zc9z)vNzl!ReKMuobp4W>cgih z$HbLf*TL|F?hoqge>h%Feim2EtG3$BW|7#%|9%10i(ofe z2KI1}z7}d!MW9K31rek)$1`ZFFC;lGJHHTDE3~Ilx0*iNy}U_u75Ae)>l8X!yq=}q zgtzaxPvf3a4?MrlAIuT^z2fY@15A(fERJP7RAkxkNp-%F*sp6U4&q0M&zRX*vy%kk z*Rui-k~Kh=O8=aeDOWAv!dzrYnWtau>l_55s@+{ZJt?CYk*hB0uMX>3?hHHm$6gC3 zlmJ%r;atXAjMqc2(}Jb@cS6LD+K!h(>ZI zq=(6guT!`_t@a7mBtsOq5Y*KcHHoI~P90@&Icn$x?Jntnqy%8e-rg3^vy2C_K-tb7 zmMhmo1a*ZxJNItmo?dBJm7H?Dx+f~ke@NM8S-8gob%e5L5DiJ)^)0YwI-HVFJVdXm zXNQrg<2qwVu$7EEJ>cjw&y6(VroN92W_SQ#U`Q5#gohh)`-|$Fk+?J}lN5uJafu4L zHd$ZcY`0Q>$2+tYey`G%Q~GwqUr~C6tNpK(%45Z79@RQK$-zv{1u)1kO4pG%GO{F0 zZjeDIsMe6wv(Sswr~^=*aHigg8Zr; z(j8sxSt{b0i|`P04PgqZr5zxrZgq$Ny;mch+RyMgq^BnQ8vW=faT#f!c#D?O(9$GNX^g4EnbbAmy9N@IA)@tNmZ zavvFG`vPsY%)$fD7~`6ec?LS6UO@a+Y2#uGJ3I#48ntivXEZiq;RouOw`kzcj%Hnh z95ULOTf3unKLur)@iCIU0V3tq|{w zJQ_%w$FH@t@wW*VuBWWAr4{r!MaY_p4iD*Q&3LXdexZMNg1g@D4%jKHDOsJM1@@}G zN(jaL=d+0Ch0X3ydP~6)!O;#hA({J&J~jzySu&vy@wf>OtvB-5C2r-vCVuZIU!!2z zq{(}(#*-&hYnAjJv|rXN0=Zm{E()fvOZZs!9zLA|+@_Ucd%ECIZgCFNSOS)Sz!T<1 zdVO2-!;N~fm}{Vyj{9I-hUihUoRXEhij*zC(Grl+iSjJ&T&;fCApdNhN5GlDJU4I= zN;Pk_yo!s=;h74ZV^Z;;Oi%;5#2vt-ZREa|UyrO@K-N59Z3h)uL*>o{~P=u^Hz1 z`tAswOQ6IQ|FD^vjNzxpmEGA&aqUth?GLSP7MlLIiSTq^C%#bD(K%-eX^FQ=s?-j; z7uMt^WfH?!9d?8#a&k$XzG`ueNGT7z6}SR^mS-!s86g#@WG$4Y{)%RWW?h6Z_0@90 zrX+r#BOnw!M(*Nn65r%n*wd9cTg?RRsf2GE3Tx4*Qj9whc)q`psarIT zZ^MJOVpNl^*~EFmHhT^)fk<}PHmTYd!kTou0nYI2x7K0&!jCZ5$?U9yW~v_8&1UBU z*9IsMX;{W>ao&Y)tZ##O0npv`=W7c2Y%A+nzzsJ<+)f)gB#VK+Q+r7&_izn3nD`MA zxAKLnMw_quZ)+X(Hd@SsmX5~RV3ox8ZSC)Hn>?8}2Q1TeO1^_K0l{wkwa%<1-s}mu z0f%MbOZMjW&Y=6)gJP1a6NG@VQrasgXXM6KW_H6Wrg2w_E%6@eBw2@Q+RB)*#`VDCb!<|hfT= zcwXFf{!p{Y=c^7MGe=q|Ch=_$bBj=>UJG_cIg{rCuw}u!Od8N8YVJ}j7RUBkZkVbx zTXV&a|DjI%(Y(UHgLX4fOr&PZBK}_W9biOso!7D)B=8H*gc*e3Bq3a!)b4DNE5hgr^n>LIL?hDcfFUmZGsQ3HY zEI@I!j~s>;eJ@_&qKg=ZOuNfz?3f8_G;jL|cT7*lR7(yK&h~YdtgsE7 zk5s&J5(78;xtR;cLdGXTZ&wtaMPyOha+CY2?1Xn#pS1$GqlrQVDhPO9bhem1qhvae zmmt9vadXT|)^%ju;q6U#6XVQT9I12VvA?R|TGtoE1^F5Qf^U{Z7q^Z30{;YzqwtIR z-15*Q=Al+sm=su4+iMyQdRVpFyVHcTl-H?isi3W`^;aNS#KLiA5R-pFFa?`-Hxl{e z6Y5J9PwDf8fjkZV+5Pf%nJ875q+i(N{ddHYGy;^_fO`j!-RU@sIQ1V*q&Yzot`m|+ zn#`5ipVP7-uF06ziXl7S?hauy(`T;SEDHk%wOX|CK;9aPQq%BP2W9EDduaDdENSxt zjl$w1Pfr`NCbtIm0a7#7B^FqrM+KY4X?10Ojs|IrcY&*CbQ|NCBP(1}NWHJ_O6fg( zqZqkm#)b}f`)-zINN>?Mb?UoGz#l#~CPS0Cmtjt~dM~Wbu1AtIC;BD*+m$TAf~XeB zy9hD2ogx%^ithQ%cPpH3C7}kpx2)_M^EEBRhafxT$kz4NoILp#gwO!+t{9rhV?@82 zL*FsRO6s@B^+S%MI~H-6Xr<(PM2`yK9Le`yoA6>8ovC;IgD!!Z^yHV02usjt(KLW?@uWu~&_N1#Qc% zsAANTZ2A~Qu`9AK%=N2d$*)3H~4 zY*AHnPykj=K^x0ue(}Ane4<3xKbI}8`+w@j?Ksy6P{8DX?sBE+i)5NlsysApF^V4h z?x!_DW2+W)W)(y}#xcW2p@Q*h^WGTeK533Wcv;>7e95S>9OpxTT0(lcx;?-1?bx(o zlq50XW%r$K+81%!H4~KRDDLaou-6Tzz`Q_oM^Fo`^gp@djniTsbhJ_91h%gQ##=^{ zOokFW8S6?&Ujke-rw^XA(IjqUQ)C)@=ok~qxTzyf!DI4dV zSiS6HN4xVQl$dQ! z%$q7xuAqWc5LEo>TD_Qk+w|DFp74ac`Q!ZI8fB@eNqh-w9el}b4xGZFguV7k^&xF$ zYk{`NO$i%&ZRcC9l?V*0B(wHh<%?sRibS}n_s|^Vq@l=VS^h>AOSJC*2g~Fq^1s#B zxH|p`2}awoi<+uhQB)bZgGY(P>-OC95ZfW zvl{0bT+hi&F^7pTtd4HDm-A%)O$}dFg*Zhg62d?(7-J_qJFHEu4u8N$mL(q?GyuPC z%9Mn!K5J`e(j4TUxGQ5hQ`Fdrp?=7jJ}61Sb%D5YnbBZ{5naLyW}n%haRrL0dD zUo#aS_-UkHss`s)SQn=5n9In&BeF}XQ-G7{=5e>fXReN?TVwsAZ&&kAu zzxtr@{Ow9v%V0eq@5`Vh6E_!uRm|dY1$C+IIjO7(BBfe4mD7j#E5mNb<)D{Iqvjc?OcOME?kVd4UrppuS#c2f>~mXXRPpvG5kH zb0#!_CeE9+6~Pia&IhSwTG~osh({t(%2-96IIS2=0bxTSWsN+$EMEJ38pEzQ$xK)( zZXzM6@}qyg6{Q`L{%rZkO02r9t)GA1r%?bzBn*OXhA(OyE@h7*J zBIV6HIyTz;4Ro<+N>*4bU>JzJNzXmbLl=s7*KIyWky$ufQe+;K|GrNRGu@FV9pLsO zypg|b-9@S+J*rE2N6Mzjyz;m!?knkRe7X9gl`eIKgVGH*GtCd=JWG8FG3A+)WUpYbvXtXW+*H)G5bGf2Iq9-*#wpPI zyj<;vGQeJSEGjjnqxgl2sT*;4EL3%_3x7(q>)&#D5Jl$JJ3u2YYX4d-Uw!oG4s0i1 zS)y!LYnYc=cjCl-7wyh;-kP`!yb&WC#WAo3cQ@PMq$y8T9obW_mj=)ScUtGY=q+JG~N0?QRiA5-ZbY4MS3Xd)ZYF@RZ$8RfCHVawADp z-9vkI(L<<{fZJL$cooO`zcjOSHTT!fRCaGh_tf)$C#`wB%~M(?9riSTC6?J>GE!2X z0Yp=Ew*C(E6ATvv94&{_be4!ZSXlVv@ty!@16ljR^K&^wknp4?p*R=~1WmczCY(%l z!hOd3+5?sOzF6Hjd(B3NrDz|qfUY;5voU$~UYdEz_}_^$8hFDFfI)#zLS{LEGq9*K zc9{5t1l8Qq8Y#k1veKe0Ge(!*aQ?khA`S&DO=gn^V~w^H z&VNE$rh%o5`qHHQ2KDmooPK7q33MCfN`YE4eom`WMQif3D&OS(Wy&sh8+;umilscd zslGC*0K{Spwe*>jvIn@D42#m^o;Qcjw|^Px^l1bMVQSmdHzhVMUFG<$8z$56eMuN= z3`>mPtx3{_{k-?9+ed)AS)iD>6Nh`!R<8Y*BC-Cih39qQ@ubb;yk_&Ue8S&|OXGH`}$&=i-+TNTi&|MtZkL=>Y-`C+9O+?)4pZwOiO=G zPrz315N}Mc^e)N|ly;i>EJJq+CJD0YV`I3w0klx%_;H^b@guHs`m0(}%-91EDQ1+a z&0%+Xe;QuQo=Clyr0OQZ3})!2#hKD&Ggif8Xacplsw%`9rn+W$ zs-iTS$vOn4!E*cwTrv}g`Q)tTzG0JZ&Sf|O#=Jf-dCdqQvf%ICzmcmxm zFO`~em~eL%$~{QAm(Xt4))X)?;TcjbbI56(bEsniNaj=xuHV$Kw@h=ZHCQJx49!XG zq@SJ}2wEjddo}T%SEud$B4q}pqD$>R@aD?WC4OVW$gF1<7*;6$eC?7ex6=;((Hd7< zn#S=xvk{6Uc8PUovd4vEhUld10yU3ujRAeCVDB#vi^??FUXtgOL$h!(b}S#k!wJ*o z&&5`fuLF5ZCB}ReFN@jCLk&M6o$=A(+6f`>Re8!JAVz9==4x{JRh1?SDT1q)Rp5d*ro)ow<951UZqO#I$bBznMW!FlO{p?jPMvRX%pL5*H)E)@*#GtF$*!iv z+Wjka~2(?dZ9|8ZmLE0I}-ku?e4ja3I$Y4k0+$`o7qRyFst}H=@o}+=+A#6@IsG zcnH5JmXdoljt-L-hEy&ysr9+V_Xug`YiNMP39voNyn)-dN8OHMea4!KbGBLYx;Jd$ z*!xnYFu~~lVv$aN*ybDrohoMfQEjwbo7PIqH{}U?_T6L&>!b|7-$NUUDO?s3sa)Bz zDl{47vHs21Yxlo=;i+am%ONcGhtgsDOz3$Fc!Td2J(U8nVqMRzv9Kfvel4z~SVr1I zp_zN5I2J7ZW{tPgn@#}WbyM9%3n;;AZa^E;j#hJ&W4LMC7%M%%U3n63&R862OVf4l zohDf>eW9H3T5Un8US%acYGE=@_cWIup`OW_dCcBXM4@*M){qA3qPQjQArBV^V}yo+ zy|)Ojbfek0A6akzt=o+iMPOTSC2-8_aciYBA0*J0&QAON>!mt5oI8?kSoic4(1+yE z$xZF8cnL2%?&Nar$^P9;ja>LXFOjWRu2Qlc%mq};$3Es={T+ynxh9tXq$^Yrq&&|` z8eU-y6Fvtg@a$pV-Yb+`Eu%WX9^Gm%RV>=%=uRm9iBm#$$|rv3%nytPeO~x?Gn^R$ z%N*_}!kn@2#>s#Rc{6x+QXal>{ab^EGJI|9P_M7}t@(&HJ(X;w=7>NFp-rL)3-fd@xB0oQKlE4pYoU2WkJVffHpMmXk{@E2r1-nI{#he@ru>Dr1TMO@vAzum7p@>raDIVqeFx;ST;O4ez5~*<#kj%@@81EawVZXKp|(!o z8A8<9P*{RjI}ae`bg^xI>UQyj0v7s6FUhOwqgMUeh30qTD9?1K&>=zjOP#?&eI2xMWucAdtQX?jt(POW0#0+<>R4&fQ??9>JA?yI%>!acpvx zZ2$q9(4C20i~IkIDsr$KPAn~68FD7=z%hQhXtKtf0Et-ZK0`(c@E5p+4`Xy zVq;lUuA7!G)JyYkopm>L(pP1zaFIDa*Z4Qat&u7PsImD&UGsfO-wZw80bVLdq7hVJ zzuLPyQ%s{~xKd5}biCR8)X-O{rJvJz26+-y+s%BxWm)=E9Z$PDzx$%lLdWG%yfp#yQnYuWk*_@6!0Tze&gMJm^Zfv(idqfQ&-+Uzk&*i zI5gw7y{P9!PK&g5#>ZLX*?1hsLgvV^1)bTpNSQuekNk+=wfrh3*jc?MS-J9KT~{WF zV^jnE;CA){b|vjgRKKcsK)D2K4?`+{ zgSn=dXK9wVfh%ZNx2Ko{)PL)EuHY`$dAR5{+wV>sDvaf}I*f|B1qLDAmK@Gi$;{Zt zlfkgpHDd-0`)=^!t|{bGKRsh0yjP3<2y@-1DnAuBv1SwVWH@SlV(;&&2kdP;r674$ zzz}cs4H?c*E;W&|W6s$$(jOnnEi2poF0G!{O}bFb3)9Fv3>{tK?{$9iQcnXVUuo+$ zAhCDcf%0aezgR+nf0N)hh?8y73-TOjYi?7_m#A(=<45W-(e0^L;f5|FDX*=WZN9}j zz=tw^;YO3Tv<8&~Z9?eRgadQ8&Kg&AuKe_-YQ~#H#m{nT)b;h7*JBkiag|XNP2&qI zKq`XWC5U;v%XL?^l^eoD<);^^5kL(y`mxTOF4g^bQ4B3xVH&2ew}ty+0{t&>;jWG` zcpX3RVBe*@{jL_seR+vZZ`m?;mrJl(n_6v{Ng+c_>$UOokvj2FR~rATFh^A%-8>JH z1%3Xe5(!4nHFHYKEYB=gzCda!1`g?E=G+}lslxM#j-uQ5CRclta+KN>uunE5kD;o{*s zZsHBDp18V_mw8galb%J{K@dw_nj4e4zs5@Orcd5)1JqGdzXR$N zO8IDglm#I{)s~02^#0J@KU5!OD+ldMPu83_OzpMwU8r&)5s67Sd-EX2vF342_o{Wn z3KpwWEr7`BDXtq3OEN=S&=QoFno<`?Z1fZ{gw>o!k;6gnN|Kjxu1COAUW?=GaXa)m zFYp|ufRO){sYE@>{BAe0h%r&Toz2qyg#|6JHkn&j&A@yB2G2>bzNf7X#} zZ~|JRWp#kALl7%p6jivMjc|GoDL)(6 zha!)aCeJJ}OPbr-B;Z=+t(13$+|cJcI5#WeEm>xr7R{a&H0w{;$U`p1$&b~4#?B$N zGn0G^h`m;s9SZwGSo03RH83{u3pr%=@NM*(J~H4~on~F_K!x(@HCd@mR+`<~hG8Hx z^)#U=4uFJ<)Vk>Xn|Mmn`#EvA+R#pXI1COj4Z91*TUP+hUF$Y?aFQs#upn(?&)?KO z4CbSF9OX>g&E)n(OF(T1X-ZVf6i(sf*v{p11FxtG&E)zq+_#cwp5!ffZL{{3*VQc9 zSPj-B3L=_qsS9ydthKh1WMT>2RMl_GC%}gUy(}D&z0-4v-2Xn@uZHp8w_@CjwS7|e zUT_>qS6NGb3O^l9Byc8}k_~+aXxnZ2_lXNTB=+@$tG$%|%#AZsslzEHBqZ_H_$=+T z=HOs|y{4>lN`DQ3 z)hOrR12@RMi{Fl4(ewwk4P=5>uMc!d zL5?s(kUXEHd4@xLER4ecRMq~u+@&R6Zc*=uKF?B*d)kU4EQ_5N);NaVfWUK2isz7( z)tzuOVz!wlq>337$nBA(rle*b_iizQ=;up0;)UmXN8mH|Z%gzV$TRNBW3|-DROj;i zOE?-y@*N_hh(W1b==>Gv?#aqXajEi>x7At17g2K4#47IEhABAq8veQyxgftHhbtR) z&(4`U+$ND=(6gQUh76*7j1*-_nKUD8xJ(T1onhZ5M>Oq&V2KngF(Vs;*eUy!T#0lLI{ztQbGq7nnWkwDX$ ztKBxbD^}D^=uad8%NB~M3J-QqTh`G?t^q&2(ZAx!U@}d7#uoEAFr{)x}nrB|d9CRM0782cJGZCDdSH z{)N8wq%>sG+>`34BKG`c3eL#diVmb9#A}Z%#Nq@W;?Qmny*kyzQkokK4;R2%-e;_^ ztHp<9XejWZm7hPch30xa`}VPN@3hFCvkSjN+^t zqAA>efz->Lg+}3Qgwo>P#^rx~VJzLQ&4vYu0e51fuU*6x*5uEe_Eur-iO_!#!QnjN z`1+7k<@5a6!VTZGjaE-sH;U0$@(3yit%#hOyaRd?D5N33!brBA5{FdhI>m5SkC%xK zCbsOq$wMSC+bsOVD$01%%ULIz$Gx<0h#~S1k9WA=PPku}yfw>8lI?DC+50sl_rQG0 z3zjmiT+DKYk&GuQvgC2WFr~+WF6qQot9nvB33aY`=)XPur}s>P>x}F_e?|cXRxTGH~KejFZ-I>w)CfG&}#;oIQmf`tRa%)>>Y-q0$Iy~R?C@_$e zzt7m{;NznXgWKg!1tK<6L-?2jn|{LsL~-%?@Kvnq#p19MT#tO3`a0|t*;oGFToZ2M zH%6T#@c~l`8jDQ~^NNy7FXC5h(<(|3tu2!?~ctG+Zv zakPYDFXdO-+k5Xsc)NbWie{%nn2g;cSJ>(5DJMnqGCv-y?(TCtu=TXwJD2fj_&tVe z7|-(wTJ+be!w>a^vhI}DZ0hna1X(*)ogz?A%Fb(#KtvWQdG#%|HX*<1Jryuq;2j^;b(a;Ni2f%Hv+yA5p);Vhj&TYv+!Rw9mTkKs~K4}YW z2CS=n9mFk`kwM=Jdo+hDpt5R1WGL}E71qe*s^WPv8YZ~*xW zHgPxptpF)`BTI%KK7<_t7G zf7zNK^UjI-H?755!&XRJJ1S+^f!b#uHg}Wb?dp_Jq~qz(Jup za>C2rWy19`$0#iBT5Qu}z*17(|4EpTJkbpTY@(^BZFmgp0VP{4?%CT`rPrj87}kC>sm$-ovVtM-REGN&a~x4X&rU<}i8s z`@8p^Zwtk~juKuLRsCPJZ8!!5;mCxt#DMtRWNs#O-2OvMGu3}IR8d6367n*l<0s3l zO2pa-vrHPrTGCf+wi64C#V3tXEDd9TPr+T$Vr7iznZ zqpXnBaPpS0qf~=mkg_=w<(g|25C0zUK}1*0P*HJ|WQX!=rNAY~F=2*VmZ6S?(x6{G z$<43>`wii?nvBP>vP;0Do(l4pub!X6cdy$6^5o(zP0y-|(c%1nK__~H#q8VHfpxNs zE!=)>t~|W44D#Mi=cjav*Ejkw<&Ad|R9CwU=l%g7yG3>nG~Hv^#% zSghgE9;ks@mi?mUh$oS^M~V-Wx1TXbYN?eLMEiOP6(LY+%w=zjS^pF|9l?) z{FP1`TkA@8{W-3vTuarzjpRvcVI}typgyYJlQr!Iy;1xRR>Ns}9$*UVtcN$Rg%{h} zfO05hh-cCob1gLyR&{;#*)ck zsxd{dhCbmBap(xpDqZKFEP-NLQaKfLbT`Lw=UNw*7kM|z%$Fn}OSiTQ$>Ul<@jp2|Xcgs@4pi}h_0!sqOj%}9H*$Uh|!G3W?rC<7oD*Vl&p9?(&* zBca?!zbo_(DicMA=Q32t%^Rp}Ct~W260dbNY0YI%GYrhIyuoKB{;LP##5m9e-%0)B zM?`6@+_q_}!|tVzwKdc$1u_3XBGgYI-v#X5!Z*gr30yN8UNN~j@FmS$A+b=$5cYCP z$vzL5Xgy=dMgk5e;k0)mN43oe4I=67mHu2HHc*S{5 z+7RoDd(aM9D%+@J{b7)s+T9LW2-{FkVX^x%LW&+Z@9Hf_t^+#Ul=mVg4@hj~A0_KP z<(q2YoxSMFQ3<*JHuc*)O#A1e3+NBzoYICkXnvYp!M!x}hlbJl$M5Hpb9V=4ubTnp zSQPBdZ(LR7xWe4TWqa3jcU+r2t+tjhB~d8{6^S9J^=Zkh$QJmdMN5=*mf?1AQAzRm z%#X4QgLYQe)75LR_A&QFx2P|&RI2mFwDQImcd=t>~2>IXJ(^J&7uk* z?Q1FRrwguT908qUJhW8_#Is&GXS6QX&S0cYXZ?y7+cWy5a48&6O?pt%M$j)aEcX4a zrB6Q6G6{f-E$c#Navm8IMo{?I0j@1Z@7(Df&^akrd--7Q`SyT|C+MH>)jU$zb1Shh zC8voBLwpi?Q~pra)qIc{Zh)nF{1cqSwM@wIbba;JN;-@+#LbU|>*d7t>@(}7S9>vI$viJH^|6Bt7&$YgQ6v4 znTecb;_~Eyd@nVK*~$1c4>wEE(D90=dI;xJ)sby^#j_4;=Tq9B4XZFM7jHt1o!ulE zAwNKXF7dZgX6ia5lU$nDvZ3b_gPf7;_0jJ+XSNo+p6f6ljZ{T=H%P7*rE!{~O7~Zn zGUn?(;JDZyeRrso^|G_AE z_+d*FUQPgs$AklW7>(!O$8o(z$R`7n4Q{{^;4c!|SD-FiRSYfeTb%u4Pcq1rAWD;okPBo=~ zQ8}r5vDeAvO)T81^2p({;`r~MU$l3OQuGZ!?!rnC!{56+wBa?E?{6>J_}3qWk{ijU z8%W=U;Y zH#zuRv0unj5IkxiX2(EWdJ1n*(o%a~hjTqwIIx z3gC$AwN))2mxWIRSd}fMWqrH@(t+kNDbWc}-$`{SI3nghikTwaI`Rh;>*v6wDf${; z{4-1F|88ruItD{ZMVdJ=n0Wi@%|~inX=Q0+?vTa6Mi%Q#qEpOTs^l>|xe9ky3nggi zytUL?nE$@Qd8q}Lg2%98h|s2_%-}^pj$Eqq&ii3D8TgtHn!W^g+^sY%v)aTKfRDW4 zfiYGoExN_=l8RAT$(qoA1Gqp(zw6pe%8Dg)V9~2|Vl&61cf9|DS_oeV1r(Z zq+g3$d~Nx$@`|&0VQ1niN3+zFoxtsuJ{=FJKhvc=HXd_Ly%89yuuUboDpU)a$%hp0mr|g^%a5gCQ3#L^<$dp z_7Pj0{LJDmcy^{IGVf&>HCcD{|Kw z4Xlhn3Jhle1CVjrwxyadKXt^XzB62$@`SWIc_S#J#kEa6^y|qjr3GY6ANPUie=3d= z79dI~Z&O>IBhv5W)Glvj4B)m|k0cI*@T_!%h!o^!oQmzIE@-B4%~{wCuj_&B#WZB| z-iifTCCxnIbLqCVg=1MC^p*7Hwg&^PV^edkgZD@0iZE%VfaQn--kiJG{HWOVBLnj_ zm5uHMW3?x0dm-x5BKt4Qf5MQ6{{Xtp{_)4?C>6*F0r~1hTSY3M9=??jk(Itcs6W{l zsgUJC%{FOC>M8W5Ff}JyXA2LE0){HKIf>cPY(E&-06~N8lAbG(ndH3f-}!v#2U%e zyl1K2%EsCL#}MEwN9O!ntzyGH$++{CiIGn2-N~&zn{ef4l%;80^?Qv+PSEr zuV`1=5Q*oKCi7X9N*_#h1bfq@xpbD{Rr7p185c7aWn5rmCbj$nZu9t_2@)2R#E$z{ z92|A$>sV5THs(Y3xy^Qe3W!Z!^|6v57(a-sto_DKJ|P#c_G;{rqH?1(y=0* zIBnpOSjGyLVUCCVI@hAy#Ru4>d6#mS9!2&bXZlx@>atCC%NjNwRRfLy!qrr!y^@1D zFu0fP9Fe0As~p5Vxev(%gT`xbPDz}7p}=B*kO|;@E1vMoYVgP;j}Eee*dL$ru8&i= ziYr^F?AHwfg6yL>BxGmsuPxDTe2DhvUy>Zm!?}(-;MVunj?nprMkD1Y$GASVlQ)GQ zOe=7XE^XZWlBo9ktUEPyT2oCPn={=;jTD1%XFRYY^u-ZhTZ@6GMK_&mcn6l*=RSwr z8ueW()3SP=W(yjr;=T2FxA%FR-PF%5l#o1thC<$idRHxL51DWdFf)@~KC$Q6&2alXJ~$j_)Yl{*EL%R086RtiroXElww)lAL!iLpHPA<>%WhUHc$R4j z<%2c?`V&jw{T|+34%%jqF3XZhsiZ=>)#Ef+jdn(d*8}`bb3!+$R$-ZYS{}(nspVeX zIX4#@6~BTN>G+y_T4kI^b^|~6o0Iz1vZ&j@#X}@YWX?J7U1=q;^VT8?ia5ci2MRJc zHC|o`7^VW`@C`B8wdRsANUl%C`sBK8#gayl%BOS56sfsfbQvF+{41m#L10CBkBoH% zzVQU&Ke+%9t~=we>P=Kt*rlsorl*s5w#%p)O`cyXm021%SQ3Z-LGN1#6-;SS|tnY=YCGic-vmAMl zWAO|t)4s!|X_w~kF*i3`A?LUa#~+PhQSOakx|Td&;eYM@A~cF$E_jG3pSs?l{#D|B zA@JU#rCo)%wUJewg17{$dSKVn1|rDV914tDrNy@@313lKJFSeAj}h?Sf^OroZAw;q zYsj5m-;cUSIW_2-ecq)#s`>Y`u{@9*f%;bT)_=NUSAo!b)>fQ=sa$kC*A6C9w&!jg zMcn8u;778INH-EQlh&%-*g+g`CRkv4)yQ0|NWFj+1@mqR7(aOP^cB??YHQH*rG9tn zR0bYRA{Dh1I)TK@pio8OAcegPt*3mlv^Ev3WlB+^u5T_=*T`=8#YO zyk*<4G)i2s(CDQG-Of8r@YbgK)E5)kNeV|;NjSWR}t+xPJ8Y-jG*GpXFULz`nJdvgWe;>f@@4l892>q$@#y%P-n6xk-y zCsxCS-PdWR#WEt45(y&%0lFIHllQKTsYS*OGn+8V-3aaM?+bi z77c^96`f|phn3XFE4Eagc_X4jcB)u0%vqUPunF7H z9CWFg<>H7B;9fcOtg|-TS>zl7#}&hkZ;>O=z{PVmL82GB@f*K5c}L_c(d4&Dt^Rn` zm4*l-HN<#DJpFPw<5RLF-y`FUo&c{#h772APiqy-p`5?(X&2s(+)RynXo+W^g zNV(p>LGqvED|+T{uvp0>{{RSP{5sd2d?b0M@iw7$>cu8Ta@gogXZ&l*p?-GFu|EZl zcUKbVF~bakEV7bPF@Ozn&3?i;Gm|0V7&iz#0Iv2g3tRZZ#TvEZTuTjvcNkNR#r}l! z9Dbs-_00>ydO}%1`g*;@k}PO9w&uyfW6w2|XHGkgq?Wd$%nli$ zC4l3$ef_I@rMZ~RIgN^~;@^2w1ZIt6O~f9UIpAmUs*+kkJ_#t7|?=~I7ZD9@WV?`0p>tj5Hk1Gf~oBaYRBn>tb;`z5qsA1&M6 zs!+&-_i>_Q&{HLq6?t{Y?V76B@SqF{Z=j{CD7Mb{$tIW|Cz_<%lS#x;XFdAVM&jY~ zmI?L1rlQ4kX&sh(uh>(LEy&9+7=6PC#I zM$}-4U2=z*jcNqrsqW=JP z6$&aIBBZv^rAhT?uv~b6qLTjrBKpeVMPl7}!p=N~ypQP$mp`9R*1VEi_nG%RjUyiJ zNk6SO?J^PapVFc8zNM;A>UVn2#7HbHyvS@WE{EuC{{{X(5CaXwVMcX6f9>$~cql{$mqq*Q!Q-bYwIVkihO?9qn5(zHvt|VY`JlToI z&~&YsZ>A9_C>im|9V;qLQo`Jw(r^G0?sg`5*!qTV=~o;g>L*p5oyM)Ez=wRx zxONA1W69zhIOS)CDIB*SF$yc4D;Pg9E%c?{qwb&w-nWGaM`9e?+}D@Hm$KwT8b|b8 zpVp{Lb9r(8x^~B-H}tBeMkDWjTvKvuRTQ=)^nVK6{i8yOGnRpjXCIjT4QR;;X#vSo z?OdmU?l#YNZpVgD;C&7c^A*}F4b`kw(Tts$HL`48{{WAAxVNnogJTg@OuHA1d)9i~ z7repjGy2xcMPolRr_{9gJWZh7SX=9{K_c9F8-I8oX5W0{743G{ zjcj9-bj}TV-l=2c{7CX~`?)VtWyeCSLw{U;6_n%D+7dleNYON{4@i$sOIYQRKxtiu zOFlUy{{TwPg2GuZCo#YTerejk{GfV%9V>pqF)Rh2_}G$7Xjw$-7{<>pB&wAe1F-z6 zs$t_*S?DNNl{?tL)O-~8`c%>+h3=Ib%Sgw3AI`i(P}A=8o7rryV3G+G{nt_sE9#po zu^X!GUOI7EUln{UABp@MEyNIqzp@^51x8tvedPQ{IQ0UeR3k|x5yG^pKY00hmj@J> zs}~XN+H=Vq)D@YftW9Pybsr#e)Yi@uJhQj{Ix7+biIAUK*w3Y9QEGJ1*^(;+KPoEw zfzRe?OBVaYpL0!7kbga$C5&+_p=5CbxSWVT`a3cWDa0_q;*0hqw)5@3&i5=BW z+%E;FNxg}4jC5?u0}ahtjA3dGr*}E2qhO;1^GMbbN$N?CgCiv4lUAg*i)6+pG8hgS z$K7sm$9jqlKJxX~%CweP``~TQ<6R6|HT0Gy+`$#3OxbrQbCI0m1Kz16ZBBZ%-#@zc zIe-Q$FIJERjemrblU|i;py_ZN$gMnwuJguzm6zi^B0K#$(JkQ+tWzAir(&l(cjzdK zN!(Q{I+5kE=D`yn?)hqSA}7o_BzCO`{5u@}V?}?&V=RUvxPDc4MDUcFjoBuAMvImC zbCd7sTDtH{a|K3i>}2Ve_m($5X|&KaOVbNCXP9C z^B~3mKA=`cm#Hq-DDQ0F7*o z3gb?Z^%%7Y7tUR>0Dkf9ob?#ZVclIBLS&HLMRc*>uB&Aew*^KVb`9%EMPEas89Nwy zwbYiHe0qAxlEW?Ry@gd4#>#CND*)uH$a9=-e1rL7v*nUY%ln7lpF({)3eeIghiJ6A zi_8x$0~5$9G8gI1RGqXjrAys%Wd11f?)6rb5+Y;_+~<%mLHw%%(gph|VHwM@AdUt* zS3%;K5*C!MK*>b}lf5=J=P7~uAQewEO2iWk?PtxOY);~!dpp$CF^rwLdu&B&wy zb#k_LlUz17#D$OGE5xOht}bMGKKrp^ea|)OUM{%V;j5jkg2h+xBv+Rh_O?$9dBtl< z8O@C-zLe%_l>((S(B8noaFsMz)UMcmB3xu;sMlXE04csz>X^goDE-dtYA z_BU~4etDoFmB+I62anRWH7^_KT6}hwGskiT!2t-5Jw`|2TeFoN9MFe2#{#)nBTH+k z*F5ecAB{g(@xtlW>oRF~FA-8&;F0CU>9~ybHJ@#A_e1Rx#$u9BnUJ5lPio`FTaj4p z!d~v@Qe=)eLonbiVnKN|&D`H;jbmguP^TjqEv5Q%dpzBc*pHt#n&*kGdLlvc`e|kct4Ulz|=~xC4xIs=@&r z#WFVzNIX`ZybZ=du4`Q4sM=;(&|_xYr@uj4*1A=^T6F&axn2HK0K{|Br-lK+JPNUM za}A}GrLxD%Y$zRgp+e0tjY-Q(oDorsh9a50l;fzTvmU)EV=JC>n)EJhVM|O`D-o*A zBNzF&?TXThK*wqP4M^7L`P>1eaoC|8NZ8ADs|MK!;Nq0pi1Jf#9+enLkZ^uq_NiP4 ztVo0A#%UeBDU8fG#(tF5k$4Tkpa-iGPXv+8Nh(Vgk#TYmGqC^;+j1)hQTT}$hlASa z_VHX^FvPP;GKv1k9qR(y#v0A}kWV@!Z`L%71;?PF#`Xx_%BY0>0J*^Y z>gp7kcPxm<96Fqa9`)y!x1Vdb+Eso1>z!Rn<4u^Qyt2jVweUYG^j{>S%>##)N!mxT z7$Yom7Q^bbDnLNVJBMcOE6sG@6>8F^8sN6m)Pb7PUleMDgbam=$K9#rVb;fY8V6Y= zm;F>LmOiGlu5D(#v}9q$PfbUVmrUVajb@LTc%AJSSxhl$4 zJ;;6L@U%VY9)IF1yX|HZZKp^PeAJFoI_xJ0`PWV1j|s=&?>TLlGRqoZ`;6!MR(OqV zu0zETCC*L)h|(gbyOSHL(uDj@|0WMfipZtUePB7)AYg9Mm;0 z98IWSFZwhES}b&bKt7#KcktK}Ll4TeAh1F+jMte2)49_ok1+Uw_WM5%lL8=uMiI*8 zu)rRG4|?D~;TqL#bOqKfB2=@KfgWGyIq#23`ci!s#`-mm?$#;MPT;bjVV_(Y@_&oE zP1JX`x|21$Hb|lfVc+}5Jd#4=rF7BvS1K}_ySe4Es8rlHZaf@UUuhnTp}n2urlBN_ zaR_N51zJ?d{ty8G=cp$*u3rx6%KW~#z^U}dpG?*)brZQ!CstW8!ND2EeXFKQ>c(lF znGc5|x0!WoiDACdE+QXkM`dR`e6NMz<3CYXJWb&trf9eZSc6Oi%^kxP^2XePJCCh(-qFx;)r)=MiFD^VCDS}UF`CbN2SqqiN3D7r zY0=MSW0z34U`GQWbNN+B^p&(KqBC(EyhuqqlzM}j&)c=Rx3Ip)o5>Sk@ru1-cR*DA zE2-6VH3;%Y84BG19V->AC$ow+mM2Aglx7&+jx`vv>bBU0Chx%8eOhdq=cNwi##~jt9(d7O%jv=TNEXvX- zZP>`qPHQD~TtE;ao&e5ktnsa|l~PP&Aff)1z}m`uoB$NB2LiF2S5vZsZCJi;t{CM; z(0#B!Gf}aT&Lp4mB-9SX?KwHmQ%(SmdscQwQbLg!{_`KM1tvh{QAgpQ=SWYjCP3*x zSaD`OSdr;T^5c<{Nt#@7*Ax!Hv=PQBgnHAD$~)q!OM4nQLCXqhH?XB89m(TwD()sd z3Bmko>D3y>bLEx~j;PqIi077RK#CpbJvlX_EvSUsQxOrRBHNgDMMB4NzmA14WSqU{iR=GaWn?ROf zfKTgILPZj9ox=Mu+gzjS`i%boDo9a%5N)ch9IkUT$9MZplUA_Ht0m2yRJy*O5B#=P z$7se)XUXDa1c;$8=vlG)3d1Q9o=I=gmv~X~pdW^7RZ0t>uFLXkdX(ANL}Rwa$LrRr zO4kyc&u~dU-fYw*-MX=)=B*@zI}tBZT>ezyIZ=q!SKk#0r5^Oak)?O)$N5t`(eFh7 z9q7k;S||{hqa4!XG`P(GYxlC^5)&tcQ~gav8m_alMbxILYje^diG_qwe;L3(PX7R_>n0!yslEcK0JWwn#-6xt+MgV?| zkH-h`t^*b7KNoHqS@f63FAIh~q!C_RPEBz&W_8}^RN>V7*3ynE5FeOU$UiP?Du(o4 z9akUTOSTJ6Z7+mcqgIm5D z)9m#bME7?j5hl?YUNiNr^ny6sbC#x^r-tu!AeRGVF~2`KkCe#8vC#%<(oLvp4v<<|vpE>~xhLoaB%HJa`CZ9$`RuH&UpB>-Xv*Of z9Oo6MZRJ59?qW#ybqmtDjZz6T%|Kh-ls71uU^xqu(~5ny^ESwBg9ZK~cmwHJPMx%8 zqY0@iqn-GZ{igTuG^)ov-9-XI~bpGeog*0=+TcYG>-ArTnumn7%P6mBx1QSKQ{_n;a=}JURPrmT)9m!6e=V0|ZpbX3Df$|kwYeRv zRXQ$8=bDW+=0WBCoryiMP-zxSHK(_pGMiMh4w%Jz8cC*G&ap(S?ha1X;2(2bhOJ^F z@eSqAm-0MEap(r_KgP1Bh;pAR(ACi8S4MFO1}F0W0OhILe!Xe>-NWhDVkl4#HHc9~BkqH900Tr6(NSb)GUdWCmg>F~M*4Uz<7iek)u z(GpA~-cA%_Ju8m19F^?ueI84Rqe_d6q?WIJ&VyU=T4-8iR`3OT*&_v4vJX>UWp}FT z7nAR}ncF8g90B-Mjc(+>h%d~5?h=&euwDTDDxhfJfNM%L)MI|9g<7MOnRdrg)GcJp zGxp>+IThRK8o|AUgMlJra{mBPTnTUp^~Nis@Wf~B65JjDO{8|~{{Ysjl0-$Fc)B>o z2iBu`3!cDK4K7A{R&JwpdjtXPViFwu!>I$l812v4_Gc<~_ zI)4vadG`-Hp(zsQo`s0wxlLNzQnB#^>edkpgBSWFQ2pm=);SeY#j$ED6p& z@%28n#$DdsX&U~ozF3k8Q+1k^%e>_P;~jk}>8(s=?qleBR;LX5uZb;~S>9MaO@pr8 z>9-|Y>Niu0s$H)Ee5xXCtflvfoYc z!+29mc_-U5$0f<&INzQ=y$yN_T(nlxUC(Y}bj!3Pf>ia#99IzcI)1yZYBOlM1+=!t z6pYIM03lghg1{V)!`n6J7aC-Xqcb#x%MYCT=dE$#)$ESkE{U@ee-WKRRmv2{-H+eY z)^V`CySq^lRbGXO=B!)WD7_<$is*FMN0>T!tMBtjGHOk0vtH-@C7%pTT82)u6cM?m%8T!?5p^rRMkxDjH;+Ts?9Q8bURtJjj zPMu)|ykv{3=GhtaUg!E(U87n{sa>pCI2^h5u2aP}%dPxWj?NMmEjrK@Q_#2hBAZr= zmtc;wQ7PZ}IV93i?P4ZW9Qw6!GZ`%%qh%`Ua6zwS(yTAprWX>Z!u+90$F^&L@uq=& zZ=Fg=&_r8fvYv8!6I_(5JK3F8DXr1Vi`H%l&o!l_=uzv}bF|Ym5W<`?jOC4UkTLUd zgUxzEgJGhGY*2@hK_qp_>P>f1Ztg3o$5tyw&O1!azndJqI|L;VQhQgO+Qvi?JjguVqbr^PuE$PO1*!p%ah%te8U0>d&r`m%ezP7_ z>}g+IOJi~*fww13t}4`em)ap+-zG;Y4|?UiL9ESfYmQ_j;12a?M!AOYMI*`(U=p$n z8q%&K+a<}Tb2?O=7{6($)mq%a3zm>LJptR1Q)+so3#HCg&PweW1Z0}4x*|B1Fuu$y zD^|IRN4K^~115Jd2P4|Gj8tP}VeMrpYKPgz9}L(dwkvi^g}1XV-}22m)hKN|po$0Z zgIT^Nz4Gmn)lbalEsnyWqL!w-GOw_!QO>Gz4;9HKq(>Rzr(H?CYsOPYTZ1~~xnElI zFN(LCY<^fs{Ow-qFSEtCR z4ac=(_^QF}tl+*tN0T5|Be*rUW1++U00^UMts|M5Pu;4Ylqfhk_v`sqN}n=q9TBds zlQpd*hD8JAKGkyXL$%i}#1{x~zj2h{sr40$riLX#f`1D1&kty%qDyqUSExhO{e3GK zP@LuNoincrarcih(!3k3cyGjzUquUDSScwiv5r+6yAHe@S9#JlOy_Chy%IG*ffYbe z#sK26^)C-Yak^VvM=2by@VEF^sTjDcHhEPVo92oCDY+Pr@%fueGAETK==nqs?b*|_Z;vqf%l5(wcP+0W%p4xnLU zQSa8HPL%2ZM`v&1IQr8=Vx3HWIiLsTjMPE!c&P!%pasn?X}G1u04GC5EgWWm4A3cQ zKnL1@6ZNAtBLv3>nvQ@MG-jJif_u;ftvR-8R;&&&GJi2%nAq%kSC+tbTt@!@-EeQ>v#5Qi#1O?G z4Q}iM5P0XOBD~h*zuG1&PVmj2u4^1Clr=_}g&BLVq3?HyQfqsG-8*_@e~mIri@2qSN2_I!rPh*fj!l#BP@2$*k0GT~Ik6QG#=x`$n;_g(PCb(4~92ZfH z1I23EHqt;n>YkyexI-B4*wSgNV~@*xlNbt5@~L+%5&S~Y#;f5et|DLFTK@o~r_gaA z{EmML@S+Y)eLbh7SFlNEFwAo)Q}iH$KOx);;_+@Cw_Xi)+v8cM6L6y`R z(hu^EdsPVXTrN7CR*G9T&zOY+*07Sfv>l4F$&A$rB^&bD$EfXFml9bE1cp0e*8z!X znY#LM=~)v+B$-@bdSZyH4syD(8|Skq~~@ZerqyKR@ED1WXk;t z#tFyts>y1y5ce`mIRp6HkIJl#h%kr{f3ho$Q`b{_m1liI;Yo=jg%5T)Z_2AKoRS>e zPb}xI);mY#T6|`kfPL##6ofxb(@oXXHnz~9c^d4jx>SbVH4}szreX6r zJaz0d+PR+t+Xb5D?598@QhR5U{DpRRmhoO%v@(EIhA~}~sjgd`m8b7=o=LB1jW(xo zF$9(?$sI;H{&GPYuDT?S*#_0d2V8SmdZIymg=UPLjAU@w^vLGAXHC>rM?EUa4(#(SUNdtf7jo`C zK^X@>*XvTjWu~O?T(;1z;1bQi=DIycLelPSWLS*R$Uab2@(W`lAlD0XJm${pUKM^5qJ29^YxlWTiNceD4ng`?ALC70#@lo;%0AULGZ_9A z9D;o_$gPNOiju$gPG9 zFhL!v9rA}(Sr{Z)jP*uCf*p=3Ie#}9aM}~&xUT!n&#jsZ=2^| z?+vH=njKdfEu$(Fl&d7nZxeV$8(p64$|F~h7K|yu;B-FqkN*G&ee4s2c_PD&$XT(+ zYV^D7=A4=c2oJ;@;HZNd0)@-gMb}BZG!mf=;&Fai+(}WhA zkUj@Wt2DC&AtKMEU-I+78K*U~HynFcRgOcMyRy2zMp-T510Q(se>$aM;fsX4N#yd% zMsfh_p4HilUA=iehNK7=w-sp+*_j=*iVow=Shh)EohOP-vLVWvv{`x0G0?E!W8RAe z&ZAA$lI3<>%@kATfX8aLJtwE!BM@&s}8|^{U$rF;$1pU}tWE1V4)Z=7Kk!MEM^dz#>6m7TF^^2nu??40) zy8(>j9SJ?T&!uiy=w24lz9F4H*u^EZmP2bucdT+`j27%ze}{Ts4rm@8)pZ?bRk?~Q zvM=5=P%@!LF@QMrKSS+Qw0(0$(0ng-V+Wx3ipy_`J|L3M?26XWp#*2j$;N$2u1i-*A-{O! zQyiap<2m^W=WpSft!|Rcc0?K5`=Dgja)f28H;kl}lRCEXmZ64`nLNf#qiDbf*XvzZ zfIMc`SLvtT5&Om^WIcC!8s#iJ3$5xfEP@S+Mn?WAbbSuzNAX^tZF>tu?QodSMjMAi z-m`8J(G|-iuFs&P8|^0pykvfrPfvr*jT0Yx0R06@^X#&jfae&gG^_}BgOiL8tz`wM z-o}~0Tp;0iriU)uLH8ncoY!*aA1)~K6QTi{YYq=i_Pa>0k!IjWz+^%$d<@uiC$fUEPL@-UMHS;T||yaJrs0W@}jRR9+aJGS=mTZx;OcYs!Pu1Xn`0n;=v?HY>};JyG9>S4_h3 z3^Lx`qDLu2Mg;9RHMMNJVpa4NmS4@d8T-uN%C%Prp7n)I5=QV^wnucqV`mAtfshAt zSsE?QsdwU76KacP%SwtffrHfcImZ=C#Cpb|YjYB5@vDW-{mZx?YOUeV6kO=OAk2C_ zvOLIMXpN5k&FA&Bz4R$gN~=;WfPmm;mWo z5b4S>#dwM*W0}uKrn`=nVm&2+85tGR$7G;%u2;pL8r4nO>)JN6_FIc+@YeSY`-f52 zJ%{<}TE-VMb7Ysk9ZUT_dsk7mR4erKuLHQZdn>7J3?UdA7Z+ORMVFH}4d1wZm>J82LfQJJ(eza*Nd&xm@*chBu>B(H_mt zX0&X5Q1t$yy3Joii%`=pEnPtlgmdd(IauFZ+E19SWs7$N#>IH|$6D5Ft>-OyGapjK z{{UJgg;JL_>>kynW1Q4%KFM`+Zs7TlxnIcFLEsC;k5HP;js&Z@N4Wn0>mSmYBO|FM zP5JA%RXN(2lBu>&OZtcws<(jZPqQO`ftv?7!N*Vy8{6-^~%j#){` z-LqB+eta?OR_hKspwk=?#}%&(%IZ0+Dni!=Z1xUdMy?9T=Z5!-YPaX+HVif-XvcrnTOG9;|x#sb|nHsmZ zG}HdH`-)|zLbPY^At2M`1#rW$=DAwb^i#UH@BpRA+e{x?Jt-6}>TWOhRb!F{4SCj^eA;fMZ+L?yCdnS7q5Uh? zbR98um0P)6Y!V&Ij^dv@V6MSal--eQ!}?QROfF_$^mxx1{v%TOpICTNqO_7VkuXSN zI(-jX*uT7sO0n5)qikaVYt64Gl6!}mPrOcRktt#)cd|Mp)d=#f%JxiB^dBgdEs_*x zjPr`-waYiK)^6aC?mNqd#y1A!AIR5jV5=jk#yx$j1IJTaHnS5zcS<1(F$aLaoPIdP zbtP>Md*4&oG+9KNZKNy?)whjR^N@Z~-lv|@t;!*we*W7_g8WLHQ1^_t2T1Fb5BFnA-jh2_j90%|oG%-i##VMda<{Y#GP6oul;kXGPJOF>4I=*a zfe0W^SvWP*!)F`-2xE>ho{G4usR_mfaXB4ICIp@owSY#=J6*FbSSZ>Q`g_&5bel*( z*=~U2(Rf#5v(&F6^QD1)(K<&N-10C`bDl}X zGzX*jdfih%TpW4XeB-WqeuA{^CU=Y@F(rOdPkK~??3SwnfwD9TKLLu(j`&A$DmE99 zwDzw?KI%yGpS;^T2%Gn)I`S&Svz3&N!m?89Y>?pOk_o}>Q_Cwntk?i|=ADyd*t0Fc zD_M6B=0BIX{Mj5HMLK(+iPMwPvTpAF$YGXKg3XdW1!)^0Sg^WlDl&=r{uX8AeQSd8 zEu`PuR|A|NRUGvI3iSBoNf7N|alyf=_gX!Up(@EdMV&@64mS^a!x>6-VQ8HgO3|ez zvMsHnWB$9VVH0R8$&zMhBT~~ngx$pH0iDQ?`j@$-3 zXK^{=xpF!6uT1b>iQ-EgDOx*tEu*_7(A*uZCqc*m0A8k27^7m7>2@qtu zkVhL2UUCnmdhUVX4I4#;OwkM5fzO#R;C&aRa~=+hQC&^q)ohjn44@z*1-<_OD)h#G zmm-9@qG5SLUS|{W2T!y9%!Xx!Bl6_({{VKNWH{YoW|(OGM%j%9E0hK;KoCiS3PVT`Kj9b8X8oRU0z0Z zfCoWcEBHt4#J!e$&^H+Rf&LWV40u-hc@{~>*`xJuzQfpd`kM7C8#ruj))-JI9Zzz7 zN$*icgzwF=9cu8mD%9}ltmL+lqmc@@;DRwx6?^~(Pr|)(#J(AeR)oz0s@*8#{i^cY zh@J^rITWiY!3VLfrFEk@T2E9|$6eVUPH6(}7+!vy)Z0rBZ_1b>%VZ9HM-^8ml#O8L z=KQF4cl|~^53uj09v!dCgvEXqXWV|Cir&eP4Se40C?P$q3&Im2gH4b}p>qO6PpKq>UzCB?BFeOCG9MtIk=YdL`?(9_Q*<8Cmq`rKt0by02Y?T1 zR$@scQ!ou~qLB@d5hC-~=}G4hdE*|novrJT=}>?U`P=TG?e1%hxbY4B!<^hQsqDN} zsJ26Ap&8dbN9RFp)#b8XT(Hb{QmZM*`KxO~@m#{qAWi&e#kdYteio z5?biSQ~v-hj{N%!ir{Unq?&Y#8;dXiitjGsaia@3NnD|iE7(=iS9dY2_D3hJMX7k+ z>RUSkVzDw6dzR|pc2Vi?T?U8n_eat+!)##^$m8dc`jh=Dqth(nwTd|7K%o!-2Wrq| z7^`tzNK<_dSokMTEC^={o`8zuz9MKg7FMx+nE;)E3ZtpO&P{rS<>Xd2vQH`b5Of_$ z>;-wxjV-QMOk2y9Cid)*5Od`xWgh;&TE)?V)gx$DoSmCF4*=+|d8tP>!{x-?NB6Os z_U6%Gyt8Xtl;8}kgQ~Zvte*&Yemx^jfeMZ4yPW#D>-`7eU1DdP)um)lH#Dw%_2bJq zEVb3Mn+#%HdmNmD`C_uPi2nf75r;)NuAAbkEBM+#rzK&G9CkSW0QFWrmjgown0ROY zRfN^#c3#jn-dCODjSm^g;;v5qOa@s9E_hMtMvnrrw5&37fl=!_RHA`gvwZbzX%BLl zPQ&NC>-;sYqDh>^A1{suYn}5RH7^+pS$vy_=TtB`ls%W;X;ocbA>N8!` zr+BF&hNLxdWSUiscCpAIf=Dghx1@5oX2}C2k@T)}#=7yg15UMkNJeB~*m_r5+imUJ zYJjM@z^@k6?^^RxxR~-;&pFLr?(ADDA73rD)7qv~0;pSRD|u}d#B2&kfz*T45t@;% zN{q3u518MTm&OHgzY%oZM^w@wipOz@9JHvv#m5=0fZnL*buqO#&}5p`R|1~c>N0v7<5K?gI*F9J z6Ze&fz^_V^bt0r;FLl_84IfZ-+}5oc`ne1I>S3j{K()r93J%K|G9S~A*0rXJ%HBUV z>fuoj8=d-}YHTrDO&F5a?H?!RQGkCA^{Rt1b227mwuH63PQ`gL&D-d;&fL4pF_GN6 z6Ov7C_=+ZuX!l4?Qy_M%t4NW<2HMye6y+O6<-wONF5KIp2aq|)@FN5&g&x)EUI4xOPK_n1plPk7Bog=mTc5kpj&s5ErHA;(SK&Pu zOrL$K{XKI~t<$CK(T%%FEyo=6&-6b0_oyItwTzvh1w4V%9kK0HFQ$&`dq+4EE9DzI zNIjJLHaGnQju^% z$hjX{dvc_AHDkn9t!1eOD}b(Xp6Ys4ehIF5txoz*@exi`bnVE7AKyy{cV^^;eAdGWVQ;}Om=yOeK zNVK`*6wq-&HAI2yNx`GN9A=OJ9Mgalo#{crpa)`;r7vo202$3FG|(w~Py*6-^`pHm zC;>$j4@wRP(tsC(NrORdcpOp^PbAO-;|g6vkQAH~jPYK-rfYDU7@>)g5;Orep1fC% zWM`8}=gN!nNXzs%sGTopjVV_4eGdNs#vUr37^1a;d`+F~>x0E{zwnT1TZ0>b$K3^F zysl&{>{$AflTHnh+O(wwPCTg2q`BqH%{K8Krr@Z$*cf1xP^TYSu7u6}0 z%VMEBm11qFf-8wtKK}rnShkVv=8kCCtH!~U zj>H}+ld~nOqunmxp57FdrDGvr17q5(*jr1fY4-^%k0ix%rH?{60D4wErmULYlWTG$ zC;+=aM86mk%hziT2&ru~8LX~V5lf+9NgQ#WYnv4&-IF>n=_RTWEz|ghXlrQ@<~GRm zJ!?kl-W7-KcI1#r^CR@R3KRk++0%8BBpuivjRG^|+%Pua`qT{(54Zx3Io!M&)01vFE021qEZb#? z0Q<}`eT5C#T&SzN9VM@c^;=6+hG|xI1FWm_eq$9z&s(|FwVx_rps6>OZezPLD=R4nD^`g3!w6^9^AVHG2bAk^{ zS0MVVHxXery6s`NJRUgwYZp_QB$zTs=0zdPd25v|=ttK+)eR=X%&){Xgxx zHmw{`tfojFBm|s-Q@_7KR^psOF48<_;l_@dw~8WpOl2ZGD~#@Je?gP^SGMZ*miHQs zv~pcUwl>J1Aj>nJ{0i}Zg>lOMEIhZD1T7;bcr0+g$eQ+RwSo(I82qZEabP=ju1Ql; z+~{>&AG%v<$J%Y2NX&>-DPhhtT{^x{b6#Wr00||(*;L#Hw~<$=1fj>hceZzEbogf{ z<#@yA*o@Ycr5P`Eky;Xu#EYb>X9qYqKU#@xVe?F$T5mmDXwTV8{?;^sIY*Ntfl8Bc2B2+@DrDrsa;NMe&c?Tg!Ugn6H?#pD3n8yN`?BQ{+eMMLEm=XcO ztWl|Omjw6p6oyQU6a(o{rP1iifOs@`%E_Fyx>ml z>*ehqh-1gKa<-bZ(XQ9RjAMIN=C5*>7FR9<2^urKN{!*Lah!T_T%C(3h9fxL6p8^P zXPj18>PihWd>r|r(b7Sx&lCk^Ff1hf?b$ztOQ%m9{ve9YorJeBv4D~Ck_K>3Vn%-| z#cW-{i^7!hftyz+;c% z93CsvmKjGx_c&o%>g=UvjTO@I!>;PJcF*>Mv9JIG6|UMt+ueCGBW{3gBn}U~Q`Rj$ z$8?y^*-%I_J$ly_G~+E!ijM0-g37?)=O0=nx{fGhm75BRSm)ZBSZ5tgBgE3i>dL!H zkU{TVH!*1QDc&x*T;Ib7503qj{#hd1U->LhAOS}M=~_bJWmF1&@ZIzkEtZL6K%f9H2SHMbXz5M4oo#I9xPQA8 zZ0GP9q-hm%!8}zU)O=;BNN`YGGJ;QK!E$=~wK~CYT>R&MLW4>s-{=zT_1j zyJIxjY&;sHb8Oagte{~eI5_NUqA-r6{uPn0TtjVd2b}v6q{i%Y;MVh3i!-MyNf+$n zSuAqWDzfF3SmSue&OLFF$E8lIj+Iv368BGrV}T94f=G!xvB4cZFil#>BaSGc@K{xT zGApIuZ7Sm=p&0xt!=+u4I3R);_3K{jxEu=d{{RvAUiM_F*0MZ3cWWZMGk)}TVcXib{?d#%1lNOJ ze6wjIvYam)vo3RU{qnz!WzP4J}#QtGT`Aj`cw;V9{&KHN99?2R%z_f=bj|C zZ9>em0`kT4CTot8+ULr3!f#b2S0Bo~XI;`-5+;QSjk=FRTn@9ROB$Ptaq}-!>t3b{ z5|ZV699W7kbaA#?m)YFC<%DyFY%G!|3Vj81v9jDMv+1_VybyMdefX@)iN4Vq+szWT zIl){3Q(L{k#A&-~j09BeUbCp0I9ypzTT^>_cPZN+$_H<)RJOOoySP6tIIRoY8SV)C zMbviVIl~YS{qi}jeGfy5@HBSQd4zW>fPWMI70q7}2tx8^wBaUWi#_qM$2j0tuYx6e zU3y4aB0J%1iPV%B=bUuoJoe3M-Dz=Yz)3PkCO-1CxU8Fdh+*+olXV7ThDmlZZ^#7k z)6)Db_<4(-wbVZ)l?8fpJQa^U# zvN7%4eL40u)V2KaE>?M0=*GG2A4`fUn&3`eJbvsb!u@gB`i`GUx2Vl4Ay|k2btb$# zRYs=yS?uAdrv=Q7eLnY8yt78OUj%xPiYA4TrG_=ji z%v7DpfzLuKoPa|l3eP03kQyzg0RZn7X#j;TQmGjPq41G+ryEm%+i+j zL66kdt-hOWY(IAj2LACGKbfTDknGvU+vyU@0a$RSjm`chu;fytu6i2vk025_$@i}> znKsD$cQtU8#VFXK$;YKyio$Q~lYRzb6M07+jx(R>P{T6OK2vcI0#ocU4|&#n(zew}4C?esSG z=w3+}C|px*THupz$En=uw)&i6-r_ZQnZFhH6@z*3^$fdg?hI^s0!~-_Yq63cZzk8m zvXOuXO0-YY4)RK3LI^zV@nmoRL0_zYqHp1i7KmPz)kN88i1w<^-DJ1UsiP|gH z=PJ?iD3CIx3Q^k-*g^FPW1GeH{_9@ zL&YFCZd_y9rZMGO6*==evEEy40Jb>o#V|nIN|xfK^I{Ex-2G^I(0Asjk|L@wn{sKU zHCN8pR>xK1p4YLcz~uU3t3Bk5FaQ+nD~@K7U+)MCYSXHGd z!mULn+?JgV{?XLR6a&CG%><-D-Lbfy)l$Y;W1M`%@Nh>o=_W5DZzHclE6_~ytW}%| z9L+G!0otRWu6FV?yM(lyZaJ;pNXZ?H&JjZ68PBg3&)rIk2F|360bYJKTBx1)MWd0@ zX$j@oKnF!ZkIJ=_fo{jt)(`fKR>WHuiZ?|VF8z4vik3}Y-V_Scha_a;wff~ce&bI< zRuXeu$&$;bS=?R6E2&%`;5Y>P)jQn|7`)CUK_Jdi$Q2CQ#iB~6W!tzDllXs)PH$57 zA1qyt1`Zpjs(7j|RU&?^RcO(RZF8sT4lS-NMbDbdRI~Cx#z#&^BDLr79OBkQv$KZR za-W~?H%>V>vN5cHwNgetMecZOOD2g%<EF`5!H1xay;f#Bi9*h{DF&^!?$>X1>73vX6`ha_@g$pFuVxZ@59V@D}Zdz9A&UzFat8+b(M^L&8VBj9r z+b!hw{u79TOPhG($vm!94A%pF3i-(k63AOD2V7TidAYnXX=;jm>%~-6?x6SlE1M55 zXBRU}bLRI_#b%ahw7(^`&*=nuBQ1HKB_nGK_FY_NHwl9`wv) z@tSz+>L@0|1YjTOT3S7$&3YnHg8rtdf-!wV!-=%ZjDxGA%iUS~bfk<`(xjiaz ztVN{4&kQg7@-ZA_Z3GN{0{~Y(!Kabsk&l}r7Gv&fj=Vp4bEcE`NWoQLK&etfc+Usc zn#!re6(5c$1h)$m&VSg6oB0aP*G7s-Gg8+}h8;doKvOWlmu06# z8YQeq$PO4U&H44F#ky&>Ma9jrSXb*!sf>*0C!JyUB=-*!ddHadbk5m!f9TKgH+ z(P+0jy)HevSjIs-=A+Pc1i!hyK{=L7d3?53Y?K(|^QT;0CYzwiaUlU>jJM@ncZD^J zEjsF3tL(=e*#V{h0G@71BX|2JBp<@Nu*tUB=VDs5k5an7hfdRExs`&XN?EI$@I-eS zrTyQCuTu>piLp=| z6OsYz?V985uHiaCw~dg^GjEbZ*Bz95cc_*p9OsJYr#U#@TbVdTJ4B%dpNK-pyI36c z%`)*KwqLsL=b}crWAqiG%*w;%E$pKwA4=P|mr1vY?u>#X8g3shy~E#YZqAS#TU^sWZ`U7C5N@|mNGLJ3fC z4mbxDylHdjX6MVX>Jzqwr^XW2>d?0CSqL0eNj04|G4o$QKj31#rLLU!sM=;>$*zTl zj8=@#+BdoEx_r=TkeRIJNmWh`)5!XY)3>o@e6nzB!)&!1y*hCI5!)t?5r*Ps8UOoA9Sd|{c&D(dJd(zcM3ImbZ4uxLbE4Vym7*(xjT>atV{iGQoXj{ad|6b;DFml=~yfjy0dR? zx+B_k4-{y2aI{yqk)Fjt=Dg!n@ot#XIJJ&R0qAj$=l&JKm+L}BEW9E!eFb#V!$umI z&a9+sO|MDP8+g!qxh{Xh^s0@hUEE|e?m_+);C@1q9WHkxa0{PPTNa)liPRY`Sf1*G z1Xh!)7_C{7jrKCdvMR8AD~@?RYgb3rVzWm1WL9qRu4-$&78`Zn0nP?Uen}r%hFE?; zUxml;)(%jq8$C-&q8TH5$=!+a!xDE&-{G3i)L=HdO5E>~JUWzI07pMsmG9z<=x`{l54>f7Gwd zAocdCZ|+tYz3Ocu#%GJnidY%n=3(Xxk81KMLDZh7r&fN>9&G5e$qkrT zlWQ-BbvZ2zENyO3v=SbeJ-(y-E6i-$Ak+f37AxBfxFgPRhr6}lKH)d%N2*z_xQcPgu+M^GZ1Rf6}mNwc7 z_Ul~MJ1#tcDW?OPLP_sTW5y^Qz?|n58|NaOo_VGg=cQ61v5(W9D>~qHK9zZnPJQaM z<2m)kX&ub!Y*rqkmlPobAkv(-L5i~hv~ikHJx)z89TaEjKn*SEKRQg4jGC0&%6VWT zAMGFMQmv|O`I(se3{VAS+e)!JleG>#aaz{ebn_3~+mY{#ex|l!(C#NdZXHCQ-UB3$ zdI05?%=Kpt$5qJ7z*I{qr`(!v}d02mV;C@x794QaW8U+Kgt_4eyDI<=9 z!}rp!&IBv|(dY6Nsb}HKb>>3yk<<<=p^_4xHy=#YiyXfx5tJRtIH+^O)J9e(0N{74&}i+sW@#H^eApf9 zI*r)4CDh2DN7JL47K+LgCnp;RN|2yal{*ih=zj{kcNQ-W_Sk`aXhUwNb}{S*X~pO% zZgP6RhV3t>d$#l9mB`-1ly2GfHG4y_w$daG3Z{CO&RG66(=sg4?NT~&0Id6sDhFJ~ z3{U>gL-oGt_4Xa4}NSjfako4)WRPu8hfUdd-alP?F? zv8z(D{i8X+Via??kBLKzt)5wJKJ83u^S zDJI0?+*(F68A}C0h3skZE2HjwD(CpPr{C!pmJj}sg^xTqPRHNR-6xuOV~{x>%Uwp9X`!2D^W}DxvBaOcEBS&C z=UPTCTM;M9apaJ2%2bc<3Ic{arZsHWAo z1n;n=Ks*dnLHV(YRb)BbKyJgasTvY-jB`jVIV1=(jQiA~n1D|dMQgXI&(fY)DGDQO zP!=Jb8IX*YX2Orgqx&lhf#wVd*MaV8TbcgOe8Rcd0ms&fE{u|I0Q3rSD;Dol(CFpm zrGDjUtjIg!0AtgN^bZbNO>d$=xlqnL!Z!Cm{cKkX`=viJlD)=j(R4|cOAC09-dMmM z;}zS7__a7I>}gq~t9Fd#3U|Wwt2ZlbBw-|LjP(_nsLZkX7U;xG(kzDuqA3TTsp7R} zwT+|?Cro~I*6nkVvzXWJZM>-M;Z@0P%yG^Ib5O>*hN~PQ!lDqz-`cf2Q9NS(#3};9 zfjIR%pXMshhP334$)%AoaQx-FA4>D(7h4f-i1d?GaOw4CZndRqZ8@6FeCSwYp$bZ$ zPJL>XhlKSj`9t0xvBCY-19|!maroD8cXVNc=~KdTG3i|o1x?SHnEi61hblPf{vX=P z=0v=?j3^)$A&E7WH-4M))h7|-dkDEOS@6B26!0f^Wwc?Fy;)!w|BjE zVPWp-bLKOHno#Jz^Mvt|PZx@;oQCqH3$d5fagWx!TWAZxqBt{viM@?5xYS`6VvM zI&D$k-y~O{J;)p@0^QrN`d6NK?%^9!)1XAiRCX<&y}0B0Ru%o)Z0W+?)lA`U{z;LL zaKi`cKmB!gPl)ng_>_TBUar`c$R8^XDFyb#L7W_l^D8S$L)D|K2R3&RftveW3Djk6Nn}a?P!?-dv@; zN?F|Y9S3vtuHpoaCG2YCa(*0W-`xzzOgd)TFt1@pE#;c+J*bD!y1ZEoh` z9bqqJ5y6&QNSNYe=rhhgO3;=we&L!1hs$D-Z+)Zv;0FX&yvdQvVqHe&_9VKE6{NaU z1y^@`=LCORom$vm!798>e6bHbG0q74r=BZ0^w{*v^LuSKnSDBhxOrsA#yBI><^rLX z;t7JqJ;3{143oTYgToJgrF4EG)MafeQMj~SrXb#DJ9izuas6wA@CrPd#m*-XA0YntjO;lT zl%>ryjUyXDv(xP&yGv`keL-VsT(t2R#|WFcXSw3J?-6S$qgp9a4Dy|^uk$~Jbds5) zxpeaY+bC_vkm^Sx@~#8oHK2jxmTR_UKk@AdQH4Lplk~x?D>%96?r9d|8#Cw)J5AH= zY~e9mLS*vSbejfP9;W zHPxQ%XAV^EVA(^N)%^_@9vi&zM~PTzuCPgE4(TRg@|iz{PrfUwO)X<_9k#}EzTSdh*W}_)}2u zrk6d|jWM{>M?PGP1=z+vm z=(Aw3QJR3>IR5}v?4#2>QYtnVaOWJ;e5^1eV>QawrpT>!mzRh4Z zZC2vd?KM43uA^naLkg}(xdNtzNK=jn>sFE;%D;)E&=laYiWJ)#1YXLbu19a46jb6B zIL<-#te+HWr$f>jSf4XV@*qCGv_)OmZP?h?SX4o&bMIPQl2u_(3=LNXoDb(k#bZ}fv+}hW<6fc2 z2RYBrFU-!*#7`` z8mxH7$N(7aR3nvyDvtZUP^^EwR>kTpdsqihF@xz|9N|_@?cC~6Rym8Q?V$x3!nipH zJt~Xq!3Z%2GWTxAtLibVw{l3UfLk@5P=lOTtxFFF70s#1Cet-1*Y56M^1evp_8k6o zl`M}h4(vy{rP+~6B4Trj>4i8$;Z`$ky~<6D@How1)1r=BrHWz1oCfr!JW3lZzzTi5 z(X8>LS$RtUp?c)>t7_-n*B*kKsC|ywc$(t&J9y-50W6zUkELNb`&OWi$V1sh%0w>(-Q>wS z?FbwYtU8Zc*?}#{0_NO<{p93Tbic8WV3Sa>P~UX<>w`qzA@KE>^Ca;Go&f&j!H?t( zcfyou#>n&MR#hc-Z(7H0xG~B90DW`(>vF{q^v3UDfPX6U#`ull3sJfCPdU9Jg^53= zD=PEjm4&G*r0Fuu_~J{a2)X|NbY!2*Rko}`z0YL3zPHnCrM$nIJ8Ni{bIT)O0beEf z(^$9HykDtmR*?BHPd4OU)tJqTT{Bs2!2a=hS<>h(1CLsd5pKiE2h*ChsomRMU$nY?n^{}!07PeLPJ5lk zslyzhiH#MGG4mj9zM1b@a-?MD+6iq`pEBLUf;wlM)JbBum8im@q3%kC(o&}34iW3L?4MRtAXP~MrQ z`TKtCdG|De#gX>_TB$f0CqFmqOZJvOeB-tkiigc!c8mi`5`djA&?AZ zY9usd>ro%HjjNN2t7?px+&K5EdZgrEF$Wt!9V)uqe7Ryd=trdp35ZSO86KjNM+uHN z>S`vqRpbHgNvcxbZ|{@s&{Y8?$CW%qSaaC=8qT-6k!Lo{z{?pSa(eqz`_6ig!m*dY zGV0gwEIOo6%XI9?9IC@{9Fe#JDXM(>2ekHjosOLYJYmG8v(-rGYVGE%`T>BqDK2yD zMK{CRankgdZKZXRSXX06xx*3Zo@&0eiDOO8$R{54$;t2Xv#C$rxwCEJtv^sf3+;XU z6(b`oU@jE!dQx>HlVXK9 z&S^5Ih_#qB{VHqQovkCb;g8~D*jG>CYt26Y0K*oyT5P^-MMm+Qvc5iG!`C(89y7gp z?qP|Y_9*#Ly02=?@c#gbwNDIc0_#q=DZ2tl*ltM|(T9JpTD&7ryq{A{W#su+L+R-! zSH{qP8bz2Pz{zhxTrQLGOT(I>b-I#GK45q!{OwPods`Ewy_IN$w_4 z-_t@dJw>MBbFU=+a{uTcK8qD!EicjHPQr*NV=f(7JC^%J%lhzH5i^dTrJa`oh1= z*PzEDG++=20CVeHe~T@W`&iShV8_}d1qcT?8Q`DRxFYP%*Bh%NrtmegS$K3@Il+@M z6ZQW9>td|KbS_uzDVA9|2imqGu(eMQ$)==%ac)o%)qY%Nx$6b;H2ZjiaIq-k>0H$9 zDLEZ=UnEpf)c9uDUTQ5NUy>9gpL4+ewcOg5Nd8~D?_76-oGjM>9!Ok++;d%Yc5z5| z$CV?nf8=Y>rKEH`+RpOSyt7C~LJ$XXCA7M7)>)LWU(l>x0d4Ro2Yz!@be9WfMsZZd4qvT6CImcVZ4b zDfY7YcG0Ie$6-%rXXkMhygn3i)L#2Dq_vVZ!5@!mg{^}Rn0+fx%x}B5K9t7?t75L? zSizq~hT#19t(Y|P1%k;KAR>%?1wUry8&@9Yw=}g4YGPrWWLGvO_D4=1OPsZ)`>U6g z$r<@UW6f+2vX}|`F^!|;k~QRDW2yB6zj~Hyb^B0{U<^4GJ>++{vfIHXV~D0qs5dZd zjAtIa*H#x;+~vgSG7EdVIPUGEZP7gE{T>$_0gi(l1I~XM(Xfs;ip_O3h+nC8jy~Z> zeD>oN9mm-8?Iz;#@+jlGgl`hE?%9r*`j6*cdwJvCO8umY2aeh)30&;U6(^5eap_&E zN;{mqu19@6#9GJN?6jMPiJ*}Sd5Be$o<~1IDQ~YA`%o-vEK?#A9`?wQMgb$7es$&V zs_Ry)?k1E*+_S`fQ|?IqwSP|WRmv_U0%hk7CoD7Qdepj!jZtKuy?PJ#<-*(Ba@@+>58*m&dD`S-6p@eZvXx2!?rO*_J&V+L$zB=UG4O6$C5 z;TOJ{^*u8$F2UzOPbUMa{{V$~&9d#kfG{A5S3aYWSx`~sx*cTs?$503m`4Orsr!f* zA3<05ZWYf%Io`NAt%#-hI@R3fu=#?8Cjg&%t)|T#`bz|m`BOyuT}}>j%8+}4F;qND zsQ&bHRH8q32LTmJyoOZ(R+uRT9ZR690qqiS536mQyljlRR=;B)YeDDKZtT&OD2h<&9pmgo^?3SZJ_q|H7EQb0$FSY zzKw34R6%B&&B&3mr*=E^KZ8{{sO~FR^2TV}9glnpbZD9RW?`RdlJ*!@?o=#SYY=_8 zr^N~%yCVDIh!o`5Z| z0~i2%)PfSHv8J@Ay7d0-Aea^yKY^eL(MLiBdFRDQU&+(uQbGo0IQ6dlF$dnV^|)cX zg(0|&WJYZ5agp_<&B^Z#MakIke3JarhXS;`K(_0xTt{sZM-m;OP+(SGao)NiE@YXU zm7R$c(w?-%VV_z;-D^TDN)AswDR&J0DlptqgWijUjeT20y72a@&`b^p$~frDP{g~4 z&3kUQqS|<)OZzp(*zZtohE?o+K9%G5HdE=>vfA7(*E6(P2)aXeXwvM|Lcg0>q zodlJfoOL=P^fX5!lTC6ljMg2;i7X+`*Lhq0fyd@+BIm`E zwnPykd!Ly90EH2uIdlnBiZDS8xYX|Mj%AsTnuygY__yJtw>Q}KTM7oz@0dQM94%J5HX)b1IW^(S`NvvT*mR!uG#NI7Lx{dY`2ND25?9 z9CsCV282Wzl(SVt;mF81^yq2we5x=wq%z>U5wQ`dQSHq|CESiSvtV_{Z_2A@Ba_8h zx6>~!kM6D9kM9wI`IT6R@@chtC=3K~uNc*aC zb6Zepw$b#BGwZjSLpcborknj-Y~Mke(~raVMM0BnW4=!{)S_)(0LMTvQh9raG_E@s z)}9};QVfOWJ#o-hyf!e#xQ-=7J(iTlszE?h)$4iVzz%T!n9WZyT%=;vw)G#4T8h|r zAB{{3j#rv9la1IFE<>=#iZlS9!knth)2Gs-AyxK=$?8d?&LcR<1HDbhV$!KRw@P>b zf^uq$-IIkR1COburNpEh55|)jb<@mJxctRu+(wYF+QqT_ImxL)%RoLS@hfv2ACidEXVm*PyO(apt5f2#1?l>dpYOA9~(3ex+9Ex`U4Gj(F8%FV4(<(B?b@0;Y^91PO>V2+Zi>JX|8{5wD*q~ zK+=#C(QqnuV6IGU@}9=B!|PFybAgle6xbxl+B3U6f+}-rWcjxo z6Vns{S)^1M>))EUr1;*-*2YU?Xu|3-CzCea*eeuI-yk1{=> zu;bgUO%Yc-5Hb1FD6+9eRP8`B$tS6%vIczezq9dz>zZN_#uGUM*b0mm z&!F|G3Y;8bniCm5p0v<_C~T4W(ll8(JkoDD034I;P69~|IVU~3esu0D5n}_Kb?IF$ zmi}g+8SDXEeGOr)q>KO~Z{yatG>C*YOo4E!$2{k)IJ>eZ8zOd$O>+!qAABFnn$pzQ z%(HcH2w{Q<>s0L1exeWe6rbr<@5`9jA|Y&bdetoq8?!!o@q6dQR}u9QVL19^{{T9` z+(%03JaU^nTXr*+Aq#Ztw0c)C<36=fR#r4_`mw?L^rr>~t_@Z>+yDfclyEtqW6Ocq zepIM_PAV>@rbwibNR1hfOaYo0Bhg`)2qQbTorC#SPLFERd_q=`};wi$u@QRwxu+BUadTFy0hH1v;=z{zvE7` zSp}>r<7o(>gV&`m+SgnV${~_DVB8nJ|VRgJ34ncUv+ z)?t!XUKs5eKaF;lGI_38rW_2CJJ$`Q7lz@3sTp3qtE0EnUfNyCH)o7jO*vTe>nF{j z*1+(-Lrfzb_yE<&(gDn8IsPU&6@zCq_ZV^lgWL@MwXlxLxCiqYu8Z8|aF;F zPMp`5_@3TG)~@aya3^pOp5Z{ruIl-wCN?v$MQpMv;1P=AHMxG(sWHw(V_@g={cE2Q zXB3&)gp*C!#anfB`H&x|th?yMHz#PwDhcPMZv_EPdwNzJm)7RySZ4d-i2ZAZrpKj0 zd)G7|47O0St1_@ZGK}M%hNe*w8{C-3z;p)n(Qx1{qY2$G_)SP5Y!) zic@1T^MIotUTQ^!LFb=(YQRU#l70=(>rM{Jxso!2*QosJ%&biuYv?|-`+0uNenxVx zGyXLaFl=Fl4{=E<90p&$ys4 zpLMF=#c6E0k;okM9`&tBS+-0e?xb{H6tvWJO;S5sBH3;n{{RSIdg--~08ghGWQIWh z0L8QneQ{l{g!Bu44B0Au`PgPj0`g62>KCz0L?+r_o9@VQis6N=;|g}jg}Em^g+=h`GPkaMK&$qg)Y~b+ zDWkYD`F8hDk_d=_^)<&`X_|Y)vdwP3V&I1y9zE-%yjkTwQV=8Cm-juZpSZV?$cge5 z+$Mgf6@)I4brR}&rM3Z1diSklY%VLG)5$QbOfjaKU7`eT+ zl100meV!q|fLp1@u*t5%LCtx`#VtDPRnx6ww2a8`$XuZ6axukc%Ny!z8A{en@lr`S zCnvsp*SPp^Rf^X@w|hI-?yP{>gZ{QG4vsqx*~j?TkT!#=qLSKu!hzJX5lL@(eWkI6 zH!!x~cu}|?uUg(wRW>4Nt;wfBCFIgDb@J(KEb3yuDFrcaUPkjE5cezT;46!&^|;*E0M!DrFvMn$OK^54lb?z z#B|Vwz2-Y$$4nY<0+=QB6*4k60O0yp3DQDs$Rv(Or6a?Y8<(G@UB^l~_31zv{#z-? zC4ZG-c#imKur18wd2611Po;G(fxWAp@pY}Ucy93B#w3kbaO^{NB8L?vW;G`lba+L*?uJ3*@{cX1u7+NAO<5)jWAZ*+`ob*XH$hp8k`yb?=sO+PM3q#0vVaoB!Grau}0&OeVpTThM#d9$_6$1r`;{Y@=w)W|&w7A-M*qz4Qv(4Ct-g7O7*JDY4|Z zkpBR)%{v9fF{nH~F;V%-r{+BRin6v>5Bwol)sIR($sA#UZO6Y9SS~^)B!E9oO+0UI zqyg@Ep)K}*sv~Ir6VveYs|q3Aw5tg}x<)E&7aP^swX}B->9b7>=WK|- zCmbA})n)Bfm+p+!2L9W$nEwD~YS$K7Jc!Q+@H|yVmj3Xh9W~aSr~3) zJiI7bn0kGG&lSw4guFqi%F|p%w!yhCdosoP9)C*i*6YL)CNA~s*+)VYEP4D?1Nzl( zvMMt;QM{a%Jd^nKu7_P(%VQ|fYgf6Or}%C?CesI0it6FmWi8a0)B*kD)cq?#ZGh@C zQUQ+q;-p-F39fm&yPL&XSd78`$vCI^lEk4w>Dj-=o%WwACnMY%id((o%cC!R;aa-x6T*o{uKEoctD!TF;Jt{v4cuH+{bi2ID*hbmlV9G~eeLZW|C9sO- z(A&&_Mg|uhSKNA4dp9V%5~`&nc-FA9#>(v<%bXL~Ry!$_agtAdg1tAydR?urhV=;I zpL?-Z-1z5d;YE4mQJ$Q)^sM6t6=rKGsXL;fww1C01a}M4m2{Zgk2r9Bs`{}&RDeE$ zg*oSdKiaFY3~E}#3V!VdG2Ve5_TwiVhf0_ra2RLmdQ(~cW0S{VwCpR55u9W;KMa~d z2IKP#AMYM1{$T?ucK$rmJj2w2Mn8&+3{RO`0fT@)8gH4naO88)aZVC}#?tuy?*sFw zzAhiWG$yBdvGZ`^I)UZR{NC{JV1iU7R@nt%+P zp8l0XKr%QnAk9+2r0Rn9br{K>30|B`O?rAaVUG5vtw%rfd6|OA=mIZhoez zx#QlnDBSSYnwKv%)x=?UaB&W?N-gEEM^u=(tFny->C^83_>bdmE#d`g; zQNn=~VO4OoGj=K}<~{C>_Ul%ONkV^%6;|8AZx%fJq`~SVLL2k0uzk?mM?un>>(pbP zYR{RM%8FKYky*wgiFT;$JJtBlPQKjKURfCQsZkpscBD~iDYiD?cmM%G{=W39FuUB! zkj8#@ZKE|&VZh69D@#O(on{mn$0sAV6yvM0jcpe(+t@-PML^DWkK#G2`R1s!5^M!0 zXvpetDk0~_3w@=SD9Jkj1MxN6dme4BCbf9B*6krYs_;Eca3titm#jDCDo>S|vRyVt^1oN?oHQ4h=M|u%z2}WE(JYJt^VY zaZcKMW0Oh%%`!&$Nh!$%Th(|p^9-xX5g;9p2mC5T8@g1%G0#eN709fo@@_ViWF+Sd zF-v!)dGB-)n{%OfhH?jTG4<#1u4}}ag!cN?vcSbkN0aDseLo6R@m%)0UZZhyDw$Rg zZN|_(U~$L4Bl4{<^d$-VOXz1>v}Ybpr=Q+SX#Oa7yxsOuF4xC60Au;rpy?1$X5sEQhy#Rn-{0NbyDnlVlB1Zu}K)N zM+Hs@s(N;ZX=Q$H9^&G5WAce*EC<>)T7n-6*3}4yTbC7sw_y_ULaK)S0 zt>b%bCr{D!d)Xn2&bWiiR>nzV(;tm_UxYR7FGhmqJITy)NhU@Rk&nPtza998`%h4} z(;|$)1*S}hUv^|Zr`UfpOt$c*l|Gpj%#2z`UCp(K=hC|=#YPXCsf_O0<>+=cml0f} zOB-&D##OP@R598A0BCBl$DBtQ-hU3Ctug|Z)*ug*1;uUnVgnR2UCf)~n*oP#y@;nu zH)x$FX>j9k7|*SBIQBgLYwlW2h`hM` z>B)x;v4d18q`Qye%_`e9l$(U8%h8uS)HKS_Wd2eiFDO>@u1{Bx7E{%WZu|vy(!&+f z5Fu`TMk|xlV~pxg8jPv{zl~v8H5;86W}M6m@?^4+$oW-E9Qq3JvdwWkmk-t0kzV8C z28z?ew~{wO3lGd7oIwv^*53;^5s62dvcES!-nby z;a)>+yR*N05n$oH+U3wUQx@n)==b%LmDZjwg(v9GEB015=pRkFIwod99_ z*I!{bnF6psQCuR7!4=F$bZ!FzJu9WqZq`Gvp0&u((ZzgM@bFU>yKcDNL{}Ic{{V%* zr{!K)v6G$+eLtyb&}v$2)>kMV8C0tS+upu*)%5$FBUPTpNLij2E1Z4d*#1;PT?;b= z1k?PKURaOB)1N7yI#h^t#&UllP&LK0e|033`nlRaAzA@2X!;_SKGfzCV?Qio{iBMc zZ|26;Ay?5xDTSq-#sOp40YDPG+>pe#<4`oGpyIS*(C=G8&A zQS+0Hr#Sl62FM)OZQ^H%G-lkOyEqB#fBjX%ed2RXJE1f?cO5Hd!m$Z8oy2{}nTOYb z{c9WC*G=HmQ*(Ch)GC4b{{Z!8l#+v{D&nUR(6USShgd!yX_w@-vd@4|+jy%Nlhf0B|Z*0hFIFXa_%9sph`y zt9A4g<20j%93H0>0W6Y|c*qq2KK}rP1d>i$CWRUG%>o4+9rUWopL861(iFfMJt;Pi zS^$vAk=LyPM+bpNmggH#{c%r@GVihYZ%p-|L^nE&R!|u(7%GmQr2AJHf3Mx$$aXn~ zGyE(4YHt@^6RZ7^Rh3{E!Sy`mx%}kT(T`FaA2g0mOED*tlTEkOuIwZ&Z8OGy*)*8y zc>F3hz&YbS^;QGWJUQZ_tDv==g5G1l0luE~(u|N5W7QilE!wroPs@T*fc#s zbqy{V$lCbZCp`%3SxsA1QY{ow@`VE(#XaIl0)n~6L)?8TLKO1sMh9AE^-#{Om0q|t zn8+m)#^)crjC3RK5AmiffQcijDdc zk&km#nn?o%BWnKuc#r8oK)@frj_uR(rlJ=R$P0X`So?IQx18K$DD)iuwKn0%{RKHu zn~nhYH0&#~wiI9=nY~EsKzPy7omik@zi(V;oK=FD$IH0VB7#pGdKzP~>Uvj?yeKT8 zg4XgFRR-0A1Ty<$8Kl*GZEj*IVhoH&-e5o<)YqCLBu4;kHPM zxYQraxRsqZ!AtjGD~Qs)d~7af6Iwky0#U zlKI6WM6L-?S0BV%r9x(4{{RX%0CP$hfJf&@1aL^l0C%Qs13c1_NFyB5bT~Z? zA&fh>aBwN%qabe^x$jIT<~gPu=L7Me2|nNjSP|RQQarxF3 zsK08vXi$Jaz;8oSXp+16s7`P}1Ja-Xwxs#$OhVnR%B5shD&USm9cly5 z(!DdrJ{gNt)Fr>Q0t<=o=Og8lj1%l~I@g=r+RbS@Lvb4}NLDkjxMrjlw;EsC^=7* z23nF>uns`RJMmTgq8oA+U!mf&G$&MzfLt*AYD8Zsl_1qYBdh-KXC1+*((ldXMtcLE zlmTnZZM%Z-JL0!|EUP8#({OgET#lUAIOMwFn{e(2O#LgW@OqPLJCz_87|+tQrSTZz zbf$FW$2kXR87H-5U0$`(AwgIdp!N4Pa(QI#e{@a>1Cd#JlfIswX-(0BIma0%is_Tk z;DXe~*6e4t)Cx$r0aUO1q-XqVJVMyxitRP2ExbXYjYin!YhEN=asl=Jb;$wp6?w?rQ=2MeEnYW1V1s~nSFMe(CXF+*i|i0r2fg?$9 z^2m2Qj9H02N#?sv3R$JoensCJ51$;A9=P=Pt{i>-eQU7r)!EbHx76Tv4+Oql!j6oY z9R9W3prxY{n@3K;Z!Thi4pZ+r{{UvY9Y1KaNK5&0AGl9L?Np@D(V;rNnHUgFpE5P) zXwM~ms@yCifGnxLE{7ex=!HH*n-MN%tXZ~c?QVj}84J)c>06NM5=KJ#h<$}(By%v` z6~LM=M{m>Gv@KYuD%Q#W005S)Gnu1M>Jm7|niuI)B<;t@NdEJTXYieB z+q{vQ=EVF5ZVg^c<~(_L!$f$qfb6UOe=73%93a51+sD^in%>+K%NHZtrEt=Xt^;&6 z=w;F6p?>Vk7VlQwi*DZ}*S% ztMJ;#3M!xqqksUX7CotqNgZeq_C@MW1$cLiFU9`=i7uWp3L|-HM|@YS_={Bj%FymE zGIwp-gVdVvxg5K*Hzkfqt*Nf0$V1H_=AFT$=zg`S14zlKF}g4@$fmI8G^*G>#*l=; zEC2+Oa6Qd^C-Aevt>NtfXShwbmq0w39FIZxR~_)T;rh&C@dl+BX#VxAN1^`!Z2tgC z_H~tG;A81Ql;wR7oxBCF$3460mpl~aJ;$p0S7x^JY1a#JZ9Omv zAN3fHa1Ju0@Nw)<>rR{E2A`%&g;8;EcqwjhgX-VmQtI9%m&86Fn%Kbrj!-uyNjL-? zewCD(i#3Hg$z3DGBZ67mY*90xVOo%QcJ5pmcErBtCc21WQ~)XfzJOyDZCBd0<`s-P ze-2tgGU23ebAeh`_EAD{6l#F_3cjc}V;QIH@6wkr`A9?q9P>&MdjcwPA92kumO9kA z6=DTL&lL4VDs##0QeaTX4;_$ zll;G}aGB3Rz^!4ioVOJ871Ve^Sap zAgc06`ct>(q7r@hqC0X9?mdl46lar@^sFvwK|QkE#PP@!ql^{DYV&*hH?X^w*~UsY z2h?`2O=3`-^&KmU@q}buK{(IMg-_sr`t@*@wjxO9m56a&UxcU4%+}18>#dZh&${KdeSNNEHc5T^w$oWpSw7EHJ4oM>W%5{5jO^bg3`q2&${NMi0%8e@gWo z6T|vuj-%}M${!fqMoFt)AeH6QCrHLiDHyJK;~ST|vCy!oM_tdHypkA#2*IxN!1GT% zmDET<*&?ys)MR7&S2=xb@XS!I2sp)dz8lbG(zNqD(=!D@N>xE(jbp~ z(*U5z!`tN_<4=+>k?b22o>d1wjZEJ#CPpDwqNo@?^%u>!VFHg+KP(Q(+!*rrXAcni0rHMLO5Bo|{7BG{bVr#77uhcW!D0m6QN5 zymlO)#+6IEWMlC(t~(6{SZ$m2>{kr-lR5=};5|<#_thaIs*hG4E2OcJG>qh6~PdP?dFJQ!DG}YIK!x ziB|_bLE@MZ=R8vKOiG?g+z@%=y(F8p74IyDeVT zlHde!_a_4-5A`0^r>p+sPqIu$8=xwY4!bzyj>FQjlI9|Ix!MTEO=s5T>toFI3mJ6# z$F`drl<>>QVc3d?ZaSRTqiR|;jrEk}+(~n^`7Eajht!_`07~<=M~T@SVU**V-V?rt zGK*S*95BUu9lH-EGFJ_>ZU?s&O*v$_tT`>9Y(!0+N$r9;S%B_ZBlaY*^^IV(SuVL4%b-D$kCYTY|KgF}u6W8lo zQF@%XyC|bk=2*_zV);*T(z#y}YBvEI#no0I;gAEKG5-M9uB?DThDRh3@=69z(2C$Z zO(xxLMRv;JQCYrFP6cgI*&NhkcG+V^zI%Jrc%>-E9Bp1lF|6!8}N|agUZ{1S`lmRbTI8oSv1FS{p&W zWn&Y04oD;uOf1CbpRFrP8k}-Z@~E2QC*%SCoj*Fv>nBiH9FRV}>B{36LA&1-U0Ix+ z>`L|84h32Vg0!Tc4GfYH6N6tsg zI%d5u!NGw^?C0IOPvjtBY5+QyQu0 z`WkB4r&fHIenTj3gyOEFhfqPQx*;*__8U zIOc*f2%tR=jx~KUT_aVxu%8B7c}g)I4m^8vWmpeZiS|?1f_BTAdLEoSFx(p`WL{{eUrj==`r206_rmpS;~6vh_L+EHMz>-&ek0^JnnT=9)hl;#|9D@kEV9k1-HZ>2wh_u z<+79g@X7qeYC+;X2IC)R)-0s@Trc?5*%@opee`*S?H|wZQ^4pd3BxXWL`G$K@ZKhqca$+on86+Npy=*+G!kl4q zg0h>Nm5O$0m<(gQ8qyQa*oW)S^{y*L&{j!eH$!r6$x-QDy2$dXh(K7e z!w!UY#dG2-!hYl(>}^Vv;qOUW<%d|)spqh&PiYm#M2W_8fN@q*AUfoqQ%~4CX1R^Z z+h&V}b!GHngZYYg+JgT8tc};-86TLd0RwZIHsjKP4M~HHh73K9YE9}zK=#qO&fTs3 z<>1l;mLu{=7ZixbkgEDULH?>L0);D$E|h$01@Sl{6wlF*&v8-Is4v~ z$0UfY^H%ZL$&x7TL$9EwJ{K&9BiPeccIbKhYVJuOZV2r_>56XQoQgM(%8P;NKMwqP ztLaR-Q*FA_$15A0G=J`&Ph(!OYvN05o21Bi*~i@?ya&OKhg59!+uprPNRv9Xzay=1 zSFJeJQCl4`qX$XJ?s)8WR`498K#|<@Ou4zb5g1XOA#SRNt#lFTj@`dnm-{2v7$UTC zM5J)Vinq#*`BsgE{np_Sa~hrr?_CRNF>b-ebLmolWx*Qm3O55u*_4|^wn>qBzl@x3l+h~L7$+fYJtw1{;P zf_|ry{Oixr3&^Y7oL8b0^5amv1ps|1R4N6XG8;xRU zSZTogYYW8;&!#Tjr!F&*&*54Nx=B_vR%T;>sxoVz9D}_33{u$m7sDC_&z-2O*HaBn($LB>=Gef}tgqnEtuF52XyBhdLomkRKhNb?1amkABVrG8T^75lUTUuo zn+#A}!ZyKeA4tckJOI22N?7<7`{b7 zNEy!xIXJ5F+}u2d&`aQIX!{8`5}omxsu znVQk4*Se1Oe-!A}xLIz|T>k(FbiCDCMll-`|E&ymBHNjcIFx67uN~917oM| z^Ys4!_2#Uf4@)nRiR26B4>uALN_WBS+nUAEg|%ktbH2u|t)j)N>sFT*aAw{q6=swU z9kMtcz-K15Z?xlMWb%20q|=>@c`7@P&{g|pM7BudMU4<(D)beZs=dO;6UbUfPs$1C zhyMVtS<{y^((GK3b z&m3dIfar?7an#g1aK(Tuxb7*%^UoajG-sR^$@CNe(8|Lkk?aLLea?YLsq0WX2;}oO zg&Rog(9si1IK*-9Qg9gXv;kl^MhuXNIpn)w3axQ+3$S^KK8P?X&)8;o{{Vd`#(v^( zDs{I=qbiXF1o8m;-&$84j5hxONZP3xQ`9$lWYMw(GUd7IdwvzR(KzlEM`6V?V*!k3 zJ;|csV*Qhv7;TAqAJO zrCW(%mMr537~{1vNP{Ta%g0(nBVC7S9lH9Ol!!{=Q%%-qDiFRiGESvZgEah817T}(8mfvInSja zxTj(OMm@1nvwCCeRmwV&%5Vp6)ft$E0kRM2MS|qvfNbv0I(yL9D~vMp>}o9JV4P>B zDFhp{)9Xcrb|c6bunmsFxBN4teYVwMuuj8{TaAuCyF_PC0fIkz-`g2})bE;m?cPEo0#R)$t1Lk4< zKb>jHjV?ULl23UYxgVWC_G+p~&mC#GHYmj-A(mHcF(r>yJd;x;lM(|rHGgWa%8Cnr zI qZgK$XMrut`DNAgy!ogUQcp%pg;v@-txEKX1(5_ZHc*r2L9*CnX)#tt=7YV7K zv?kw_74*l~y<Z# z8}e`ntZZv@NoPWVn04cdUq74zIXx>SWx|n?Pi&fBJn(t?(O|jJ>5%=l`C`Wd=5O~! zb}^hY6+1>T#dDqrK`Pzc?H?#u!T0KarF2;dxSjy6x-hhF-1BiYIjUQ*$z<7)Gw#RE z91ehW&3G4yd_#4uUOCH0Z@jWbJM+Ol_3U0Cx$?AI`I%Ug5sU?Gr{?*3{x#*gevM%4 zTJ!Bwj)Exm#y=OyKai`8prv^dp*F1^rCkF~dF|Tv`dMVKjlv`?A=@kCIQjwC@~)}1 zIi`h7hzcOuf-!_q>^oJF1UDA&M3%{GX>O!LZiJ8t>cpN{pHo&=*K&-Xr?qobpFFS7 z?WIqWK4xr0PKCDyQPc{yhyo3{2kF+VjXgLZ?PJxH;L?4bcwCkqa)H%G59Ls35g-5! zr=L+%c|iQAl#t5I2?^x60DEG-r177H?R;&ZHKyf`_8h&mr}$gY`yPk$?O!zCX}4Nc+_pEc zyf+XzSB<+L^WM8?ROeQEoYf=Csa8C%BO;zMPf}04G&Va_ihgfO=`lL*18FODaW%|? zYM|Js-1V;0P>t>FoT*kZuIw?uIQOg%3|)PS#?ApA41|(>Pg>dW1?{rfPLjH*bTRer zMS0a_&r@BGPMkR@XmU4t{2GML(#T^Fj3Pb<*P5xP>#Fxc&LVud*dwqRI48AMgHO|U zk2))xoO%$7YepG#`wPo??Jc9XdwAE`Cb8d05IRn+C3Sw)Emege7u zfrp_LX7=OlSM$t&<)gEHwGPswBD#_=Y>2MW(xN2P&NwyE_)Edl>(>H#)TN3MxsL_? zg-o<59OsuLdwkSn+%mg{GAc4~YtlX>>5>eD5V51W+))rC@d#sx_lt~eAEwEVq0(76WRq9Q4Iv0oR zwS7HqQQe#p9e=vs;8x|fhXs~4TgI-v9|twj8ZGM1Y2qTZOH=17%NzKmXS=nYCURK$ zh(>eus!V}-HS{L2WvOZoU`rWi2e?m`vn{?E!*b>I3n(H7oO#N>%C)-L8L9U?O3g{U zfN%$xI{j<6|^j)9k?9QEwDqm7r!F2MV0iM`LtX0mFz{tFNJ&$nMbvEF?hR5 z3Azg##(zeQQ}rkMS16lY@IJLTUw_KART@@Ca-|tNvqMq5y4LSwnoFszVTk;e@XSU} zt2YFnrU0mw1P1^p2b=@R_o_Q~lh^U5B&>&U>yD3yyf^qDaCxJ8|eeYUFOsfXj}0 z3P42#ua@IzESo>{GPCm61)*;A}H7da5xo*E&1Xc_fA1Wpz`^Svc zsGyTSJFo@UJ7r5V zbl_9la6dUK{>~`@-S|!wx3H1dtgoNSvixJDI!XPjZ64u}W&met{`b??wsZvY^lNU! zf`2k=VSr^Szz{|VJqWI~O;o4K&Sh28tIUrrH__?>K|I#Vt~RoRmOh|*RfuG^A1r&7 zK;SZwk?op~R@0`_^$CTtpR<=caJecyDg$;<0Qq(UjEq;0QCZmbl$4dxp)J}*i4u{$ z&jZ@GY~zk=UoJ&fMi?qaL9RC1HFsj6&*@fVl~D7Zy#VP+9H}Cyjg*2i zHq?PY>wsz*LaEBTE&{2TEAxA#-4b978P~c;sQP!9Za`wTM zGI{%>`czABtVRL%1k#(bIFWLr?&Gyln{1>N483|9b_;~BwJc$ z$o0ipR$#ki+{xQKes#-uiag#IRaF?{6yp_rNtCoQWu2N79chI!a94`9u>e5N zIp&q1JT69kDhenR#{=@G2{{UJ!K+_pQgR8-ITaH^ys7E$NMw;dMjxkIdu#*~-m0iz zK#S)3^rsEEA=@lJgpNP@^uoi8fPf<%_Bd^N{kNVR1cX>Lmt`1 zI|a#PWw(gz!Ei?2KdooZ(-+!6T>1)?i)SoyM@qe=-04Yj!%(<*B_lD(A`=6h0^rPw{*dBqd9DT2&B_4B~Sv!{(FCX#8xr%Q_CN&2eGd*P_I^3YR9=6_*xLO9m*OHgDl}o zi}n#%Suz!ChvRqqC=XtJD>-eYW+4}jobg)HTnS&yxZgB@U^|?UJqMun7^m(EGEiZ- z>MOpK92J|pJeu{rtgmY$9c{?L2V)-nsaf1Qw1YSp+`Rskd6`Mb90T(M&lL;AvTkL3 z0DGF+$3HO>Nfe4Uo;%WT$Dksjc!uMEK|JJ9nHM2Y%Z@U=saQFiJuAT$;WP)p#_5@X z^%w*C8qvNxM$j@z?MH`zwzi7GNQ^}g08d9fN&Rb4?N$QQxDJ35k;xU@m7Qs02T2l`XTMF|dA}>(8n0R63QT0$&BH*vl2I z)Zt~+m9G*RP6;_1g8cy%+s$N>t`2L=z9Zj{4)~+O8j7wWw=xMZKY*21{SA7oJ7tMv ziT+jqf;tginu4{DF73OR@1+$yF~uO$2pP*C!nZzg*b(i;G2I8=45!x>KFq6_Qdp3r z?qGeza=s_CH`ccLPUa1NHy#iws3z6Zs{`k;=xZP0&x^jpI%zkO0TZ9}c4N^00C(~2TUYTc_Ojc{_a#E7=yN|c_#QZ+-%$GYfy-Gr6QHZ5u{;+xiCENNb)Y(t$fED71gvJYssc)GjgNFV{{ZV! z;^H;qY=iPOd@N+=`O`Lxo;dAHWnpz1;7Uwq{k{k1(yRvr3}BB@Ol9_{{>rQKMp`m@ z?L24c(t#E_*D>SU@38o0P^?4DL}Lfnt#9v|FT4bU{hklyUVrf}`DTXB*>?gMyDJPH zoK?eF6D<+vAoH4=IbNL9YBrvnQ}dnxuA>HQcz46oc!N?{lx=|sX;}6ipN)HEwYpkc zM+7JqG*F>OKr5f{UXnDe3N~cK^L+1~ze?#h8s?m?&7!P~Ul&@-ci~M!D3mk#dw@RN z{{TwxL-&t8Yu@}>G_iPFP3S`= zjsqM;0~KS*Zsea@;r<$UFiPWB)MNQ^`;|Y;*Q;>}Dj0cf-G>#uIA2$>%~E>V9%Ted zW5^jJrfH<|aw)Mn5&WkO?^A+HayaW=WsgGSVm8k|fTkE2w_a-Q+%Fj2)|A3P+!T7| zsS?>(Myc|UbJz-d!@z83Bb-#qhj#weC{vHUj`Y~BXKmSn4n}zuWg}97hf~m0vMO$#aEHro_G~8kTLlbwI`7yaKI?^6o90+_ZD(_w(~qve|Z?G z{VDC^-xMZN-%_2u3n*{Yim=vEb#w*0XPRWNq5-uAY4bjUa8Gk0_gV2aur}T5mnAdG z2iiX&im4um^Pk?#=ifQaZzZ`cj-KFFgf@UEZm149>rUh4Mn0Y4_#l)tP=g}#mKm*i z5TPhj8%H@k>rU1cjGfufQA~Ke=Ez^`NtCoRLa;gNzLbD(ErrKCbJnJ1@|>%&12o3R z@Du+4)}#ezU8E7g>Gi2&lW^*N#Y(?V(xZ5ZVT=z-U_5d_3z~Z-?%1ZI54vy|cBwZ3 z)c#b!bO;IBP>?+VIFrlTA|2R_niRUHI6>3p6w($zLdF;#H#b^#5C}Zi{wV_X9*QZ3 zZKYHnl}0+$k;#x*6-w|yG+0+8$I38(4!I(bvXFjbC)0|IlEnPwsWT#vm4avUpkf$z z=giux*bcQ^D7RwHq{pAVR+R8IsbVpbY2>RmALbr|p`-*z#1P3NDe4OLr*3jLw30WW zmCIx4nv2V5ShPw?pW)9>rB7;1tQkh~rzAH|#+8ESrrE||u(pi#Sd<^GD&K17Vew2e z%@j8gq%Q0bZG4QH=oIC8*JLEEp~)z;^)R)YUAI198CeGJ$Gvm2+N_dglp60Qw@b*3 zEO6aW`Wnt28j+tbJZtpHuR9Zmo1Zim=b?wh!9vn|7@B5__S=MpH4C0uy$7v!sQ@UJ zcB5siTr>Xwc>e%eOC3HPE*4c{?w=u7Y=9jy%W(1BU z*&lBHwVUF7e%Hf3AG*~phUAE0V+WUGz)|(9mdEENyuagATAryMo2J-V+)WY_G_4rO zB;+W^Q;av~U7VKD7%AUYc>V67b*}0X%HXO=A@c3x&KDlyT9s_ItKBZ>i+P=rHo%SBDn)uuo2^Hw>CxTCfTxBT)nj^Rfg=6{Kp!6S&IozW8lRBx+KtcU! z*jE`cvP2a&2Of*bs7B(Ld~WJXALJ>0k!g!15RdCN+cg&1S&E3rKD9J<(&2Z0UX;M6C8VeTP+d=_ zHCP7ZE`7lq*1N$d0a#;c>UvY2&}{j@ZgP12b3hpD14#HEF{ejrz~^pA>BU`{Ol3|1 zR^uRG(^dX+v3j1A4HCvSMSvIF8f?;S7*@iAz!Z-r;JG6w)2&8ETy)^{?L~u{#4hdF zk`L)i=DTgn9D~91rIY1gFbOpo!sHetlh6uwCgi4VyI63<^Yo%MdEab#EISS{PKMYL z9%Hn7inK0bgb3is13VsaK*=Mp@{qK}7?Jl%c_&AJ_o7Gm-|V(&1hl8EOQT-klpiB02z+n zn5Ri-o@N+l+;D$7ZpT70cdzSSRgY%mp#v{HeYvM>!0LXTsgan={JVkfr>#%C4rmp| z<2fabI3VB*WYTY3gZYX^d>$#-1>s3n7#*l%gMmyBkXwO@g=Lq_XM~mqyM59AG*~Wb z#W0Q{&4nJUY8dXNxtS6ifLjHab@~dT;JfAA$o}Jj^cAh4LFQir{od7`Npi)q;I7W7 zBFAE?dIhe3MDZq-{{RW3`gP^hR`RMRk>)l8g|mokx2D8LXv@5+zP zr1)W|-_PN<({(tdORX+uv@YsEbzT7;xas&;g6KaGyieiKsb{H3_Ur!uEGPZqW7r-? z;6-}hg}y0CsoGC}s8|$LWPnF%Oomw)pSrzQfzD11cFFSUaHY`odJa3+H(}zdd+Zpj z^#ugu5?Zp6_2Aa1)PbBtZEgW0A&hOe>6~?^dy3?@vO{TQJ=CD=hBlG5CVMI$&PIODA=8nzQi ztOoDI*9GGJG%eCzL=+^M3;_Nvx%@}+uGPpjgRL^&>9-NV8%(i{?I8Q9x8J$;6wxOU zZXP^3lB67hI)h%R;J+U>Y3)# zb-fO29YCu=a~Vcn=jrY`*N?&B`Mf^!TKR1Fl>FsNmK~2i*guVId?xV(4lW;3nWejn zWKeG8Z8_)b?_QIyhrZLAMYW1_+mZryhp z9PoOcl-KPy_aWGEWr3E697#5WD6=yz9g zI{Etw$DV+nO7Vtfb}Ywp9!WiGdQs|SQS=1^J!@;i+G}ch>Z9)httcCT6R zHk0-p7)vBTOK^9V?~cN*q^yXFyE@2ags_cR0teh~4OrP#S{0E>ItLz%Gf1#YANq-z z0R9&Fm-XhkjjUz(*%bIgP|Fql-voN*yn9B9$~HH5*mOo1i;!E_kzR}O8&Q%y1#Klz zuQQAkZU`N6E6kohv)d|@^SI6rVOlMR7~b!(^tO@qS+u)ZtadfCMha~?7^@8yRO|qQ z{46WL{3YX!Pr_1pVNJ!0oy~0_BOmQ4?0-XEzoU5b!@BaQit%l(PgrgUM$e}_AHdgr zDv^@2Ji3(U2XlczxrZFp*wOy*KQ3sA0xwLSxv5OMKrBG^uOXhPCm1Ijj+|5_r6V8^ zx$i^V1kY2CN{UOFB>_MIM@m6rbUs>Ti02g?@22k&L_Gd-Xp`jJEkd zb`IZGsrJMhgbK&GJkitv#(f1l1;`9Yz?NWJwmVW=%<&Ne;4$PrI@N-teb51@T?6DF zo6!5y0?tD=PGmj%RG4+)$WS=|V>M1Uc<~z_m;2r7;;JzL5t)Z}GzfO=XUk3OJD!w` zrLtO2z);d8C;?)`@D*r*rE+l=)&B7#r2sfZkZ^+->S?T|HW*;>j#~n!*tyFwaz!{P z0OxA2>S=)*VWSKRqMo!7xcsU~sH9RH81hKx3_AW4$GHr<LndY0mt71-{Ti(|HMDKU={jl&zHZg2-CqmVmFWmYjW3=pU9eQH2P zFt}dB0)<|ubt{g-+q78+8)RO`u6?W4t>4UoC;svS`d5)Od$0~eb|a{-QqiO`==N^M zEMZVjxEQUdJz0%)qAxJ#HDcLIH_Sm%$hfaO_%QR zP2aSPo`#t$fF^2^u!KpArL>!Y!S7x} z@wH0aX%c_4DE|P^X*t<8i;QmU&ob9Fy+c{JNw4QJx+cW=LWN&I0Ibw@2N|Zw+n9YS zAow8GPEG2~R8&=##g9FccQsk&GCwDpBstD1Abx5^Bic04HO7}|JV*$V6lV@_GqiO4 zYSO-XmLzpkz#oM_!*DwIdrm|D06Y_~(zGruBAo!*z_0Nr#w(cA$@3dLatP>YWMU8o zZaQF8q9bq2@t0giPx7mprK0kz;a&*QeL< zti)ZXrF!$&!**tOUFK7k%~6ZNHd9NxX;uE{HM}utD@f;_67SqOZw}~}SIIrDIS5 z)-a<*2r@h_rCF4H?XB(9WOLHE5#yaGS_eTZ$OZ%Ne+mHmqrc-%UoIjTrz_AN zN99qn$sZ#QJ%7YggYN*K9WZl1$4u`s?%Wvn9Vv=IWo(6Rgq7)2EMeFH25Q1us3R!U z0aoVTOPg|t;5TMNRIN{ptpdjevP?Po1;Q-5NBjsATOtRI)wl$7HF_x!pf00uIpUrB z3zGYg#UF?-B30Ej$tQ;&>LO7bXX+FT{xr<_c>s6CX(Z^%N&}JYRVN}}zY$qIxfHM5 zXO`PBC4e>%F# zyLdUO%-AOc6X}WuLS-F3>wk39Jn&y)g6-|leQIwdWjK*TFJ(OiMjP|M1o(J&biiu_bkl98)rmQ@xBxxF=jB;DOD~%!sZg}7iO6hzv7ZOUt z70qBA{J>+qb$WZHwvi)7dGg86zAK))_iSxRYjd8}Z+_EzeK_PotP|)5TJW|d+a%=I zu-;D;I!*MA^1qPhrhkOjj*RDnThLZkIqSQ-n*JWpZ@fdQEtRKSZemYtfCto)1$`so zuLJ2C4T@>+2$J`PX~N}Ix{QDI`tw{Tz-=4Md%aRZtg8D!?idUY{RzJ+^{q=?{?F7J z-8AWDWsFGDju#omGmf>pFtJNU)zy4qtLO`8ut#qiPZNx@Oyv(l08R0=A+v#4L9b zy0|`AcPAK3^&a%DhNE-Fej)g-Kb1|%NBnzLn91(8iwakOK=0RXJhilgDl^LW#;1c zK-LP-L!NFJeea_8`qkS@WwOpuosTUsJDI+};YgxrtY^Op*RAD1f8~U2$^QU&{{TT+ zli9>Hx$V(WV=*#-I0K5W1gOMev+O%m8ujmyqFxZ#-5$k0=Za`Ym~YEwn9fUJjAogW zoDP*FN@SH&9FxKARzc~01hY1JIJuEXY2SLE_D2=$HX4*`6k*tHal!SkJJUzmwCihr z0FE%Y9jj)}@-Y#$xm?2MA-kdMMRM1Jm%NUuS}QZ4)-)JyB73V$B!Fb6X8!=|*CDiy zwbAKX%$C<5Wu3!CovQtD>G)P=s~RSwI7Pzl&OIqpZ9Z01*HmYYPZX^fd~O*dzkkk@ z4%FetO4!UmvJdrdk?8*b<4Du&{{UKrJ}VNiJhPldl#jfJztX3MG=WG(>(uu(1a~M- z(n=niSYoH$o+uH-{7Sr0V{2*{7k4j@VP0#I`1{vY2_~@s#{wk9qZj-kIRWu7zzLt7foq# zz~JQOpSWY1izmHiYW^Ux(4&R#H~#iOiWGB<@_zwYmh?HV8tM0%)}*o8TS+=u!e^go z`>HE<;kSi!Emun#&F-A{6A22Bw2Yv~_ZOl1nvN|iT+==)+uUi>eWu3tKQKz?F`ImH zryiVR>MNu1M~Xk;4ZhJWAhHo@vdToRsq%&NUvN%;3eK}_s%wAHx^)_ZeG0S3;lGKR zw3ZXtYB0W?m1&Vb3Aa7KJFUw6#20 z=al78;eT3HSCIVOO)4nI9MtQD?OHJnSV_+AwIW8pjY=>%6ftG!22ZHzNC6`oxB`%= zX?Bj7riW!2Ia~wleuw#06m5?O){!rC~d>3C!wOi7L)y%_DP@iNa}s7L1*(ZxjzcV zcMQON?D}@4TbqdXq=cY79FDYDE^FMpTMGggXvrt2stI@p%Nmx?r(D%?ihlNAJ&jB- z2m}QlqMAW+Lp96CfZkgEWYlISf0*ue&jpTZT<`+(?@khI$0TF0s<4XYHsmSizokAk z$je~2YO;qe2pIG&Ons~Jg1fWMM@j&VqudVR^L52Bm6U)KAns3ko6j<^hybJxO)*>N zJCr;68USF!0(oQYO<35Dy9cFA%;S;*dkStq2WsYlk(5)wQo!-eG;V>!Oh}9#Ph3=) zIm2$CrjgR}#m;;EKQva`8tbON(kjk#4q8b9Ju)O*yC+?N=O3a6_z zFf;^(at_sA=eef&z=N?j6Uc1vDx@)&bH6ToZs1k4iHhY;?mP6QaIs%A$WHkVG0|y4 zr~yDx>&G>5V*x&6g$IHMKaDAhJ)w`w^ffmbSgt|Xe(26=7Buphl1u*pN_zwI1yi@U zs8vXPs&Y?iLmb1Ro|)__7C2&e9CoSP3;?4nI*)pb=4QqQKTlo1&9UPoMO6O=O{?+jMZ74-2exGbB|h?vH{OgQM!g)uLhKgL>75zCCNB1 zxC7d}%j1fFp=lC&OCtViKf=92*pqr!)d~DXdEdtf{{Z5eu#o=%4cf#aI3w_+ z63Dq7RQuG@Lzc+MW=np3rYZhC=B9lFTdrD1@0TCkAq;0yvgQxOH6BwV2& zuqOloo-2sfT6@nLJZUS%^QZ+07yxI1UW)>eDiuQeU4EYeNefFZ;S@wE$5ZSphP5AkMmstk_odlAYz{Wp$YzW8v z1lNOW`lt4`yLEUlBWVbYS2zebTxZy4@vlaeB~Co2pE{0i?9;i`F7=DIy1uxQdx;KN z+~ofN^@p&mtIaap$R#c#IL8ft>g|*B=I7g8oO_IGHZr*l(fx>Hy*hjzmTs> z_@Cg54JOt-b4i6DwUxq80uPclZj7U_Ju&J>t$B@uLveo%tnRVyI|Rr0bL-QL<1_*5 z7I(>aWfbXvBv9Ky&pwpdWyb`m&u>cen>|8pHu2Icfh=*mBcjQaI5^1v01;hXma}hi z5QYYDxDb)^eSOEhV&h`dxwfge0zzyDIXJ5KH;KIw6L?-ifyp&mN4TGsVyaI;QbwqS zPe#GW#Rx1W$!B8nzR}J==m)(^96*hc#gzlFrQ7GX2zm8C&aTFRV)+Kmqw}UK8Ltsi z%2DiY8SSQoy+nt>dHF(XwiV9S?rg0Vs?U!9MiwlxF~gOg||f zF~`!4;a77WJ?Vh{`5ESr$cR2?=RTa~t4T9D;xI-}4o5Uc7$-R=+?qf+-bWw|s^{05 zRTu;iKgOgPQ@9*~#}wB9kVbJp#^!^`z~Y0~fyFm)86@U_A#?x(~?LmjGFUsL4D4JDT+C{L3WF+y`UNt$Ccv zVx5rnRwMGR+G@z)o4Y=V@Ex_bli_c&>0V9U(z>kCwnpbFbCcME!1^Cb^eHsQ);v88 zp02)J?ucd(zHs9N;P>XfJGIqsY?!^Z-0(@ecKD(4dWSbR$h@C zkC1tTJa;uwjkjbUNL*mQ zibMeXzGnqr9gTQ}{{W9XYN_PUtlviz%eX5(11b9Uti|!4iclzzU%ybQ6?R-O!0b4u zWG{O9A4geFtX+82P2vckf-5+gt?lF7V_%p3X5s*2)O&WV&1UY}@4}ZiuBJaWGMih} zg1G1FUntGupAlUGw;JZ7AZ6M^T(Jatb*_uTHu}b;cWZlhb3C?CI}zs0%nM_T0(#LT zn$%H?Pjj|=c0~Xv$E8D(I0GK_bDWTJa5&9Ym4{BBN}HObSCFp-h~xN&2DyI~U$@y( z)IQ+txcvogT$15}f&s1%Rk&-5)&;i&`4{LZ$8kua8wR#~HEH&}UM=5sPUHAiCJuYo zci`PUg}dCI3Hf?;^{aam6?S$}x$T^aun7tY&MK-BqvlRMM>RUgTjo>p`&MSfgp+EJ z4oUT?TIr^6Smm*e$G$3_81*@)1{lBpx{(X*Pdz8$s8-W+X9&+ zpq|4&g;$byVp0GZyS8{A<4`lpbDi6whp-3cZ>>(e0HXj7IG_Yb!V=`}ko9J%azNlx z4o4i)@&KS?3Wrjx0}nzeg5Bf)08=9;`z`Y?t8Cty(?WK+LvWUn$r-$vSi=(^zCg`$ zz7*7S{{R?7`WK09x9S#`EpZBvKw;64{XIP~=~Vn}r|Ndsc3vXW_X({rl7sHxQ}#p4_C+4w93PpC7}YO6IFx5v1gvrje&7Q6{uHEW@V5Z)a+4 z?VZ7j(I14}f9E3=ksi+h826sN3g4oErv6P_{aiq`n0r)qb) zHLGdxO!8Vy=DNfV@x}od^sMVo7Bprfg1#ptTePng%RD!V=ND6{ z=O?580C(%sx%Kd-xp-Gr@V|>!Seg|WSne7*&<@*J^!BD)c&^h@VWxaQ(Bv0U{G$FC z{Huij0C{>h`Qp2wN-CO%V~$lDlaeNhvDT-^Y-hDr$;cflRp>dXItu~FTnc=0{Mm0x zgtj@)N=?JAbM&IXTt|&g3gn;2QhAI|Bw!xZQDq7V%{?V{Ct?xDQ$>VmiBFbQuovE; zat}K&C{{VG9LVB7_s^<;Qu%`v%=VE<@I0C&F z0~yKhPQW4z5HYlSRC2~8QmvEe^{EdH>&+_$0I@irWDy@J{HNWJIAhS%jLn~w3D0vw z#w8degWK5Damv7$T;piZ)`1omDh68w)4>~lTocfmhz+a8PrW$hK+Ap<0WRWp!=)gF}ev6Xi3ltxJ4FD9)=Wfi+Cza$=6`R0(yaYZEK_Ysr+({L(J zvN0!nlY@ezfGX_Dt`tDyY2=0CoeD=15bRhVm}Y^9Eg?&G<7*MtsXR#=86!`dA2N=I z+i$sBWv>6D1k=5888wSI!Prv|k-s?@u2v0bF|uH=5p!qlX0 zT#A7sG31`-H74U9&DBkGBN48-=e<&pnA@)%s^hj>IO3t2FzQI8>@;Lu++SNvE^Va; z0|gjmNUu5gz2XGbGz&XXZt*y|4o>WYg(Ldcq)d|FqT`;7-K)obD6f+hw0X|i*kpIW ztSULht8C7iag391ba}jBvpMv|Ojjc~qRk6=t=REWY%7p{v;kG4!w*AK+sN>^bR;?u z!1@u*B!J*!=}nD*anIfqnMv6Cf8n;F7QLoLsqG5kb~ZaW;E(5B_&FTc%%2JM>&q>D zwA*PL{UnE2!TapJSL#N+sycW+~(@g9#2v|nts zOh=!cdoQ^5u4KEikdcBp<;Dl)-n66f9Ftke1o64vg=;cN&fYIR22@G-ds|mo*JP*2R zl1gA~y7cJ4arO4A83%C$n84(o^Z{qfW_`+VI6JdYCEd1{XR9A6{@JMnjJf{c9gnA9 z#QRdl?B_9@06D2GqP9Bbp40yTsa2VYE68G){6=y5Rl{>RuBKM8HL*HivN`ha3}PL) zWh%Un)N}7rHm|6)>}DHVATE#!_v6DqQt z{KR2>G07F<2a5JDh}tCfUJ{n!q#H%T1C=1RJ+s(Xj-ScX zJF-J2BWD;d8Rbd$#~$_X+Fqw~9sIg&#qFiiOd2t6>xO6P&+A@Q@ovgk^|ZaZfh}$) z-3r2uxH2b42l2;GPkIw6o@OPI-ey49!Q<&#cKV}ApmMWBa;y$N{Z*NArPXB~m^B0~ ziI2FdB=y}BP&%%e9D&Jjc1WP}xPOSj{A*ny3Y(cp#c-Ykz4C6aA!0D`q<|k$kWcci z`C3UNQpXBM;43GiHCm!nc2S(;>)xFs0KD!bo+*5>{7Oq6nC>bzWMj@lb{M1<7HIfw zxZZg_wO^IOu*P~5+O9@`fOp|~fBNZ-wQw*to}~3Y^c0G#8!VfH5$q~`soaB+-kQOP z%7g94YD5`4AFmwH1mv7!2cDF9nH&Sa!8FmidV)`_FPZ#2Py!MMJ9FrLD!i7;JH4Y8F@(}t0GQT}}@b-}~4 zZ#{_VO^p^b`BVk$Jq0wt6}Lp*TX!9g;XoE}LCL`Mq}szH`BhPLV_+stymbd7{{XF1 zEX1)pe;NgHOzrGOeQ7wrU=BUcT8Oydj?~h_9G}8~xg>|=>rtEx6Ow5+F&?z-*aMvX zXbYN}2A-O(uLZQ*ytV|s(}RF3r@rtNv$<I|jzA)*C1V*$-$Tu&@CDRE%v<|;G5i=Hn!pby*{*>5(3L7jVbG7(y=v~;_sA6B z9Go9o;kAtn*-1Q*D;2onM91Ct9^EP&Q+oLq3X+QXHgM9S;O7LI^F4YmCtABqU~kAD zUTf53Qyh$}2w-q)&wODP&3T&ud6BDqdw)vVXvt?fX=8}ZDn_F$>=b@HQu@`MF>|X~ z_WkmIF-#3F7TLX|_JT>?XE6{|_4#K(7T}O@hk@IO`2i|`EZA}O?lfKn(X{-IZIn;6pT9-I3Az=y5fOSI618& zu@c;JF6RTUJON&_quHB1HV6phC@u9pYl85lki6AaM#jiS{Cb|%=_tSvN>8D1T}jYw zIi)BYowz5sI5gED9GZ8S1`R=DkV?ISkI0I2aw8tNZ$rVPR{(Biz#aNxp)wn#DG>eF z{Pkhi{n7ns0!gkF8_SGul=dA5)}u>yGI^6e`3Fx$`cRgJ5&n-R2V6fO{{UK$$WUTv z%KregIbY6z9myPuZgKUc;nyqE){Ui4T+lJ--!=~hoJjZw9{=!nT8vJI{uZ1 zb@3A0O?8IJ+TsL21GYhC_s=60=lYJZHluasId=r$0oZ$c)^+3~dwrJxoDh<>TkD!% zRg9OsOsh4q>zW6}D?Mvfk4=)o;ybAh6_JCfKcN-V>Y7H2ugBzSR*=un8@OP7K^)i5 zI#e$N#(jj{DzJ@IzHu z=zZ%~L-FOk@kpT67Ru;w);hL!A7z+wKj2kgSMdjq^lPh$H4PRGLq(a(EHWUtg;)KT zUPt3w;k~qE)kk!5*L0+lg4E01wBf%Ba=FGq;L}rZcl1@o~i&|I?w_TPCE*TqDNjczp?bGGOEWv zH4BZ$JYs+u9EFLExFV&>Z5$u1R-bIY>d7(=gnuen{Kv|d2uD@#Koh*H7|A$aLTRBM zP<*JkJsG&G7BE62%Y4o`1-YxHH{07JoO@Alu~rKNG4Gv;&+zmB)Q8K0MT#_3{{U!z zDww9&$dDX&IQ~^mVk8+s(B%IBjRPfXfZ(z+F5C`JCa9TYl2RP*3Qt4DI{@-R;EWD> zQ{aTLCvH@G<1_&HBzYNCKm#21&0L98rq%->^vxt*R8i+YJ9PG>k^H6^UIid6Mg)H- zzyrNba>{n+k4n!=!MI1aty>bgZXc)3bfH@fSUiF=KXNMA}4!q!s(jl-jg#(lC zOfm)N+nXE%-hmV>^$6vV#zxmsjz?d5mG-v;l6&Oz6%>%f#h58Z>&LBENsiDQbKhy} zKohd}QbcDXZfUJ?@?lQU1`c|kr65L?e)ht83}oV@j@csM1u}MGa%l{#H@7&(dCPD= zD%{X?XmvZxT(~esyPOXG!l}G|Va`FouGhhOYMV)}Tpyjnf!l%q0N1H)^c~XGolUF| z+gL{w5Uf#!`B6qORP}jQ-$Ots~Hbb zjB;zH?HqFVnQ$`85MVJ#z16xJq2cSlwP@CpG0Pa0Tk1HiDIsCXVS%lp`IX3eI}nxF zan))$5f28WEh3x|k?3j(S9eYgJFSAut29H98oWp1o)zttM+R94Ki(J|{&}x)cyb6h z$JV@S;|fG|Ei%$2!4R=*cgu17jb&0AT;7zmvBYQ^XWEt8Msab*5vn8`p>3WM_*^)=)M$lTr~paeO#kR}ILML7KS z{Z1}4edQzMN-_H<3EifArauWZg{Ls1Vgof34_z7b5a&Sr#UA-g+MoMA#=d& zdeldAcRYL30nZ|kR0D(W#YOWVAxJ8C+x`^dppXbFPAqp5g&Fi1paMI9LZW8_f)85L z(l)_&EyU4>fo&yX7-VPVtZ^5eBB~5Yo-x$cn=86r+bdy(n~=ZW#tHoarF%&v4k^Lh ztqtb0BeUJv#I|V=klY5`NODPUeu}vEuAU82Io5ft?ib8fF`&-e?IaA2y#T6~&x>0z zj3e5`6P$DmuaDGgrpI8^wxbnaeKS!C!gKdE&ZLING@vkM3_TydixIv9Sxb zMcVlTY0q)bO7T05LTN451z@oZe5j#w^~YNG3#}VexVnv{nlC6s@yQSeY@56KkEM98 zh;&)B9a>u%h+sZw*?KabpW*mZa!jbJXnH4xH5nJfT5NHMkcjzdWa@Hy0nhTUPVnX2 z*SdP7w+|pgw`H_MasKhI5AY_oB1Y3>l%oeP7dXg0M+5QvYuNldeHP~vNRPa9d0oy& zqT>~cc9EpDiJh-#>1E<0aRf@cfS+V8cN}#uCt;D$Qv6!de$n8WFKylA@(gTNLP=EJ z^Dqae-e160)~gkT_4Ty3b4nn!V)1TWh*+-E!0Ui})obXRT=1jmGETFZ5=f7go#0`D zl6W0L=bmdu;~iTj9MahzHz`uAr=hA&Jh{|+)Up;rRly7n2tAEfW@eFjJ$R}a+IR_7 z_32hcEDfs+4_(V%iWI~@(bss+3Nbu$*!KKuknlE?Bbwh(hC+PHNd&=&`CE9v1F#?* zXVBMo8hK=b7Y;GEu&kxIq;@ICcnSfwNBanK~U4}t8{#4avWGshwY>t$TgMpEX zKuoN~GoQW36)LN5I}{QJUeyd`NogiIc!GFe>7U#G~l}nWk z=m#h02U^aRq?lv_5);REKjBCRl`dCtZiYwXpZ>SftwSl3IKw#p9`wXiOLj&WOm62L z>SX{NV<7XzOv}pp=ZyEMPQ_f7%M<)PC>6@QlY_>3Eh>zkyZO}5C@u#^9lHv6I5`}T z(u)bQxq--IfzqnAvoA1^oN^D{tx`c;a&hP?2OHCs10R{DP0a5!t`<-cvDZ8w#*}3J z;hL66!93uPLsg_!kNsJ|1MZWN{{YsY35Eb9ZAKqh(RGg3NB>UqFu3)WK{kG zl4-&Sr&H!2VTu+L3{AL##MF@RhLS>T%acH`x;UD~39ykBQ_gY7#d`hLx|SE;W)QTFS3DocR$s&&HMQ@Cnw)-76C@vKMeZ36 z@8k%rTx6o8%&6C=I*yzj+T8g&9qQhi9WPaaDVlYa#-Vhu+Nm$sBlvyuS17iuor#M{>W;#14BOaV+kKuk54bxAa99EKu9n_K zVn%*p+NkMTLh1IwT0X`C(^b2h}{7s51YaB@|aqsfMw$~~5WR1BV$JA5Ui9Iwpd)+qsOqj!WX*?2+q0o%h)`JsST1!2|o>`L^S3SLP z>s^fgKi5{->~Agf+u8Q~lgSne{YOH7D)Y;^uIG@?G^@FohWu!o=2jgjwE3B;-XXY# zLa@6?7;r%AS+d^GJBcI!u{@VGpSi|Ck&7wdDD6fLPwwO@{HZ4#)F`E~t8JIj*_X*O zs`HQ(RO}gTccKmBtG8TMaxXGd9XsQmkx+Wq9L}zG+ve zKDE5!%KkW)#3=Sx31J)P6?`bE^cA3lRWE03?| zRj>7H2DvuAH}H#E-RB1K#e9hmzSHuz;YEnt)G?rK&c%FH9bB;yzrOzwtJC{n1$2Q_6`REEy%^q>jCfUE)esll5W zBmin94#$EAxv2n9yquZ_JORR}$uw=q%ba$_AajkNh0l6vJ4nt;Py@-x0dtS%M?KF@ z)SJ8HBGgRVn+OX5-xL8clfDPk)K764d~VOUr7_7SJi-9uIq1WvJ#$)+OBB(QxCiP-(z9laNw!tt@H+RYtV;zlD8@K%W9dK^CU>1c z=L3)`2~t&$cFnlsZ>?I~yLUO~H~?{)kbI+Y!^wVw>*1gXODjjPG0M{w?^%7`lT-u-|5Q~_#v znWO`FRT$1{tKFc&pz|0L&p;|9wJgMv&iT%B#Y!#m#8MFhi6)SFg#8zRO(;I<=lzm9 zRpNmLPu(1G*A)OxydP}Rxa>g<$w%^J$#KG-#;=E0A@>A3af8s+DIr+^R$dAGX<~5^ z5y|&6fW&nA&;`&=1NwHaMew!tul77~2^2~RXHMfLyzDxK`3cy2)OT~-S*pBgA_6j! z;N#nx6KUNOD5)(^WfCLwIj%Fs_wg;&t%E*1=>cF5RR^cya4Ue5$2$I>Z@FcV9N-yZ z$oA?h55(H!v-pQM_iBKk7CdwHkM}Jvau`lt)-Ed z7(5=CtX%@bO-4^h^fasAWyirVH@u~tY<)AM}Z)#kn`(T=h4BSf`S-jP7lToymTu0Q(UoqBwQ z5gD=2SDW~P=I+DCS{|y1o?NA^m@^Q#b_3=<*$42f>ZGD=O6}-+4!9P5FhKtRa!F%S z2XVL)`U74tmW^xT?Gjg0iD!N>*x>&FCa-F{x0U>~RT3&OVUvTvAb&dH{6V2lCy0DH zcPvUBm*mJGg!TL@8I?`xW5F-sHg{4)+4&r){0S9Bx!vzw7mMxUiuz|x8SW+wK8!!a z*El)+sTu-u2*+youYwniH-|6nGm{0#a6OrS#Z7oJsSJ4SUbXNl;g?s{ErB?Db$opr zAJ&ejZ)SUzqi!y=Tetwp@*5veGCxsXP)H;*4X4<3uVjKUEo3fnxF5?E;_|wWmmKvJ zRp=^xNw)xa#~dGeiH>+&Q58pP0g64>RAj>-8u-~2n+~Rt8E7ygV&IG9gRBUPqm)%Mv-J4sd_ zE0PGy9)`NwJo}2;Tn{E1;_=rU?wkJrtz7SkyfI~aY@*c+Zls?%i8%WER_bZ)$XV|$ zN@uK7LQlRy0~KE9Pl8w(jmvpRbBl|TPr2C`$ESLidk!sJ@#*yYs|(Cax)x8EI*jz_ zMS9PMHK91Onmd>zn8qYA?)kDfY_D#3u7AYV@vfC9f#LFmXYNO^pXPrvTxFfL-JB6> z))QjpIf@9^mOKvK$2h5XlGAfo+nUc~=t(7e*#xs=X#1!S3;rkGui4xzL`KA_2C%CHjc-Kc>7KoZ;bUwRd&YndGUZd zINkiou3V`Ox$oM(hVdKd5Nc3dY8HB9+|3DJKKtgvuRx%kxyj`9uO{(4dMxlmKAw;z z>1Hz}w=t@ZyUqaIeFkysUd;#1NlBhPZ7R6f*7$XD8tckeZR>3!$e7@+^(*!2-%9jD zC?f?q>(;!>;jr7O-^kh8EyDh7{{YsnMKa@+;c{t9a-*?kCQRpy<19~ar72~QF+xE4 zb*h5ow&9cj4o=>+4A+ou2!Z8fJvx4Vf~2Cw#hg+WkVs>bA9;Dt)OW1Ibs{caFw47W zn;*Ml^{XZP&<2+iMO2JA5=#5da=73R!lpqe z{t^dZr@dBqJc&2M#(#tOe-BEQJBbEkB*yspK|jdTu&z!bP}^lEetKe@s;c-@P`Klq zpTeOk+W~jsyN(SaOo)d7k)E{dE00VK!L+F5*c0y+>M%&GQcU^*Q3I25{ea(}*qe6W6z;F&BQ+232~WdS}|O zPT)&trB+lbPhV`Fg05TJ7n36y{!|5qiA0BP-g^;I$q@%E20pc*u&Bu-=QP=3N8OeA zy{TMj8Lo;>2*{{aB@O$g$REOLc@(2FF(ClwuoaIktt9ib8dPFw6yfG$`?UW66H4Q; z?4Aj2-XFGNrAcQvKkpI!E2o-R?cix-V5oy6n&f^T5B_94$JbF90Sy6@v2Vh=)tQ>%*}0L-Yoikc1GRJ81XDX^& zj^h=FnSR4@b7XMp^1O&fSse1pN3k4wj8>kURrJL99KZmEV0QGjH8029^j5^AHxiXNL54kU6lTHx!Sg<{pO4?QA&(o zeST-fI>nP|n$`7#0u8;yk$>MTMRcAIg{|%9xnOUXlO!%8*)_)SUs6hV! zS#jnM_!tfSYH;6YhB%_&BI2`K9i2{6O&0B9^Ro#?Vn5b^K^<~B*HhuBE%f`Af;-5r z9%zHHjY!zQ{oIky2D1FABGe_h5vemDmz-y@Bi}X9>AHG7;yJyyV+LO#_OfJS9X@ZM ztY?UfWoB&}Eyn2EQb{M01x(U1kC%*&p0!kFH)S~trr^YY{Jtv4fKC6KZlXC3od z*KuxMdiz$98B%eQPpuzj-b&*li&#yzMWkw;9FiEVHCvf1E)jP5tr=+B(C4*n*Xakyni&~~dz?1%Tr zQ~j~Z{&bBAgm^%x0049p;xL>JX*VNu=zD`r-bY?V102LKqz4DB7|u@^rW|LlGyy1M zf<`!~%j9*X-Um@p$Obdp)Ke9}NI3xZ?@eho`czE8Mh0N)a@yPY4OTP+(KDAaN<6Gu#Mt!Ls)GRF^ZJ|`R9@TXs1P(awnr-A{SX`u7-eYZ` zp1f4Dk;1_&g5XZ;{;-_09yzI9#Ic3_(N#PEGxfzSg_O4u=VIbDEX4M~$0DyyYAkIP zWst1o??^_8BP85cb=3~bg z3On;mmJ4{|ONp*B!MOvyG>mFVEfGN@Nn^}b*>pU!RFm1q7A1Rxgh)A6=QPWWRw$&E zAc*ZNoSMorZ&30cgQ=m|$5Wuu?r$`x9{&LQ6zW<&OjkgzK<|)8Do3q!R{B+jwIb@C z5Y<>-sL6$xEy{ng%6OH1`u%G=Lell9<6Tq1I`mhP>>p^b$Vmb23~j>?de7B$T{`xC zy55C0iL4>d+N@NdE$Y!6erjkjZtR;>7FRbWEpteF4M7jhd#5JQ$JOCD!S@)g{U%Qp zn-dnDd#3oeO;OFI&Kgx7n_CP&TFbE3yhAOaSBBTcdURvvS6%YPKEHi`S|f|XTFk-h zb$h=J>WX-Z`z;tgfFlT5;cgk1ZYQa5q9FSw~XN4C}< zJ7>1qf;TW7y+(dolPcUOB+)8~6Kk*-9D7p{E=vKqx%Q_kk}^J(N-LJ!Q_IVa$BLXJ zkIOa(U8_iB##HMgW1bXpO+G&|R={5^dHI@_BuK_pKt8zZRsp~r>0EXqiWt=V zV^U|4fzEgop^|d?JROnCXfVChWVJ`SChp#pI{*4Byq-RuFml) zk<*3kOi{KaBoij%k?q=mBzN3}dya$Fq9vg|Y?kQEz$+t?2N>wI0b%tB;6nrmUmY<}n5VqSk@gJz#P3ri zKo6Bi$N>BKXE=u{ z-~rqY)edArS~3ph;hVQTX$-FF2MVmm`BOq=$}kM4gX#3CmOX`}l0qAhO+2cwQN3~e zM0TJG%_CZ2WMX0iXgzj+^{UpXD4)SsstNx9M`e&>&@eyaQ4)Z%kPZ)1$uy8cqQgiI;&M6!& z_){PTIbeSFbIxnE&@N(63Rv9A`%x{FyLG?=9jZIgr+)tA1d}5G9n!JvsHRN;rp4N3rf=^0i&_!P)+2vJ_?NgnQCpj!6~8Y5q96KoVmW$sJe? zg#KK9m8AYPgf3pnO#4Vb!n-4iruv>tvbGT_`FYzKp*a8#ra3&;Rk09pYA;fu zgR4cUR8~DI3sY|tXn?XtyH_VXS2f}*NhZCpBrsT{cNoq`%Z|T|VEAjt8sg72z4f#+ zT19~q0J5S#__Z|lKo-)F3=AHgzVtFF`=_t3YMmk62xFXjQya>K)V>P!!ydKFXhgp; zP-M9^4Z$EK^O-d{{Z!=X10>n&|XeF zt8&=$<>5*H0C@5EQ`BeW1HXFO)Ah-%ZA=mX_R6FbY-0zp2Z9f{p;^5}CepevJW1jm z2H--L_mRsYpz}5(9!G5B2D$k>S7NZsvKgdf+~G&lsjk0R@W+GR(mhg7J4%vF?t(rQ zzKV0V@Zzkmh-@w`qlO(K2^FvsAp(yudW9o_?r~E2X{)VG6s2EF*yOaEYYXY)pGnji zH6xPN_f8{nTMdM9^0yL}Ti4YC7<033mzty0FrW{lnnYg?Q>zlj;w&7L_twdc%{ z)Ss^_%||x7KBVlheW8XvSnwYwu1EtPrFQAzpAN?1 z02MRE!=eabCDihd5NUeTXcu>vF_>*)QS!Uxjz}E`1Fda*G0`k#@atdA8ocd0E?b?T zW2I~O<5;u2@P?&r8>x1T6$8{TKhn8BhrS?NEgs^@@<^DvfIUF}01D1kB~p`U+7#mo zsXoW0T|s8?>}Zsc(2fZFYc#4xepkRA_0MXaE-^uH@j}kfu31Ok>7VIb4d=ug`-Nq; zj6@swnC?H9O6GB!STac+t;Ai+xy(9dLz!lEy@z5HANcXZxK3D0zI{q}oSl9%l zmSDrBOkvJ2IX}zRsePhIPTP<+_LWB;kEL3QQdG2OgXm}h{{UoCGMu+zj;4jSagz84 zz7A>2aQQtBF~Jow$@fRf3G_5vEJ+Qjp5HDrRyLw9-2(%GP12U6KzSYm0sk2+#o>I*lmj3{J{{TwAqs0otPa9f;cSZj%PJV>O!jF=|k@^)fC*s7FKSLGSdg!pFe# zLn6TNOt32hle9;-xXwu@lY?9ph?d&!JD4PBW7}^K7Cp7K5lx`qnijgAd7=!Iy1SC2o z`2g)waS&$=6<3^-(xGPvLMoS#{n*F(;--!d`Dt;6lQ~r%b$<@$-ldk1wB=^qjP#=$SaiojPR0np zpa!+7vCM7du;=CIextoGNa_BvGlBeR$oxlInBZ~VoY++a9Qx1%G0GY}%0V8YlmJEv z!96J+-X}g_3>=ObP7nV8UZ6i{j||EsQT@&c{3sDI1Rj~F8c;g$dx{OjZl`OUj@=DJ zKyFTRRFMsi0{vq;^~rCN9+kqSrysp<&gvsCP?ZtJ(I(BXfy zyvfDPfXwVzuuv3a`U=duYsq}TR3P1v8Vrytw!ZLqw2{+YjVTUPh}HE5en*R(rr zctSpFYuLcJ3+j#4zN0ndXysTku?!AD2LxALrTj+m2A+|_HNxM)8OkNt+)(=kwP@RjZpmoW`UwZjzlg2d?Nd;O)klqdFKez?k%iSPupEzF z>%shSP(Q@K4r}m;d{5!eKa2@cfsuZX!TjoNBF{i+yx#_R;qMOPoVe=rY_ z{o(2FNDVT;9tRY=vT(V{s^NjbCp>XiBH8zk7~?pk0xFa9bAi&H=m5tY_7tGGPM_n7 z0x=l}fyp!h5)h}6o_WnIoT~snP=4z7s|jq3ftkQ=diSW=)i$hLkSG}pP&AEHp(DuvyLZ9jfen*BP*T9;@##@BU=SO3rxj>M*Ab$zz&zk_kH(}CLXl+@ zo}lDV26;v!a0Gs}5JHc*e8;Hn zF(7rOq}zaE$AU)Edelf5?&pv==cm0$h>rV6A1TQ>;}n2!^BZfGAr5%$PG(OiC}0K& z>B*}_`5{=g00+|^)mlRlkdUAr$3J)Qq%spKGVWvmezg-q<@f&p4gu@@>QdujK%fvf z-BC$8$Acq&@bVQoGyw{u1`7EHoSKq3V}a46U

|$US(d4AMJeX&wHp8pfKE^NkIJ;XBdJY&71f>d`J@ft=R3O|e*sv~ z*nO)~y<;g>7Ukkzo?~|(rB=~yE~C{g#7-kSh++p-?lD%b-Pxp+c|G2SjocWTMdOAb zR!r*9GfGsj+l&^+>s#8Dv4Z85n707u*0HXYInM8G;Nq*8J~QzTgEi%` zp3?Sanjz(wg}W)++mnDR$splnKkWYiQA=K=C@Wa=WhZ`Py0-D$x&$!CrP#)^z^@pV zbGs_XI4UvF`_=}j;*C$l6Yte71f{xR0)_ek=~+|eSmT@56zAT~3QKbMixce zA?K$D>0Abco;X3KQru4ofiCB3nWR+4z`B@oLhj7a|gFv#Qslf_J*W6-Tdu9zO` z;fV7tT<$DS9e={9vnr5DHEU0vD6ip>$W%w&xQvA(kbgW>I(*MoYO30asLtucHd zV|TCV_aX_Mc1sI%nTf_g?s?8X8o)M@OBPw3pBW9%Df(6XB*k-TaGH03t|f5SH;RZl zn3SK$p1+lJmYx{WtjPZWM`_)98UAn6r|DZBdmzNv3V0F$kI&MUM3Gc&1g~6!pVz%6 z?pjF=MkwP74mhW=EKgmBsT@>G5nO!32l35Vc-2#Yee>3Vk!}6nFb+5;{ApEHo8?w3 z-H%FqWpYXRP=QQfS~VEL0zk?9zsR6KNDOfMi8%HBX(V8ev=RUrJ^ug-e0J;@vS29n zr^gxxV$84W%`pmO0yK;&ERCF;^dHmz0IyZzj%Uu$NCz1`s{a6##R=LKS+SgODrUAj z2H6JOU>x)HqT(}*1hB?PE0PClYedRO83UllCbZri;~z7!j0~O!y$f#79!>yW`2A?O z%=C{tmC=VC_{~<9)W%EN;W7UJEZgp}{)g#WWDX88!=WC&v^wo!hT|N8-heXVx>F~Y zBABQC<98mQ_o>w7hf$wOec#z4w_|uwVa^LM=zYEZm4&-fyo|?a(?ep%?>GWh9^QlN z%>Z7y)+4mI7S{2#-P;eA7|&03Jpia;(&WFHZZ(NS#2lrpc@Zc702NbN+gjS&v{s3< zgO(@n6YrjC*6qVA+4dDJm15dBQ?vjA3=dDrqiMhyCvg7&W})+P!0kB)r8U9;2q!15 z2TB2oa;GGW;MI4JBXAf#{ppfdk+M0dVbpFd)5{Z+5y9D> zgmg7VX+n|-s!wo1;MB<`tj3#8==NU?Yr|Kz1}W5CDE;gt;Qc`EE1|c(k}Hst@U7g8 zV!n2Y`g>c1o(WzlV>wqtf-BQJIq??O`ZTfEip40vOQ0}^{qOFdexFK|n%K%wJyc^E z6-wawvxV(UfankbJ})x;Os-UaY!+ zjM?wfl}drhJk=0FZX=bOXxhUcEAVN@DtH}w(nzZ|QIHT185J5sENAymxF6wv@unhA zn+K;fecw~>O*2Or+d8yQj)$kWw3z0Ub*dzA zr4-swXb}F~(Md*V1J@U63b^Ul(x;Dz%8X+*T}UVZ8NlYB9E1;;{Jz!BZcJ5F{KuzC zgs?l0naRLBA45qC?m&m{uLMv9&`1EV;Evw)DVb6*x#y|Ks8zO& zgp3ZjrfpUPvH5Y{fF8~ZgZHu8f~50~y)oXa?^FW-pFk)QNhjU;2X9&cmu6%ffO}MA zvCh>Q&m_{UTUY&}dYYMHbDgW49&tzrL}2H6EOCy!Q{zGd0tN}*|n zagR|*2$!OOcpT!KGP%x1I#Z-yGFRJ>2N@!z2t2&7+($t`6{G<`^6~*Z(9nWQhRXq( zbD=4{$x?I2sicKS1)S~3pa{m+8&4x2jTYA}XDH@4pk;7L1pXCb zCy#K90^5!oBZ{)RegHBMr~oWU#ZkDsi5ttb+TP%NznT7&ST0_gV5}4z5zhk^n=Yde zCv1Dpe_F2L?p2G)l7EDB`c!EPVZ?)66Pz3U@L1oB&+iVR>3&!S$aL!k*EZ@B?w{(BcZOU;>5~PQ*2I$aBN^!4y3KT zo2wykF^@VY!;Ey#9R5{NWnV+3)>$NyH$o0pur5mh^57p_RJM2NeGin90$>3_C+2bM zk5P;oC0yh%u4>;A^f(8433f8zcZT#cw(b7_Dd+O7Lm(L4$^QU)vgbqQuGJ$A*SV|E z?shImUTY(=`jOn^##x5r{3(nsf2~lMwEA4ybd01DTsyWheFx)Ob1v>Oew9M%E(Ztj zri|-TRU>9q)oodH8w)uZT2R4AR^5T=pQjaXSkv0;!&ep&9LeRb!Rl0k2kBH%Je%8m z05c%|b<&g@aPAdHa%*Z`&kq|8OGNQ#)QIK;5I`Istsop`uWA<}T|-cCtFX+(0zcZS z>o`^Y+(@VU$L2q+dN99Z&616lqwV5(B9RL1R$a=?z{uzMW}*^D5z7AYy-47HjWkZ; zWtEt-DF>5DItb=;9RcI%+N#d_jmdJS9)sbN_t|s9{H#yCAXDZT$>)%28nWYSdF_xJ z1##T>q39@k+V(wO#r)km+5Z5)5}rNORLr?utfa4|Y8^T~x@D5?&+Zea`=YKAI|1xF zSDyDbWnIk{LmbDSz1)3&#;1&dgOGFI){(Qg9)$JHL=$5MkNsY9%ir*zMbJxRUkffdXIwqQ*kDlX$y#M8W|(=2JNn^ z7~uPjxaZi`r#z8%PZire_1nFR;LDIwPTN!t-;GSZC$S&$#^itBC;HKMsA-a2M)$g0 zh*||C><#=*0qxrq=v%`wf)dfBkPcOC-H+)`$bpmimd&=hLpkI4wvX#v3+mEXd{DF1 zAdErgvJc-j6z&@dC)4~Ze^K#ghwLs7*tF2|N7Kwa1N6`Jt|M60ZahP%{jTOH(_-vJ zPJwwp)YRKkRk70e!(N}oJ}9=bNz&%tIaNfFf)&qjqx++g@5!z6xerW%)C%GJHEQ!} zo@$&3P`C8{wcDX)BRI!OOwt-4UTl~BYp zk+r==A(2~1E+p8QR$OIUf-1;6BqV_34l~fzX&{PGyDS`V9eBk#E~K|Amsk-J2J{>^ zrhO;@?HMF7UJqbtSw8SBzyRX}fBNcj(aghZDdQaUs3r3Y1q?_BJ!yzyVOJ|A0qc&n zW;7fst`vc}k6itJwOTS(LdCfQ09Ctk%HW&>jofwoC<8?lz6xNjGDZpZpp}emnM(nX z6-FvEF;&QP>A?QDs*fo-8;5od!Y)8B1T;qzP zd#YR6!Ww5gkK9l)SoFXY0e*Q^Sd4;CMOwv|Sb6Rq+f9XTCLA#h@<;yje?d%!()(Lt zZnxT59z!`N3vZDR$T0tg^TqfUyQ&%FR+Tj?On9oC~WT-*VXk@83W>h}89 zsJ9?@?cStL9ETVqoKuXL!3S~l^q>k7+D2S6g04C`oDu#%N`*^X$ju}MRA1u&@%10! zSJ~LP&OOC4Rt$FojB$zpq%I?NB|r)7NxyI1A1FEJnq;==GjL2Se~ffLQB_((gej5kmPOry{Q5-HCe+T;PG1T10fr**%|pyN`b9Qjz$UZ z+K|c;>NnYs+aexS!*#$Ut~2;n zHPdmoLaDT+bB)#PqmNJ$!0#ktFp|j3Soxrk-Fjq?#}xnwjkFL?Q;O?#i()jZl3C?T zRRl{h%VTi)Pq^LC^Y~XW6{F4bI{9(>bY}YcRnTqIR~+rlpv2uQyCLTv#Ci|@y{2!A?nT4{XHs3AViCW3+YypJREfx?^=lvn`q?unMOx`hwwC^XN8m!W=wSG!}=Pu zLSq0Clg4OV0CUxQRNSi(q=_K+AHsMe>MC7ocIN;L?O%_M`)gQ&t zfckf)gvrVcx#)XS2syzA9<;Gek`>s%k;hJd5l$<>8OZdeCMEzmrD8xItpE^MfxzI> zuwjBg#Ze$P1<1hk?@72EoE(AB_oot=9m(zKQSazFo}Wr&wj#N5aHJ8&4F$cXnoXG}*Xz=r#Hr&L_NH@?deabrd0Y{ZNi!@8&c_O;uf0ci zav(v4C7Xp&)~m%6mh-3O`!8gsAQ4M^gUmZ(>Lbr5-2VWDD6^FmeVT2H{{TGSl^@~w zRqJKg#em2onwC$LV*}=(EN6@30`0nl#!nlwl|PSf#-&D9AdDZOr?h*cz%^U!n+E7l zmj3|kkC*bK*wQW}fCUVs_Ul%V@s6L)v0C<6n5^;TM@LRa)Q+O2j_qB8th^uM2Bzhs zNv8>nl1DY=9y^C_zxI&0TZ9CCM+f>>q02fJ`^LDR946~~SL#Es2k@Z@vxzjBpwf4% zBzQErsL7`T#Q;g#ie_nZ#R5HaFUkPTJ!3`K0A>rx9)_n_iY{~4r8Sofp?+@p^sZx| zLm+Z;xB%c(ix%W!20LEngHMf=FU{JTakKz&fsD`q$MvZhYb$`+IO|U{ z0=Nyg)~PhDyJ~-lvM~wu}xsU{y3yLW5H%01{8j-heK7 zz#+=u-awV6W~{K)cwJj(zE7jzxu3akK() z)YC2U$f`lw8+J*eX^CSHUgpbG4A2*!EuQEuDwFgzO=M`ehP?Fk6t>$do2H>8l{{YsifFh^`u*dT?KiDRcGDuj%58^zZ!kvMe^2=`N zyt+n;Fd2>q=ULKf>Z4_?y2uAf9AFP{I-1C9d8}jtVZ1QHGjoq`YLzm`><$PXooKL$ zP0W`sx@I#l`>c8opr!=GiZ^wnjq{i2IOsamo1g?o(%&g`#DX&F~RNAH9AGI zTrz>%W;4w|wblbSM7|%@hq%m%cXXRW2^7YSZ zhC=(kN9P$n^PJIT6X#{)QV@OI za77`MWwk*f{E8(}xG)$$ome6lOt;?0YLN_S1GJ#C1GHy?b6K)GWwx9zvB))@NONp- zQLid@Nr>VudS~lP3uXnt80nKywY+4KrOC!mJu62~fN!*RcKj=YyN)|pVZD&=K>V-2QJS%D<5q}|nX#VrSCiH! z1}=2xD;{0q9X?Gf#Colapa+Up&JP@rN9aXmEp)y%yNgosJh#d*_K1|9_ki2|1#@9a zk&%k^sLJt8o()*p*#${I%^Wg%uG94&{cKW*A82N5#(Wz)burD&?f3w`u%O{wTlg0s12VtN7y+lbrF(jXQdZA=S>~lj}iAzS2 z>Ru9QCh+B}Ips?r9qFiX+35C=xn(ODEIkz1xpBPyewu<*|85qXL)E-Cc{Ogmqk;8nfRmVnCgNpRK>jllz~I)gW>Qu);MVnfay+d@eJzZBzIR15_hm`NsrQszCeONcA;#Aa9mf@tSgm-GSer%`eFy+M87V0;Pfy#72Oz z=rP>V8aodO+HBM0pXJ`|yC)xXdS~&jh7d%U7~Fbfbmp6Drmme~ZERc@L;!{9oMiO( z%}EqzYvUmEyJ#mq)th%_t76i}<)Mw2=G}~j9RC3IsZjwRAlz_AJc^DX834!tDh>uZ z(yN72^9&q(pXcj95j=G>UBa`2k(N1cLG;Boc+Sj#Ngb5t zfHM4ZekrQO18;A$*4#uK< zteo^chi__PY&U$Qx7}WYH3Ko^56bxXJI#j+&24qq~;~B?OQ>vCEXQ}Dh znIJGi&$s07k;Nj4MHyyRiJ2ru1zk@ehe9jTG*5|obe7`BU4}%R7Y`~#WsUvF87h6T z_}2$%ZF6&%1VtBJ-N^p{XZ&kc8)dhEs>Tx>k&aKly*r7<=T+h_6KQ&ml5F&Wl#)PY zVF8R_eL%^`AC+=jP!e{N?NT!`E>vwEpnkN8w5Z7koY8TmVF5sSTyw}il_+ATg+8O2 zVLeD0_ow{LjCylHM2WSrk<|3*OA;t7KvRRr^%RO1UPLUXRDws^#r~Z*sHKQV18gfGKP!$1rmI{=r+WFQc=>+t z{5#V`vE^7CgPc*w$0H}L5nUvTHn1bt)9XlaoSb8|Q7j8EJqAauMS&sQcmkfoJxwsF zQ^qzC$)+IP$;ihQQs(rl7@$!jpKrTNmgVlpm2dY~A?5yq)}0hW-Y=DS2dV3tViFs1 z8xqV~M(X5qRtC`~Kqrc7ZIVOu=bp79s|G3#GwVgbnTnM8Mm=f1X6C>=?~vKuX($0XrOqhD27o=S zerzj`ekti9g~k_-X{Jb{ka>p-ilHsTO(78>0CXm~k+GvI#*7~zDIEZ+jWCj6$T%k< zM{0^Fv;MH>J*seoN%DXP@~Qx69oQEkLF_5<#0Ef7ob{>p6ePLe^v8NZk1KKe+quD} zAyp4^*NQ?it%7n6I(DfE+CwPo#yaAMRR80~G|5Z&A49`qKh=u3V^5fC#_?wL`Zj=E3d9 zYKa`X6-8$4kTXw6OR@7G1aY)~jWmQedIRt2N)?VXg&lqAV^l;W3=!!|y9Atb*q+o_ z2cja7%DsA%)YY>l*z%;3Ip(RC*iKlH&CruLQiD zl8kzqR{4(JIp%-_X?&&#rIehU3V+%OOmm#{sDWsUV6fac_Y|Su7-ds}dW;kGpb3kx zKQfR{BWcclwIH3=HDzu~XC%}ch*HCrKa~#XgKDD|z~r6Oa( zUOb)Tf={hg!?6Xhq~!hsrAYWz^BuPkGEg7S^rc~sO{O_Ebk7axnr7!kz}&bLJ9U^bFtOQd$}2yox-f+A)I8I{I-)xLFAeb`&C%>VCDg z9=9;Wwyk4{@m`MB`fAIt86lNfk9CcPFn)(`=Ui-Z%OA?FzHZhiF6Bl`M}>2p^NoY@>-twzr4+Sha8H>SST4XXO6|Nm zpgc&oH;NaK(Zb!2_mB8jRcr9>%S_SNP>q`A@lNO7K2=}#N4Ke{bHLk*=ACsVXqzfd z4^qH<40g>(g;}wjW2IFMxMXqPBBW9Z=XYAbqHsoeCkNZL1%@&XG1EMgP?i8IHv_dX zrc&5aS-ON#%`q0Y0GK)5)ML`1S7O8<-S3_U<4CczjhRnh;T`^`y;xlI+kuUv9R&ju z$QV;85_=Ja{HcZ}wb~&C;vAUI@cli#s$8&A2<`Q$b`=jH8OAa@&>{@04e^y^JOV4v z{6}T>ZC%(8mSQ9yk6+g{-%SZD(n5T>4Np(L z#p*=Xha?kY?gfJm_^Crj!DTteam7uvb?s5efE@L$Ze$o~Fb2h6KyzFE9JTYcNLjf^ zzj(!m9CZC_0wo?`#~(`XJT+)y(_Vfz%-jt26s56oM(3Jcl;yM6J?c{CKb#)i0otHn zFfjux#EhQ9=|UrJIc6Jr5A*oeWs1^cc_8J60PYVQ)QgcCmixfqkSUP>M#sS!=nYC( zMw=rfd34&tu(0q({MiX4kJ&ie?dSRcUKnHc7^WbjySp& zVfT-wJDS6WO-lGXSfsYLXB%D4BqKbO2c=%K(bvounv{a_?2nKz`655;uk|&tu!bSI z?dw3Ta-EH}t(**&$1j{dPFSB-C%suNTphe~`TNvn*fZxFTn-!_N4L24sLYKEDqL*? z{(3LnKVL)oP&*e8vo3dS$?NE8k{#G!ijAMTIL$SaIL1NjO+8m)FqD$a26)fwKo5og z0KChA+z#~O$ z!;BRfqQEXNoF-3RO*!F72rQ%(91+1EjZR|;7$c8UoQkC_%UnA{V4=w2rp8r%o~D5i z%{jDm3_Q|7DYSFPsHxzyH&G?U#EU3yQYg>Lzh6^QT3Dn%FfAm{_oQxE{{Vogq_HKI zSjmb;>dVgWsP9tCM5+`3K-+=_Gg1X`LoO7KcMfq?e$ga*67nE}APEmU<1z; zA~DI~O8V4|^GEXi)6dtR(xMS(EC>guG{jFFYqw}%c=}RFyJ$EV91IF*kOs*B5$jJ< z&%s=C&!sU9XogAT^UX(-ox3tRim(X`!y^Zw#YPpgz~iRxS_0x@b~cp^J$uu@;DDnX zWc0-~@(xGSG`wvYkaN&cV7RQ?R~YNhH3%mM2dVt3Qzrv&2Q?l^mn<@v>L>ySju}D7 zD(8U9!Kk*+2Q0p9W4e9dJ;#5-t>6r>7*c&IJG;2jFiPZsfE03RqFNzqsC<@`NT3c$ zJ--T|=50Y2?m~7&heOa&i2sz>$?y$W9Ma=|BjOBzP*pm>vcx{#j<+ zNx|*~Cz^nQ00jR45j7+-q`YUE1T2t%q>K!Y&a3%IS7;o8oadUb6tN&=_Q^D^yGR>I z!0kYaPi{hIC2F&-#o7)>YSfqs03ot5RAk$pa53sBp%k3(25?8|RY}<2Kh2)zuCxPj zJw2+s#=(i&Fe!+G?gw%}KnW+(ywg;?H}~?+#~`1VJ(sn2z9hIdnjOOf^GF0>k8}PR zuQ$&$!3_4MjM&pg!08ir-scOQC? zs6mDtfr3RnVKPcEF_GWWxy^`#ML@vLPagF$CS0fhdXw6%wYQYn&Q1n#=~0MTocy?4 z=O&nlm==?1Zj{$07XS*h2PN7g=qjl}ob;dvd7D>iuRYB;hQ@duvr;r+3b8z613l>& z;NTp9e>wnQKXhe|Mh1FNQMZ$v(ziWx&rEitZ@Y%he@XzQC}fk(QOb@-sHqWzfH3b* zISN#qgY8wB{Dve9BOVW-pe|Wqw*>U`rV?2Cb4XZlN$XA|Fv1bQXr( zC^3c@XFG>#U{4jJx`c(4Dg3Im$lKopGXu$|ET%kd2j%KT5-B8*bAgg50vMJ;jPkDB zpOgrD^yjoDmu z80M_bBAQ?mc-k;2vZIn4r~{}RkF5c61PrcsbRCJSa0c@L%WcMfQJhpO7;rcRhdptc zm?=mZ0DPSAJ5U8{wvJbFpoGprs;ca#X5y`kgZ<&{PA!nb7~Shg;aN(s$O9^SieLtI z-n))=kIJ9+g3^M^#yLK|=Ac|{D%?Mz{x5oH{n}%x=QM!THhyZXkg~As7p7{sg<=dQ zIbq4#bInzX;%LTqvVt%$X`)dYvo-i}bqlIo<0C4$2l1=AjKHo3>sk6~5$Y0P z5xoZ_^~Fu8$6~oO0u}WGlSNpZrYAnS9DC0Rb znCLd2loBg?O322myo~!_yYyHd|;z+r$Y52)+@eXE*#VjFyG^LlnQX3_9> zDJsDA?hRC-bSTQlqQDa$DysbkDI~Ydfa99U()B&UG`aa)@E@~%eJiGjD=0Y4a>+Qo zPMEaI+Ir>?r=~DL?kk7aH1nok%LoKUet6HZuIdK~iicU#W7aH606x(l%@ce2eGOeH zS?U!dbb00o3PWeqW2GsON6WD1ra2`0R3y8zCc_svCB`~+Jk!B3WFP^t&Um0@N4I>Fvlzxe>OTs$mgHM1 z;{~(3j!5mAUCi(E0DJVQrJKu{K@3-ED~|LCrfor1W@F0Z1ZNe_c(GO+C>6PPd@1Te zfzR}=o_T!0LLQtP4wc0Cx^{g+7!bqLZ^dNl9F;a0$i-T4wAiCNq=YH5%ORC5Suw0qaMYT>Sihvx>~Jb&v>CX$sv6 zfz$s0uThys?XGeLG5-MTsPpoixcUxiHoAfs$!(x7QpdN}fF^13E%F*c~B{o~ksS4C;2MP&}lc-3K0FaQ|(n!N-Gy#_vDGx+tZO(E#)Hm^lq zn5nqzRahekr#pJ|98_-;lw+Pf7rjpGj-&?2LOARXr683`kO#gy;(?G9RgmE4u+2uQ zCN01lK*>ES%VoDQJPv@5sJ2EyNag`-1e%MHmK|wiw79m|C%m>THGHHi!$8OwJ z62@hYSPjECCp7aWTZAo>(>bNi%!CJN7~+ASyto+2Clm;5gofGx+!LO@m1bEXw72soU+19tcJt1*)5q>M9uMv&4g2tqoKl=IMcsRg zt3>QD6}$ZpLsQJ=MG6=YIRy2hcvfiH90IxNimD^Iw$8)n!aA!ToqnFbjRP%=sZqC_ zat%)EuA!4~#|Ej}+}lWZt3E)(D+~~Q#asl;1`w!GjQ6Es++oq3^{f$8f? zU@@>}IU}l?889*fax>3L0CrTqF~Gqz&^am(Pg;0SG7Pc9e>#l;0fPMa6wsOi3C7-^ zr9|-l^V26B8kEigfw%pfe_9);!5=}|sF9h;&ONi~LlYQ3F9)%wHqZtRGgPOZOPR&3 znGH66cO4LaQ$>Yw*B9Pc5k#g*+oJk=pFu^flf?dQ!-*wsH+BQiQ{uLY);0>}M`M?e z{{R(P+!`^w5V+!t1;)tRTLf-B0W{J#F#|h~KT4Yj{Q%A~X{mQwQSJxwqjj50?&W3>V;nFp7-cVPAu$p%msT&+VD%BwJC2P2Mp zQzf0sZ3ib9B-12u+wzxS0jE4_-~}U*RSzrk9(^inX5G1adesQze0MLEXKQXU3&>pl z10UA7GI_4I$I`oM*72O|NC5=*$vOU2$dSzefbwgv@Fkw>yNlKwL;@-IJfF~hwa5;5 z73-cK)6+`PVYZb(k(YZD{p0>|O>PXp@#?g;cC&8V9Bx!6sXhMyO5pG_UB{0u5XWb3 zq-S$E_TvNjn&L>!HY6acnq9ZsH9Kp4>YikPZexHss)1{J!dCIm;%f-qcQ2W{A3#X` zMN82P^#nkSRPx7yPioMP*m+)O-So)h(Hur8PB6rD_7yyfAzv|D=nY_OK#?4h&yv|B zcNH@2`=~SZq>X^w9H?+UU_Zv9n(Y*9Ok+JqY5RCLZwDksQu_eXD90E8UMH+;jOrE>k+>MiH5q85$C&(+_eB6da*{#ooRd+^Rn&&zO#xlpkiJ@;nd)icB~P4; z`<~{2DM0V$c_eZN%1?3htFYS)4ac8)ig?R_$S~uOYHNAcScCLo!OvnS0&<`(%;KFI z<+osM7^DR^t0~S$KJgDcA&Y#z$HJ zMQrXugzh{yrAUw@fEIDbZ1YOLx{M~^PdTRK1Z}t{j2}!;1*8c90UJK&o@!Sk%>WSF zc)=LStnj>VXBp4R2UAy~Vm0ssMhu|SIMgtG2XsxrHWtxRw&+sor8l^@SFUSTLK zR2_^)Y082&Wkqrc;2!h=8lZK~aj;~mJfHBWrE)?AIPK|HB-qc4eb50p#Z$PKVeET~ z23L|TxqcaPam`1&1UK`h%*y(k$2XNBK4B_PD&B^p59TRG`AYR2 z2iBr#fh5E^Kg8StQA&)*acs-jo`c&I>`P^h&d$3rfPagnG!3VFoy7D10O}NwH<*p) ze3Ao(8K;ni`j7`16f_Bu<@a>RCagsPW5~k)0D7gjjTDw6a9)EoEX{zsKpVP`N|LS- zrMzD;{pJrqI{q~B+qW=K034H0f>dyS`qbup{D=oUzkPk-w5NRZJQyFE)Uk0{{RGJa%%R0X|qk1;z5rz@t;B7vv0wYV;p@{6N=@QvpQq& z7`F=2jp}&L1y?XJ>T&B=T$N3V03Lu-cH|6XpK9q#LkGEA$trz^z6D1lDqI-IR|A8N z57w_bgB}6N>x^`&<)mgPgRfeoCP8m0JCX7qKQJ}dcym_Ve#J8!@?$5T!ntXn6F7|& zo2>kc66EvRy4^FtI{Mo!_3J}nvyiUYZ^(aADrXL4RXJKo9SD4Msi3%G{G3*7VpT#3 zXI10O+kn+7$Uz3U^z#M>N%)$wZ<#Hf_iQzapd(0IW&swuFM8#Kdzzjyw-!)_HTT6Y?IRlQh6HZA` zsuyA8cLx~jO(120rzCu%uL6K2-y_MD$Sux2y((FzktSshIL3JCRn?1*PnKAWkUG-L zP)M$)+doPGQdyK>@y;99)K?ASY=3P&eo?V;`B!kVAGw?^G6?HjH;WrO<12D}wCC2U z8<9PXR+2S5w&Un3Fz+LSx2H;Xl(KXxr_!a00Kf9oA3(h3wKFTHTV* zqp*{$Xlk}z|)~u!z*;~v=K4LEH9{%+#vIS7UVY*-%!qIhm zYkd_`c|>gbh1teVu>Jqhl$Wr|^K7;B4? zIEo>Y`1%TsG<%Ih_u7oX_MEM%97ezIdYZLiWp7~)+jz?4WV3S3?oWEqjBX%~NgM(= zrE%DfD^Do!003YBF~_w{3Z~@+M;(~cfx%qzdoN>BDyzDIjs5+pT%uTC%azo=GCs92 zkVMSmATj6-RXJBV2N=Qi9@Q@QbA>_*oDgxI1p+J&EX0R;rs8q)Jo@69CAVg5u;6DW9mODU`QIC5e}~X|Qotk6%P6K+>&<)dTrpQoYyD)fr`{#L*|sHA9%$W=!mcBUe% z5{VznWiiY-D&296ej=roJ_ZWpjF6_Imf2^B2PBYktT@5;G}W4GQbotl7BS}q8dXB!f`-WI2TF2zPKe;;O5}6F6&>Bmz{si@q<&U4`^5b_4|7Zf46wrO zAwapwEOAxT-D)bU3G$&i`On^OpzZaf`y96tD_y!BysS(2Q2zkz$KJN$fo)?Omtnvq z$E6nw73{R2k@-mw2r$M*?rT~LQ4?gcpk4{j6+z^ibMVYD(N7f|vol~y<&$VZr^WE=-mhF|mPO-b%7SQ&|yHU9vXvGpG0e}z~w25^{OypQiu~fiDYi`Y6**G@@*Ip zW?{^n^d8>zavONd#y*t! z3b)-DZ1?DC43(u=qdNgSj!C6*Syb-p&Iqc?#dD98`g+u`&ArFW6cdBEbQD+#7a>6* zSw~-b6rG17o9)|%L+nkBq+&G`p=uL*R0Oq0t+w{2h&^MgMr>8p*n2BQ(MIj9R@JJv zMryZ4)%<<={)IeGlIy;&^E?i$NsDvA`H^AJC>j9epYzW-PGn6y!eyh&A7mT8iE*pk z|K)dqJu^-n*LMCAnoJR~m>4aw)cO~_jNE9nmSfQaVo?8P??}xJ@bi_xMzzhWzDPDN z`IGD2FQheaetG@I^5#^Dr)tNzOP|Evw>aHz8$k`+6;4h~nwXmpZKG=a$8BXRrYdDJ z!SC+l{De+{=mI#|yPEo7J0k3|LcPLh&`t8sl@BQ(nyR&onSJ)A0F@?U|0$ zi}b98(rvVilA?nQz5g2fRm$@63WMj&;UR|4!T6KQe;k+j>)m{J1oqp7$}7yjGaY2j z-9De}x>9_|3*PcRmRobYuIfp0oKiArWN)~aF{Usm7ipmHu+W~ue-fb+hDwJBU9-}R zusv$VKQUZKTEAdhw)0Kdk3Py&t!l7)68w8+dZcP~6_+sxrCh}t9#`WA)SqdG2BJ2r zwZK!RJ4VrNBj}p%#%7QYGz>a>CPSY0rz!Rq$g^bT($ZJ>-~(D6@14vGn@6ZLVZXy? zX9NU>(}+jJ8-0nFcMo&&m_CcCWG6l*1;9z)DMdZ|?;B7szvdhcH}1fVO&gIX6ZNn) zPhw=Ls+yi`?BQa+AX;4&i8L0YBpOpp!0dT_M zY*J%e&8L%6##tRs>x^N5;B2eU3yz({qlZ_`BYuE*H;u1vC1r&NbH+&2O^*->rypV3 z+~Q{JqDIB^{1(^dmae|MjB`g*IMc~{`5hstOEb)Yj!J=C2}qqcZ-LuGfZ+cD?p5_y z&=zw-pm$mepX;{9H}Zan>@p>F5^(S-3Jb1a>Ax;W3o|{zbq?*Bh!&;gF@zZbp*mEP zGuln1eqR3>st)M0FfG!pb+9go*2cYi_ls2Pp8i(yFbH&3D^Z4Jhci%}?!#IowusqT zs_HdAZVmrMr5pEToVg{^;R>yU_8_WykwX2BnQ`qKR#PG)1%H$P zF0YBmsHK`*6?>uxi(5LCcT;nOr{9pG$cgTIdaepCM~AxkNQAW5hzB(*15$e~$#wld z01oC3#gjoAHTV6Dy&m~xhMZO)IpjC}!^6xT7Nk|`_CS2toHG17H@Hyd4SdeohdTvn z9-6!khjOt%?Cqz)^3?e1O!2-Wu}~gsI}I~75VS(^#zW4^xV@6`T8${tE~2LeXwCR5 ztN6^z^lwb?(=|EH7882R+OktystHOP7v`OLU>)$tc6qk;2pNm^Y#Z^1-+yIuGu)Kv zw^eqI@p6xXX%0`<^TPDjI*$9xg=yF@(%@@z#fG~l{fel6bdoNlV&=OP_z*WRky(iOH+?+RcVlRyG5h{Xtcr|8!JwBFvkZ;YWJ+l{a`9 z`%_Np4f}Dz7Z%=X2+qpvuIOA5&AyN2TXRg`>t?6I&Pfi-Gomclxs#fel|$=bABqF8 z1rJo+Ehe=+8hT?)EgA#17arWRn{l&ds6gL>}Oydsvot33JTrm%` zk)N@+c0<`$3X9J!Dc$9u7sLJ%h5tzjySQ_xQD23GkOQ;A^# zDTegVg#Ra*R!M91Dg?>#O8+Jugu5^y;P5A6G1Pc@_D@}El;cfSFlWOSUWr?3&p%6-@U42l58A+9Vb!hJDlx;VD-~4>F$)EHpElQ+PXS2n-J)tbV z;KA_0e&6LcXi3dLxL2<%dgI55%nv)b_j*A=z`<$VwmW^22*uO&i28Gb5E%u#Y9Cnw zX^Vc{U%F@CAEhV4lNyx2UBkB8mY!;Tl$4lm{KM#ImCwG?QcWYb$&k&<*T(=l_S0Z7 zD{#e0Oq=E#7%=!vW2KNzvFejLCya(385FlVWLeXJs%ZrCX+;F+x+v3M;b(=$ckJ({;k4o3zB{bzU_r_;*aE9T5G^fhsZ zj>Ni!A2*C1?_g0$j zRo=8C^q0cr%sYH8J;qL$dtRz)MTbVJcgSxq1;7_{u9ShDBjG97@3(7GPSwpsnq{>& z8md1Y#D`3MF0N=%xHF5BP^+_E;VH4hiYp7b2fvrVllX^h&LOv2`cIazbCgfZ0#*A4 zB*=!XwK87Hb9Ypsy(H`tp;;DO-x8eph`gA*8%}<|#c6bW3nnIYFK$Bj4Zrv;b4E(C zwA$g2GP&@BE|MT2Ws(ZuxQC>kRDkO3N3V=Gl0-E5p%n7@w(|;QbiGdc9XC6wgOzED zu%!u~I&T{nZ3(%RYsD1Q)8P+^#-0*<{o%-UiJhz?{f|A)S^En`dZ@IwSIueBzbgkE zSlD%F^|lK}GDDR=!_-9Cy`{%cL7L-$#kj*EBJjJF6fl-f@3Or;rDCy{d~y!D-c?SE z9y+PbZTDSkmXMWQm*fUVm9_yz<%JM{<@|)#4HSlK(pbv_Ahen7ZbTg`5tOs`?#qC? z{f{IiCU_ZOgP}_ULo^VzC+CWm;<{(pNEjiw%4TU>3*no7C}0z-ZEU<19MRo$KIk#> z@*|8`SZ`je8**Qfui(pL$b20{kCi{r<#4hgp@j0z{ypG*xFY!ZMZH8hb8Jdc*7SB> zl8CqBhfR~#m~5fF9h`@Zg`J)qll*B#(b~cAy3xIW$dpM`N5D~jLi)iIz=j}HenLcC zjm_%ii-kh=$6j?*fYT#2=73(_a8jwqF0c;MA;J92M?!f2?_80>=!m{bE`6Zn!-M&Z zQTEi~-sR~V?XZ(nlkW$IoY1(HfJT##BEJ|64RwE>jj8hmQu+c8%6;87k_q{l3rn6| z+3hiU!vp|L2Gp2uje*i*?RUgI#fyhd2&R&31`4(RPhV;}Aa_rdxgWqLSnbOPt#0;$ zKx0kMsFjIX{g5ai`f*!w&|BnY=mpX|5JBWo%{gba=9N zbE#tx#skM55I=bq`ReBT<4ZM}P|IGeU7bDivQMstD;tf(ced)oCb~cC(8G+kr8aUp z+!WNu`B`#UM=4{e5|G@C21w5g)~TxN={I{)Y(Nyr7Lli)4*8V4YeSM1(3k;Hbi{L* zbMi2dm$<$Ikk%A~H{j5Px2+h=M!~x2pKHyM^RtSr?r|JP6)E~2PD0e#aE%A&_Wn&? zsL$sfE}f%8*yAG?p0t_Ti97Z#nAF|SBHE78glF(QBx3sW^O=&Az|2)AGJp7UUB8U) zeuLB9E@(36Nrd4-!Hn$2AVBHF_{HrGB)BYNv@&mT`SLEHVElRmQX($(o8PWf-noBP%|3k5_gtJ0*i3E8+o zn=5Y$&&xJqhIAdE_?7te#>(n)EW||{{1{Uh)&?Qt(Sk^-RryC!U6z|&?%g# zr6Aw}lYPDLLE%5k@!U>j^riBU`vnB#F$Jf4da&ie|FQ_M&xt4l zUaDLPOylFTOEaN5^$V3J>se~dc|XQOd2Llz$KNFny$G}95?CgxE7~96hSnObL+`28 z1)pF}DT;Gf_XWeN(vfKy{F+T%x2;sK!mB04*YC7X&l()#Ne-!%r7UoKeq;t^OntOO z=0rudn-h`fAL7Kbg>+;lw3|4%SlG{nL(JOm#{;BaaLEGX+3cn8QSDC@1WQA&Eu@kBsc9-8`Ns|IvJ!Lz+EJscN zpt6V|a+YSwmCG>G%urrurV@#422iBO3&Ha@ngze9h?+L{tbH>VUW_Y%iAow*WHZyv zSNreFmKf;}eVOR!EF1p0k!p8j*ArKcq&r&7mNsJRp_0srIX#9wfomK+t%ZHL8vP<) z_>9{OVSdBVZ}8fOtTFk9P+}$wOWEK)iGj^-^~5n^a=A-iKNwy3^PonXigFw#$9xUt zy15FUSG+M0i54gANIM#VdOx}>d~QGC_^))3{jRk%>2ukL=<0+|j{C@~QleF2zK!0{ zro-x?)A4?osLuJFy<3^`xhJA@4`7)n-2Q%B8s~$V5$6iztkiX;0fK_jDDKDHf&$+y z`$h+w!Q?z2K;+5+6#{8Y?9X}5{YufXVst0d?bxO8Rh;b}{=xau1le2VL^H$m zhKvC<>28&l*AR*?lUyL2@DRLvSmTCeluXJymZr`x!}G^jGOtOgH;HVvmMU<1Tzknh z^wIF!JZ=&~3PDL5)vlM}{~1n=Il;^xb*tUW@UAbFoEfzwf| zk$&Tw{%c~PrXHC!Jwin62vLejqV&1kztW9Zzj=L78ta@7-dt=UDx@&ii|ZtNkH5Co z-!YDN?WeAsR5=aKi1-}GK(Fs!{M56~G0rXVzD2$ck4Fii05|`Aq%y?#;j@Eh@|$>o zHDaoXB&HQdg5Q;}nzUG4DDXLJ>;4~L9i&(3zK~zQ&K}I-!Er2Eko*DmkW$eQD$f

~BgQ0!N;Khq&_lT>BN6+t40B^*8ipo^GMC_!o`V4lwVd?&tmt@48Cr z?s^?j30SuSD3EiZ4~#=~cD?51wR6ReK3^~~nQg|!6>xv4v0XZhZdH5q1SSq*;QXEx z_PISBVF)1&i*(QH9U%|dyh}G4^u&g6EG9uLPpv3ZotduHb;UBs5^s-{cit}z`Ed(8 zP7mQdR_?9Q2=&SnnB-5u&UXL{|8W$bSK8aoy+U&;H9AOTm^?fpCg-mT4HWJ+bUlq( zO^^p%nH7Ny)Gl;3T{DSmtWb}-q7Qm>PT1^jxyOl=l*$9-eY{3Vyz8iM(wlZ>%|;wR z@NOh=csb&{a2}1__Z45c`t#4sUbme#T>`HR^QuYAQ`vDs^+VbyB!4!{sS6L@RW%zWnsaE|}W@ zX0xgw>?rZLRxEUQw{ni>xt~?BPrwICdX!Hz=Hy;lrQhYW6jBr+Inl@XF>Dhz2L;V% z3&W4riuCd}lg1O~zcy(r!z@P+Rx=Vul#`3dZ9!HVOqR!yHTNLmy((ze;_6HsCy~q@;Pt7>5*^R*=x|ouTQ7fW3|-}J}N%`P2H#S zFNiHjn)cX^s&@jQ6xlC5?z6j24YLJV1?I%f z9hb?U$6gd{y`zOg?HkrqnlXtWD4aW+%HDy>N6(S8BfJc%-zN-f__LDg|7^C z9UcY@4{h;mXpmy6w1rSOaSyUZO%XI_1tX}Q>unO0&abduGURIvLnkx_Tk>we0d3%4 zxyov3P=ODQEPNU??RmWkjRx6^ysPd0TbGAtMGsNJ0qRl- zG8j(_iWWG+*2|ye7-f zn=;il)Fkq0q-)h7Gz1iUT`#C-;;+Aa>oy{3_e(ss` zn=NfI0uJdQX?U59yNx^LsSnus;WyReKe75QJEsI?Jd|tU*wdqr{KYmnLj}yrcat~b z*g89}B)eb)JD|Q5)rv!AHhIBE`59%}=A0T;Q@|cI6463&2+Gj$JV$%=m)f3MKX`-7 zEodwgZfhbo8@t?MAg=6wg7+@tOXdYcwUXRzCQ!0oql5n{)8AjWv;zRPvx{NyOA``M zRyrfEX?i+h3p{o2wMUJwL3q9oeD#~s9W~klDH6PD<4i14KpR~Akm8Y@p>jdd#vX)ue;|%L%N=6VF=YH>c5_oOV@XHjG4=^SJUqo^T?xO?bH=UE< zZyrRrr()u=GCgda-fbRJ;}3|jWhtRW1KGILPbGDI0W5-%)yK@O61%GP#%rBG#bn-; z6f2tCDKZFJ2&n(iA!M|Zx#7Rd`L`|!uNC`H$5{)VEdzy>l%2JI^{%GJm(3}(i;L)m z2ghT01T$jvGmOEx8i+$*!?2XyoosTKQM-L&j9)cDlWXo3?cs=EC*in=Hk|o&)Os01 zWKH*50EO(j+$u44IHx0;DR^XNpQIsQ5t_iUi4jx4%#ifv>lFO=E~TJKc~^q7MZ(zy zy7~Mq=&ZWLHNhg{ngEA>{vbR?tHa+JaeqIR*w{5C$*U<+jDdn{B}OD(mP8P$xtb?Y zdnOkozkdPXOUP>LMZ^GSy5I0=5#5OK>9r2^{gr~@B3IX9o`wAPFAOBLZE@-o?vGim z#R-C{Ldsf>%T(SvHZPr#)RSz%Owkg9LVygeE5Zvu^QQnlq$r`Jk$)kMJ?loJ)sHF4 z1s-%wNt!kMl7Y#ea!uKAW9=388_T!;yFEF{ZR{V;*@v}rTS&RZ{53jwF`F}Eq4gdf z7KaBpPQ?83$i?Gbh0vr@aS(XW1EF6-dIpm}yM`Uq)Cm+EBS98iJ0}wUTC|lrK}G)6 zR3f8<>+qRX*7&@88X(yXTnCZnF&~_>xhbX#k-R*sa(P#lN-{VOah#pbx8V9>MI$?# zNxlBFfaqfhu zmbqUO^L5v4z&FA1DYY=tP&NTR4xu-zXgDNL=GoKChm!{TzqO<5Eu9(gS?ydXeaqwT zLzI{_Ye4-ARHP%HY_Q~``bpM5`I+H87XNZS&IzUiQ;Uah&6O--&ZWecamrKWNQb9H zKY^$r=_@{>OQW)cJ&VThWa1f9L~#(T^3Mw;8Y9LvUT;b;X}sE1yDWt|p{Rx{MRUB_ z#jd>m<(LqpvV$8p9|{%#jztu2Sq9vD6DuiIa8fXD#jd9|^Qh(HU~} zxi;6(*OyH<)8TrEJw6`Aw7=RO5!No>IOH-TYc?UNeN=^qX?xZxmRVG*AQZNBU+|uY zjiM(gqRWm|Ty#9t@l1KIaA705&AE{GHu2-|S9`6Oeyx6K)`V-btWY6G4ZGb*j^cwX z1H}x%;z?10EW((p@YI#n1gLynBh? z#$C^-g)c&XV-la`5pyScn(dj&vUkLuE6e%&-H7-mnLVve7!p*nFhn^wX@PtBe5K}a z*+fR~AzXgy!SYPLbC7o({?1&+j-fNnx}pd{wVW^GzvN*EkAd)j?k71rZurrWzE!ye z{NNXH-}EPTc#(`#4tgbVYnPV5eFx!`(*MFF*ANb}ehFa&~_hre~X(k}E` z)9U4s)n+{7S%~2P6)pO8K?@thj9Y6n@yxULYi(99TI&3>$VZQjVq%re&e>4R{{cx+ zSy|K5q`VJfr#cv7-w86NR{D@)qMpB#mI8G77s(>%Ej+6aE10d)gFet5j8#^d& z;RhDvesk@f?4?eZry2}l=#UY|#CwnH#_bG7x;Kk!Sf(%jGPKUq>t-78t!yeHF}~+; zXh}Fc7!b)+^DY!p*8XFn(@TmN0!_;QEr0Up2Y85!@mXXw`Yp$Z*bcS>`YmlTr@}GU z{0jjzNsjm*AUszPWA}Hsn)Q=`F91Tw!WfaIr7sx1WjKu=r=qqgy{U zuw4H0$!``dp^~^V#+Zf7@F@3u$stk8^Oe>n=Jrn%uJo)n=sexF z;xr4sesEiY7W>bMm(()u{QVSHHJ$EQ0m1#~=*U=`k7l$0TZYRf9^GFqV-=@K((!bE zFtLhXQ)?Qr3Rwq_-f~5m^C@dX<~h4gJZ>M;2)z4enWXwsIYEMD?YWK3qi0^24Ug(E z_+PenTxIR1^#lLpAwz&)%FA=}uk_@k_|G2_p0eQ`&yB)JmRf1N`cyb6e^2n z5k=Olhr>$D( zJ}n|&C)#$xZiIs(dZ>-0CitU6Sm49l^ZYfV;y;2rht!c&w1DN5Ekl5rSSK8MFIz|q zA~#H_8)}-)2%DqBl(c$bfUa{+=IW4zqPN-?+V?rGKF1t9`Tfbr2LLf8f+dYlS*ft} z2(Xh`^r|jgLKSteXQ2)i48He%yB!-S3Y%-{DrZi9BF>hBK<=`bBe#Yb6HJWf`F5f_=WWiCs_ zi3k9pO|$zAh-64j*fxkaBi_lC@T_ydBD)$Po|e6l(mD{TbV^`*=Rdl8*l95t^-MxT zAgAGZ*!l;xJ>2~3Komxk-C7&O%Mdp=dw8U#>WWEBKj-??YO_q0*BsHQDpAjz#Y?np zTm``4^+LQRH-^B}HjqAnjM4ftyS#S?|6TnYHvg?6GqUDMz|P`Aw}z}@hn+SXhN-lm z2`28bWbpVU8gN#}kF8R?t5&eV_UhCxN!xNmEveO{9koEc9ANjxlLW#=zqc2ev) zjgR4YDC_*|u{00?V)~rli2%Q9rSqsc964C32$G~zyo2jJt@^;#=R*nK#u*Gb=$uKT zU4ZYo--b-OZ@}@29eX>EB`Nkp_f_kWmdxp85sOnFgaGNMORXC@oMF}Ajc=@jy@Hfj z;m#;SN+mlZVAggv1Ku7~n zL)jd?ZvxtQn@NrTw4z1gV)gj@q@odUZ6kw)e(fef{T@ZSbo?j9pvss}M&Q`Eg@S@u z1Bu01k|-~~p{skvALI4G=45h4Lks0BO!3vcu33g#DG8ey`5G`A)q$ut5OzFM9qd+H*`!ZN7f(=Xu0FKs*}Q*NG1&o z+w?i#h7j9gIvX>24dzex5d#7r%`z7CV43?*$S{rt3kluLWUHKjabI;dXUh^H7fA`Q zl0&-fXv(Av)IZ&_!0ETLunCaLZRNlrh=?^d1O-6=8~}0|*q2Q9Al~Ld$~J(DsCP$> zhVIGKpTG&RiDa|bA-FsUxYXLNb<(iHy?)O2w47Nay^QY98d}6)xs2x3xtWt=xxP<| zlUP5THtXQXd@Jwh;x~{1urGn+@zbIs;Zn3Sw&S<)ZGa1?{N*ettcNixHbVj2?|Q4* zti>(qhZRr;C!}A>nCEf<;YK9c%uSON`_~m!Q-bMTboh}jx?m^AJ@f6Z{J`&wQ->6+ z!PoAuyojGHQ66ZFmrjIAzjxjctzlfV|Mm-J|AIyP^O3}Q?ymbd-d}0}{x%HVQD$tU z^Y=b+TT$Z>bTV3~X_B;-2PeZ)Nn?~IdZ(FJxRaJ@`=5PX0ISY-azg(8l^*c_!cK%k zZiWRA+qu$p!-Yq>jau4dFM!a@jyPUShd8gAxSr85Ez}rrla+kl$D+Y-6z? ziK8&`S2T0R%=r74qP4+A(j1c`CjW-lvD@XCx|%1HL_Hai@~75i@YC0C?!5Y5UKNkn z?;<<7({ej51zEKb9yuUgTF~Qf?6njHHtn zzUT8nA*UP4(KPQ$EmMk|ma)iRFvR0W&48}uZF%VoMC!DJUkNx0T$V%P_gi&w=ajF*XT^dQ2_LKOv=l&O3rdFe~ ziIzC4Y(k`3ibK}1!~;L@VeZx|A=*iuoyU&~oMPezqjGc)kJkoJA{kKMs#4LRMRWH+ zc;llC_ffD;@4MGq&(9KHzx_Z$L;4!E5EoscI4whO6w&Atu4`;#Zv`D*t^U{6n~G1=RcPSa7> zjzKL^{}^6WTlCWT-wM$&rSK*l`a}ez`n=4JVNC)X|3F-tw>|&RERpR8?w$XpqYlj=Av>s-H*R2lf_^IMX>qJHv6GssweN2Yy@S6Y`RbD9(jAWX#pb2^lZZKu3T)}s|&WMaT9=?j#50C*4acDPwLs;+qYG% zB6@=PoR!R>Unif4hC_mg0kOuSf0P!pVhd}RmsT6=10F4zrSo|K8|?&Ri|Ho{z6bLW zjnRAlZG$=Y-|#bKT~*)_L#7xLIwGVBV3xA{hySa7I9pp6rJfmJw6gYmz$w{kcrA@v zSxwb?Kz5;GrX|RRxi!kpyFum9NMh4(LQ$gH0y%k%SP9`0rcETr(*|et1bD>&IFQ zoe6mpv}%L?1}0fYJ*mk&seN%^qGSha<`JXfJ=QK4BU`qL6B-m(%@OBKgOR3-_wg4R zxNj?w{`Yx}|LZZ@7f&2AzCbtMA8^V6EdL4?M23`++teZ(m=1669JjPXSMJ6gFgh|k zl*<+r7XP&p09|7?tQ0f~ z8MD~da-FL}apgb6L{&`!-`g=!woqPf+}uHJZ09mAvWkR1O%aWE+N_4(&p|S8ZeaFC^QV}4^K)p*|q)CT!mB_|Ps}o(L&ELx9bQI%_>w zw-2R>E*UQ+k)fiIJ0Cr*rl@RG&vfP-)58zXYuDrP#+VaaD?MOjh%94nu0koc9+MXK z$~~BuzNjES?<25HN69Z=^Pa2SD4A2OoM2O>layMLvK8|z{9KTZks5eCjlBzP2)>B zr7W&$=lSS2buWE*L54<`;7)i^E_W#VG|J~oxfdUi%v2=9k#F@WaBp@g<|Pb52%eeD zT!K`^lp}XYEVfslX~G|{_Nvh(5wU2@yC&)}RaI_te$tL{XKXI&J;)u5QIiP$BqEaW zv`qZY8`2TF5@Z+igy`q{-)@fyz&2MslGJ=p*JCW|{Xfgg4E;0{9l0S?a-Ki6lHBx6 z6N(*Yj0u38?^!Bfi@rQ>UaoT2J=T7KGMtkk9k6fZ+lr!o_Udush`odV(@Ek}n=f-& z9d>Bf3xIPL92+R)w{!%%EMox!s&*0UkV}(6pk%Hfj--N`F;-k>gFkG3D;@OZBT1UB zcas{Gk0F~Bq9e!vjmn>RZ&@NgEC^PjTc1n0NE56ukT4d_<4i$^nf8&s@d{iw%eG8( zvh%}p(W9cEO(8RS1F%2hoO5Sne|qct;dJs~$_(>HcEu|F?$S+X!b!EYWd6tIt>A@@ zXc^nq$oF&M7HM$=%Lwy|_p)EN69jjuZriG>Lc%<)Xsz1|?*wAbh_`kN`W3H2mJ0O9 zB>gvlWNbuH^=FxRY(Rn4;oBfWCmS8c7j^F5v?2nlg|H}OEfjx@ASD^z8L%swoN}x8 zCr)@sB{X&KMUy;RenP3m%U;o>G(Vd}z+1$OW}r%5jRJ<-=t@^ev~-(t=7TnvL_5AR z@2y_Igmjk#6y$YwT6t4t8}*lX>{da~M(wc}QkF|3(bP~F`nyko#P*gyFj@z=rGoFP zan7>FGYgr7+uc=@AWg+%$RXkRw8F;ka|tT31>!Z!i5ts;^)f+CWFH%elsCZQ*D+yK z#++%{h16X5rJ13>2G9HIK**f}67wGcGjSpWV5Pm((cr(eOiP%|QCinnZ8m2Ud8d($ zA3XpQStXu_+s<6nyKZhT?FPl`F>KszF5Bz=#_=IgT-lZRlP)3DAd)!2KjB{Ry0=z# z+!trw;waCBx>nXQ{JXGx5<`3=;%+>LjUd3mNFiEGV4K`ah}Y$bPu zhJ18U9SpGUjMibOVw4a3K-tPd1F^}z$#2ga@uk;d~PPyd@}`0iZ*PL*Ky5 z#-gb-|79l>|V(N@TJkd7p z08kVE?d#;WkUoaGS`UXez?+vwq&Y19thqmN#TpV`hxC~F&}}t19NKQuw^aPV4~Dtj zm0ILB+)(4d7A3c6zaLL;Agwf4Df?TqM>bRQp7@|e)qYq0SV-P5lh>jtZR~l>Krz1= zlUCtw8^p!x!PKyZ$Cyaj$vv$@wM*X-P3HMX7;SKd6Z0f8oi6eA-x=$^NUnKs zj=9(SK(e#4pOUDl_Hx}_rPePQw+jjcy$dL!9D{I8>~JE_4HM3K}qk z)_E8shnlL?odq}{aaCMKHGDhHLB2q{xQGH1wW1}0xC%dzk^nj$#3lKjFpmWoSx zhZXtW{#64B09{cQ%8p$0?mOf6r8iy8_O2mOa%J5Ttpra&E@?zL9>6j{UTDr{$Q05ubJU;cqp$?IeBO} zcK3Mbn%wM4^xG!q@gucAG=;Bk$-;@+rq8aG{CWvwCrRlAS^wtG&t&nDiJ83Q?v|RT0s^n> z*8f(3*0lH^PAuNdOECSCkuj+S&uO9n*fFU1RAdQ5Wi2`Eaz*R$N7tgt^oFI2dJRu$ z^_-8(c=mLDl`g#X-;b}hc#wg?g@1ec`^t1C1GnG+{fJ8Mzd5q=D9O3`=^^|2W8g>1 zqyuR^V^(ND($Og1$D_LO-y6|}PhS|b2ZF(tey0t&rrs~W8H7hha8bW!evM13o|nJa znF@L7KcPs0k(8zv7Ke4vF8I!WyJlhco%u~XaeOJ^#%#?W6tDlytur3ko!8rZW?p#? zu3ryjS)>C?olGj`DkrsB=CaNP>_|VA(4Q>Zkn#dA_MAHwA7JMT0=zW_zYAo3FQ)UA z)Sejh!=8=7!*x&rif4b`;`8h+7uQqhlbbTa*F_%qV|Yc+Dw$KngI?r|vW(1&54Tmv%rEkphg z|1XNj$O_jWP7z(9o9EK~rfuIsc>4XrZeUinWwrOvN&nDb0+B|Ml4!FR9@70Dkojkk z%?PzOs7zcF069gIi@U$GnNtKyIsU0}(0jG`oq37V0Ulw$5*`+5D82O|UngR(1$ ztaSRAlny>aFlC!bM=a7MwC=lw=Gb7=-v0nG^(@AIS53&?y@fnmomrZ`RQwE}=~Ig7 zy>xaXieD@1akp%~+^ki$1dwvA*!HTJk^F^VT6W-i+8^%u2R#?ByxZ2ba?-|Ng*mrL z&3`MPIzE%!-=(;*q80>!la8(-@;(>}xu8CfAzaQ3Wd75$0%)YIo z`<>6(gU&s!rY*joK`klF#*KY zmhlxuE4mLy`yB%T&TH1xK97I1$Nmb)SGgTmO*Vc}GkPBxNN!2>pYtu{CHK(c`~_2~ zHw?&VkVE$2r{x;~E#D{Up%iE_zrdr`=Q~RgppUdY9S1tA#pPYF1 zv#wL!EIpW?4o}eK(f0AJSD!-L0B#65;&1^UP)c7BS|o-^X4njL><`sPDue`@Y#Yk~ z$Ym$da-G9)Gfau}2x!Uy_TU$FI^=_|_G8ogbVL5c%P@N4g`ljg?8+u;CMWHPl!YNY zMLWx(%vq9?w=gVWk@;MwxA(lrX`B{EL8KX#I0Q%BsNqQyzij2J0qvNSVZttCIjO2a z6u}ak`Jx;kb0-IekaaWl$51!N0s^>9vWrOr%~-A=i9b{d9yxlfnMlz1A0W6*=6zPC z-%^3K>}cZP;rAnb^GqGxH}XR&^P*xB*hw|Tt~jIDE*EcO6&&hx$1v+GD~#pp3U&<9 zNp0-3ISu!q*bK_Rh{@Gt$Y$5&S-7^?}*##0%EUym)jM z`%?lx13tgJC)ulsV!;9^a{WA}*3Y)AhLjXjnvP(I+}Z7Rs^UMLP*-x$9*by@ zNP1RM(_dljN{5&oWAIzbUWZ*53JFqFy)0|9?+LC_-DCCF?s=a06YhPU*-da`dvN!@ z6;|!Doa3}f=mi=K6~oFlJVUC$H}iQxJ4uJo;SbRg_ma2Rd|fj((dOIB?=r z_+}vZbLg;bQ z5u>s3mq{ilPu`yt6cblAU6Z_EQ(cO9j>XJRJy&q=D33=@OO%nMu#A0@j#S>Be;d)-|;{w*R3DfYV@65%)g1}iB@phnlPCQmMCX7biZ%T30 zFuQy*)F%Z7<_*>8YIZ(UU>t4a?-X#-%=(hR#$d_xBP0%ALH@)6KoS;o%KBgPcY|3B zYQkEmMgl!^r=!s$z=+iBSo}M_NKBU2?K|EZ&qH`p=(3F=Een+1vh4Gx9^g}~7`ck8 z5T;5W0VfI1F(g(=6^g#kYUj^n6udsM7~P0)1AqaH)>GM%u^9jTDC6ZnMKs6q$K$U? zPvI41+?}E~s+Lm|+b$&vM zrRn*F;@gjCkL%E0x?1eTt{X}2)`nQHot+PNNtNvMz^jP();1K4L!sTx=+^;; zz~V9*&4?kmva;|f+Y?b=Lk%}l{VQLjU?)Z1#Mh>>bW{e`ePl>}Gyq(+;v_*9G$0NN zX0aq>E##=%Avj(<56daYGuAQce2_Mi<<`4|G{(7+6n3S1LjjIU84jGU{5p!TX<-YyJe<<4AD{|JtLV28%N6vIWMTazDQTFou zXH1!K+q<#63e}!x{l3J`%aXrDI%s3ZXvIc)mtV8RTva4Xzx|`ju*0Kasz7DcOCo^H zB=dQCP{2k$6O8;+P}Nv}MP8mThsw_$y2?TI;(ICOOZqxUb&n<$4!A1{p%E$H`QEmi zS1oqCnK4F1m&=r6Tt!X^Q?h8Ryv!HxH%z=A1)2 zzzye*n7C#0Us#j&t}6BL_f@#cGU)$cW}|bz|6Aoaue0gDC%=!*WFluP3OR9tN{*e_A%X@iPs#cn zN}*xpA5OIYVD=@10Zv$*%P=#yFDjp<*axYqWqHJu--?G38a!m^RG#St_Y%IbCQ2f) z3nWqMQ=?%hb4b%IuLPbOcjf`761cO)WpH>qH^tw)o=N>*r1CFj4QIa57tdYT(jvVd zlIu|@Z3ZcP(K*Q+26K6jKMYbXY6<}TbObJyY(1ZL;})h)!8D(DFws}r zSr%S<-GW5wL`I)a{L`A-KQ?iN#=DHxY5f**?IM4-I@I~qlH#T8lf2&?9oVLf|IMm1 zO}|1SU5(L(#pex)%g=gZfAaZ0a|>fpqpSP_`PrU>9A)=3Lxl?qI^E24St?oBtqqRw zZJGanv1~W<_BYoeo=1#Ap3T!_?uSX9LUF1Z9#{}_<57~dNP05ll{20|T@< zP%-B>#g<)9NcBBaC_o)d{?Tcp=h5&NDX|5U71yP*mw8oV_ECEzYgLq`Ob4Q#=V`=C z$n3=}bIlV+XrfY->boZ&-`Kr$^5-B+;99w)N#G?yQtq#Ve&R()akkN|+Q()1Oe=xx z3yZ#35;;0~;7y@*Gaa+_k(0>;FWIpFcr^FCbVv%h87IOJU}xkS{Md)4C|zQZx=f-B zt-iMm@Y4lZxW3Id>Vl>C!-f!{sjSu5$ zY8az`I8E$^8SZCL^Q^Cn>D6-FsOIg11B*IQhxch7965_wq7D9&kvc^*e{&x~et2|c zO?CO5Aj25K9v0OL@RCfbmwA_An1@)4m+c@VKd^IvX&ruc`61Hm+W*XviXX#fH;g-9 zdZD9-%M7A+cp21HI*TZidyg-9j(|(0&Ty|AaY|zYlBRJRvcG4g657XXBLLUs88qiT z3)qV`uxuK_a`{cdA*+Z^S-Xf<5QR$&D{Gq`#?^KI4?>=nFmHF zvZZaoN#HcEqD{bEV*PHY_C$;-L(dg=1F8>eP>z{}?Yzbz5_ffnVey)NGhuYVcAlyN40OLMCF*f=%yX z&x6geNLlk<-yV7RPo;btfvsN7x1A;vtTj5MF7k0rdRq{+6c;4fxT`$xVH~Vkt9JND zo}=f;XyD&@)x(=W{&No2$-<8>ZsC}(oM^nK5^UECk^=%%{Mg*u&QND5W)^&=(4D`2 z!zpR~c7m}lrC_9UW1&n1j#XZ!r)QhijkFg<_2wzWSdzSEf}l;&a{q2DhBKJY1c#ZQ zvJ5O{6#YA&L^Ra(9}i4n=ELYN&rWR3=U7Z>zx8*9&J^NqAQqR2O`+mzmXC%6u6L1>!~ zvfEdh%}g40d`w#w`_y{8u(g`2S&oedDkweSn$8z&K$^Mf$?w@)HKxh#0tz-|23;q~ zh3)du+7DXqtj_kppn<2=@v+~6=S(?M%S|)W7(~;@l#@JPMDL9JwzWU&j|AeD&T7LQ zwpJ9p1dFe|vd@fpgb;i~r5-~^kEDKS*7P1eNBQx^Hat@szP}7jlzFyjWTB|W?*FJw zH5|sDEZ;u*dA3PedhD@;Ctxw;xBX26=WC46445Kj;?2|1w2FPD)tZm46&2}^dVtTZ5dMH?%z-Jb zNic$cbH6?&=DK6?*^X_A`qwa{x&COWfj!^=bk&OP(d|Q}WhgF29YN0-z3Fn= z`_-6&_|r_5xKmL`mhNPxU!s0X$JLDbTwL;itth4xN<5b~0EhW5QcGDk>QM{6>)W%}wFv^lb= z%5kti(oS(CYZmJY#%vlf@Zv065BS>@A9q!rm%oq}Khzf9GdcaW#5h-UYsY$4AtjfQ zVP$tIw3sn#J}t-M4<4~(*F}RF$ILJVpu&LBpWFi3{AwI}&I19vh6D#0aCbRQQDNoAte#O1oTu%f2gIwrPC z79eZ*I#OrUV$l#8(WfoUp=yANHS{o(SeK0_8Yo8la&#Tl3T{+Rl%=AKM6EOx{oaQh z7Scaf-e*7{>F1A-Ad&ccAef3;XtiPk#xP_QY&JgZRD8BFZZ%B)4nt5`g2Gs`!ja(B ziF}Obd9Nq(F`)1`fLU<;25v=ky0AM0Y#*-6H*Q$8!l*N!qQg<)eRy+6D^@&C-FpI{?N}${c-98lNX-r$Qpgj5fs^H(vD|2E-=UHxXok^e>wlDW7vsFz7yz=rj zYPNCNc@O~JRde6(IWKcXFw;iaKJ!h2xz@Iv@XRT~u7nf4(6{z_%Q#GsCoyPJR8p&k{W1)X6 zB&qdj#qsWjQ9&F}$mqBNKGRr4nxgsuh=G8GQtnzo=}>9;csLIb>hLq3K-%lK!xx#b zwmG${Q?Cdi0_ILK*Q$d`Mb|{B026sLBZXMR+}GEY^#J5J80>o<1B*-l_qf{Q?c*|l zr07Gy$w0l6fkIEF9b~CZEh^g2BnI=PMOdRSTWH0|k3&+>Q5eWdjJmWf!<9C=V{1b2 zuW}vH-PP8R)y8d$@L?24TVmLF+vpWHSO5F$xQgKr>t=QG*Gy;m+6v($-iH?`erwurmZu!$T*+?;UDRh!t*FJD* zJ`!=Q>&qY+BP3#^FrJ16>W3yxT|yh3_JymAg)&Zm23v6H33tzGN=-F)KXdpN3-!Z& zE+Xy8!uW5r*4-v?N1R``a{)K%0piX@Pga+>NrI#>-pCs{(u|IQ=pAdo52V#8Rq5s+ z0Kb!v87Zi6QU>C)()z-uM*_c*q;oUi@QCPBQ(k+qoO%br(*);tjT2rfq)7590xHEa z@9sgpiae~8F$6*Tkdv{gi6POJm+lcO>nk)RVX&I2yU0#D;gr9NhrpF8@?f-L9y=qkV$_){x@7}mh1WWtyEdwWu%T$%?JKko!3JTA#qRYyvE zR0H?|JYWPFSl24_NcXh_+=f0VlFe`LgK{v3>BgF8$V8GfV;ZJE!XLmJ_XG#f98-S5 zA#31>_XANMj9U@c-`l?6A+3bF9prjSx_v8sH$?CG9xS5sC$*}&?hV&bG;uFL2}591 zzdL`eKvimSfcRK5!J*>~{wJgyr?`OtWnKtQG0cpC35Kt&e@{_g;#t;*VITmOY&v-C zbNHXoaH|9}8gJP0EX6-!FjevD0^ce$XsXv_x6Y2o>Z*p%;UB+d42f?wWXFt<#B_<= zD0h4}CES{@{*IuJRdoJj0%&F6RDWMJ(>V$k81dFz{zBX2a-KF<^m26bL-^eHgDO4C zI=~sj_)Oy8d08z#&F^!wZ+KN2c$Sfu#nj}9I3~?ihqb&B%fQPa2&jxpocUFJGPfmV ztYxjfPdeC@qV+EQ4?`Z^^QYBpmZzaZ4QMTFep5HPk>0iRsDh6d63n)oe^ODfElee7 z>X|W$n-vi@L~meIKwN;)@8h;_SxafrgMt-Qx!Asz?fAIDuL1RTfkaj%%HhnOC;RQn zdf_KQK~Y|l`$E@8U1GRYhS(SpBmnB5;)hPbPJW@n=AOwY=hydwS{X~%KldZqS<`ec%1Ia(^CL;soh~9kuMunwpwGdvDc%wKAamMS9 zn6@a7w7Dz7(mL}$z{gwH*DjNMZ_FsIWPx@kAGqt&rgxCnQX|87$q!&o$=Z>+ z!M$g-k{_z?H&Vvar_Ptl7v_e8wRE27e^yr1DR@rqM|10&_+V@x^r4epVrE z*Ia*YY-n)PW^ahaBQfe!Q%cZ{4O<3KRL_+)C!6wW0wwAt#})(2jx4&1^5V>$zGY{M zPTj*KYjBzpD8?G>_3{In?PDVvnhJkR|J@8S;5=M57Cav@6Qxb5TP8HPr2_m{oFZCT9sWa8r$yGHeJtIp)n1d!Ifez&?VR zt%&uRr9X%}izyR{;LVV_OYM)Hc!cpNTp40xIH}@|{|_24tfyj*h^_romVla#fnN8P z5XEPgbGNS0dN{^Ni%Q;Ddw_tCXj(YEhJ%Gn+%m24fIWcTZ4QG0>e|hQ+17}eAiDSv11{F>z(mUDG+~lf zk8+|s>jJ~TbnQr>mo&FDC8XWHnC5*VbOmaY#G2Vf1=RyyKZFytvso!YJ@9JGQbatL z|C`xXoAW9#WD$(EIMQRS3I~yuNsTz3OwoMw`2-E*Y)sGi+P(!NRV@88wp4CqG@^{A zMFBo;; zFgDHAR;mI$h>AdA!+QhtrOwXdM2jC0TCIujtXXa}EFZI}WVq>>hspm03B_$$W-e+hd9=}q!Ij7IkkN2Ly1Ji+0i{-J2D~|cWv6L z=--9IHI*iLS$k;~dZ{xBSWyz1ind@86d23`RVtIRL}oZ!ROVAN9#NNElTA}>yao&c z=v;dTJ2&b=U1(!}Js)37wBk&OXWu}wq^XddF;bY;INUWJt%v}5>sGlJuK{=?p=BR=Wo%??RQNIGt1 zuDy)5nNWQg^OyuKzDZXVA+E#`ERV)E^J>gz33o`okND$P`)Isvvb7RUXArR-)A(cP zNJ=J-OdT(6Z8RtYY+Gt_hU$hcScltvOl-eQ4ssLC6DYOx;DWy&GqAEF3O?6EIv)2c zyOw{GQe0<}w8F1&f)`>~C>9@*kNDPC;FC2$DYML;@Fl`w9S%!n2 z;oqYPSD&m8RU2{B&nF?zK6H#KO(%J4$of32l88gHu~AgH$=Weh3uH4@`01vZA5ATn zJn7_=*T8k~wNGfP@VkfEcraB&kj-Tr!UD(9XBn&-s1^Ntb8qPW&9FMYr(13=yvpIh4N?t$?Pvc&OZ_4Q`d+!|p3^P?=cb^?$}DXiEJn~m5RV|%etINrnzxx!pxMsY9_#hnL`q?*}Q+P+S^6Fx;zH4+x z3|xL%c3~CCQ8~bcYNp(~y=6durLm-NXa9=EzUIs+4`Y_t0bP0LD>u~0^)@k>_qR{) zQ(dr&l>5~-XSPhk7C-B|D81U%XEnylmnHwrP5ctjCyr#E^M5fcrqVG z4MeVsuV^iu@${w$P(}v%26*ZWLI|awI&X^G3fch-jI|SBPV=%UMU*r>fwt5lF;!7W zrM?|8pEw_4VNL8B13Xe*lh%766kdE>0h1W=?$=bgu-byn^FDXWWnA1+f&OP}ZbxQn zAAP)04uJQQWavhz44F>J!RLux)UjgffSH%^4wgqN)V&Vb?8n{k2A|s5c$&qIM!-qNL};zux8~WaFkmaTG^wPY&{(EpNKwO+oYw* zfab9Hq&(B!DpoxdT(7|Q`TNklMf(dJXM6!W<>|-7`hjF*PxSKCH3RR5nd!qU@*KF- z{uV*LU*Dti0#gybkmJ3>BC23*X!4buYi3~Yqq^F;SG%w`LzMX|zgmY@M_=S)Wwb-nvQ1z46dG5 zuUZYK5fpH{BYJCc%<`jbYLh?@`&=vMw%2-vU&M%4u55 z7Qbi^)hHl(KC+>(a^V7+#XKG z?#x>7US3@L!1ShT;SAm9n92Jk18doPWXHK?Dmmc(Nchu z4&T+?zIXP&`EI+)0iXxl&Mdzlfn^I8-iL&Gorx0|*HV>G?P4eXy6@@7bu@okfqbS5(%)cD1*9<0CW6NB zzf9AI-x_#8hpJ*1LM9X5vQ^9~=--~Dg}jgE%ya?0fneWl43a=Qs%XqhSC`sO+Sq_f zo$x-;!MiilU0L&##2%Bijt!FF+hP!D_UP8@}063O{wrz6gUVBSNT6(S0nXzB{QV&jyLEIPO2yWrVV7D%64wjn#6)e=N)(YG?3JkYIwP z|BI0b9%gSv<8cC(~eZB1{-}mJhmSk z7@2-jTk9=qFe%>U z?(A8&FQcc}a`wy^7Az?!Aict_x#}Yf+c(7;1{rFN2Bxo9Yk3Km;Mu|0$5nT&hD`1k>svKlS7Zbc44;{# zfo;!>0t6#QtmXrDff9eO=qij|wEW}sZG>NMh9mxEuoRR@L0F6-C+kGXAsYJ_%^+U@ zrHr*BaO2#rv6zcUF=avFs1_KJCE}>jtRmyyxqaq;VnUB-jC$D2^(oCQ#GeKV`O&{@ zB8WQdLoKX0&RN0`XD!c=iW8fq4S1r&+gl=Bj62q7V{|V7Aufjs(b=K z3V;)wI%O)8JX{f#koo@U<`vR!C;%yI2tTVvrl3MYzc)gG)N24hEes)t#u1U5_CAG@ z9^C4M7BsKL{(|f6a(`zX-pKe6cX;>gQ+J(V;PWS6|E?B?F62i6@j1m_?6hw`9UPDG zApbP^5VdEuCmPDEUr{3nJLf_0KT-xl`L%*&&{(M+O+`gSKfDhhF2Q!)8x$UCZ>8t% zkxGkF{t=d681Kba;uk~EgI;jwTPgJM$yYg|-ByJrni~Sj+Hzjk4eTjCnX#OI`GZImLo(olEeA?t*TqpreTm#XdsKvc&baIKDOOW$~8O z6X*Sw{9m5MVPhv%aS;)>Y>YQv+uEw5JZ{WYs9|yHio~+xt*b%_@`7y3@8Kj-`b-1v zj^Ici>}xjp;Q_#-uFEf8J6=n?vWf~wh8Nkt-Bxf@|8x>ev*24m|A$#5*yxWQ99ssF ztLYb;Q#6%ObjusvdHPmkchmZ+;x8fT1UJrYa~23q1gT#wM4wo*9F#o@-$8Qz+lx$Z zPa!SeQujZ{_XcjPA^Mb?rE?%V)_=T~qRl-!doPauW(^kd>snkA5QK2XpVW-QjlSNm z>6&f%TepLo%_10{RoBa~YN^XuN{e`BueOj)sf9i25vuyMEy(YbTHJrjk4ohe2>E?h zT^ijC#COb6r+{Q$Ad}9;YYWSAht7%my<$d5@1S6!aF4|JmvTa_hhr2K$m*YI0+h(6 zB9tX7!0L4Kxv2WW5}5QrfTQT{r!z5|OSQ84JH;6%uR&S2HAUE<+_c&t4UfuI$9cJT#a zzx;7?*SHvE8)CwfI8d^U&7ix(YXk`L=c-Fu3%H!@$>JYW4|-e--WUPULxbdf)!P(V zyX+2u{g%^dfKZ*cpo(az?Ka&Sie2b7vR_M}f3mf(RJ$DEqHopZ)VLLbV^S%8fK>!g zQ_yT(IHkbd#S8|#E(z+9jHnd3N-Hc{$X>7F7HrQfWa;lTL=O&gZP1hicr+P-#g5D8 z)=Ci`?Nyr=o

P_R|3qtY@dOhRS0BU&wMItrieP^i7ZC;z^uE@0cw*AH)DEa)N_G zV$ppx+$8sK%j%Ha!+E15L|LZHq?iAU}(XgL=UX(YV4G>LSnNI_llt zuaAwoh{EC)j067Np7<3QYklVWN5)sbSnA0EX&^FT1@Asi$zf5*zn7i(;Ni<~7`5S) zqNmVxM`y=6I`;=To!%=xH09mviOQCC?7D%>W^84wB#m{vL@*9RWQ9H1^}wVNu4K5L zV24Ob+ENpS=7=ap<)F)VO3C$F@7w&YAVF$k>rvkeK65(Tu1IoSM zzy!eJj4dwVCJT-^zuJ=leDj`nxOzKklGo8stk#xR3n@ki2Jzm`nZ&VvTC7fA6NRmG zR8p{}uvOGinc@dWrw7l^CCF~!g(l$uz24B^{ z0^$ax|FC^4bF!;Kb!jF=HZpEpw*DRrSc;Y^Q6$R!O0?13Le$?_oi$QVGO&%+zBciA z&?ekxwq-ZlrQ-Cbp^%GLwxr6(rHE=Cb@fLQ(7)1>U#EiD#Ve3Cb&RWcKtqVNHODR= z33OD=BY%4(MlEVelJZR^H(MRvBghlR(`!Idfcyk+TQ^>}d@CwmfokP`3qGF&#p-yF zi$nOM zQbpUy`cr~t+zojEOUna9-5t>0fBsIlA8cqN_zj2t`ZL}$5?7Fme2S>*Rh;Lrq^KVa z7BgQBUkrGUT75kX`1b~+e?-35fx@@*-jfME#PEqaL;5gE_E5K9FssBHOWy-?$(eZAvVpmrRDk7%lkd=u^=dBL+drEVo{;I0KhLwcbEIdREyw zJN=)%tLlMxoZDbxdgiFPwl=n~C2F6;Qhi8B=}Da$s*}qE`zs|sbg+hNNcNA8pnR~{ z)42UTb72+D2GsMcG%~oj-}r**?S1O&NOtn+?v(t(wf%uDe{~sOVTZJ-p&+|BQNjBT z)8U81tPq*0rOCzyXS~eg8>AF_4k%R+6ZXf?EkJfD1-~LO{^&jLK-Vy#s(-(9j zKCiE9lskudZmFW3*`oDBd%XhQ`?5Ikf`NgiZZ zN&ho9%-(@=c`|bI1IeN%;U^k703ee@UfOGZGMg+Hza?u*!xm)DkYAJfQuAm!H5VCj zxC>$FsgR^$6g8B&0<pH2nj-uhgbZP#rg8UG1ldc$md7YHc z{=yFU@ljV_|Lhs(%b;1F5#&uem3^k+Ae;A ze(-?&5X@|i+ik-@`{3JSMuvC)H5N0=9&|I4e_^j81T*#d9?tyIGJ~&ODE*b1o_>H1 zZvIY}&Nm0qmcG2qZ5>LKR(RVaAjRT~mlS1Wq6LJE-3_r3@3|r*{4Q?Kr2Iw0uH)OM zQieB10o2SXz*{{itT6sqp;{ERRWGZV%K&n>!M-PbFg38Q>F;zx@ijQBBjM}}r!n{}NIlH(`f-d1}Myo|8 z$yWX4Uti&iA64$nW1hVLtYSU6IIcc6xLuhj7G9&88)a61ByBNF2 z%S;mBA1~~sC@ryLbJv%zpmIwli2BeR4)+bBSOlx{YWCfgEeq+eK>RFP|D1+5J#RQ5eD9$aCL*3y?I2U&wsc1+Ba=L3wANX)g(XQYUY38Z zCn7JYCf2h4H9BAbR5e%<^7SFYSQLOx>6FB43}%0gBhwz(Anpk}++M<41Od;=XFY8y zH>`AI;mQp06{lix9|{YkeOT)FUd9xo`-B(C-WwoIR=9L)lyl4&wZi~o?V#)f1}g(u zIR5j%;h64MNRWF}%>+-N<(U`+0!sBAb5uf$UXDYrRYsv7ct2b>mB_*>I;Bo-Ti@M6 zyGR}#9u|ru(>s66wD@ZDVd^33!xU1w!*}O@fJE((t*5QM{n3Scu7`^c4*aNpsK?LI zXS1iL4OnIms1bk(D_>%u4cs(O;Zg}mg#V#aGhRKMlG4ccnaEC3uA1P8p&vCmt6+#T ze`iSE{2Jr$XepZvN?D1+bSXDB>@5%!(W7csplb-uyulth^ca0P^Cz^G)=#p>_?h;d zH!>P!nL4fbm5)-_Y+Dd^p20Ia117Xuo=}f2u(q6k>zMey0s+i)%Sp~>ZfX@D4UWVw z@7wiTI6e7TDc}p@TG9W|`sp$-^^J3*_uD|xXZtrj=d8JFZ^=GnopLj#ja$Eaz+3$g z-6qT(Q%?efYo;#So!pjF&*XmJo-LGx?qQRJtj^EO%NyzyzELHAfG(YFrq+A)JkxA< zdLx;3`=edEP_X6xL;tT1Da$th14sg5Z-LseeN=UZ&S*3pBD-|i!m+83V_D6$&*4q}DVi|zyd?4nhB&Td3+rGm0 z%E;uzVYz9REkK1#gmu&9Fq}z!@=ted=RX^IDl#$2WFSO1s1$KW!7N3!_usuRe0i14 z@JCd>;J?b*((o_{TzyP$B1Y)&ftJg)5&u zkFid5I&wY-Qs@!OG+TL1^Vr{_%9yhxV17}S!ar?F`Ipd$J9?-Jn5x8D_!uYq!9RQP zywbi(q{^Se!69;FQ!!Cf)il*2UQ)}82>no5d##1Xq9Ia+Eu1QKg9v4CIle77hu)PO zRg6GX2)~m5Ye@Ugu7@XRPudwD%(&wq3ZYX=GhI`dhb*P~)9N%8G(R5oukX*t=}{-o zucfb_y{`9@)d@0&*pA4jeWFr9(vZX14cFV$|D*4=`e)s3{{5#u+!P7h_B(Dn~@xi&H`d*15(#&G&^v5FLHbPp$fJN=J0{j>1*0hWvsqrHJ2Ljy#zwJD(nRQBQ9><2I>{AjJ!qyb3^hGD^wypxZ#T}(Qm(*fRq5`iW)pVyqa$A)Yd3)G06!Z zGVIQeoWdmPfN1fV{0g`S13D80oLnlO^H($`>1D_@IU4T4_aWymZF9W9zRPsC-#u$J z5`h}9i-q@b>8utqc+BzS$rWhTtyAyLjtWmST*IJjXzMPHo)FyGO%XYy7xh)^X|(gv zMa4j2lBen$$ZzSSEPd_z(=*G_m$V{P{J~MSG*UUErl6CGDn(v_EklSHG?#{83}vF1 zeO{CI_(q|E{hGmxvl7dpvnE?UUbjXBOt4kC;U^Vk+8}RyT&!;_{cCZU>EdD{G3TtA zwaQ4n$^OBVPMAkV;DA(-%LHDRHzNjwTgTwqFgmh8S;On^ayw|T&8$Q1$9*Eu9|p9X z)I$PFWg23g)@L;6+;K0JsP>~wTiJiE*^th^D$V*M5}i))w@;%K$v)aM0-G&*#n(K5 zp?s@oq1DuHUS$>;tgQf7R#NL}hf)_@$3X7hiIt(9QK^qs9zz`8x_I`Dm-0h${o9X% zLW)cPD3(U(WX2T4g5vhu_sm#+3e=Y! zLcA?6UQsE%F4kmwCsf?mO}mCAXB5e^ESSBdX0tlLKnd=klPK8wOC~oNX46wWW@&<5 z|EEk4L+9KJqK z#IyT%hWdGvi}g^1B&V6PQJoPt{F(tM5XQY1w`~VCm=qs?2bP^yd#HQK#5z4HX1;fO z!kwe6(6Fh>hjUP9s2%Fj^7STExD0Eyw2aL<+eT(HtRMh1L5`DClR%XRq;x+xnbxlH zS*;KH@HM4#voN?yu09>ls-J-KCUE1>RMOq%z78dAG-6kp9riv!QH~#XTI+koUJUq_ zc(uLH;PW0+dCUL+RSr^`Y0gTv|7hD)Yqf4TYe@kMuD^a&EzLHLJhxE);{bdcC6tFpMgk>|(NBW$M53{UU^qtQA5DK#Clt~Q^`y)@XH2|;1H&)Q30RVZSLy)oD#XaDY~ZKplBrRJPeB(&CQn`* z5wgWnmsjn?>aZU&0}j7ci$16$m;LEsF)ov5ZWTwzDvh~K1X}%8PYU!iqU+w}v&|R$ zk+!0swc$a~+KU7bIQgaK2$5VLt_4)0A-7`=uL}5IS05C8W_B=C z^I4&}8DrRS*sr-o&sg(y+WJnnee&)4jI_sITCeIWR?T8e3fb7Cw0trGNza8Ud_f|k z7rt_+=1$3mkY zA|jbMj80v%c#tC3&9ImMFQeObnB}k9Utdny2FEo6zzXqko%(`NkD{yV`j0~k2iRll z`KDh|FJ1D_-u8dBpfNP;Z4qjZ)B3cy?GpL)<^9UK3fdd8l~E>CAM!=|CbGeL@{uiX z&+WyI_7CZGb?x?4$waga>RxxvFawl>j+JxvF0H=t-uRcSl#I2PFgERa1#KYl zyJ`K>|4BBn@UJ6C+t?@1zgdxrmh~y_HRQNf*mCwcfS9tahKE+mWWHZo?M09eL^SfN@Pd;ktQX7 zNCMKj6PSH2?CSYRpA*qz{4xE{#>#1A3^Q+6u_+bVNH;C*`F1f6jHRD{*}DnK(7l>6 zk6TUEv(`}LnO`>6#B3ZCd-<}U1D{%<@ls+aX#-Uvk__q7GGM&6dl{G7|H?U@nBdS2 zh&-5x5zdA0dJ?~-U3Y|d4e-@va&O|6UyM~&ksR76bRqJbKEF&o=LaD@j*AM8d;V&a<<_@gn^ zEIM&PGk}wmnuCG0!4K}JX$*I(M_A+R23(AVO9o{3MNDz9!f)pOH2mKGALi`ts|}C+ zcN#{VYsgE!KETQT`t@P8@{ZySwfX4vw1E+cFs*-6fE{(CB=fh2GcSvNmuosrV-1^; zyNWeRSfhQO)|`IDkj31)XC7mET|B*73P}i(7`4=^uDH^+=bvdGv8AL}mMD}NlDMLk z0jL_Saef}~=tm_O1z26)r!r86(#F-6n&2A8)vtt|0-vY`jKesS#k$1)i!>IQF&P-s zSg-K5Upkg(@GFJzZGXl8$mYc=ctZoN3;pVNeEe3R`S!^;uVoO3z^W`3Apr+f_&p!Z zvoI#7DM4fQH@j+L!`reA!wJSetj{dJj!#p_ylvENr`GY;csCg(kqn7-+j%PyKHhRB z9-12XfWN2%<~iTsrafy}8~4@bA)Wke*D7j;gX*X+)YMh~fH}+fLowiseqQYLKIi*7 zLere3lqG>(lql_$5aEf0chMJC=NR1q?iszpe}AbS_I4t{rr^ET#>DGvMig1_ca>N| zsjTBVWhu_cBapm#ctT%pG`aA&oAU`>v-F%JHWms*Z(tFIHa*4Br!t|>ROQ}J9vgo(>>s^2R z^_}p*bHj)Jsh;oCAiz{6b`M*5t5W>@fzc09PZv+ai8603m*%1(jg7|IWSYTiMkYA6 zq7~=h(TNUwKC8!|PJm{aEx}N{BqEzwdjtotBc+B|h#UfsYejgG>;TkBvA0eDO;Xwm zZrjv?<_J*bL_P`ND8TkKV-5E%K}argg3sb&zE#Ai>xkpP@=eRWiB8s2XsxRwR2Tg~ZCV+0b9xn}Bqg)SJKOIR{&eNFwgIf#d@b%0ZY$j@D0durG^O^AlAMyn-Rb=0~V0_=zhC+U+H`tlqwv|E|I>CgjEBB zsk)J1ll`Y=2Zq_P+1==*zHPp`If+3=;D7f_Z7BIx0VVGJuS3Jx2ICZ!y&EHUl{bU!8iu8yjeLXD!vo#ieXlUo_yo<^+raSGcad}eEjv5J zQfv}5yvjjCdUupzQ0z;3>qqAv6oH9af^}XJi!~%i+5%GuSYxL8!9Eo`ovYH{^kO1G zmgJ}~8xo7Pv(bKD-NKi)u8(9>I_5I^LHRvIYw-Ccu=2}L3`gb&Yh7!lAg z&7V@j&@ zGt2s0*N0UO-sXn6r&e|X96vN(+A*$~d>$=Qliw-}Kf7>v@bR@+@~QXpzb$As1(H_B zuJqvjypzKz{wR+@^x1r5v{bqu1InSb*O4-ijhng{r^QlmKMcqNmEy5Hy%ma*Eo4b+I;6x_C!xqp32Hsl<5c6kJk4ThevAf!g4^m&`!B_Et_U zyehBe3_jQ=(!^Fuml#oe>$wv_5qXC+S>aA*YGNs%ZltZ_nH34ThffcRcV_tRR_ItG z*?Jfn8M;mcZW&&0wbi%r?S0 zEo)QQbRi>TsplGN30G2;4x#|Jq@^Y#h66E~F>G~;TA8t4c|AMrQt1{U39Qpxk_zRnZb9|`-gyluM(*rk@mGpmGriNkJ$T7){p|VW^f41w zB*6%o)BqoNqZp_*Y5Kyl3E`KLG zvv{RgfOZr$wSOM9dqs+)8oZl6^`jt&7=h6P4+wTjTP{Qwl%;)@{Pdruq1j)5hT$}f z8o~iAtuuHr2~2C~`}Niw0zuDyaMh#2NsN2eRfhsx_C{zY-ZY-V7MQ&=& z?A{#^&s}~R&c-l@3m=$H^Ln%hc}vmh8j%-*&9m`!AOg;69Mu9;+p`d6s>a=B1vTMkoyR+HPCV4 zkVJ{&+t8E(k6KVoThOUe^@A@>Unejezi5@RNN2~DUIBDW&G^#kTv0{!Br$x_VwT=o z8?WvG{#gM)!}xkV=cWH(QEW&##RekXVcnh%3riwsd35kjGm+ z8HcZEDoPt9_wYhbMtF8)3x?#4B;=s@Q4Q=y(#OJ*<7-iR+&fns- zH(YpVaol?|Jvbd7-Ra+U?5t67dHI;uJ-4N^JCl}v#s9PH{ZPe^^atJPU$FqASF#-v zaLsCFcF7hwp3rKSIk@yH3$DTU@xCfIDe&)4P9EcRNq2d`nC30pcN3nO8Vi@+qYnB@ zsb&|`u{9nUM4*sK_{)^nn773|tPsDD>&bSXY!fvsO4qdSnoQp=SFL7gUy8FaX>2vC zi;va%x!%H>Fp|0_>(uH~)MC8(E6Rqi8nReN75%mjxDgD+id^d1ieJiou2L}SVCQGy z@SkEx39S8)ALKNy@EXFkH_LwStRj%pAU>s6kmpNGo^2vXs9s4rsGiCXL@H!Fn1`G! zUBgD$EU+VJ_m!ymQZbBUh*6ilqk5`|xz_dU=$cKHCP)Ii`?KfcYhHF9!$a@fXdu@j zviY(@{*jM)11OneTOndM*5*3IywrU<@5xo8xjh!jte$SxMYv9F+l%s2!U8y*X#UR- zJpHp_E=E*9x-chP;Fk<6ErQQ`OwaKE;=r(+H*;L(S30%S8~-i=$*vnMH9;YVCCm@Q z1z){N!ltgcSdrYGu>#O3<(YzsK>&0TBM&ZLmOI z!q13-di`w$ln$A`7{A3al0nhILU__qZ!vf%DgFN&m6$(EmkW{247GwG4esnSF}x^|^8JTE=+Vv_l# zc&>V=pBRB*fHrq1YOdZuPka;M?ewji=Oz zK_nH@7AsLAn2yZ1A(&13KNSS!IJoV2q!nF6{SfJMH1+yNoVe4(Y5IR7lkHz+5L?$G+JWhe9ODrzyAPUof0a^`@@0#+NotccMy!G z4tC=_hN~f&uo<({tpH9_Y58%m9sr~h2wbovp6iND`v{X8pCn@(Q$n*p`$MiM0(F)* zknjcp+Q+}Kq)8pnFU)_0pFvDzA0SrazC}o_iAwJIbI^342^pJi!k;iqWOeIR&=z0- zJAm7R%}j|TOzg#+4(HOLK;}?Y&#vJ>6Oy2a!OE!4LFj4f0F9WoS%w#m57wFBQR`$ zGC%tC0U3-cuqQlVF#eRQ3la*I^2coQMM~=NnA$`EFn9!X?~0L@D5YJU0PK6vGOL}B zwFWRV>q-N&s|=9GB#xup`%yIL1GBc$GD%}ut$#eM(b>6Gzzr@4p;X1E)Qy zep2A?AxBmNngWvv^v~%_vxNxTdFMUo13LP}*@eZg7YZ}a8;5XeYdf^RggURzK30tN zQ|VK<+P{CVBR%Ub=GEY1eQd-|6nU}d0mrY_ftqEQ?ZG1*0Hq2D=djKPbL&;0x=Xl3 zs>S#j?0D>I%y>jB%mB^?dm2DQ@3iHcKS7Fda8v`FREJ}@V9M+5`q7cJk}?4F#wZvE zRwRxA=K`WQNkcAM2OyezF6AnNZsCpC>F-R~#zh6!j^m-A0z%C$;s_*UkSX#rIxJ); z0Pv&v(CIR=6*vc+(}K8Q%eN$S>N7wQU;qFk91nVXsaVT^S3Qj@Z+VW@1fJb9^r;9E z6il#SFn!GcMCEk7$41;3*xuQV@nEHz#r2xY`&RYA#$ZFmv+z zRE3k~z`}vYJ!lTn6(EJ%(;cadYbydUSJP-b3QPtHx%@pTSSzSSBP8JGy;8rvFiLH$ zBj0dFHjY5|C;Ct!Z?DqMJYc+};bX>GkF`{{vTMm!=T!NeXApysp#K0mlFr$nnP!#z z+oE}LKJT!tGF_tO6;G(9Bb&a4O(o2>+uqTebHW>xBe?Yyy`@f?g|M9tR32T9063|K zHk%naX3lY3_N`}kVy!lrxC6|PuN#M`9<;>HxI=DY4;-rU9=RN5o~Y{(5BHd5xXJ$j z>sCgCbS^aL{?6$+Ba_b<_4LJWqeuo>L*s$Fx%Td9h>01n!AHmj(UFoxJg87YNx4-< z?_zsW39`$x3c%p*Cm-Ze?^G>-IL}eRpa(FFEMb0Z51bRw_vubRikTZ(0qWeb2iln% zK&7)7E^t6vH?Njio67~h?mqGK_Mi!pL{L>p^i?Am81gdWtyPchkvBiIfp0OB3w zR{Q*8Za-Rdf*A5@8-dep0n zj0R9LGtU_{7>_DM1=Gp@+pA4hw`>4AaX&eJi=fQrz|>>)RAPj02V(f z6(F}*u5zS~4(vBd0BKg(n89MXAXF|$D6IT*yLa-bcZGIlB&ZqUqRE#p<>h){j!gh& zJ(T)$nKaQY`!_A4RC$dXWc zk3fAr>Z15?7*_>Adb}wW;|}dQ__`kLE0BM>Q7ps z6C$YlpTp9k8EwU|dFFz|L^lzDbMH>i%s}L0C+SE9i9)epyMXF@RKZ*?n1xg+#xd(n zAWU*TLxIIf=4@@OFu?Sn0iPov{M|SdD-Zx+Nx5qB@S7?R-OnlvVrrD>h1Of>DA57F&*GASnfI%l29`wMHHq4HCxB%_X6+X!1AQOTIHA2oAW8W;K zmXr8%&{g==C0Jh`419gON^1$y^`48>s9lT%*7Qi??qik3mb6iYEE+3C=sx z862?`U^qGCV9+BuP{-zO1vz<88A!*zkUbwMI1-)*M^LB#Zuj8HNfEPpP5kfhOINDi^+X2mU=` zY;bsy{{X;M$?gnxS(p)UTjwBl?TVkwgUDE6K>?2nxaoi^BT>F+?h6DC!>P|5IINp; zXF^X-j=z;A zqizI77!zXv6@VSF)YNlpV!3oVC$YG)A>))C!#^RdZ3|e68D1;CAG3C+K^DLq@<#_X z5z^zoxR+rOJ~A=T)wZ7ENylPUhhR~_>@ic2$b@H%^Y2V|^3hmx=qPCwP_uRGjMNws zD=BRK)$dTg*H;R9(9BvUCjfQ)DT$1NqazvXObA-vCkjXy%NleRq(!)L?d$1Qqsa}^ zkxz{oB@fhg6o7~9$s9cZ_NhW7P4X$u0s2!UxkNc25uPzm48-nHxDt8vpa`dr%#81B zC^^MdGkH#~rz6sxJf*NtC%sm12;ZCx;}iiE%p&eKMm&*_eGNgU$kWFtmA`u*4Z5yj zSxYN&22E#O18H)%DBml5u0O3b47-K|6W_fwhR7p~b@!*EBjA#LovCnKu;YqiB`Azg z1IQmR^)%Owl!h3{^rAA0AY{p9Akr3nq?3`H@kn4YNaLqkPnb#2kQ;FoEE$l6BRjn* z&NlQriCnN9r~#oPZ~rZr2J#B$?*YdhY8%VA ztx!!nfsf4Aea4{oGmCow9l^nn{ner!H_MdCbtl>-&r{HS1u+pBueCNIH(Of_9G|?x zu3K5fYJ|YJ$5uUC>sByBj1IKFK0w>Iu>?~ALY$Vy20Buf+(;Z|lq>wq!wNX3C>Y!q zE6{`9fErP}u*CEk=hB|9f(Qyh;{<-SGok28j12Aes4`ws6dl8kDTuJ8077HsN3Y(h z&hEL{@~G-FOOmg*u6PtW=1c&29sOtmB6bDqob;kbA%dP%ahz0$${6S8^vxqWkVXq( zarn>$2^Z|wBI#G;E%T@Ktr&JY2T}}Y^x~jkBu$K@WO6EHy@E zPYxCKgYxtp%>YVdk$Gg?y$3j?3$+6VD(%2rWBi%~K|?k@Z)$oucF6;d=Ae8tD}lfpd8hJ_kO9tdKnh0ehaWKGtyx!uD$5^IIbTCnq!hMPHxkL*o{Bx` zV6z0Yv*JmZa>ttjKp8hy?_)RHEW=2R#PMSp{{U!Kjk8BQQOg{t>f;~-`P0w<#O`7+K&+`0 z=~q8#jR~@mzk41t<)7~lY5>+qp^pi$s3#=mq#$7J!5rZF)Nsr8fQ=h3myR>v*cy;^ zAaooOX$(v>XdJL0oM6-PSwew?>$jk!ga$h?!Q(mLQ=}g*&~V1anhf+ zr1xC|=u@KsN*ueB=-5spdw!yPHb&;tq#4c$k7 zdR9%%<`o%Vl#%@C0}|%S>fY*2PfAd;=l9SbyDW|uLfOEbCWIYC%j zR<#}MQd`Z0ojzG1lwp+mk5O9ba6FKtU@xchrXoPixr&ul;-$cNiZDDJY-J#*q?N131X)Y4T3Rw#Ik>0@&v?vLdK`rAa?91m~Kq zBHU&*7{dJ69qEXf=8|8L3aC6_j)Uu17n+>cKrYk$mVPnLeN9Ow8YN}OVtEt+8b9xF4gmRT zP;xlr9-iLy1LKJ|9I3(M6)MTU1Nc}{0$7IhVUfpwT5t;EgVU$ALe0oT2OmsS@eWv^ zJ9x!kM_l8jL|3fXmbI z^sE&VO}LIroZ~g<(#9Fw$W+3=cC*t;iP7e_w^iifJ0EdEo`E`A-4J*rwY9|d*ANgL zavZPRBDzWOy_c_i`qdpjQdlz_(Gj%sfymEuS_~tYWpRe&5n0LJR|KqzQo|IGnAump zIIKBs{E3-SzH`FYK=CkERmS1a)*IVhiNnhoEKYY0l~G{PTzu#^cIirxB)hTz$9il0 zicvG~Q6Z6X9FPxcXBbGEu(>!rNTyqsT#dU|->pluZqIJi^&5fX=}cj?7{D3pns{Bx z5Vq2wu{rBb7!ZI6C%?5*xRe7LH(vOlKrM{Y?koZ+z!hMEuryJeW}}JLP*yzd&T7Lc zOq14tDNk&|HBbf@f@-GUot1@MX@Fo@@Oi+h&$Xm|y}8W*LKGkhNy9cX=}}z5rXr3q z7*xeuu+MsHe9e|Usewk)P=vtQEfB!^RkY+1P7hqwTgd#m7Ueo8-X5Z_z+-5DKXClO z`%@5*w_(Yqhb79M5T3N}mnUK3m!Bx191%dpR@*U=8v}wdQ8c7JRQBeXad8c#JEd@` zSq6K(Q{}csiZh?yp8o(!EGw7qH3*>sHTmW}5qTulHMF|CY~HboA%o(OX$S_`kUs6odC91}_lxI3H)L^AN*U8Fv;c9m4uYd=aHIdOpBFz-MU#&THa=I@iisEOo1 z2il~{U^edf0QaXL9%*(_j1hy{fEuhzD+M2P$XVq1WJN&rS9I}~)|*wXF8l1>k;47lof9WhQtW^u+x_h6rO!9L706OoF7$Enl=88R`_Z;>Z{OUd4VyGkx3=U`m5E*oKbY5&6 z@JY{>-k(ETkq~0si=2$MPin0231f6%8aC=W@#$Gs*H5I~EwrSYM>vJg@c#g^O#pN& zG;$4rb`M^oXZ|F`WgTpCYTMRZ0VCyK@pBH(j4QCltQelumV@8_WIPRmz0t>0iFq< zLb8P_=NUZxDWtYZY>?eeH161PFpJS>Wm0%!>wstiIPW8s+Bl^IC*A|7rrlm7Xjyh$TpPRW83)`fViU8ZA&e4n4weh!=78* z)~q{3YKVxSaJc?dh4CPgExXC&V~Td^9o)p|rJERcI1rXf(Hhw`I(r5 zk74}i0$7UNF3e?yK*8-%Iv|xws1)?LRUin8;N^M2#WD7oH)m;XiccbdGj&ZWJGo=H zg@iY;a+4^}9*jHHoi_Prk!@#T4bM`fbsnOuZCnti4tP?1Yc~G?OIwswXLO!EVwnyK zAL=Lra!A?}@~~_k4{|B3I*C(ytADb6D<4nR95}GlqX~Gwb^0u;>ML7peqQVWk8cALC+1+sa~@4FEX~YIfK=z^ z1CfesO_@BUb}UXoYH1~;-;=cdpM^SEf&e3!h8<6*tpG6QJCyl(C#C@vJ<8}oCAV^Z>3re{m!vpDxmCSbXx~5e~;E|lt!U>Os zC5J#b0D9G7Ig;R+9Lj%$lbjDqU|WP402@!s%LEv}^(LCJ1xI9Z+!OvqR%?)$T$N_| zwt<|Vb5`C+jA+p?49Y`-af9e-0bQa5hD_vjB-RDqr7iO{j-Eo`o%0Ukq36C;_^4gceY3jTxK~yq+rhl8nO{2jW$8u zF`e%0!G+Wuf=0;?p&!<@eoPh%!t~>u(YeZwxb&n3U52E$rLE?S!*Zt$8}ay$YS)i^ zDlpl{91bf#P}3v7Dy*!sN6Jc^GM`-4YkTRm>z}deUoLR5&cAj40Km`&?!;ibhbJTq z)GZh@9lNj(K9wMJ$irm&RDj`zNj{>05mN^U=AY&jO{0!_joV{+TQ0H(%Q%{vh8>Tyhvw$&;C?V5iI14i7c_4lR%`N3o13a&`O6$Qu>B5z!6 z?l`G;JmfLz4r$`x2whqu*S}-!NDAAA0;=Sw&f`eAbjATvb51IJtOer$8Zc5KVjpyL zpb29HbDljZ(j#vu!0zDw6<^Kr;{kbWXBg-zRd+aLZ<{$Lfsr@>iJ~P>PHIJCu|p6~ zsHFZpvCkypnt@6ASgCGyo`RTx%t-^Zi~t=;{OTPh#if_+_h99})kc4Wdi`oEVg0Lr z<}d=v6Y_!5RvrG8zW)F^^{i@7qB`hO`%INV3Nko9%9O7`Q2E9@A>^KwBx8HNNlXFB z?^sytxQy8>q-ADW!^5ju&MdGXapj>la*|B8_Jf0W@cY9mP@; zG}YGylz=nds9tM;J6qYIl5Bnp?^Lap<5nvpzmsbGfDa^ArL~+E@wU~$JxS_mh^23) z$vnwD>-mmL0z|8ZupxTcvFd65_t zROI8@ftA^JmItcvL8dlDYC!LUR<2bpuCo~f(;ev$D-x+0Vy8Lwpawx3;Z+U_^G~{9 z>~h&<_Ul%qD>O=D#!Cgr?T=cgG$eVOhCmy453L{}3amr6N6mn~)UzUoY$;vGBQ+#> zk&Z{o1vT?3kl7$`4KORQ4H-bDs6``b(-vX-nNA=5lj~b9!9YX%bqY+ zfe%;95y4a2HE2X51=RWw)rV&96vozG7!0Ex;b^( zr@lo@6KER%!0V4n6dy3-)`1el0LfqudV14O&mh~3e#4Ca!5Odc&YawEzU<6G}hmjW?hQEcu)WU-;Q|deW-l9@Nzq2wKvTXjHm=2 zK{WWrWG5Vs-n0PO0b(-WHhK}(rcd3?f>}l5ySpgy$kMWN~?kij> zJMA(6z;WqSZ!K-^?OG>Zy%-)6dY10g z2sFz!ftul*$#OxR2#S#sTPP5j2dkpoK`u=XmNxEMZWFEOx5> z(blVlF|Z0Z7Vh0DWRalxBVhZ6PTrqNU`Zrp6cm|cGICTNLHD9X^9rw;Q|0-A;G9$i z8*;YYxjZ*&mM~UK?Ip9rXMsQw`EIxbe1v6r&M6D3X9>XIp0zXOa;d|E!3Tp?$YQL_ zakGu2^gZYS0XyPl11p|>wGb@rmRz=YBx4mca!1&+D;^Xc^)Q8q-MO0`K^^@l0yc?R z6Sj{0fEWxD+00^W;>`CB|axqZEFymkf+<-BY zS3W@=&FzLc=ltcB;C zGNWnk#}x8c<{uyc_x_a4@;_Ao9-g#@Nx;DN>s8w48By{$=uJwlAN#A0r!>}qWpJ!< z&Oj6Z%X_AgT)*BJ=dq`-M{ampX$+%o+$igTOpp{|kU%}XX@NXws~~l3FH9b%-jjJ( zW4k%v^HDSHOsP9S$)X@+RnJbe0cXq$8SCwV+NMIPfqI;G?^K+w-J(o$o+?ygBn-Ly zdI|u#h)7@?1P*Gg#k6-3yb_|cZn$7-cWl1%oc{ni8bD7g!0$*56BX`-wTAnf0#Ccb zKd7y1Te$9RgwZm(F#EvNvkZhUCQf)FvZT`<+1`CK0!D1SJrVx^z|aQKIN*@MvC^e& z+f>FrZkY5nlcs7gUb8D^DOcran0(4>P#kXDPY0>=paz#LxMlj)fe2&)Kn0hDs2q|Q zu;BCnb*B*CQlhw00N|PgNx#U8mI2qO#WpEYR|;FH>S{8Zf-zn?bHzX;NMev}4i}IQ z@n(US=EMxil`3(~MU^CN&pa@yw$~w6KQNMkN#u9WYOs!nEw}~ingCfM0ZAFhLr`w= zNM*|tk%Lo9CRYl>7~C`OP#vb&08g3xSfFBwGcf`7IT<%1XPm5B5MY@kIkCDevYnHpxuXKBlIugMLMjzI+(q5-Ka;N5w z-ED=$GRDLJM^jn0_7mE^T20a89Hd|## - -

+![output](../assets/csrnet_output_tensorrt.jpg) diff --git a/csrnet/config.h b/csrnet/config.h deleted file mode 100644 index 1bf0a17e..00000000 --- a/csrnet/config.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -const static char *kInputTensorName = "data"; -const static char *kOutputTensorName = "prob"; -const static char *kEngineFile = "./csrnet.engine"; - -const static int kBatchSize = 1; - -const static int MAX_INPUT_SIZE = 1440; // 32x -const static int MIN_INPUT_SIZE = 608; -const static int OPT_INPUT_W = 1152; -const static int OPT_INPUT_H = 640; - -constexpr static int kMaxInputImageSize = MAX_INPUT_SIZE * MAX_INPUT_SIZE * 3; -constexpr static int kMaxOutputProbSize = - (MAX_INPUT_SIZE * MAX_INPUT_SIZE) >> 6; \ No newline at end of file diff --git a/csrnet/csrnet.cpp b/csrnet/csrnet.cpp index 7ef08bd4..9c62d7ee 100644 --- a/csrnet/csrnet.cpp +++ b/csrnet/csrnet.cpp @@ -1,536 +1,404 @@ -#include "NvInfer.h" -#include "cuda_runtime_api.h" +#include +#include #include -#include -#include -#include + +#include +#include +#include +#include +#include #include #include -#include -#include #include -#include -#include -using namespace nvinfer1; - -#define CHECK(status) \ - do { \ - auto ret = (status); \ - if (ret != 0) { \ - std::cerr << "Cuda failure: " << ret << std::endl; \ - abort(); \ - } \ - } while (0) +#include +#include +#include +#include "NvInfer.h" +#include "utils.h" + +using LayerConfig = std::vector>; +using WeightMap = std::map; +using O = nvinfer1::OptProfileSelector; + +constexpr static const std::array NAMES = {"data", "prob"}; +constexpr static const char* WTS_PATH = "../models/csrnet.wts"; +constexpr static const char* ENGINE_PATH = "../models/csrnet.engine"; + +// for simplicity, always use BatchSize == 1 +constexpr static const int64_t maxBatchSize = 1; +constexpr static const int64_t MAX_INPUT_SIZE = 1440; +constexpr static const int64_t MIN_INPUT_SIZE = 608; + +/** @note: we use this value on purpose to simplify preprocess in this demo. + * To set your own H, W value, be sure to make them divisible by 32 */ +constexpr static const int64_t OPT_INPUT_W = 1024; +constexpr static const int64_t OPT_INPUT_H = 768; + +constexpr static int64_t kMaxInputImageSize = MAX_INPUT_SIZE * MAX_INPUT_SIZE * 3; +constexpr static int64_t kMaxOutputProbSize = (MAX_INPUT_SIZE * MAX_INPUT_SIZE) >> 6; + +static const LayerConfig frontend_cfg = {64, 64, "M", 128, 128, "M", 256, 256, 256, "M", 512, 512, 512}; +static const LayerConfig backend_cfg = {512, 512, 512, 256, 128, 64}; + +constexpr static const std::array mean = {0.406, 0.456, 0.485}; +constexpr static const std::array stdv = {0.225, 0.224, 0.229}; static Logger gLogger; -static char *kWTSFile = ""; -std::map loadWeights(const std::string file) { - std::cout << "Loading weights: " << file << std::endl; - std::map weightMap; - - // Open weights file - std::ifstream input(file); - assert(input.is_open() && "Unable to load weight file."); - - // Read number of weight blobs - int32_t count; - input >> count; - assert(count > 0 && "Invalid weight map file."); - - while (count--) { - Weights wt{DataType::kFLOAT, nullptr, 0}; - uint32_t size; - - // Read name and type of blob - std::string name; - input >> name >> std::dec >> size; - wt.type = DataType::kFLOAT; - - // Load blob - uint32_t *val = reinterpret_cast(malloc(sizeof(val) * size)); - for (uint32_t x = 0, y = size; x < y; ++x) { - input >> std::hex >> val[x]; - } - wt.values = val; - wt.count = size; - weightMap[name] = wt; - } +ILayer* addBatchNorm2d(INetworkDefinition* network, WeightMap& m, ITensor& input, const std::string& lname, + float eps = 1e-3) { + static Weights none{DataType::kFLOAT, nullptr, 0ll}; + const auto* gamma = reinterpret_cast(m[lname + ".weight"].values); + const auto* beta = reinterpret_cast(m[lname + ".bias"].values); + const auto* mean = reinterpret_cast(m[lname + ".running_mean"].values); + const auto* var = reinterpret_cast(m[lname + ".running_var"].values); + auto len = m[lname + ".running_var"].count; + + auto* scval = static_cast(malloc(sizeof(float) * len)); + for (int i = 0; i < len; i++) { + scval[i] = gamma[i] / sqrt(var[i] + eps); + } + Weights scale{DataType::kFLOAT, scval, len}; - return weightMap; + auto* shift_val = static_cast(malloc(sizeof(float) * len)); + for (int i = 0; i < len; i++) { + shift_val[i] = beta[i] - (mean[i] * scval[i]); + } + Weights shift{DataType::kFLOAT, shift_val, len}; + + m[lname + ".scale"] = scale; + m[lname + ".shift"] = shift; + m[lname + ".power"] = none; + auto* bn = network->addScale(input, ScaleMode::kCHANNEL, shift, scale, none); + assert(bn); + bn->setName(lname.c_str()); + return bn; } -// clang-format off -/* -CSRNet( - (frontend): Sequential( - (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (1): ReLU(inplace=True) - (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (3): ReLU(inplace=True) - (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) - (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (6): ReLU(inplace=True) - (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (8): ReLU(inplace=True) - (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) - (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (11): ReLU(inplace=True) - (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (13): ReLU(inplace=True) - (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (15): ReLU(inplace=True) - (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) - (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (18): ReLU(inplace=True) - (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (20): ReLU(inplace=True) - (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) - (22): ReLU(inplace=True) - ) - (backend): Sequential( - (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), - dilation=(2, 2)) (1): ReLU(inplace=True) (2): Conv2d(512, 512, - kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2)) (3): - ReLU(inplace=True) (4): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, - 1), padding=(2, 2), dilation=(2, 2)) (5): ReLU(inplace=True) (6): - Conv2d(512, 256, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), - dilation=(2, 2)) (7): ReLU(inplace=True) (8): Conv2d(256, 128, - kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2)) (9): - ReLU(inplace=True) (10): Conv2d(128, 64, kernel_size=(3, 3), stride=(1, - 1), padding=(2, 2), dilation=(2, 2)) (11): ReLU(inplace=True) - ) - (output_layer): Conv2d(64, 1, kernel_size=(1, 1), stride=(1, 1)) -) -*/ -// clang-format on -void doInference(IExecutionContext &context, float *input, float *output, - int input_h, int input_w) { - const ICudaEngine &engine = context.getEngine(); - - uint64_t input_size = 3 * input_h * input_w * sizeof(float); - uint64_t output_size = ((input_h * input_w) >> 6) * sizeof(float); - - // Pointers to input and output device buffers to pass to engine. - // Engine requires exactly IEngine::getNbBindings() number of buffers. - assert(engine.getNbBindings() == 2); - void *buffers[2]; - - // In order to bind the buffers, we need to know the names of the input and - // output tensors. Note that indices are guaranteed to be less than - // IEngine::getNbBindings() - const int inputIndex = engine.getBindingIndex(kInputTensorName); - const int outputIndex = engine.getBindingIndex(kOutputTensorName); - context.setBindingDimensions(inputIndex, Dims4(1, 3, input_h, input_w)); - - // Create GPU buffers on device - CHECK(cudaMalloc(&buffers[inputIndex], input_size)); - CHECK(cudaMalloc(&buffers[outputIndex], output_size)); - - // Create stream - cudaStream_t stream; - CHECK(cudaStreamCreate(&stream)); - - // DMA input batch data to device, infer on the batch asynchronously, and DMA - // output back to host - CHECK(cudaMemcpyAsync(buffers[inputIndex], input, input_size, - cudaMemcpyHostToDevice, stream)); - auto t1 = std::chrono::high_resolution_clock::now(); - context.enqueueV2(buffers, stream, nullptr); - std::cout << "enqueueV2 time: " - << std::chrono::duration( - std::chrono::high_resolution_clock::now() - t1) - .count() - << "s" << std::endl; - CHECK(cudaMemcpyAsync(output, buffers[outputIndex], output_size, - cudaMemcpyDeviceToHost, stream)); - cudaStreamSynchronize(stream); - - // Release stream and buffers - cudaStreamDestroy(stream); - CHECK(cudaFree(buffers[inputIndex])); - CHECK(cudaFree(buffers[outputIndex])); + +static auto make_layers(INetworkDefinition* net, WeightMap& w, ITensor& input, const LayerConfig& cfg, + const std::string& name = "frontend.", bool use_bn = false, + bool use_dilation = false) -> ILayer* { + auto* tensor = &input; + int32_t idx = 0; + ILayer* ret = nullptr; + + for (const auto& _v : cfg) { + if (std::holds_alternative(_v)) { + auto* pool = net->addPoolingNd(*tensor, PoolingType::kMAX, DimsHW{2, 2}); + pool->setStrideNd(DimsHW{2, 2}); + pool->setName((name + std::to_string(idx++)).c_str()); + tensor = pool->getOutput(0); + ret = pool; + assert(ret); + } else { + assert(std::holds_alternative(_v)); + const int32_t v = std::get(_v); + auto _name = name + std::to_string(idx++); + auto* conv = net->addConvolutionNd(*tensor, v, DimsHW{3, 3}, w[_name + ".weight"], w[_name + ".bias"]); + auto d_rate = use_dilation ? DimsHW{2, 2} : DimsHW{1, 1}; + conv->setPaddingNd(d_rate); + conv->setDilationNd(d_rate); + conv->setName(_name.c_str()); + tensor = conv->getOutput(0); + if (use_bn) { + auto _name = name + std::to_string(idx++); + auto* bn = addBatchNorm2d(net, w, *tensor, _name, 1e-5f); + tensor = bn->getOutput(0); + assert(bn); + } + auto* relu = net->addActivation(*tensor, ActivationType::kRELU); + relu->setName((name + std::to_string(idx++)).c_str()); + tensor = relu->getOutput(0); + ret = relu; + assert(ret); + } + } + return ret; } -ICudaEngine *createEngine(unsigned int maxBatchSize, IBuilder *builder, - IBuilderConfig *config, DataType dt) { - - // INetworkDefinition *network = builder->createNetworkV2(0U); - const auto explicitBatch = - 1U << static_cast( - NetworkDefinitionCreationFlag::kEXPLICIT_BATCH); - INetworkDefinition *network = builder->createNetworkV2(explicitBatch); - ITensor *data = network->addInput(kInputTensorName, dt, Dims4{1, 3, -1, -1}); - assert(data); - std::map weightMap = loadWeights(kWTSFile); - - IConvolutionLayer *conv1 = network->addConvolutionNd( - *data, 64, DimsHW{3, 3}, weightMap["frontend.0.weight"], - weightMap["frontend.0.bias"]); - assert(conv1); - conv1->setStrideNd(DimsHW{1, 1}); - conv1->setPaddingNd(DimsHW{1, 1}); - - IActivationLayer *relu1 = - network->addActivation(*conv1->getOutput(0), ActivationType::kRELU); - - assert(relu1); - - auto conv2 = network->addConvolutionNd(*relu1->getOutput(0), 64, DimsHW{3, 3}, - weightMap["frontend.2.weight"], - weightMap["frontend.2.bias"]); - assert(conv2); - conv2->setStrideNd(DimsHW{1, 1}); - conv2->setPaddingNd(DimsHW{1, 1}); - auto relu2 = - network->addActivation(*conv2->getOutput(0), ActivationType::kRELU); - assert(relu2); - auto pool1 = network->addPoolingNd(*relu2->getOutput(0), PoolingType::kMAX, - DimsHW{2, 2}); - assert(pool1); - pool1->setStrideNd(DimsHW{2, 2}); - auto conv3 = network->addConvolutionNd( - *pool1->getOutput(0), 128, DimsHW{3, 3}, weightMap["frontend.5.weight"], - weightMap["frontend.5.bias"]); - assert(conv3); - conv3->setStrideNd(DimsHW{1, 1}); - - conv3->setPaddingNd(DimsHW{1, 1}); - auto relu3 = - network->addActivation(*conv3->getOutput(0), ActivationType::kRELU); - assert(relu3); - - auto conv4 = network->addConvolutionNd( - *relu3->getOutput(0), 128, DimsHW{3, 3}, weightMap["frontend.7.weight"], - weightMap["frontend.7.bias"]); - assert(conv4); - conv4->setStrideNd(DimsHW{1, 1}); - conv4->setPaddingNd(DimsHW{1, 1}); - auto relu4 = - network->addActivation(*conv4->getOutput(0), ActivationType::kRELU); - assert(relu4); - - auto pool2 = network->addPoolingNd(*relu4->getOutput(0), PoolingType::kMAX, - DimsHW{2, 2}); - assert(pool2); - pool2->setStrideNd(DimsHW{2, 2}); - - auto conv5 = network->addConvolutionNd( - *pool2->getOutput(0), 256, DimsHW{3, 3}, weightMap["frontend.10.weight"], - weightMap["frontend.10.bias"]); - assert(conv5); - conv5->setStrideNd(DimsHW{1, 1}); - conv5->setPaddingNd(DimsHW{1, 1}); - auto relu5 = - network->addActivation(*conv5->getOutput(0), ActivationType::kRELU); - assert(relu5); - - auto conv6 = network->addConvolutionNd( - *relu5->getOutput(0), 256, DimsHW{3, 3}, weightMap["frontend.12.weight"], - weightMap["frontend.12.bias"]); - assert(conv6); - conv6->setStrideNd(DimsHW{1, 1}); - conv6->setPaddingNd(DimsHW{1, 1}); - auto relu6 = - network->addActivation(*conv6->getOutput(0), ActivationType::kRELU); - assert(relu6); - auto conv7 = network->addConvolutionNd( - *relu6->getOutput(0), 256, DimsHW{3, 3}, weightMap["frontend.14.weight"], - weightMap["frontend.14.bias"]); - assert(conv7); - conv7->setStrideNd(DimsHW{1, 1}); - conv7->setPaddingNd(DimsHW{1, 1}); - auto relu7 = - network->addActivation(*conv7->getOutput(0), ActivationType::kRELU); - assert(relu7); - auto pool3 = network->addPoolingNd(*relu7->getOutput(0), PoolingType::kMAX, - DimsHW{2, 2}); - assert(pool3); - pool3->setStrideNd(DimsHW{2, 2}); - auto conv8 = network->addConvolutionNd( - *pool3->getOutput(0), 512, DimsHW{3, 3}, weightMap["frontend.17.weight"], - weightMap["frontend.17.bias"]); - assert(conv8); - conv8->setStrideNd(DimsHW{1, 1}); - conv8->setPaddingNd(DimsHW{1, 1}); - auto relu8 = - network->addActivation(*conv8->getOutput(0), ActivationType::kRELU); - assert(relu8); - auto conv9 = network->addConvolutionNd( - *relu8->getOutput(0), 512, DimsHW{3, 3}, weightMap["frontend.19.weight"], - weightMap["frontend.19.bias"]); - assert(conv9); - conv9->setStrideNd(DimsHW{1, 1}); - conv9->setPaddingNd(DimsHW{1, 1}); - auto relu9 = - network->addActivation(*conv9->getOutput(0), ActivationType::kRELU); - assert(relu9); - auto conv10 = network->addConvolutionNd( - *relu9->getOutput(0), 512, DimsHW{3, 3}, weightMap["frontend.21.weight"], - weightMap["frontend.21.bias"]); - assert(conv10); - conv10->setStrideNd(DimsHW{1, 1}); - conv10->setPaddingNd(DimsHW{1, 1}); - auto relu10 = - network->addActivation(*conv10->getOutput(0), ActivationType::kRELU); - assert(relu10); - // backend - auto conv11 = network->addConvolutionNd( - *relu10->getOutput(0), 512, DimsHW{3, 3}, weightMap["backend.0.weight"], - weightMap["backend.0.bias"]); - assert(conv11); - conv11->setPaddingNd(DimsHW{2, 2}); - conv11->setStrideNd(DimsHW{1, 1}); - conv11->setDilationNd(DimsHW{2, 2}); - auto relu11 = - network->addActivation(*conv11->getOutput(0), ActivationType::kRELU); - - assert(relu11); - auto conv12 = network->addConvolutionNd( - *relu11->getOutput(0), 512, DimsHW{3, 3}, weightMap["backend.2.weight"], - weightMap["backend.2.bias"]); - assert(conv12); - conv12->setPaddingNd(DimsHW{2, 2}); - conv12->setStrideNd(DimsHW{1, 1}); - conv12->setDilationNd(DimsHW{2, 2}); - auto relu12 = - network->addActivation(*conv12->getOutput(0), ActivationType::kRELU); - assert(relu12); - - auto conv13 = network->addConvolutionNd( - *relu12->getOutput(0), 512, DimsHW{3, 3}, weightMap["backend.4.weight"], - weightMap["backend.4.bias"]); - assert(conv13); - conv13->setPaddingNd(DimsHW{2, 2}); - conv13->setStrideNd(DimsHW{1, 1}); - conv13->setDilationNd(DimsHW{2, 2}); - auto relu13 = - network->addActivation(*conv13->getOutput(0), ActivationType::kRELU); - assert(relu13); - - auto conv14 = network->addConvolutionNd( - *relu13->getOutput(0), 256, DimsHW{3, 3}, weightMap["backend.6.weight"], - weightMap["backend.6.bias"]); - assert(conv14); - conv14->setPaddingNd(DimsHW{2, 2}); - conv14->setStrideNd(DimsHW{1, 1}); - conv14->setDilationNd(DimsHW{2, 2}); - auto relu14 = - network->addActivation(*conv14->getOutput(0), ActivationType::kRELU); - assert(relu14); - auto conv15 = network->addConvolutionNd( - *relu14->getOutput(0), 128, DimsHW{3, 3}, weightMap["backend.8.weight"], - weightMap["backend.8.bias"]); - assert(conv15); - conv15->setPaddingNd(DimsHW{2, 2}); - conv15->setStrideNd(DimsHW{1, 1}); - conv15->setDilationNd(DimsHW{2, 2}); - auto relu15 = - network->addActivation(*conv15->getOutput(0), ActivationType::kRELU); - assert(relu15); - auto conv16 = network->addConvolutionNd( - *relu15->getOutput(0), 64, DimsHW{3, 3}, weightMap["backend.10.weight"], - weightMap["backend.10.bias"]); - assert(conv16); - conv16->setPaddingNd(DimsHW{2, 2}); - conv16->setStrideNd(DimsHW{1, 1}); - conv16->setDilationNd(DimsHW{2, 2}); - auto relu16 = - network->addActivation(*conv16->getOutput(0), ActivationType::kRELU); - - assert(relu16); - - auto conv17 = network->addConvolutionNd( - *relu16->getOutput(0), 1, DimsHW{1, 1}, weightMap["output_layer.weight"], - weightMap["output_layer.bias"]); - assert(conv17); - - conv17->setStrideNd(DimsHW{1, 1}); - conv17->getOutput(0)->setName(kOutputTensorName); - network->markOutput(*conv17->getOutput(0)); - - IOptimizationProfile *profile = builder->createOptimizationProfile(); - profile->setDimensions(kInputTensorName, OptProfileSelector::kMIN, - Dims4(1, 3, MIN_INPUT_SIZE, MIN_INPUT_SIZE)); - profile->setDimensions(kInputTensorName, OptProfileSelector::kOPT, - Dims4(1, 3, OPT_INPUT_H, OPT_INPUT_W)); - profile->setDimensions(kInputTensorName, OptProfileSelector::kMAX, - Dims4(1, 3, MAX_INPUT_SIZE, MAX_INPUT_SIZE)); - config->addOptimizationProfile(profile); - - builder->setMaxBatchSize(kBatchSize); - config->setMaxWorkspaceSize(16 << 20); -#ifdef USE_FP16 - config->setFlag(BuilderFlag::kFP16); + +auto createEngine(int32_t maxBatchSize, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, + DataType dt) -> ICudaEngine* { + WeightMap w = loadWeights(WTS_PATH); + +#if TRT_VERSION >= 10000 + auto* network = builder->createNetworkV2(0); +#else + auto* network = builder->createNetworkV2(1u << static_cast(NetworkDefinitionCreationFlag::kEXPLICIT_BATCH)); #endif - ICudaEngine *engine = builder->buildEngineWithConfig(*network, *config); - printf("build engine successfully : %s\n", kEngineFile); - // Don't need the network any more - network->destroy(); + ITensor* data = network->addInput(NAMES[0], dt, Dims4{1, 3, -1, -1}); + assert(data); + + auto* frontend = make_layers(network, w, *data, frontend_cfg, "frontend.", false, false); + auto* backend = make_layers(network, w, *frontend->getOutput(0), backend_cfg, "backend.", false, true); + auto conv = network->addConvolutionNd(*backend->getOutput(0), 1, DimsHW{1, 1}, w["output_layer.weight"], + w["output_layer.bias"]); + assert(conv); + + conv->setStrideNd(DimsHW{1, 1}); + conv->getOutput(0)->setName(NAMES[1]); + network->markOutput(*conv->getOutput(0)); + + IOptimizationProfile* profile = builder->createOptimizationProfile(); + profile->setDimensions(NAMES[0], O::kMIN, Dims4(maxBatchSize, 3, MIN_INPUT_SIZE, MIN_INPUT_SIZE)); + profile->setDimensions(NAMES[0], O::kOPT, Dims4(maxBatchSize, 3, OPT_INPUT_H, OPT_INPUT_W)); + profile->setDimensions(NAMES[0], O::kMAX, Dims4(maxBatchSize, 3, MAX_INPUT_SIZE, MAX_INPUT_SIZE)); + config->addOptimizationProfile(profile); + +#if TRT_VERSION >= 8000 + config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); + auto* serialized = builder->buildSerializedNetwork(*network, *config); + auto* engine = runtime->deserializeCudaEngine(serialized->data(), serialized->size()); + delete serialized; + delete network; +#else + builder->setMaxBatchSize(maxBatchSize); + config->setMaxWorkspaceSize(WORKSPACE_SIZE); + auto* engine = builder->buildEngineWithConfig(*network, *config); + network->destroy(); +#endif + std::cout << "build out\n"; - // Release host memory - for (auto &mem : weightMap) { - free((void *)(mem.second.values)); - } + for (auto& _mem : w) { + free(const_cast(_mem.second.values)); + } - return engine; + return engine; } -void APIToModel(unsigned int maxBatchSize, IHostMemory **modelStream) { - // Create builder - IBuilder *builder = createInferBuilder(gLogger); - IBuilderConfig *config = builder->createBuilderConfig(); - - // Create model to populate the network, then set the outputs and create an - // engine - ICudaEngine *engine = - createEngine(maxBatchSize, builder, config, DataType::kFLOAT); - assert(engine != nullptr); - - // Serialize the engine - (*modelStream) = engine->serialize(); - - // Close everything down - engine->destroy(); - config->destroy(); - builder->destroy(); + +void APIToModel(int32_t batch_size, IRuntime* runtime, IHostMemory** model_stream) { + auto* builder = createInferBuilder(gLogger); + auto* config = builder->createBuilderConfig(); + + auto* engine = createEngine(batch_size, runtime, builder, config, DataType::kFLOAT); + assert(engine != nullptr); + + (*model_stream) = engine->serialize(); + +#if TRT_VERSION >= 8000 + delete engine; + delete config; + delete builder; +#else + engine->destroy(); + config->destroy(); + builder->destroy(); +#endif } -int read_files_in_dir(const char *p_dir_name, - std::vector &file_names) { - DIR *p_dir = opendir(p_dir_name); - if (p_dir == nullptr) { - return -1; - } - - struct dirent *p_file = nullptr; - while ((p_file = readdir(p_dir)) != nullptr) { - if (strcmp(p_file->d_name, ".") != 0 && strcmp(p_file->d_name, "..") != 0) { - std::string cur_file_name(p_file->d_name); - file_names.push_back(cur_file_name); +std::vector doInference(IExecutionContext& context, void* input, int batchSize) { + const auto& engine = context.getEngine(); + cudaStream_t stream; + CHECK(cudaStreamCreate(&stream)); + std::vector buffers; + +#if TRT_VERSION >= 8000 + const int32_t nIO = engine.getNbIOTensors(); +#else + const int32_t nIO = engine.getNbBindings(); +#endif + + buffers.resize(nIO); + Dims out_shape; + std::size_t out_size; + context.setOptimizationProfileAsync(0, stream); + for (auto i = 0; i < nIO; ++i) { +#if TRT_VERSION >= 8000 + const auto* tensor_name = engine.getIOTensorName(i); + auto s = getSize(engine.getTensorDataType(tensor_name)); + if (i == 0) { + std::size_t size = s * batchSize * 3 * OPT_INPUT_W * OPT_INPUT_H; + CHECK(cudaMalloc(&buffers[i], size)); + CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); + // since input config is dynamic, must set input shape to let TensorRT deduce the output shape + context.setInputShape(tensor_name, Dims4{batchSize, 3, OPT_INPUT_H, OPT_INPUT_W}); + context.setInputTensorAddress(tensor_name, buffers[i]); + } + + // we have already known nIO==2, so make it simple here + if (i > 0) { + out_shape = context.getTensorShape(tensor_name); + out_size = std::accumulate(out_shape.d, out_shape.d + out_shape.nbDims, 1ULL, std::multiplies<>()); + CHECK(cudaMalloc(&buffers[i], s * out_size)); +#if TRT_VERSION >= 10000 + if (!context.setOutputTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setOutputTensorAddress failed\n"; + std::abort(); + } +#else + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } +#endif + } +#else + const int32_t idx = engine.getBindingIndex(NAMES[i]); + auto s = getSize(engine.getBindingDataType(idx)); + assert(idx == i); + if (engine.bindingIsInput(idx)) { + std::size_t size = s * batchSize * 3 * OPT_INPUT_W * OPT_INPUT_H; + CHECK(cudaMalloc(&buffers[i], size)); + CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); + context.setBindingDimensions(idx, Dims4{batchSize, 3, OPT_INPUT_H, OPT_INPUT_W}); + } else { + assert(context.allInputDimensionsSpecified()); + out_shape = context.getBindingDimensions(idx); + out_size = std::accumulate(out_shape.d, out_shape.d + out_shape.nbDims, 1, std::multiplies()); + CHECK(cudaMalloc(&buffers[i], s * out_size)); + } +#endif } - } - closedir(p_dir); - return 0; + +#if TRT_VERSION >= 8000 + if (!context.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); + } +#else + if (!context.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } +#endif + + std::vector prob; + prob.reserve(static_cast(nIO > 0 ? nIO - 1 : 0)); + for (int i = 1; i < nIO; ++i) { + prob.emplace_back(out_shape, DataType::kFLOAT, -1); + CHECK(cudaMemcpyAsync(prob.back().data, buffers[i], out_size * sizeof(float), cudaMemcpyDeviceToHost, stream)); + } + CHECK(cudaStreamSynchronize(stream)); + + for (auto& buffer : buffers) { + CHECK(cudaFree(buffer)); + } + CHECK(cudaStreamDestroy(stream)); + return prob; } -int main(int argc, char **argv) { - - if (argc != 3) { - std::cerr << "arguments not right!" << std::endl; - std::cerr << "./csrnet -s ./csrnet.wts // serialize model to plan file" - << std::endl; - std::cerr - << "./csrnet -d ../images // deserialize plan file and run inference" - << std::endl; - return -1; - } - char *trtModelStream{nullptr}; - size_t size{0}; - - if (std::string(argv[1]) == "-s") { - IHostMemory *modelStream{nullptr}; - kWTSFile = argv[2]; - APIToModel(kBatchSize, &modelStream); - assert(modelStream != nullptr); - - std::ofstream p(kEngineFile, std::ios::binary); - if (!p) { - std::cerr << "could not open plan output file" << std::endl; - return -1; +static auto mainImpl(int argc, char** argv) -> int { + checkTrtEnv(); + if (argc != 2) { + std::cerr << "arguments not right!\n" << std::flush; + std::cerr << "./csrnet -s // serialize model to plan file\n" << std::flush; + std::cerr << "./csrnet -d // deserialize plan file and run inference\n" << std::flush; + return -1; + } + char* trtModelStream{nullptr}; + std::streamsize size{0}; + IRuntime* runtime = createInferRuntime(gLogger); + assert(runtime != nullptr); + + if (std::string(argv[1]) == "-s") { + IHostMemory* modelStream{nullptr}; + APIToModel(maxBatchSize, runtime, &modelStream); + assert(modelStream != nullptr); + + std::ofstream p(ENGINE_PATH, std::ios::binary); + if (!p) { + std::cerr << "could not open plan output file\n"; + return -1; + } + if (modelStream->size() > static_cast(std::numeric_limits::max())) { + std::cerr << "this model is too large to serialize\n"; + return -1; + } + const auto* data_ptr = reinterpret_cast(modelStream->data()); + auto data_size = static_cast(modelStream->size()); + p.write(data_ptr, data_size); +#if TRT_VERSION >= 8000 + delete modelStream; +#else + modelStream->destroy(); +#endif + return 0; + } + if (std::string(argv[1]) == "-d") { + std::ifstream file(ENGINE_PATH, std::ios::binary); + if (file.good()) { + file.seekg(0, file.end); + size = file.tellg(); + file.seekg(0, file.beg); + trtModelStream = new char[size]; + assert(trtModelStream); + file.read(trtModelStream, size); + file.close(); + } + } else { + return -1; + } + +#if TRT_VERSION >= 8000 + ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size); +#else + ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size, nullptr); +#endif + assert(engine != nullptr); + IExecutionContext* context = engine->createExecutionContext(); + assert(context != nullptr); + + // read and preprocess input data + cv::Mat img_32f, rgb; + auto img = cv::imread("../assets/IMG_1.jpg", cv::IMREAD_COLOR); + img.convertTo(img_32f, CV_32FC3, 1.0 / 255.0, 0); + img_32f = (img_32f - cv::Scalar(mean[0], mean[1], mean[2])) / cv::Scalar(stdv[0], stdv[1], stdv[2]); + cv::cvtColor(img_32f, rgb, cv::COLOR_BGR2RGB); + static std::size_t img_size = OPT_INPUT_H * OPT_INPUT_W * img.channels(); + std::vector data(img_size); + for (int i = 0; i < maxBatchSize; ++i) { + // to NCHW (N == 1) + for (int y = 0; y < OPT_INPUT_H; ++y) { + for (int x = 0; x < OPT_INPUT_W; ++x) { + const cv::Vec3f v = img_32f.at(y, x); + data[i * img_size + 0 * OPT_INPUT_H * OPT_INPUT_W + y * OPT_INPUT_W + x] = v[0]; + data[i * img_size + 1 * OPT_INPUT_H * OPT_INPUT_W + y * OPT_INPUT_W + x] = v[1]; + data[i * img_size + 2 * OPT_INPUT_H * OPT_INPUT_W + y * OPT_INPUT_W + x] = v[2]; + } + } } - p.write(reinterpret_cast(modelStream->data()), - modelStream->size()); - modelStream->destroy(); - return 1; - } else if (std::string(argv[1]) == "-d") { - std::ifstream file(kEngineFile, std::ios::binary); - if (file.good()) { - file.seekg(0, file.end); - size = file.tellg(); - file.seekg(0, file.beg); - trtModelStream = new char[size]; - assert(trtModelStream); - file.read(trtModelStream, size); - file.close(); + + // run inference + for (int32_t i = 0; i < 100; ++i) { + auto _start = std::chrono::system_clock::now(); + auto output = doInference(*context, data.data(), 1); + auto _end = std::chrono::system_clock::now(); + auto _time = std::chrono::duration_cast(_end - _start).count(); + std::cout << "Execution time: " << _time << "us\n"; + + const auto* data_ptr = reinterpret_cast(output[0].data); + for (int j = 0; j < 10; ++j) { + std::cout << data_ptr[j] << " " << std::flush; + } + std::cout << "\n====\n"; + + if (i == 99) { + // write output to jpg + DummyTensor& t = output[0]; + const auto out_h = t.dims.d[2]; + const auto out_w = t.dims.d[3]; + auto stride = static_cast(out_h) * out_w; + float num = std::accumulate(data_ptr, data_ptr + stride, 0.0f); + cv::Mat density((int)out_h, (int)out_w, CV_32FC1, t.data); + cv::Mat scaled, heatmap, save; + cv::normalize(density, scaled, 0, 255, cv::NORM_MINMAX, CV_8UC1); + cv::applyColorMap(scaled, heatmap, cv::COLORMAP_JET); + cv::resize(heatmap, heatmap, img.size(), 0., 0., cv::INTER_LINEAR); + cv::addWeighted(img, 0.7, heatmap, 0.3, 0, save); + cv::imwrite("../assets/csrnet_output_tensorrt.jpg", save); + std::cout << "approximate people num: " << std::ceil(num) + << "\nsave to ../assets/csrnet_output_tensorrt.jpg\n"; + } } - } else { - return -1; - } - IRuntime *runtime = createInferRuntime(gLogger); - assert(runtime != nullptr); - ICudaEngine *engine = runtime->deserializeCudaEngine(trtModelStream, size); - assert(engine != nullptr); - IExecutionContext *context = engine->createExecutionContext(); - assert(context != nullptr); - delete[] trtModelStream; - - std::vector file_names; - if (read_files_in_dir(argv[2], file_names) < 0) { - std::cout << "read_files_in_dir failed." << std::endl; - return -1; - } - - std::vector mean_value{0.406, 0.456, 0.485}; // BGR - std::vector std_value{0.225, 0.224, 0.229}; - - int fcount = 0; - - float *data = new float[kMaxInputImageSize]; - float *prob = new float[kMaxOutputProbSize]; - - for (auto f : file_names) { - fcount++; - cv::Mat src_img = cv::imread(std::string(argv[2]) + "/" + f); - if (src_img.empty()) - continue; - - int i = 0; - for (int row = 0; row < src_img.rows; ++row) { - uchar *uc_pixel = src_img.data + row * src_img.step; - for (int col = 0; col < src_img.cols; ++col) { - data[i] = (uc_pixel[2] / 255.0 - mean_value[2]) / std_value[2]; - data[i + src_img.rows * src_img.cols] = - (uc_pixel[1] / 255.0 - mean_value[1]) / std_value[1]; - data[i + 2 * src_img.rows * src_img.cols] = - (uc_pixel[0] / 255.0 - mean_value[0]) / std_value[0]; - uc_pixel += 3; - ++i; - } + + return 0; +} + +auto main(int argc, char** argv) -> int { + try { + return mainImpl(argc, argv); + } catch (const std::exception& e) { + std::cerr << e.what() << '\n'; + return -1; } - // Run inference - auto start = std::chrono::system_clock::now(); - doInference(*context, data, prob, src_img.rows, src_img.cols); - auto end = std::chrono::system_clock::now(); - std::cout << "detect time:" - << std::chrono::duration_cast(end - - start) - .count() - << "ms" << std::endl; - float num = std::accumulate( - prob, prob + ((src_img.rows * src_img.cols) >> 6), 0.0f); - - cv::Mat densityMap(src_img.rows >> 3, src_img.cols >> 3, CV_32FC1, - (void *)prob); - - cv::Mat densityMapScaled; - cv::normalize(densityMap, densityMapScaled, 0, 255, cv::NORM_MINMAX, - CV_8UC1); - cv::Mat densityColorMap; - cv::applyColorMap(densityMapScaled, densityColorMap, cv::COLORMAP_VIRIDIS); - - cv::resize(densityColorMap, densityColorMap, src_img.size()); - cv::addWeighted(densityColorMap, 0.5, src_img, 0.5, 0, src_img); - - // write to jpg - cv::putText(src_img, std::string("people num: ") + std::to_string(num), - cv::Point(10, 50), cv::FONT_HERSHEY_SIMPLEX, 0.5, - cv::Scalar(255, 255, 255), 1); - std::string write_path = std::string(argv[2]) + "result_" + f; - std::cout << "people num :" << num << " write_path: " << write_path - << std::endl; - cv::imwrite(write_path, src_img); - } - delete[] data; - delete[] prob; - - return 0; -} \ No newline at end of file +} diff --git a/csrnet/gen_wts.py b/csrnet/gen_wts.py index 68ba51fb..cd6e29f2 100644 --- a/csrnet/gen_wts.py +++ b/csrnet/gen_wts.py @@ -1,31 +1,185 @@ -from torch.nn.modules import module -from model import CSRNet -import torch -import os +""" +this file is modified from https://github.com/leeyeehoo/CSRNet-pytorch +to make it compatible with python3, and support wts file exporting +""" + import struct +import cv2 +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision import models + + +class CSRNet(nn.Module): + def __init__(self, load_weights=False): + super(CSRNet, self).__init__() + self.seen = 0 + self.frontend_feat = [64, 64, "M", 128, 128, "M", 256, 256, 256, "M", 512, 512, 512] + self.backend_feat = [512, 512, 512, 256, 128, 64] + self.frontend = make_layers(self.frontend_feat) + self.backend = make_layers(self.backend_feat, in_channels=512, dilation=True) + self.output_layer = nn.Conv2d(64, 1, kernel_size=1) + if not load_weights: + mod = models.vgg16(pretrained=True) + self._initialize_weights() + for i in range(len(self.frontend.state_dict().items())): + self.frontend.state_dict().items()[i][1].data[:] = mod.state_dict().items()[i][1].data[:] + + def forward(self, x): + x = self.frontend(x) + x = self.backend(x) + x = self.output_layer(x) + return x + + def _initialize_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.normal_(m.weight, std=0.01) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + +def make_layers(cfg, in_channels=3, batch_norm=False, dilation=False): + if dilation: + d_rate = 2 + else: + d_rate = 1 + layers = [] + for v in cfg: + if v == "M": + layers += [nn.MaxPool2d(kernel_size=2, stride=2)] + else: + conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=d_rate, dilation=d_rate) + if batch_norm: + layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] + else: + layers += [conv2d, nn.ReLU(inplace=True)] + in_channels = v + return nn.Sequential(*layers) + + +def preprocess(img: np.array) -> torch.Tensor: + """ + a preprocess method align with CSRNet: no resize, no normalization + input resolution: (1024, 768, 3) + + Args: + img (np.array): input image + + Returns: + torch.Tensor: preprocessed image in `NCHW` layout + """ + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32) + mean = np.array([92.8207477031, 95.2757037428, 104.877445883], dtype=np.float32) + img = img - mean + img = img.transpose(2, 0, 1)[None, ...] + return torch.from_numpy(img) + + +def visualize() -> None: + """this method is for anyone who wants to see the ground truth output + of the input included in this demo, just call this method directly. + """ + import scipy + + def gaussian_filter_density(gt): + density = np.zeros(gt.shape, dtype=np.float32) + gt_count = np.count_nonzero(gt) + + if gt_count == 0: + return density + + ys, xs = np.nonzero(gt) + pts = np.column_stack((xs, ys)) + + tree = scipy.spatial.cKDTree(pts, leafsize=2048) # build KDTree + distances, _ = tree.query(pts, k=min(4, gt_count)) + + for i, (x, y) in enumerate(pts): + pt2d = np.zeros(gt.shape, dtype=np.float32) + pt2d[y, x] = 1.0 + + if gt_count > 1: + sigma = np.sum(distances[i][1:]) * 0.1 + else: + sigma = np.mean(gt.shape) / 4.0 + + density += scipy.ndimage.gaussian_filter(pt2d, sigma, mode="constant") + + return density + + def visualize_density_cv2(img: np.array, density): + if density.max() > 0: + density_norm = density / density.max() + else: + density_norm = density + + density_uint8 = (density_norm * 255).astype(np.uint8) + heatmap = cv2.applyColorMap(density_uint8, cv2.COLORMAP_JET) + blended = cv2.addWeighted(img, 0.7, heatmap, 0.3, 0) + return blended + + img = cv2.imread("../assets/IMG_1.jpg", cv2.IMREAD_COLOR) + gt = scipy.io.loadmat("../assets/GT_IMG_1.mat") + gt = gt["image_info"][0, 0][0, 0][0] + k = np.zeros((img.shape[0], img.shape[1])) + for i in range(0, len(gt)): + if int(gt[i][1]) < img.shape[0] and int(gt[i][0]) < img.shape[1]: + k[int(gt[i][1]), int(gt[i][0])] = 1 + k = gaussian_filter_density(k) + img = visualize_density_cv2(img, k) + cv2.imwrite("../assets/csrnet_gt_vis.jpg", img) + + +def main(): + # load pth model + model_path = "../models/PartAmodel_best.pth.tar" # partBmodel_best.pth + model = CSRNet(load_weights=True) + checkpoint = torch.load(model_path, weights_only=False) + model.load_state_dict(checkpoint["state_dict"]) + model.eval() + with torch.inference_mode(): + img = cv2.imread("../assets/IMG_1.jpg", cv2.IMREAD_COLOR) + h, w, _ = img.shape + data = preprocess(img) + output = model(data) -save_path = os.path.join(os.path.dirname( - __file__), "output", os.path.basename(__file__).split('.')[0]) -os.makedirs(save_path, exist_ok=True) -wts_file = os.path.join(save_path, "csrnet.wts") + # this is the postprocess part + # 1. convert the output to heatmap + heatmap = F.interpolate(output, size=(h, w), mode="bilinear", align_corners=False) + heatmap = heatmap[0].detach().cpu().numpy().transpose(1, 2, 0) + heatmap = cv2.normalize(heatmap, None, 0, 255, cv2.NORM_MINMAX) + heatmap = cv2.applyColorMap(heatmap.astype(np.uint8)[..., None], cv2.COLORMAP_JET) + # 2. blend the heatmap with the original image + heatmap = cv2.resize(heatmap, (w, h), 0, 0, cv2.INTER_LINEAR) + overlay = cv2.addWeighted(img, 0.7, heatmap, 0.3, 0) + status = cv2.imwrite("../assets/csrnet_output_torch.jpg", overlay) + assert status + # save to wts + print("Writing into csrnet.wts") + with open("../models/csrnet.wts", "w") as f: + f.write("{}\n".format(len(model.state_dict().keys()))) + for k, v in model.state_dict().items(): + print(f"key: {k}\tvalue: {v.shape}") + vr = v.reshape(-1).cpu().numpy() + f.write("{} {} ".format(k, len(vr))) + for vv in vr: + f.write(" ") + f.write(struct.pack(">f", float(vv)).hex()) + f.write("\n") -# load model -model_path = "partBmodel_best.pth.tar" -model = CSRNet() -checkpoint = torch.load(model_path) -model.load_state_dict(checkpoint['state_dict']) +if __name__ == "__main__": + # Uncomment the following line to generate ground truth heatmap + # with the file in `../assets/GT_IMG_1.mat`, check the result + # at `../assets/csrnet_gt_vis.jpg` + # visualize() -# save to wts -print(f'Writing into {wts_file}') -with open(wts_file, 'w') as f: - f.write('{}\n'.format(len(model.state_dict().keys()))) - for k, v in model.state_dict().items(): - vr = v.reshape(-1).cpu().numpy() - f.write('{} {} '.format(k, len(vr))) - for vv in vr: - f.write(' ') - f.write(struct.pack('>f', float(vv)).hex()) - f.write('\n') \ No newline at end of file + main() diff --git a/csrnet/logging.h b/csrnet/logging.h index fb09a20a..4e2bd80d 100644 --- a/csrnet/logging.h +++ b/csrnet/logging.h @@ -17,486 +17,450 @@ #ifndef TENSORRT_LOGGING_H #define TENSORRT_LOGGING_H -#include "NvInferRuntimeCommon.h" -#include "macros.h" #include +#include #include #include #include #include #include #include +#include +#include "NvInferRuntime.h" +#include "macros.h" using Severity = nvinfer1::ILogger::Severity; class LogStreamConsumerBuffer : public std::stringbuf { -public: - LogStreamConsumerBuffer(std::ostream &stream, const std::string &prefix, - bool shouldLog) - : mOutput(stream), mPrefix(prefix), mShouldLog(shouldLog) {} - - LogStreamConsumerBuffer(LogStreamConsumerBuffer &&other) - : mOutput(other.mOutput) {} - - ~LogStreamConsumerBuffer() { - // std::streambuf::pbase() gives a pointer to the beginning of the buffered - // part of the output sequence std::streambuf::pptr() gives a pointer to the - // current position of the output sequence if the pointer to the beginning - // is not equal to the pointer to the current position, call putOutput() to - // log the output to the stream - if (pbase() != pptr()) { - putOutput(); + public: + LogStreamConsumerBuffer(std::ostream& stream, std::string prefix, bool shouldLog) + : mOutput(stream), mPrefix(std::move(prefix)), mShouldLog(shouldLog) {} + + LogStreamConsumerBuffer(LogStreamConsumerBuffer&& other) noexcept + : mOutput(other.mOutput), mPrefix(std::move(other.mPrefix)), mShouldLog(other.mShouldLog) {} + + ~LogStreamConsumerBuffer() override { + // std::streambuf::pbase() gives a pointer to the beginning of the buffered part of the output sequence + // std::streambuf::pptr() gives a pointer to the current position of the output sequence + // if the pointer to the beginning is not equal to the pointer to the current position, + // call putOutput() to log the output to the stream + if (pbase() != pptr()) { + putOutput(); + } } - } - - // synchronizes the stream buffer and returns 0 on success - // synchronizing the stream buffer consists of inserting the buffer contents - // into the stream, resetting the buffer and flushing the stream - virtual int sync() { - putOutput(); - return 0; - } - - void putOutput() { - if (mShouldLog) { - // prepend timestamp - std::time_t timestamp = std::time(nullptr); - tm *tm_local = std::localtime(×tamp); - std::cout << "["; - std::cout << std::setw(2) << std::setfill('0') << 1 + tm_local->tm_mon - << "/"; - std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_mday - << "/"; - std::cout << std::setw(4) << std::setfill('0') << 1900 + tm_local->tm_year - << "-"; - std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_hour - << ":"; - std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_min << ":"; - std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_sec - << "] "; - // std::stringbuf::str() gets the string contents of the buffer - // insert the buffer contents pre-appended by the appropriate prefix into - // the stream - mOutput << mPrefix << str(); - // set the buffer to empty - str(""); - // flush the stream - mOutput.flush(); + + // synchronizes the stream buffer and returns 0 on success + // synchronizing the stream buffer consists of inserting the buffer contents into the stream, + // resetting the buffer and flushing the stream + int sync() override { + putOutput(); + return 0; + } + + void putOutput() { + if (mShouldLog) { + // prepend timestamp + std::time_t timestamp = std::time(nullptr); + tm* tm_local = std::localtime(×tamp); + std::cout << "["; + std::cout << std::setw(2) << std::setfill('0') << 1 + tm_local->tm_mon << "/"; + std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_mday << "/"; + std::cout << std::setw(4) << std::setfill('0') << 1900 + tm_local->tm_year << "-"; + std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_hour << ":"; + std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_min << ":"; + std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_sec << "] "; + // std::stringbuf::str() gets the string contents of the buffer + // insert the buffer contents pre-appended by the appropriate prefix into the stream + mOutput << mPrefix << str(); + // set the buffer to empty + str(""); + // flush the stream + mOutput.flush(); + } } - } - void setShouldLog(bool shouldLog) { mShouldLog = shouldLog; } + void setShouldLog(bool shouldLog) { mShouldLog = shouldLog; } -private: - std::ostream &mOutput; - std::string mPrefix; - bool mShouldLog; + private: + std::ostream& mOutput; + std::string mPrefix; + bool mShouldLog; }; //! //! \class LogStreamConsumerBase -//! \brief Convenience object used to initialize LogStreamConsumerBuffer before -//! std::ostream in LogStreamConsumer +//! \brief Convenience object used to initialize LogStreamConsumerBuffer before std::ostream in LogStreamConsumer //! class LogStreamConsumerBase { -public: - LogStreamConsumerBase(std::ostream &stream, const std::string &prefix, - bool shouldLog) - : mBuffer(stream, prefix, shouldLog) {} + public: + LogStreamConsumerBase(std::ostream& stream, std::string prefix, bool shouldLog) + : mBuffer(stream, std::move(prefix), shouldLog) {} -protected: - LogStreamConsumerBuffer mBuffer; + protected: + LogStreamConsumerBuffer mBuffer; }; //! //! \class LogStreamConsumer -//! \brief Convenience object used to facilitate use of C++ stream syntax when -//! logging messages. +//! \brief Convenience object used to facilitate use of C++ stream syntax when logging messages. //! Order of base classes is LogStreamConsumerBase and then std::ostream. -//! This is because the LogStreamConsumerBase class is used to initialize the -//! LogStreamConsumerBuffer member field in LogStreamConsumer and then the -//! address of the buffer is passed to std::ostream. This is necessary to -//! prevent the address of an uninitialized buffer from being passed to -//! std::ostream. Please do not change the order of the parent classes. +//! This is because the LogStreamConsumerBase class is used to initialize the LogStreamConsumerBuffer member field +//! in LogStreamConsumer and then the address of the buffer is passed to std::ostream. +//! This is necessary to prevent the address of an uninitialized buffer from being passed to std::ostream. +//! Please do not change the order of the parent classes. //! class LogStreamConsumer : protected LogStreamConsumerBase, public std::ostream { -public: - //! \brief Creates a LogStreamConsumer which logs messages with level - //! severity. - //! Reportable severity determines if the messages are severe enough to be - //! logged. - LogStreamConsumer(Severity reportableSeverity, Severity severity) - : LogStreamConsumerBase(severityOstream(severity), - severityPrefix(severity), - severity <= reportableSeverity), - std::ostream(&mBuffer) // links the stream buffer with the stream - , - mShouldLog(severity <= reportableSeverity), mSeverity(severity) {} - - LogStreamConsumer(LogStreamConsumer &&other) - : LogStreamConsumerBase(severityOstream(other.mSeverity), - severityPrefix(other.mSeverity), - other.mShouldLog), - std::ostream(&mBuffer) // links the stream buffer with the stream - , - mShouldLog(other.mShouldLog), mSeverity(other.mSeverity) {} - - void setReportableSeverity(Severity reportableSeverity) { - mShouldLog = mSeverity <= reportableSeverity; - mBuffer.setShouldLog(mShouldLog); - } - -private: - static std::ostream &severityOstream(Severity severity) { - return severity >= Severity::kINFO ? std::cout : std::cerr; - } - - static std::string severityPrefix(Severity severity) { - switch (severity) { - case Severity::kINTERNAL_ERROR: - return "[F] "; - case Severity::kERROR: - return "[E] "; - case Severity::kWARNING: - return "[W] "; - case Severity::kINFO: - return "[I] "; - case Severity::kVERBOSE: - return "[V] "; - default: - assert(0); - return ""; + public: + //! \brief Creates a LogStreamConsumer which logs messages with level severity. + //! Reportable severity determines if the messages are severe enough to be logged. + LogStreamConsumer(Severity reportableSeverity, Severity severity) + : LogStreamConsumerBase(severityOstream(severity), severityPrefix(severity), severity <= reportableSeverity), + std::ostream(&mBuffer) // links the stream buffer with the stream + , + mShouldLog(severity <= reportableSeverity), + mSeverity(severity) {} + + LogStreamConsumer(LogStreamConsumer&& other) noexcept + : LogStreamConsumerBase(severityOstream(other.mSeverity), severityPrefix(other.mSeverity), other.mShouldLog), + std::ostream(&mBuffer) // links the stream buffer with the stream + , + mShouldLog(other.mShouldLog), + mSeverity(other.mSeverity) {} + + void setReportableSeverity(Severity reportableSeverity) { + mShouldLog = mSeverity <= reportableSeverity; + mBuffer.setShouldLog(mShouldLog); + } + + private: + static std::ostream& severityOstream(Severity severity) { + return severity >= Severity::kINFO ? std::cout : std::cerr; + } + + static std::string severityPrefix(Severity severity) { + switch (severity) { + case Severity::kINTERNAL_ERROR: + return "[F] "; + case Severity::kERROR: + return "[E] "; + case Severity::kWARNING: + return "[W] "; + case Severity::kINFO: + return "[I] "; + case Severity::kVERBOSE: + return "[V] "; + default: + assert(0); + return ""; + } } - } - bool mShouldLog; - Severity mSeverity; + bool mShouldLog; + Severity mSeverity; }; //! \class Logger //! //! \brief Class which manages logging of TensorRT tools and samples //! -//! \details This class provides a common interface for TensorRT tools and -//! samples to log information to the console, and supports logging two types of -//! messages: +//! \details This class provides a common interface for TensorRT tools and samples to log information to the console, +//! and supports logging two types of messages: //! -//! - Debugging messages with an associated severity (info, warning, error, or -//! internal error/fatal) +//! - Debugging messages with an associated severity (info, warning, error, or internal error/fatal) //! - Test pass/fail messages //! -//! The advantage of having all samples use this class for logging as opposed to -//! emitting directly to stdout/stderr is that the logic for controlling the -//! verbosity and formatting of sample output is centralized in one location. +//! The advantage of having all samples use this class for logging as opposed to emitting directly to stdout/stderr is +//! that the logic for controlling the verbosity and formatting of sample output is centralized in one location. //! -//! In the future, this class could be extended to support dumping test results -//! to a file in some standard format (for example, JUnit XML), and providing -//! additional metadata (e.g. timing the duration of a test run). +//! In the future, this class could be extended to support dumping test results to a file in some standard format +//! (for example, JUnit XML), and providing additional metadata (e.g. timing the duration of a test run). //! -//! TODO: For backwards compatibility with existing samples, this class inherits -//! directly from the nvinfer1::ILogger interface, which is problematic since -//! there isn't a clean separation between messages coming from the TensorRT +//! TODO: For backwards compatibility with existing samples, this class inherits directly from the nvinfer1::ILogger +//! interface, which is problematic since there isn't a clean separation between messages coming from the TensorRT //! library and messages coming from the sample. //! -//! In the future (once all samples are updated to use Logger::getTRTLogger() to -//! access the ILogger) we can refactor the class to eliminate the inheritance -//! and instead make the nvinfer1::ILogger implementation a member of the Logger +//! In the future (once all samples are updated to use Logger::getTRTLogger() to access the ILogger) we can refactor the +//! class to eliminate the inheritance and instead make the nvinfer1::ILogger implementation a member of the Logger //! object. class Logger : public nvinfer1::ILogger { -public: - Logger(Severity severity = Severity::kWARNING) - : mReportableSeverity(severity) {} - - //! - //! \enum TestResult - //! \brief Represents the state of a given test - //! - enum class TestResult { - kRUNNING, //!< The test is running - kPASSED, //!< The test passed - kFAILED, //!< The test failed - kWAIVED //!< The test was waived - }; - - //! - //! \brief Forward-compatible method for retrieving the nvinfer::ILogger - //! associated with this Logger \return The nvinfer1::ILogger associated with - //! this Logger - //! - //! TODO Once all samples are updated to use this method to register the - //! logger with TensorRT, we can eliminate the inheritance of Logger from - //! ILogger - //! - nvinfer1::ILogger &getTRTLogger() { return *this; } - - //! - //! \brief Implementation of the nvinfer1::ILogger::log() virtual method - //! - //! Note samples should not be calling this function directly; it will - //! eventually go away once we eliminate the inheritance from - //! nvinfer1::ILogger - //! - void log(Severity severity, const char *msg) TRT_NOEXCEPT override { - LogStreamConsumer(mReportableSeverity, severity) - << "[TRT] " << std::string(msg) << std::endl; - } - - //! - //! \brief Method for controlling the verbosity of logging output - //! - //! \param severity The logger will only emit messages that have severity of - //! this level or higher. - //! - void setReportableSeverity(Severity severity) { - mReportableSeverity = severity; - } - - //! - //! \brief Opaque handle that holds logging information for a particular test - //! - //! This object is an opaque handle to information used by the Logger to print - //! test results. The sample must call Logger::defineTest() in order to obtain - //! a TestAtom that can be used with Logger::reportTest{Start,End}(). - //! - class TestAtom { - public: - TestAtom(TestAtom &&) = default; - - private: - friend class Logger; - - TestAtom(bool started, const std::string &name, const std::string &cmdline) - : mStarted(started), mName(name), mCmdline(cmdline) {} - - bool mStarted; - std::string mName; - std::string mCmdline; - }; - - //! - //! \brief Define a test for logging - //! - //! \param[in] name The name of the test. This should be a string starting - //! with - //! "TensorRT" and containing dot-separated strings - //! containing the characters [A-Za-z0-9_]. For example, - //! "TensorRT.sample_googlenet" - //! \param[in] cmdline The command line used to reproduce the test - // - //! \return a TestAtom that can be used in Logger::reportTest{Start,End}(). - //! - static TestAtom defineTest(const std::string &name, - const std::string &cmdline) { - return TestAtom(false, name, cmdline); - } - - //! - //! \brief A convenience overloaded version of defineTest() that accepts an - //! array of command-line arguments - //! as input - //! - //! \param[in] name The name of the test - //! \param[in] argc The number of command-line arguments - //! \param[in] argv The array of command-line arguments (given as C strings) - //! - //! \return a TestAtom that can be used in Logger::reportTest{Start,End}(). - static TestAtom defineTest(const std::string &name, int argc, - char const *const *argv) { - auto cmdline = genCmdlineString(argc, argv); - return defineTest(name, cmdline); - } - - //! - //! \brief Report that a test has started. - //! - //! \pre reportTestStart() has not been called yet for the given testAtom - //! - //! \param[in] testAtom The handle to the test that has started - //! - static void reportTestStart(TestAtom &testAtom) { - reportTestResult(testAtom, TestResult::kRUNNING); - assert(!testAtom.mStarted); - testAtom.mStarted = true; - } - - //! - //! \brief Report that a test has ended. - //! - //! \pre reportTestStart() has been called for the given testAtom - //! - //! \param[in] testAtom The handle to the test that has ended - //! \param[in] result The result of the test. Should be one of - //! TestResult::kPASSED, - //! TestResult::kFAILED, TestResult::kWAIVED - //! - static void reportTestEnd(const TestAtom &testAtom, TestResult result) { - assert(result != TestResult::kRUNNING); - assert(testAtom.mStarted); - reportTestResult(testAtom, result); - } - - static int reportPass(const TestAtom &testAtom) { - reportTestEnd(testAtom, TestResult::kPASSED); - return EXIT_SUCCESS; - } - - static int reportFail(const TestAtom &testAtom) { - reportTestEnd(testAtom, TestResult::kFAILED); - return EXIT_FAILURE; - } - - static int reportWaive(const TestAtom &testAtom) { - reportTestEnd(testAtom, TestResult::kWAIVED); - return EXIT_SUCCESS; - } - - static int reportTest(const TestAtom &testAtom, bool pass) { - return pass ? reportPass(testAtom) : reportFail(testAtom); - } - - Severity getReportableSeverity() const { return mReportableSeverity; } - -private: - //! - //! \brief returns an appropriate string for prefixing a log message with the - //! given severity - //! - static const char *severityPrefix(Severity severity) { - switch (severity) { - case Severity::kINTERNAL_ERROR: - return "[F] "; - case Severity::kERROR: - return "[E] "; - case Severity::kWARNING: - return "[W] "; - case Severity::kINFO: - return "[I] "; - case Severity::kVERBOSE: - return "[V] "; - default: - assert(0); - return ""; + private: + struct TestInfo; + + public: + Logger(Severity severity = Severity::kWARNING) : mReportableSeverity(severity) {} + + //! + //! \enum TestResult + //! \brief Represents the state of a given test + //! + enum class TestResult : std::uint8_t { + kRUNNING, //!< The test is running + kPASSED, //!< The test passed + kFAILED, //!< The test failed + kWAIVED //!< The test was waived + }; + + //! + //! \brief Forward-compatible method for retrieving the nvinfer::ILogger associated with this Logger + //! \return The nvinfer1::ILogger associated with this Logger + //! + //! TODO Once all samples are updated to use this method to register the logger with TensorRT, + //! we can eliminate the inheritance of Logger from ILogger + //! + nvinfer1::ILogger& getTRTLogger() { return *this; } + + //! + //! \brief Implementation of the nvinfer1::ILogger::log() virtual method + //! + //! Note samples should not be calling this function directly; it will eventually go away once we eliminate the + //! inheritance from nvinfer1::ILogger + //! + void log(Severity severity, const char* msg) TRT_NOEXCEPT override { + LogStreamConsumer(mReportableSeverity, severity) << "[TRT] " << std::string(msg) << '\n'; } - } - - //! - //! \brief returns an appropriate string for prefixing a test result message - //! with the given result - //! - static const char *testResultString(TestResult result) { - switch (result) { - case TestResult::kRUNNING: - return "RUNNING"; - case TestResult::kPASSED: - return "PASSED"; - case TestResult::kFAILED: - return "FAILED"; - case TestResult::kWAIVED: - return "WAIVED"; - default: - assert(0); - return ""; + + //! + //! \brief Method for controlling the verbosity of logging output + //! + //! \param severity The logger will only emit messages that have severity of this level or higher. + //! + void setReportableSeverity(Severity severity) { mReportableSeverity = severity; } + + //! + //! \brief Opaque handle that holds logging information for a particular test + //! + //! This object is an opaque handle to information used by the Logger to print test results. + //! The sample must call Logger::defineTest() in order to obtain a TestAtom that can be used + //! with Logger::reportTest{Start,End}(). + //! + class TestAtom { + public: + TestAtom(TestAtom&&) = default; + + private: + friend class Logger; + + TestAtom(bool started, TestInfo info) + : mStarted(started), mName(std::move(info.name)), mCmdline(std::move(info.cmdline)) {} + + bool mStarted; + std::string mName; + std::string mCmdline; + }; + + //! + //! \brief Define a test for logging + //! + //! \param[in] name The name of the test. This should be a string starting with + //! "TensorRT" and containing dot-separated strings containing + //! the characters [A-Za-z0-9_]. + //! For example, "TensorRT.sample_googlenet" + //! \param[in] cmdline The command line used to reproduce the test + // + //! \return a TestAtom that can be used in Logger::reportTest{Start,End}(). + //! + static TestAtom defineTest(const std::string& name, const std::string& cmdline) { + return TestAtom{false, TestInfo{name, cmdline}}; + } + + //! + //! \brief A convenience overloaded version of defineTest() that accepts an array of command-line arguments + //! as input + //! + //! \param[in] name The name of the test + //! \param[in] argc The number of command-line arguments + //! \param[in] argv The array of command-line arguments (given as C strings) + //! + //! \return a TestAtom that can be used in Logger::reportTest{Start,End}(). + static TestAtom defineTest(const std::string& name, int argc, char const* const* argv) { + auto cmdline = genCmdlineString(argc, argv); + return defineTest(name, cmdline); + } + + //! + //! \brief Report that a test has started. + //! + //! \pre reportTestStart() has not been called yet for the given testAtom + //! + //! \param[in] testAtom The handle to the test that has started + //! + static void reportTestStart(TestAtom& testAtom) { + reportTestResult(testAtom, TestResult::kRUNNING); + assert(!testAtom.mStarted); + testAtom.mStarted = true; + } + + //! + //! \brief Report that a test has ended. + //! + //! \pre reportTestStart() has been called for the given testAtom + //! + //! \param[in] testAtom The handle to the test that has ended + //! \param[in] result The result of the test. Should be one of TestResult::kPASSED, + //! TestResult::kFAILED, TestResult::kWAIVED + //! + static void reportTestEnd(const TestAtom& testAtom, TestResult result) { + assert(result != TestResult::kRUNNING); + assert(testAtom.mStarted); + reportTestResult(testAtom, result); } - } - - //! - //! \brief returns an appropriate output stream (cout or cerr) to use with the - //! given severity - //! - static std::ostream &severityOstream(Severity severity) { - return severity >= Severity::kINFO ? std::cout : std::cerr; - } - - //! - //! \brief method that implements logging test results - //! - static void reportTestResult(const TestAtom &testAtom, TestResult result) { - severityOstream(Severity::kINFO) - << "&&&& " << testResultString(result) << " " << testAtom.mName << " # " - << testAtom.mCmdline << std::endl; - } - - //! - //! \brief generate a command line string from the given (argc, argv) values - //! - static std::string genCmdlineString(int argc, char const *const *argv) { - std::stringstream ss; - for (int i = 0; i < argc; i++) { - if (i > 0) - ss << " "; - ss << argv[i]; + + static int reportPass(const TestAtom& testAtom) { + reportTestEnd(testAtom, TestResult::kPASSED); + return EXIT_SUCCESS; + } + + static int reportFail(const TestAtom& testAtom) { + reportTestEnd(testAtom, TestResult::kFAILED); + return EXIT_FAILURE; + } + + static int reportWaive(const TestAtom& testAtom) { + reportTestEnd(testAtom, TestResult::kWAIVED); + return EXIT_SUCCESS; + } + + static int reportTest(const TestAtom& testAtom, bool pass) { + return pass ? reportPass(testAtom) : reportFail(testAtom); + } + + [[nodiscard]] Severity getReportableSeverity() const { return mReportableSeverity; } + + private: + struct TestInfo { + std::string name; + std::string cmdline; + }; + //! + //! \brief returns an appropriate string for prefixing a log message with the given severity + //! + static const char* severityPrefix(Severity severity) { + switch (severity) { + case Severity::kINTERNAL_ERROR: + return "[F] "; + case Severity::kERROR: + return "[E] "; + case Severity::kWARNING: + return "[W] "; + case Severity::kINFO: + return "[I] "; + case Severity::kVERBOSE: + return "[V] "; + default: + assert(0); + return ""; + } + } + + //! + //! \brief returns an appropriate string for prefixing a test result message with the given result + //! + static const char* testResultString(TestResult result) { + switch (result) { + case TestResult::kRUNNING: + return "RUNNING"; + case TestResult::kPASSED: + return "PASSED"; + case TestResult::kFAILED: + return "FAILED"; + case TestResult::kWAIVED: + return "WAIVED"; + default: + assert(0); + return ""; + } + } + + //! + //! \brief returns an appropriate output stream (cout or cerr) to use with the given severity + //! + static std::ostream& severityOstream(Severity severity) { + return severity >= Severity::kINFO ? std::cout : std::cerr; + } + + //! + //! \brief method that implements logging test results + //! + static void reportTestResult(const TestAtom& testAtom, TestResult result) { + severityOstream(Severity::kINFO) << "&&&& " << testResultString(result) << " " << testAtom.mName << " # " + << testAtom.mCmdline << '\n'; + } + + //! + //! \brief generate a command line string from the given (argc, argv) values + //! + static std::string genCmdlineString(int argc, char const* const* argv) { + std::stringstream ss; + for (int i = 0; i < argc; i++) { + if (i > 0) + ss << " "; + ss << argv[i]; + } + return ss.str(); } - return ss.str(); - } - Severity mReportableSeverity; + Severity mReportableSeverity; }; namespace { //! -//! \brief produces a LogStreamConsumer object that can be used to log messages -//! of severity kVERBOSE +//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kVERBOSE //! //! Example usage: //! //! LOG_VERBOSE(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_VERBOSE(const Logger &logger) { - return LogStreamConsumer(logger.getReportableSeverity(), Severity::kVERBOSE); +inline LogStreamConsumer LOG_VERBOSE(const Logger& logger) { + return LogStreamConsumer{logger.getReportableSeverity(), Severity::kVERBOSE}; } //! -//! \brief produces a LogStreamConsumer object that can be used to log messages -//! of severity kINFO +//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kINFO //! //! Example usage: //! //! LOG_INFO(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_INFO(const Logger &logger) { - return LogStreamConsumer(logger.getReportableSeverity(), Severity::kINFO); +inline LogStreamConsumer LOG_INFO(const Logger& logger) { + return LogStreamConsumer{logger.getReportableSeverity(), Severity::kINFO}; } //! -//! \brief produces a LogStreamConsumer object that can be used to log messages -//! of severity kWARNING +//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kWARNING //! //! Example usage: //! //! LOG_WARN(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_WARN(const Logger &logger) { - return LogStreamConsumer(logger.getReportableSeverity(), Severity::kWARNING); +inline LogStreamConsumer LOG_WARN(const Logger& logger) { + return LogStreamConsumer{logger.getReportableSeverity(), Severity::kWARNING}; } //! -//! \brief produces a LogStreamConsumer object that can be used to log messages -//! of severity kERROR +//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kERROR //! //! Example usage: //! //! LOG_ERROR(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_ERROR(const Logger &logger) { - return LogStreamConsumer(logger.getReportableSeverity(), Severity::kERROR); +inline LogStreamConsumer LOG_ERROR(const Logger& logger) { + return LogStreamConsumer{logger.getReportableSeverity(), Severity::kERROR}; } //! -//! \brief produces a LogStreamConsumer object that can be used to log messages -//! of severity kINTERNAL_ERROR +//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kINTERNAL_ERROR // ("fatal" severity) //! //! Example usage: //! //! LOG_FATAL(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_FATAL(const Logger &logger) { - return LogStreamConsumer(logger.getReportableSeverity(), - Severity::kINTERNAL_ERROR); +inline LogStreamConsumer LOG_FATAL(const Logger& logger) { + return LogStreamConsumer{logger.getReportableSeverity(), Severity::kINTERNAL_ERROR}; } -} // anonymous namespace +} // anonymous namespace -#endif // TENSORRT_LOGGING_H +#endif // TENSORRT_LOGGING_H diff --git a/csrnet/macros.h b/csrnet/macros.h index 05551039..ccc3b106 100644 --- a/csrnet/macros.h +++ b/csrnet/macros.h @@ -1,12 +1,28 @@ -#ifndef __MACROS_H -#define __MACROS_H +#pragma once +#include -#if NV_TENSORRT_MAJOR >= 8 +#ifdef API_EXPORTS +#if defined(_MSC_VER) +#define API __declspec(dllexport) +#else +#define API __attribute__((visibility("default"))) +#endif +#else + +#if defined(_MSC_VER) +#define API __declspec(dllimport) +#else +#define API +#endif +#endif // API_EXPORTS + +#define TRT_VERSION \ + ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) + +#if TRT_VERSION >= 8000 #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const #else #define TRT_NOEXCEPT #define TRT_CONST_ENQUEUE #endif - -#endif // __MACROS_H diff --git a/csrnet/utils.h b/csrnet/utils.h new file mode 100644 index 00000000..a5225dad --- /dev/null +++ b/csrnet/utils.h @@ -0,0 +1,157 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "macros.h" + +using namespace nvinfer1; + +constexpr const std::size_t WORKSPACE_SIZE = 16 << 20; + +#define CHECK(status) \ + do { \ + auto ret = (status); \ + if (ret != cudaSuccess) { \ + std::cerr << "Cuda failure: " << ret << "\n"; \ + std::abort(); \ + } \ + } while (0) + +static void checkTrtEnv(int device = 0) { +#if TRT_VERSION < 8000 + CHECK(cudaGetDevice(&device)); + cudaDeviceProp prop{}; + CHECK(cudaGetDeviceProperties(&prop, device)); + const int sm = prop.major * 10 + prop.minor; + if (sm > 86) { + std::cerr << "TensorRT < 8 does not support SM > 86 on this GPU."; + std::abort(); + } +#endif +} + +/** + * @brief TensorRT weight files have a simple space delimited format: + * [type] [size] + * + * @param file input weight file path + * @return std::map + */ +static auto loadWeights(const std::string& file) { + std::cout << "Loading weights: " << file << "\n"; + std::map weightMap; + + // Open weights file + std::ifstream input(file); + assert(input.is_open() && "Unable to load weight file."); + + // Read number of weight blobs + int32_t count; + input >> count; + assert(count > 0 && "Invalid weight map file."); + + while (count--) { + nvinfer1::Weights wt{nvinfer1::DataType::kFLOAT, nullptr, 0}; + + // Read name and type of blob + std::string name; + input >> name >> std::dec >> wt.count; + + // Load blob + auto* val = new uint32_t[wt.count]; + input >> std::hex; + for (auto x = 0ll; x < wt.count; ++x) { + input >> val[x]; + } + wt.values = val; + weightMap[name] = wt; + } + + return weightMap; +} + +static size_t getSize(DataType dt) { + switch (dt) { +#if TRT_VERSION >= 8510 + case DataType::kUINT8: +#endif + case DataType::kINT8: + return sizeof(int8_t); + case DataType::kFLOAT: + return sizeof(float); + case DataType::kHALF: + return sizeof(int16_t); + case DataType::kINT32: + return sizeof(int32_t); + default: { + std::cerr << "Unsupported data type\n"; + std::abort(); + } + } +} + +class DummyTensor final { + public: + DummyTensor(Dims dims, DataType dtype, int dev = -1) : dims(dims), dtype(dtype), dev(dev) { + total = std::accumulate(dims.d, dims.d + dims.nbDims, 1, std::multiplies<>()); + if (dev == -1) { + data = malloc(getSize(dtype) * total); + } else { + std::cerr << "dev != -1 not implemented!"; + std::abort(); + } + } + DummyTensor(const DummyTensor& other) : DummyTensor(other.dims, other.dtype, other.dev) { + std::memcpy(data, other.data, getSize(dtype) * total); + } + DummyTensor& operator=(const DummyTensor& other) { + if (this == &other) { + return *this; + } + DummyTensor tmp(other); + std::swap(total, tmp.total); + std::swap(dev, tmp.dev); + std::swap(dtype, tmp.dtype); + std::swap(dims, tmp.dims); + std::swap(data, tmp.data); + return *this; + } + DummyTensor(DummyTensor&& other) noexcept + : total(other.total), dev(other.dev), dtype(other.dtype), dims(other.dims), data(other.data) { + other.total = 0; + other.data = nullptr; + } + DummyTensor& operator=(DummyTensor&& other) noexcept { + if (this == &other) { + return *this; + } + if (data) { + free(data); + } + total = other.total; + dev = other.dev; + dtype = other.dtype; + dims = other.dims; + data = other.data; + other.total = 0; + other.data = nullptr; + return *this; + } + ~DummyTensor() { + if (data) { + free(data); + } + } + std::size_t total; + int dev; + DataType dtype; + Dims dims; + void* data; +}; diff --git a/shufflenetv2/gen_wts.py b/shufflenetv2/gen_wts.py index db632d81..37416ea0 100644 --- a/shufflenetv2/gen_wts.py +++ b/shufflenetv2/gen_wts.py @@ -1,83 +1,85 @@ +import argparse +import os import struct +from pathlib import Path -import cv2 -import numpy as np import torch -from torchvision.models.shufflenetv2 import ( - shufflenet_v2_x0_5, - shufflenet_v2_x1_0, - shufflenet_v2_x1_5, - shufflenet_v2_x2_0, +from torchvision import models + + +MODEL_NAMES = ( + "shufflenet_v2_x0_5", + "shufflenet_v2_x1_0", + "shufflenet_v2_x1_5", + "shufflenet_v2_x2_0", ) +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parent +MODELS_DIR = REPO_ROOT / "models" -def read_imagenet_labels() -> dict[int, str]: - """ - read ImageNet 1000 labels +def require_cache_env() -> None: + missing = [name for name in ("TORCH_HOME", "HF_HOME") if not os.environ.get(name)] + if missing: + raise RuntimeError( + f"Please set required cache environment variables: {', '.join(missing)}" + ) + print(f"Using TORCH_HOME={os.environ['TORCH_HOME']}") + print(f"Using HF_HOME={os.environ['HF_HOME']}") - Returns: - dict[int, str]: labels dict - """ - clsid2label = {} - with open("../assets/imagenet1000_clsidx_to_labels.txt", "r") as f: - for i in f.readlines(): - k, v = i.split(": ") - clsid2label.setdefault(int(k), v[1:-3]) - return clsid2label +def build_model(name: str) -> torch.nn.Module: + suffix = name.removeprefix("shufflenet_").upper() + weights_cls = getattr(models, f"ShuffleNet_{suffix}_Weights") + model_fn = getattr(models, name) + model = model_fn(weights=weights_cls.DEFAULT) + model.eval() + return model -def preprocess(img: np.array) -> torch.Tensor: - """ - a preprocess method align with ImageNet dataset - Args: - img (np.array): input image +def write_wts(model: torch.nn.Module, output_path: Path) -> None: + state_dict = model.state_dict() + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w") as f: + f.write(f"{len(state_dict.keys())}\n") + for key, value in state_dict.items(): + values = value.reshape(-1).cpu().numpy() + f.write(f"{key} {len(values)}") + for item in values: + f.write(" ") + f.write(struct.pack(">f", float(item)).hex()) + f.write("\n") + size_mib = output_path.stat().st_size / 1024 / 1024 + print(f"[ok] wrote {output_path} ({size_mib:.1f} MiB, {len(state_dict)} tensors)") - Returns: - torch.Tensor: preprocessed image in `NCHW` layout - """ - img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0 - img = cv2.resize(img, (224, 224), interpolation=cv2.INTER_LINEAR) - mean = np.array([0.485, 0.456, 0.406], dtype=np.float32) - std = np.array([0.229, 0.224, 0.225], dtype=np.float32) - img = (img - mean) / std - img = img.transpose(2, 0, 1)[None, ...] - return torch.from_numpy(img) +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Export torchvision ShuffleNetV2 weights to tensorrtx .wts files." + ) + parser.add_argument( + "--model", + choices=MODEL_NAMES, + action="append", + help="ShuffleNetV2 variant to export. Repeat for multiple variants. Defaults to all variants.", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=MODELS_DIR, + help="Directory for generated .wts files.", + ) + return parser.parse_args() -if __name__ == "__main__": - labels = read_imagenet_labels() - img = cv2.imread("../assets/cats.jpg", cv2.IMREAD_COLOR) - img = preprocess(img) - """ - NOTE: comment out the model you don't want - """ - models = [ - ("shufflenet_v2_x0_5", shufflenet_v2_x0_5(pretrained=True)), - ("shufflenet_v2_x1_0", shufflenet_v2_x1_0(pretrained=True)), - ("shufflenet_v2_x1_5", shufflenet_v2_x1_5(pretrained=True)), - ("shufflenet_v2_x2_0", shufflenet_v2_x2_0(pretrained=True)), - ] +def main() -> None: + args = parse_args() + require_cache_env() + model_names = args.model or MODEL_NAMES + for model_name in model_names: + print(f"writing {model_name}.wts") + write_wts(build_model(model_name), args.output_dir / f"{model_name}.wts") - for name, model in models: - model.eval() - with torch.inference_mode(): - output = model(img) - print(f"{name} result:") - for i, batch in enumerate(torch.topk(output, k=3).indices): - for j, idx in enumerate(batch): - print(f"\tBatch: {i}, Top: {j}, logits: {output[i][idx]:.4f}, label: {labels[int(idx)]}") - print(f"{'=' * 32}") - with open(f"../models/{name}.wts", "w") as f: - f.write("{}\n".format(len(model.state_dict().keys()))) - for k, v in model.state_dict().items(): - print("key: ", k) - print("value: ", v.shape) - vr = v.reshape(-1).cpu().numpy() - f.write("{} {}".format(k, len(vr))) - for vv in vr: - f.write(" ") - f.write(struct.pack(">f", float(vv)).hex()) - f.write("\n") +if __name__ == "__main__": + main() diff --git a/shufflenetv2/shufflenetv2.cpp b/shufflenetv2/shufflenetv2.cpp index cfc28449..3e94f7c2 100644 --- a/shufflenetv2/shufflenetv2.cpp +++ b/shufflenetv2/shufflenetv2.cpp @@ -1,46 +1,40 @@ #include +#include #include #include +#include +#include +#include +#include +#include #include +#include #include #include #include +#include #include #include "logging.h" #include "utils.h" -struct ShuffleNetV2Params { +struct ShuffleNetV2Variant { + std::string name; std::array repeat; std::array output_chn; + std::string wts_path; + std::string engine_path; }; -/** - * @brief choose one below as the model to be built - * @param v2_x0_5 - * @param v2_x1_0 - * @param v2_x1_5 - * @param v2_x2_0 - */ -[[maybe_unused]] static constexpr ShuffleNetV2Params v2_x0_5 = {{4, 8, 4}, {24, 48, 96, 192, 1024}}; -[[maybe_unused]] static constexpr ShuffleNetV2Params v2_x1_0 = {{4, 8, 4}, {24, 116, 232, 464, 1024}}; -[[maybe_unused]] static constexpr ShuffleNetV2Params v2_x1_5 = {{4, 8, 4}, {24, 176, 352, 704, 1024}}; -[[maybe_unused]] static constexpr ShuffleNetV2Params v2_x2_0 = {{4, 8, 4}, {24, 244, 488, 976, 2048}}; - -constexpr const std::size_t WORKSPACE_SIZE = 16 << 20; - -// stuff we know about shufflenet-v2 -constexpr const int64_t N = 1; -constexpr const int32_t INPUT_H = 224; -constexpr const int32_t INPUT_W = 224; -constexpr const std::array SIZES = {3 * INPUT_H * INPUT_W, 1000}; -constexpr const std::array NAMES = {"data", "logits"}; -static constexpr const bool TRT_PREPROCESS = TRT_VERSION >= 8510 ? true : false; +static constexpr const std::size_t WORKSPACE_SIZE = 16 << 20; +static constexpr const int64_t N = 1; +static constexpr const int32_t INPUT_H = 224; +static constexpr const int32_t INPUT_W = 224; +static constexpr const std::array SIZES = {3 * INPUT_H * INPUT_W, 1000}; +static constexpr const std::array NAMES = {"data", "logits"}; +static constexpr const bool TRT_PREPROCESS = TRT_VERSION >= 8510; static constexpr const std::array mean = {0.485f, 0.456f, 0.406f}; static constexpr const std::array stdv = {0.229f, 0.224f, 0.225f}; - -static constexpr const char* WTS_PATH = "../models/shufflenet_v2_x0_5.wts"; -static constexpr const char* ENGINE_PATH = "../models/shufflenet.engine"; -static constexpr const char* LABELS_PATH = "../assets/imagenet1000_clsidx_to_labels.txt"; +static constexpr const char* LABELS_PATH = "assets/imagenet1000_clsidx_to_labels.txt"; using namespace nvinfer1; using WeightMap = std::map; @@ -49,172 +43,143 @@ using NDCF = nvinfer1::NetworkDefinitionCreationFlag; static Logger gLogger; -Dims debug_shape(const ILayer* l) { - Dims dims = l->getOutput(0)->getDimensions(); - std::cout << l->getOutput(0)->getName() << ":\t["; - for (int i = 0; i < dims.nbDims; i++) { - std::cout << dims.d[i] << ", "; +static auto getVariantConfig(const std::string& name) -> ShuffleNetV2Variant { + static const std::map, std::array>> variants = { + {"shufflenet_v2_x0_5", {{4, 8, 4}, {24, 48, 96, 192, 1024}}}, + {"shufflenet_v2_x1_0", {{4, 8, 4}, {24, 116, 232, 464, 1024}}}, + {"shufflenet_v2_x1_5", {{4, 8, 4}, {24, 176, 352, 704, 1024}}}, + {"shufflenet_v2_x2_0", {{4, 8, 4}, {24, 244, 488, 976, 2048}}}, + }; + + const auto iter = variants.find(name); + if (iter == variants.end()) { + std::cerr << "Unsupported ShuffleNetV2 variant: " << name << "\n"; + std::cerr << "Choose one of: shufflenet_v2_x0_5, shufflenet_v2_x1_0, shufflenet_v2_x1_5, shufflenet_v2_x2_0\n"; + std::abort(); } - std::cout << "]\n"; - return dims; + + return ShuffleNetV2Variant{name, iter->second.first, iter->second.second, "models/" + name + ".wts", + "models/" + name + ".engine"}; } -ILayer* addBatchNorm2d(INetworkDefinition* network, WeightMap& weightMap, ITensor& input, const std::string& lname, - float eps = 1e-3f) { - float* gamma = (float*)weightMap[lname + ".weight"].values; - float* beta = (float*)weightMap[lname + ".bias"].values; - float* mean = (float*)weightMap[lname + ".running_mean"].values; - float* var = (float*)weightMap[lname + ".running_var"].values; - auto len = weightMap[lname + ".running_var"].count; - std::cout << lname << " running_var len: " << len << "\n"; - - auto* scval = reinterpret_cast(malloc(sizeof(float) * len)); - for (int i = 0; i < len; i++) { - scval[i] = gamma[i] / sqrt(var[i] + eps); +static auto addBatchNorm2d(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, const std::string& lname, + float eps = 1e-3f) -> ILayer* { + const auto* gamma = static_cast(weight_map[lname + ".weight"].values); + const auto* beta = static_cast(weight_map[lname + ".bias"].values); + const auto* bn_mean = static_cast(weight_map[lname + ".running_mean"].values); + const auto* var = static_cast(weight_map[lname + ".running_var"].values); + const auto len = weight_map[lname + ".running_var"].count; + + auto* scale_values = reinterpret_cast(std::malloc(sizeof(float) * static_cast(len))); + auto* shift_values = reinterpret_cast(std::malloc(sizeof(float) * static_cast(len))); + if (scale_values == nullptr || shift_values == nullptr) { + std::cerr << "batchnorm weight allocation failed\n"; + std::abort(); } - Weights scale{DataType::kFLOAT, scval, len}; - - auto* shval = static_cast(malloc(sizeof(float) * len)); - for (int i = 0; i < len; i++) { - shval[i] = beta[i] - mean[i] * gamma[i] / sqrt(var[i] + eps); + for (int64_t i = 0; i < len; i++) { + scale_values[i] = gamma[i] / std::sqrt(var[i] + eps); + shift_values[i] = beta[i] - bn_mean[i] * gamma[i] / std::sqrt(var[i] + eps); } - Weights shift{DataType::kFLOAT, shval, len}; + + Weights scale{DataType::kFLOAT, scale_values, len}; + Weights shift{DataType::kFLOAT, shift_values, len}; static const Weights power{DataType::kFLOAT, nullptr, 0ll}; - weightMap[lname + ".scale"] = scale; - weightMap[lname + ".shift"] = shift; - weightMap[lname + ".power"] = power; - IScaleLayer* scale_1 = network->addScale(input, ScaleMode::kCHANNEL, shift, scale, power); - assert(scale_1); - return scale_1; + weight_map[lname + ".scale"] = scale; + weight_map[lname + ".shift"] = shift; + weight_map[lname + ".power"] = power; + IScaleLayer* scale_layer = network->addScale(input, ScaleMode::kCHANNEL, shift, scale, power); + assert(scale_layer); + return scale_layer; } -/** - * @brief a basic convolution+bn layer with an optional relu layer - * - * @param network network definition - * @param m weight map - * @param input input tensor - * @param lname layer name - * @param ch output channels - * @param k kernel - * @param s stride - * @param p padding - * @param g groups - * @param with_relu true if with relu - * @return ILayer* - */ -ILayer* CBR(INetworkDefinition* network, WeightMap& m, ITensor& input, const std::string& lname, int ch, int k, - int s = 1, int p = 0, int g = 1, bool with_relu = true, int start_index = 0) { - static const Weights emptywts{DataType::kFLOAT, nullptr, 0ll}; - auto conv_name = lname + "." + std::to_string(start_index++); - auto* conv = network->addConvolutionNd(input, ch, DimsHW{k, k}, m[conv_name + ".weight"], emptywts); - +static auto addConvBnRelu(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, const std::string& lname, + int32_t out_channels, int32_t kernel_size, int32_t stride = 1, int32_t padding = 0, + int32_t groups = 1, bool with_relu = true, int32_t start_index = 0) -> ILayer* { + static const Weights empty_weights{DataType::kFLOAT, nullptr, 0ll}; + const auto conv_name = lname + "." + std::to_string(start_index++); + auto* conv = network->addConvolutionNd(input, out_channels, DimsHW{kernel_size, kernel_size}, + weight_map[conv_name + ".weight"], empty_weights); assert(conv); - conv->setStrideNd(DimsHW{s, s}); - conv->setPaddingNd(DimsHW{p, p}); - conv->setNbGroups(g); + conv->setStrideNd(DimsHW{stride, stride}); + conv->setPaddingNd(DimsHW{padding, padding}); + conv->setNbGroups(groups); conv->setName(conv_name.c_str()); - auto bn_name = lname + "." + std::to_string(start_index++); - auto* bn = addBatchNorm2d(network, m, *conv->getOutput(0), bn_name, 1e-5f); + const auto bn_name = lname + "." + std::to_string(start_index++); + auto* bn = addBatchNorm2d(network, weight_map, *conv->getOutput(0), bn_name, 1e-5f); bn->setName((bn_name + ".bn").c_str()); - if (with_relu) { - auto* relu = network->addActivation(*bn->getOutput(0), ActivationType::kRELU); - auto relu_name = lname + "." + std::to_string(start_index) + ".relu"; - assert(relu); - relu->setName(relu_name.c_str()); - return relu; + if (!with_relu) { + return bn; } - return bn; + + auto* relu = network->addActivation(*bn->getOutput(0), ActivationType::kRELU); + assert(relu); + relu->setName((lname + "." + std::to_string(start_index) + ".relu").c_str()); + return relu; } -/** - * @brief invered residual block - * - * @param network network definition - * @param m weight map - * @param input input tensor - * @param lname layer name - * @param inch input channels - * @param outch output channels - * @param s stride - * @return ILayer* - */ -ILayer* invertedRes(INetworkDefinition* net, WeightMap& m, ITensor& input, const std::string& lname, int inch, - int outch, int s) { - if (s < 1 || s > 3) { +static auto addInvertedResidual(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, + const std::string& lname, int32_t in_channels, int32_t out_channels, + int32_t stride) -> ILayer* { + if (stride < 1 || stride > 3) { std::cerr << "stride must be in [1, 3]\n"; std::abort(); } - int32_t bf /* branch features */ = outch / 2; - ITensor *x1{nullptr}, *x2{nullptr}; - - if (s == 1) { - auto d = input.getDimensions(); - Dims4 stride{1, 1, 1, 1}; - Dims4 half{d.d[0], d.d[1] / 2, d.d[2], d.d[3]}; - auto* s1 = net->addSlice(input, Dims4{0, 0, 0, 0}, half, stride); - auto* s2 = net->addSlice(input, Dims4{0, d.d[1] / 2, 0, 0}, half, stride); - debug_shape(s2); - x1 = s1->getOutput(0); - x2 = s2->getOutput(0); + + const int32_t branch_features = out_channels / 2; + ITensor* branch1_output = nullptr; + ITensor* branch2_input = nullptr; + + if (stride == 1) { + const auto dims = input.getDimensions(); + const Dims4 half{dims.d[0], dims.d[1] / 2, dims.d[2], dims.d[3]}; + auto* slice1 = network->addSlice(input, Dims4{0, 0, 0, 0}, half, Dims4{1, 1, 1, 1}); + auto* slice2 = network->addSlice(input, Dims4{0, dims.d[1] / 2, 0, 0}, half, Dims4{1, 1, 1, 1}); + assert(slice1); + assert(slice2); + branch1_output = slice1->getOutput(0); + branch2_input = slice2->getOutput(0); } else { - if (s > 1) { - auto* b1 = CBR(net, m, input, lname + ".branch1", inch, 3, s, 1, inch, false, 0); - b1 = CBR(net, m, *b1->getOutput(0), lname + ".branch1", inch, 1, 1, 0, 1, true, 2); - x1 = b1->getOutput(0); - debug_shape(b1); - } else { - x1 = &input; - } - x2 = &input; + auto* branch1 = addConvBnRelu(network, weight_map, input, lname + ".branch1", in_channels, 3, stride, 1, + in_channels, false, 0); + branch1 = addConvBnRelu(network, weight_map, *branch1->getOutput(0), lname + ".branch1", branch_features, 1, 1, + 0, 1, true, 2); + branch1_output = branch1->getOutput(0); + branch2_input = &input; } - auto* b2 = CBR(net, m, *x2, lname + ".branch2", bf, 1, 1, 0, 1, true, 0); - b2 = CBR(net, m, *b2->getOutput(0), lname + ".branch2", bf, 3, s, 1, bf, false, 3); - b2 = CBR(net, m, *b2->getOutput(0), lname + ".branch2", bf, 1, 1, 0, 1, true, 5); - debug_shape(b2); + auto* branch2 = addConvBnRelu(network, weight_map, *branch2_input, lname + ".branch2", branch_features, 1, 1, 0, 1, + true, 0); + branch2 = addConvBnRelu(network, weight_map, *branch2->getOutput(0), lname + ".branch2", branch_features, 3, stride, + 1, branch_features, false, 3); + branch2 = addConvBnRelu(network, weight_map, *branch2->getOutput(0), lname + ".branch2", branch_features, 1, 1, 0, + 1, true, 5); - std::array cat_tensors = {x1, b2->getOutput(0)}; - auto* cat = net->addConcatenation(cat_tensors.data(), 2); - auto cat_name = lname + ".cat"; + std::array cat_tensors = {branch1_output, branch2->getOutput(0)}; + auto* cat = network->addConcatenation(cat_tensors.data(), static_cast(cat_tensors.size())); assert(cat); - cat->setName(cat_name.c_str()); + cat->setName((lname + ".cat").c_str()); cat->setAxis(1); - static_cast(debug_shape(cat)); - - auto* sf1 = net->addShuffle(*cat->getOutput(0)); - assert(sf1); - sf1->setName((lname + ".shuffle.1").c_str()); - auto d = cat->getOutput(0)->getDimensions(); - auto dim_sf1 = Dims{5, {d.d[0], 2, d.d[1] / 2, d.d[2], d.d[3]}}; - sf1->setReshapeDimensions(dim_sf1); - sf1->setSecondTranspose({0, 2, 1, 3, 4}); - - auto* sf2 = net->addShuffle(*sf1->getOutput(0)); - assert(sf2); - sf2->setName((lname + ".shuffle.2").c_str()); - sf2->setReshapeDimensions(d); - - return sf2; + + auto* shuffle1 = network->addShuffle(*cat->getOutput(0)); + assert(shuffle1); + shuffle1->setName((lname + ".shuffle.1").c_str()); + const auto dims = cat->getOutput(0)->getDimensions(); + shuffle1->setReshapeDimensions(Dims{5, {dims.d[0], 2, dims.d[1] / 2, dims.d[2], dims.d[3]}}); + shuffle1->setSecondTranspose({0, 2, 1, 3, 4}); + + auto* shuffle2 = network->addShuffle(*shuffle1->getOutput(0)); + assert(shuffle2); + shuffle2->setName((lname + ".shuffle.2").c_str()); + shuffle2->setReshapeDimensions(dims); + return shuffle2; } -/** - * @brief Create a Engine object - * - * @param N max batch size - * @param runtime runtime - * @param builder builder - * @param config config - * @param dt data type - * @param param the type of model to be built - * @return ICudaEngine* - */ -ICudaEngine* createEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, DataType dt, - ShuffleNetV2Params param = v2_x0_5) { - WeightMap m = loadWeights(WTS_PATH); +static auto createEngine(int32_t batch_size, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, DataType dt, + const ShuffleNetV2Variant& variant) -> ICudaEngine* { + WeightMap weight_map = loadWeights(variant.wts_path); #if TRT_VERSION >= 11200 auto flag = 1U << static_cast(NDCF::kSTRONGLY_TYPED); @@ -223,89 +188,82 @@ ICudaEngine* createEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IBuil #else auto flag = 1U << static_cast(NDCF::kEXPLICIT_BATCH); #endif - auto* net = builder->createNetworkV2(flag); + auto* network = builder->createNetworkV2(flag); + assert(network); - int32_t in_ch = 3; - ITensor* input{nullptr}; + ITensor* input = nullptr; if constexpr (TRT_PREPROCESS) { - // for simplicity, resize image on cpu side dt = DataType::kUINT8; - input = net->addInput(NAMES[0], dt, Dims4{N, INPUT_H, INPUT_W, in_ch}); - auto* trans = addTransformLayer(net, *input, true, mean, stdv); - input = trans->getOutput(0); + input = network->addInput(NAMES[0], dt, Dims4{batch_size, INPUT_H, INPUT_W, 3}); + auto* transform = addTransformLayer(network, *input, true, mean, stdv); + input = transform->getOutput(0); } else { - input = net->addInput(NAMES[0], dt, Dims4{N, in_ch, INPUT_H, INPUT_W}); + input = network->addInput(NAMES[0], dt, Dims4{batch_size, 3, INPUT_H, INPUT_W}); } assert(input); - /** conv1 and maxpool */ - auto* cbr1 = CBR(net, m, *input, "conv1", param.output_chn[0], 3, 2, 1); - auto* pool1 = net->addPoolingNd(*cbr1->getOutput(0), PoolingType::kMAX, DimsHW{3, 3}); + auto* conv1 = addConvBnRelu(network, weight_map, *input, "conv1", variant.output_chn[0], 3, 2, 1); + auto* pool1 = network->addPoolingNd(*conv1->getOutput(0), PoolingType::kMAX, DimsHW{3, 3}); assert(pool1); pool1->setStrideNd(DimsHW{2, 2}); pool1->setPaddingNd(DimsHW{1, 1}); - debug_shape(pool1); - - /** stage 2, 3, 4 */ - ILayer* _layer = pool1; - in_ch = param.output_chn[0]; - for (int stage = 2; stage < 5; ++stage) { - int32_t out_ch = param.output_chn[stage - 1]; - std::string lname = "stage" + std::to_string(stage); - std::cout << "================ " << lname << " ================\n"; - _layer = invertedRes(net, m, *_layer->getOutput(0), lname + ".0", in_ch, out_ch, 2); - debug_shape(_layer); - for (int j = 1; j < param.repeat[stage - 2]; ++j) { - _layer = invertedRes(net, m, *_layer->getOutput(0), lname + "." + std::to_string(j), out_ch, out_ch, 1); + + ILayer* layer = pool1; + int32_t in_channels = variant.output_chn[0]; + for (int32_t stage = 2; stage < 5; ++stage) { + const int32_t out_channels = variant.output_chn[stage - 1]; + const std::string lname = "stage" + std::to_string(stage); + layer = addInvertedResidual(network, weight_map, *layer->getOutput(0), lname + ".0", in_channels, out_channels, + 2); + for (int32_t block = 1; block < variant.repeat[stage - 2]; ++block) { + layer = addInvertedResidual(network, weight_map, *layer->getOutput(0), lname + "." + std::to_string(block), + out_channels, out_channels, 1); } - in_ch = out_ch; + in_channels = out_channels; } - /** conv5, mean and fully connected layer */ - auto* conv5 = CBR(net, m, *_layer->getOutput(0), "conv5", param.output_chn[4], 1, 1, 0); - auto* mean = net->addReduce(*conv5->getOutput(0), ReduceOperation::kAVG, 0xc, false); - mean->setName("global_pool(mean)"); - auto* fcw = net->addConstant(DimsHW{1000, 1024}, m["fc.weight"]); - auto* fcb = net->addConstant(DimsHW{1, 1000}, m["fc.bias"]); - auto* _fc = net->addMatrixMultiply(*mean->getOutput(0), M::kNONE, *fcw->getOutput(0), M::kTRANSPOSE); - auto* fc = net->addElementWise(*_fc->getOutput(0), *fcb->getOutput(0), ElementWiseOperation::kSUM); + auto* conv5 = addConvBnRelu(network, weight_map, *layer->getOutput(0), "conv5", variant.output_chn[4], 1, 1, 0); + auto* global_pool = network->addReduce(*conv5->getOutput(0), ReduceOperation::kAVG, 0xc, false); + assert(global_pool); + global_pool->setName("global_pool"); + auto* fcw = network->addConstant(DimsHW{1000, variant.output_chn[4]}, weight_map["fc.weight"]); + auto* fcb = network->addConstant(DimsHW{1, 1000}, weight_map["fc.bias"]); + auto* fc_matmul = + network->addMatrixMultiply(*global_pool->getOutput(0), M::kNONE, *fcw->getOutput(0), M::kTRANSPOSE); + auto* fc = network->addElementWise(*fc_matmul->getOutput(0), *fcb->getOutput(0), ElementWiseOperation::kSUM); + assert(fc); fc->getOutput(0)->setName(NAMES[1]); - debug_shape(fc); - - net->markOutput(*fc->getOutput(0)); + network->markOutput(*fc->getOutput(0)); #if TRT_VERSION >= 8000 config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); - IHostMemory* mem = builder->buildSerializedNetwork(*net, *config); + IHostMemory* mem = builder->buildSerializedNetwork(*network, *config); + assert(mem); ICudaEngine* engine = runtime->deserializeCudaEngine(mem->data(), mem->size()); - delete net; + delete mem; + delete network; #else - builder->setMaxBatchSize(N); + builder->setMaxBatchSize(batch_size); config->setMaxWorkspaceSize(WORKSPACE_SIZE); - ICudaEngine* engine = builder->buildEngineWithConfig(*net, *config); - net->destroy(); + ICudaEngine* engine = builder->buildEngineWithConfig(*network, *config); + network->destroy(); #endif std::cout << "build finished\n"; - // Release host memory - for (auto& mem : m) { - free((void*)(mem.second.values)); + for (auto& mem : weight_map) { + std::free(const_cast(mem.second.values)); } - return engine; } -void APIToModel(int32_t N, IRuntime* runtime, IHostMemory** modelStream) { - // Create builder +static void APIToModel(int32_t batch_size, IRuntime* runtime, IHostMemory** model_stream, + const ShuffleNetV2Variant& variant) { IBuilder* builder = createInferBuilder(gLogger); IBuilderConfig* config = builder->createBuilderConfig(); - - // Create model to populate the network, then set the outputs and create an engine - ICudaEngine* engine = createEngine(N, runtime, builder, config, DataType::kFLOAT); + ICudaEngine* engine = createEngine(batch_size, runtime, builder, config, DataType::kFLOAT, variant); assert(engine != nullptr); - // Serialize the engine - (*modelStream) = engine->serialize(); + (*model_stream) = engine->serialize(); #if TRT_VERSION >= 8000 delete engine; @@ -318,7 +276,8 @@ void APIToModel(int32_t N, IRuntime* runtime, IHostMemory** modelStream) { #endif } -auto doInference(IExecutionContext& context, void* input, int64_t batchSize) -> std::vector> { +static auto doInference(IExecutionContext& context, void* input, + int64_t batch_size) -> std::vector> { ICudaEngine const& engine = context.getEngine(); cudaStream_t stream; CHECK(cudaStreamCreate(&stream)); @@ -335,18 +294,22 @@ auto doInference(IExecutionContext& context, void* input, int64_t batchSize) -> std::size_t size = 0; #if TRT_VERSION >= 8000 auto* tensor_name = engine.getIOTensorName(i); - auto s = getSize(engine.getTensorDataType(tensor_name)); - size = s * batchSize * SIZES[i]; + const std::string name = tensor_name; + auto element_size = getSize(engine.getTensorDataType(tensor_name)); + size = element_size * static_cast(batch_size) * (name == NAMES[0] ? SIZES[0] : SIZES[1]); CHECK(cudaMalloc(&buffers[i], size)); - if (i == 0) { + if (name == NAMES[0]) { CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); } - context.setTensorAddress(tensor_name, buffers[i]); + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } #else const int32_t idx = engine.getBindingIndex(NAMES[i]); - auto s = getSize(engine.getBindingDataType(idx)); + auto element_size = getSize(engine.getBindingDataType(idx)); assert(idx == i); - size = s * batchSize * SIZES[i]; + size = element_size * static_cast(batch_size) * SIZES[i]; CHECK(cudaMalloc(&buffers[i], size)); if (i == 0) { CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); @@ -355,74 +318,97 @@ auto doInference(IExecutionContext& context, void* input, int64_t batchSize) -> } #if TRT_VERSION >= 8000 - assert(context.enqueueV3(stream)); + if (!context.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); + } #else - assert(context.enqueueV2(buffers.data(), stream, nullptr)); + if (!context.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } #endif std::vector> prob; - for (int i = 1; i < nIO; ++i) { - std::vector tmp(batchSize * SIZES[i], std::nanf("")); - std::size_t size = batchSize * SIZES[i] * sizeof(float); + for (int i = 0; i < nIO; ++i) { +#if TRT_VERSION >= 8000 + const std::string name = engine.getIOTensorName(i); + if (name == NAMES[0]) { + continue; + } + constexpr auto output_size = SIZES[1]; +#else + if (i == 0) { + continue; + } + const auto output_size = SIZES[i]; +#endif + std::vector tmp(batch_size * output_size, std::nanf("")); + const auto size = static_cast(batch_size) * output_size * sizeof(float); CHECK(cudaMemcpyAsync(tmp.data(), buffers[i], size, cudaMemcpyDeviceToHost, stream)); - prob.emplace_back(tmp); + prob.emplace_back(std::move(tmp)); } CHECK(cudaStreamSynchronize(stream)); - // Release stream and buffers - CHECK(cudaStreamDestroy(stream)); + for (auto& buffer : buffers) { CHECK(cudaFree(buffer)); } + CHECK(cudaStreamDestroy(stream)); return prob; } -int main(int argc, char** argv) { +auto main(int argc, char** argv) -> int { checkTrtEnv(); - if (argc != 2) { + if (argc < 2 || argc > 3) { std::cerr << "arguments not right!\n"; - std::cerr << "./shufflenet -s // serialize model to plan file\n"; - std::cerr << "./shufflenet -d // deserialize plan file and run inference\n"; + std::cerr << "./shufflenetv2 -s [model] // serialize model to plan file\n"; + std::cerr << "./shufflenetv2 -d [model] // deserialize plan file and run inference\n"; return -1; } - // create a model using the API directly and serialize it to a stream + const auto variant = getVariantConfig(argc == 3 ? argv[2] : "shufflenet_v2_x0_5"); + std::cout << "Using ShuffleNetV2 variant: " << variant.name << "\n"; + IRuntime* runtime = createInferRuntime(gLogger); assert(runtime != nullptr); - char* trtModelStream{nullptr}; + char* trt_model_stream{nullptr}; std::streamsize size{0}; if (std::string(argv[1]) == "-s") { - IHostMemory* modelStream{nullptr}; - APIToModel(1, runtime, &modelStream); - assert(modelStream != nullptr); + IHostMemory* model_stream{nullptr}; + APIToModel(1, runtime, &model_stream, variant); + assert(model_stream != nullptr); - std::ofstream p(ENGINE_PATH, std::ios::binary | std::ios::trunc); - if (!p) { + std::ofstream plan(variant.engine_path, std::ios::binary | std::ios::trunc); + if (!plan) { std::cerr << "could not open plan output file\n"; return -1; } - if (modelStream->size() > static_cast(std::numeric_limits::max())) { + if (model_stream->size() > static_cast(std::numeric_limits::max())) { std::cerr << "this model is too large to serialize\n"; return -1; } - const auto* data_ptr = reinterpret_cast(modelStream->data()); - auto data_size = static_cast(modelStream->size()); - p.write(data_ptr, data_size); + const auto* data_ptr = reinterpret_cast(model_stream->data()); + const auto data_size = static_cast(model_stream->size()); + plan.write(data_ptr, data_size); #if TRT_VERSION >= 8000 - delete modelStream; + delete model_stream; + delete runtime; #else - modelStream->destroy(); + model_stream->destroy(); + runtime->destroy(); #endif return 0; - } else if (std::string(argv[1]) == "-d") { - std::ifstream file(ENGINE_PATH, std::ios::binary); + } + if (std::string(argv[1]) == "-d") { + std::ifstream file(variant.engine_path, std::ios::binary); if (file.good()) { file.seekg(0, file.end); size = file.tellg(); file.seekg(0, file.beg); - trtModelStream = new char[size]; - assert(trtModelStream); - file.read(trtModelStream, size); + trt_model_stream = new char[size]; + assert(trt_model_stream); + file.read(trt_model_stream, size); file.close(); } } else { @@ -430,29 +416,26 @@ int main(int argc, char** argv) { } #if TRT_VERSION >= 8000 - ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size); + ICudaEngine* engine = runtime->deserializeCudaEngine(trt_model_stream, size); #else - ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size, nullptr); + ICudaEngine* engine = runtime->deserializeCudaEngine(trt_model_stream, size, nullptr); #endif assert(engine != nullptr); IExecutionContext* context = engine->createExecutionContext(); assert(context != nullptr); - delete[] trtModelStream; + delete[] trt_model_stream; - // Run inference void* input = nullptr; std::vector flat_img; - cv::Mat img; + cv::Mat img = cv::imread("assets/cats.jpg", cv::IMREAD_COLOR); if constexpr (TRT_PREPROCESS) { - // for simplicity, resize image on cpu side - img = cv::imread("../assets/cats.jpg", cv::IMREAD_COLOR); cv::resize(img, img, cv::Size(INPUT_W, INPUT_H), 0, 0, cv::INTER_LINEAR); input = static_cast(img.data); } else { - img = cv::imread("../assets/cats.jpg", cv::IMREAD_COLOR); flat_img = preprocess_img(img, true, mean, stdv, N, INPUT_H, INPUT_W); input = flat_img.data(); } + for (int i = 0; i < 100; ++i) { auto start = std::chrono::system_clock::now(); auto prob = doInference(*context, input, N); @@ -460,23 +443,16 @@ int main(int argc, char** argv) { auto period = std::chrono::duration_cast(end - start); std::cout << period.count() << "us\n"; - for (auto& vector : prob) { - int idx = 0; - for (auto& v : vector) { - std::cout << std::setprecision(4) << v << ", " << std::flush; - if (++idx > 20) { - std::cout << "\n====\n"; - break; - } - } - } - if (i == 99) { std::cout << "prediction result:\n"; auto labels = loadImagenetLabelMap(LABELS_PATH); - int _top = 0; + if (labels.empty()) { + std::cerr << "failed to load labels from " << LABELS_PATH << "\n"; + std::abort(); + } + int top = 0; for (auto& [idx, logits] : topk(prob[0], 3)) { - std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits + std::cout << "Top: " << top++ << " idx: " << idx << ", logits: " << std::setprecision(4) << logits << ", label: " << labels[idx] << "\n"; } } diff --git a/shufflenetv2/utils.h b/shufflenetv2/utils.h index fa6010a0..74a1da19 100644 --- a/shufflenetv2/utils.h +++ b/shufflenetv2/utils.h @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -63,7 +64,11 @@ static std::map loadWeights(const std::string& f input >> name >> std::dec >> wt.count; // Load blob - auto* val = new uint32_t[wt.count]; + auto* val = static_cast(std::malloc(sizeof(uint32_t) * static_cast(wt.count))); + if (val == nullptr) { + std::cerr << "weight allocation failed\n"; + std::abort(); + } input >> std::hex; for (auto x = 0ll; x < wt.count; ++x) { input >> val[x]; From 3051cf02e1327a576b82ec865dd67d7f18a2f49c Mon Sep 17 00:00:00 2001 From: zjq Date: Sun, 14 Jun 2026 15:44:40 +0800 Subject: [PATCH 5/7] support TensorRT 11 SDK, fix some bugs. --- alexnet/FindTensorRT.cmake | 10 +++-- alexnet/alexnet.cc | 66 ++++++++++++--------------- alexnet/macros.h | 11 +++-- alexnet/utils.h | 48 ++++++++++++++++++-- googlenet/FindTensorRT.cmake | 10 +++-- googlenet/googlenet.cpp | 66 ++++++++++++--------------- googlenet/macros.h | 13 ++++-- googlenet/utils.h | 49 ++++++++++++++++++-- lenet/FindTensorRT.cmake | 10 +++-- lenet/lenet.cpp | 64 +++++++++++--------------- lenet/macros.h | 13 ++++-- lenet/utils.h | 43 +++++++++++++++++- lprnet/FindTensorRT.cmake | 10 +++-- lprnet/lprnet.cpp | 80 +++++++++++++++------------------ lprnet/macros.h | 13 ++++-- lprnet/utils.h | 49 ++++++++++++++++++-- mlp/FindTensorRT.cmake | 10 +++-- mlp/macros.h | 13 ++++-- mlp/mlp.cpp | 37 ++++++++------- mlp/utils.h | 48 +++++++++++++++++++- mnasnet/FindTensorRT.cmake | 10 +++-- mnasnet/macros.h | 13 ++++-- mnasnet/mnasnet.cpp | 64 ++++++++++++-------------- mnasnet/utils.h | 49 ++++++++++++++++++-- shufflenetv2/FindTensorRT.cmake | 10 +++-- shufflenetv2/macros.h | 13 ++++-- shufflenetv2/shufflenetv2.cpp | 68 +++++++++++++++------------- shufflenetv2/utils.h | 63 ++++++++++++++++++++++++-- squeezenet/FindTensorRT.cmake | 10 +++-- squeezenet/macros.h | 13 ++++-- squeezenet/squeezenet.cpp | 68 +++++++++++++--------------- squeezenet/utils.h | 47 +++++++++++++++++-- vgg/FindTensorRT.cmake | 10 +++-- vgg/macros.h | 13 ++++-- vgg/utils.h | 48 ++++++++++++++++++-- vgg/vgg.cc | 66 ++++++++++++--------------- 36 files changed, 819 insertions(+), 409 deletions(-) diff --git a/alexnet/FindTensorRT.cmake b/alexnet/FindTensorRT.cmake index af75bfe2..61203d17 100644 --- a/alexnet/FindTensorRT.cmake +++ b/alexnet/FindTensorRT.cmake @@ -73,8 +73,11 @@ if(WIN32) endif() if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) - set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 - nvinfer_dispatch_10 nvinfer_lean_10) + set(_trt_lib_suffix "_${TRT_MAJOR_VERSION}") + set(_modules nvinfer${_trt_lib_suffix} nvinfer_plugin${_trt_lib_suffix} + nvinfer_vc_plugin${_trt_lib_suffix} + nvinfer_dispatch${_trt_lib_suffix} + nvinfer_lean${_trt_lib_suffix}) message(DEBUG "Using ${_modules}") else() set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch @@ -139,7 +142,8 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") unset(TRT_MAJOR_VERSION) -unset(_modules) +unset(_modules) +unset(_trt_lib_suffix) unset(_trt_include_candidates) unset(_trt_library_candidates) unset(_trt_arch) diff --git a/alexnet/alexnet.cc b/alexnet/alexnet.cc index 887edbd1..b2c4c22b 100644 --- a/alexnet/alexnet.cc +++ b/alexnet/alexnet.cc @@ -18,7 +18,7 @@ constexpr const std::array NAMES = {"data", "prob"}; constexpr const char* ENGINE_PATH = "../models/alexnet.engine"; constexpr const char* WTS_PATH = "../models/alexnet.wts"; constexpr const char* LABELS_PATH = "../assets/imagenet1000_clsidx_to_labels.txt"; -static constexpr const bool TRT_PREPROCESS = TRT_VERSION >= 8510 ? true : false; +static constexpr const bool TRT_PREPROCESS = TRT_VERSION_GE(8, 5, 1) ? true : false; static constexpr const std::array mean = {0.485f, 0.456f, 0.406f}; static constexpr const std::array stdv = {0.229f, 0.224f, 0.225f}; @@ -41,9 +41,9 @@ static Logger gLogger; ICudaEngine* createEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, DataType dt) { WeightMap weightMap = loadWeights(WTS_PATH); -#if TRT_VERSION >= 11200 +#if TRT_VERSION_GE(10, 12, 0) auto flag = 1U << static_cast(NDCF::kSTRONGLY_TYPED); -#elif TRT_VERSION >= 10000 +#elif TRT_VERSION_GE(10, 0, 0) auto flag = 0U; #else auto flag = 1U << static_cast(NDCF::kEXPLICIT_BATCH); @@ -142,7 +142,7 @@ ICudaEngine* createEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IBuil network->markOutput(*fc3_1->getOutput(0)); // Build engine -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); auto* host_mem = builder->buildSerializedNetwork(*network, *config); auto* engine = runtime->deserializeCudaEngine(host_mem->data(), host_mem->size()); @@ -175,7 +175,7 @@ void APIToModel(int32_t N, IRuntime* runtime, IHostMemory** modelStream) { (*modelStream) = engine->serialize(); // Close everything down -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete engine; delete config; delete builder; @@ -208,7 +208,7 @@ std::vector> doInference(IExecutionContext& context, const st CHECK(cudaStreamCreate(&stream)); std::vector buffers; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const int32_t nIO = engine.getNbIOTensors(); #else const int32_t nIO = engine.getNbBindings(); @@ -216,7 +216,7 @@ std::vector> doInference(IExecutionContext& context, const st buffers.resize(nIO); for (auto i = 0; i < nIO; ++i) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* tensor_name = engine.getIOTensorName(i); auto s = getSize(engine.getTensorDataType(tensor_name)); std::size_t size = s * batchSize * SIZES[i]; @@ -237,7 +237,7 @@ std::vector> doInference(IExecutionContext& context, const st } } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) if (!context.enqueueV3(stream)) { std::cerr << "enqueueV3 failed\n"; std::abort(); @@ -299,7 +299,7 @@ int main(int argc, char** argv) { auto data_size = static_cast(modelStream->size()); p.write(data_ptr, data_size); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete modelStream; #else modelStream->destroy(); @@ -320,7 +320,7 @@ int main(int argc, char** argv) { return -1; } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size); #else ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size, nullptr); @@ -331,36 +331,28 @@ int main(int argc, char** argv) { assert(context != nullptr); const std::string img_path = "../assets/cats.jpg"; - for (int32_t i = 0; i < 100; ++i) { - auto _start = std::chrono::system_clock::now(); - auto prob = doInference(*context, img_path, N); - auto _end = std::chrono::system_clock::now(); - auto _time = std::chrono::duration_cast(_end - _start).count(); - std::cout << "Execution time: " << _time << "ms\n"; - - for (const auto& vector : prob) { - int idx = 0; - for (auto v : vector) { - std::cout << std::setprecision(4) << v << ", " << std::flush; - if (++idx > 20) { - std::cout << "\n====\n"; - break; - } - } - } + auto firstProb = doInference(*context, img_path, N); + printFirstOutputs("alexnet", firstProb[0].data(), firstProb[0].size()); + std::cout << "prediction result:\n"; + auto labels = loadImagenetLabelMap(LABELS_PATH); + int _top = 0; + for (auto& [idx, logits] : topk(firstProb[0], 3)) { + std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits << ", label: " << labels[idx] + << "\n"; + } - if (i == 99) { - std::cout << "prediction result:\n"; - auto labels = loadImagenetLabelMap(LABELS_PATH); - int _top = 0; - for (auto& [idx, logits] : topk(prob[0], 3)) { - std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits - << ", label: " << labels[idx] << "\n"; - } - } + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int32_t i = 0; i < kBenchmarkRuns; ++i) { + auto _start = std::chrono::steady_clock::now(); + (void)doInference(*context, img_path, N); + auto _end = std::chrono::steady_clock::now(); + auto _time = std::chrono::duration_cast(_end - _start).count(); + latencies.push_back(static_cast(_time) / 1000.0); } + printBenchmark("alexnet", latencies, N); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete context; delete engine; delete runtime; diff --git a/alexnet/macros.h b/alexnet/macros.h index 55284a01..8b0eacdb 100644 --- a/alexnet/macros.h +++ b/alexnet/macros.h @@ -17,10 +17,15 @@ #endif #endif // API_EXPORTS -#define TRT_VERSION \ - ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) \ + ((NV_TENSORRT_MAJOR > (major)) || (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR > (minor)) || \ + (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR == (minor) && NV_TENSORRT_PATCH >= (patch))) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const #else diff --git a/alexnet/utils.h b/alexnet/utils.h index 2d5089f9..da343a81 100644 --- a/alexnet/utils.h +++ b/alexnet/utils.h @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -27,7 +29,7 @@ constexpr const std::size_t WORKSPACE_SIZE = 16 << 20; } while (0) static void checkTrtEnv(int device = 0) { -#if TRT_VERSION < 8000 +#if TRT_VERSION_LT(8, 0, 0) CHECK(cudaGetDevice(&device)); cudaDeviceProp prop{}; CHECK(cudaGetDeviceProperties(&prop, device)); @@ -183,7 +185,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo ITensor* in = &input; if (input.getType() != DataType::kFLOAT) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* cast = network->addCast(input, DataType::kFLOAT); assert(cast); cast->setName("Cast to FP32"); @@ -228,7 +230,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo auto* trans = network->addScale(*data, ScaleMode::kCHANNEL, shift, scale, empty); assert(trans); trans->setName("mean & std"); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) trans->setChannelAxis(1); #endif return trans; @@ -236,7 +238,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo static size_t getSize(DataType dt) { switch (dt) { -#if TRT_VERSION >= 8510 +#if TRT_VERSION_GE(8, 5, 1) case DataType::kUINT8: #endif case DataType::kINT8: @@ -253,3 +255,41 @@ static size_t getSize(DataType dt) { } } } + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} diff --git a/googlenet/FindTensorRT.cmake b/googlenet/FindTensorRT.cmake index af75bfe2..61203d17 100644 --- a/googlenet/FindTensorRT.cmake +++ b/googlenet/FindTensorRT.cmake @@ -73,8 +73,11 @@ if(WIN32) endif() if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) - set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 - nvinfer_dispatch_10 nvinfer_lean_10) + set(_trt_lib_suffix "_${TRT_MAJOR_VERSION}") + set(_modules nvinfer${_trt_lib_suffix} nvinfer_plugin${_trt_lib_suffix} + nvinfer_vc_plugin${_trt_lib_suffix} + nvinfer_dispatch${_trt_lib_suffix} + nvinfer_lean${_trt_lib_suffix}) message(DEBUG "Using ${_modules}") else() set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch @@ -139,7 +142,8 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") unset(TRT_MAJOR_VERSION) -unset(_modules) +unset(_modules) +unset(_trt_lib_suffix) unset(_trt_include_candidates) unset(_trt_library_candidates) unset(_trt_arch) diff --git a/googlenet/googlenet.cpp b/googlenet/googlenet.cpp index fd4e590b..3cc21062 100644 --- a/googlenet/googlenet.cpp +++ b/googlenet/googlenet.cpp @@ -22,7 +22,7 @@ static constexpr const int32_t INPUT_H = 224; static constexpr const int32_t INPUT_W = 224; static constexpr const std::array SIZES = {3 * INPUT_H * INPUT_W, 1000}; static constexpr const std::array NAMES = {"data", "prob"}; -static constexpr const bool TRT_PREPROCESS = TRT_VERSION >= 8510 ? true : false; +static constexpr const bool TRT_PREPROCESS = TRT_VERSION_GE(8, 5, 1) ? true : false; static constexpr const char* WTS_PATH = "../models/googlenet.wts"; static constexpr const char* ENGINE_PATH = "../models/googlenet.engine"; static constexpr const char* LABELS_PATH = "../assets/imagenet1000_clsidx_to_labels.txt"; @@ -130,9 +130,9 @@ IConcatenationLayer* inception(INetworkDefinition* network, WeightMap& weightMap ICudaEngine* createEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, DataType dt) { WeightMap weightMap = loadWeights(WTS_PATH); -#if TRT_VERSION >= 11200 +#if TRT_VERSION_GE(10, 12, 0) auto flag = 1U << static_cast(NDCF::kSTRONGLY_TYPED); -#elif TRT_VERSION >= 10000 +#elif TRT_VERSION_GE(10, 0, 0) auto flag = 0U; #else auto flag = 1U << static_cast(NDCF::kEXPLICIT_BATCH); @@ -203,7 +203,7 @@ ICudaEngine* createEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IBuil fc1->getOutput(0)->setName(NAMES[1]); network->markOutput(*fc1->getOutput(0)); // Build engine -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); IHostMemory* mem = builder->buildSerializedNetwork(*network, *config); ICudaEngine* engine = runtime->deserializeCudaEngine(mem->data(), mem->size()); @@ -236,7 +236,7 @@ void APIToModel(int32_t N, IRuntime* runtime, IHostMemory** modelStream) { // Serialize the engine (*modelStream) = engine->serialize(); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete engine; delete config; delete builder; @@ -253,7 +253,7 @@ std::vector> doInference(IExecutionContext& context, void* in CHECK(cudaStreamCreate(&stream)); std::vector buffers; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const int32_t nIO = engine.getNbIOTensors(); #else const int32_t nIO = engine.getNbBindings(); @@ -262,7 +262,7 @@ std::vector> doInference(IExecutionContext& context, void* in buffers.resize(nIO); for (auto i = 0; i < nIO; ++i) { std::size_t size = 0; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* tensor_name = engine.getIOTensorName(i); auto s = getSize(engine.getTensorDataType(tensor_name)); size = s * batchSize * SIZES[i]; @@ -286,7 +286,7 @@ std::vector> doInference(IExecutionContext& context, void* in #endif } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) if (!context.enqueueV3(stream)) { std::cerr << "enqueueV3 failed\n"; std::abort(); @@ -346,7 +346,7 @@ int main(int argc, char** argv) { const auto* data_ptr = reinterpret_cast(modelStream->data()); auto data_size = static_cast(modelStream->size()); p.write(data_ptr, data_size); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete modelStream; #else modelStream->destroy(); @@ -367,7 +367,7 @@ int main(int argc, char** argv) { return 1; } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size); #else ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size, nullptr); @@ -391,37 +391,29 @@ int main(int argc, char** argv) { } assert(input); - for (int32_t i = 0; i < 100; ++i) { - auto _start = std::chrono::system_clock::now(); - auto prob = doInference(*context, input, 1); - auto _end = std::chrono::system_clock::now(); - auto _time = std::chrono::duration_cast(_end - _start).count(); - std::cout << "Execution time: " << _time << "us\n"; - - for (const auto& vector : prob) { - int idx = 0; - for (auto v : vector) { - std::cout << std::setprecision(4) << v << ", " << std::flush; - if (++idx > 20) { - std::cout << "\n====\n"; - break; - } - } - } + auto firstProb = doInference(*context, input, 1); + printFirstOutputs("googlenet", firstProb[0].data(), firstProb[0].size()); + std::cout << "prediction result:\n"; + auto labels = loadImagenetLabelMap(LABELS_PATH); + int _top = 0; + for (auto& [idx, logits] : topk(firstProb[0], 3)) { + std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits << ", label: " << labels[idx] + << "\n"; + } - if (i == 99) { - std::cout << "prediction result:\n"; - auto labels = loadImagenetLabelMap(LABELS_PATH); - int _top = 0; - for (auto& [idx, logits] : topk(prob[0], 3)) { - std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits - << ", label: " << labels[idx] << "\n"; - } - } + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int32_t i = 0; i < kBenchmarkRuns; ++i) { + auto _start = std::chrono::steady_clock::now(); + (void)doInference(*context, input, 1); + auto _end = std::chrono::steady_clock::now(); + auto _time = std::chrono::duration_cast(_end - _start).count(); + latencies.push_back(static_cast(_time) / 1000.0); } + printBenchmark("googlenet", latencies); delete[] trtModelStream; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete context; delete engine; delete runtime; diff --git a/googlenet/macros.h b/googlenet/macros.h index daae7bfb..6d851752 100644 --- a/googlenet/macros.h +++ b/googlenet/macros.h @@ -16,14 +16,19 @@ #endif #endif // API_EXPORTS -#define TRT_VERSION \ - ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) \ + ((NV_TENSORRT_MAJOR > (major)) || (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR > (minor)) || \ + (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR == (minor) && NV_TENSORRT_PATCH >= (patch))) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) -#if TRT_VERSION < 7220 +#if TRT_VERSION_LT(7, 2, 2) #error "TensorRT >= 7.2.2 is required for this demo." #endif -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const #else diff --git a/googlenet/utils.h b/googlenet/utils.h index fa723458..842b7131 100644 --- a/googlenet/utils.h +++ b/googlenet/utils.h @@ -2,7 +2,10 @@ #include #include #include +#include +#include #include +#include #include #include #include @@ -25,7 +28,7 @@ constexpr const std::size_t WORKSPACE_SIZE = 16 << 20; } while (0) static void checkTrtEnv(int device = 0) { -#if TRT_VERSION < 8000 +#if TRT_VERSION_LT(8, 0, 0) CHECK(cudaGetDevice(&device)); cudaDeviceProp prop{}; CHECK(cudaGetDeviceProperties(&prop, device)); @@ -181,7 +184,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo ITensor* in = &input; if (input.getType() != DataType::kFLOAT) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* cast = network->addCast(input, DataType::kFLOAT); assert(cast); cast->setName("Cast to FP32"); @@ -226,7 +229,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo auto* trans = network->addScale(*data, ScaleMode::kCHANNEL, shift, scale, empty); assert(trans); trans->setName("mean & std"); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) trans->setChannelAxis(1); #endif return trans; @@ -234,7 +237,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo static size_t getSize(DataType dt) { switch (dt) { -#if TRT_VERSION >= 8510 +#if TRT_VERSION_GE(8, 5, 1) case DataType::kUINT8: #endif case DataType::kINT8: @@ -251,3 +254,41 @@ static size_t getSize(DataType dt) { } } } + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} diff --git a/lenet/FindTensorRT.cmake b/lenet/FindTensorRT.cmake index af75bfe2..61203d17 100644 --- a/lenet/FindTensorRT.cmake +++ b/lenet/FindTensorRT.cmake @@ -73,8 +73,11 @@ if(WIN32) endif() if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) - set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 - nvinfer_dispatch_10 nvinfer_lean_10) + set(_trt_lib_suffix "_${TRT_MAJOR_VERSION}") + set(_modules nvinfer${_trt_lib_suffix} nvinfer_plugin${_trt_lib_suffix} + nvinfer_vc_plugin${_trt_lib_suffix} + nvinfer_dispatch${_trt_lib_suffix} + nvinfer_lean${_trt_lib_suffix}) message(DEBUG "Using ${_modules}") else() set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch @@ -139,7 +142,8 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") unset(TRT_MAJOR_VERSION) -unset(_modules) +unset(_modules) +unset(_trt_lib_suffix) unset(_trt_include_candidates) unset(_trt_library_candidates) unset(_trt_arch) diff --git a/lenet/lenet.cpp b/lenet/lenet.cpp index 09aeba7d..dc6c1fb4 100644 --- a/lenet/lenet.cpp +++ b/lenet/lenet.cpp @@ -36,9 +36,9 @@ static Logger gLogger; * @return ICudaEngine* */ ICudaEngine* createLenetEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, DataType dt) { -#if TRT_VERSION >= 11200 +#if TRT_VERSION_GE(10, 12, 0) auto flag = 1U << static_cast(NetworkDefinitionCreationFlag::kSTRONGLY_TYPED); -#elif TRT_VERSION >= 10000 +#elif TRT_VERSION_GE(10, 0, 0) auto flag = 0U; #else auto flag = 1U << static_cast(NetworkDefinitionCreationFlag::kEXPLICIT_BATCH); @@ -131,7 +131,7 @@ ICudaEngine* createLenetEngine(int32_t N, IRuntime* runtime, IBuilder* builder, prob->getOutput(0)->setName(NAMES[1]); network->markOutput(*prob->getOutput(0)); -#if TRT_VERSION >= 8400 +#if TRT_VERSION_GE(8, 4, 0) config->setMemoryPoolLimit(nvinfer1::MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); #else config->setMaxWorkspaceSize(WORKSPACE_SIZE); @@ -139,7 +139,7 @@ ICudaEngine* createLenetEngine(int32_t N, IRuntime* runtime, IBuilder* builder, #endif // Build engine -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) IHostMemory* serialized_mem = builder->buildSerializedNetwork(*network, *config); ICudaEngine* engine = runtime->deserializeCudaEngine(serialized_mem->data(), serialized_mem->size()); delete network; @@ -175,7 +175,7 @@ void APIToModel(int32_t N, IRuntime* runtime, IHostMemory** modelStream) { // Serialize the engine (*modelStream) = engine->serialize(); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete engine; delete config; delete builder; @@ -192,7 +192,7 @@ std::vector> doInference(IExecutionContext& context, void* in CHECK(cudaStreamCreate(&stream)); std::vector buffers; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const int32_t nIO = engine.getNbIOTensors(); #else const int32_t nIO = engine.getNbBindings(); @@ -201,7 +201,7 @@ std::vector> doInference(IExecutionContext& context, void* in buffers.resize(nIO); for (auto i = 0; i < nIO; ++i) { std::size_t size = 0; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* tensor_name = engine.getIOTensorName(i); auto s = getSize(engine.getTensorDataType(tensor_name)); size = s * batchSize * SIZES[i]; @@ -225,7 +225,7 @@ std::vector> doInference(IExecutionContext& context, void* in #endif } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) if (!context.enqueueV3(stream)) { std::cerr << "enqueueV3 failed\n"; std::abort(); @@ -286,7 +286,7 @@ int main(int argc, char** argv) { auto data_size = static_cast(modelStream->size()); p.write(data_ptr, data_size); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete modelStream; #else modelStream->destroy(); @@ -316,7 +316,7 @@ int main(int argc, char** argv) { img = img / cv::Scalar(0.3081); assert(img.total() * img.elemSize() == SIZES[0] * sizeof(float)); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size); #else ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size, nullptr); @@ -325,36 +325,26 @@ int main(int argc, char** argv) { IExecutionContext* context = engine->createExecutionContext(); assert(context != nullptr); - // Run inference - for (int32_t i = 0; i < 100; ++i) { - auto _start = std::chrono::system_clock::now(); - auto prob = doInference(*context, img.data, 1); - auto _end = std::chrono::system_clock::now(); - auto _time = std::chrono::duration_cast(_end - _start).count(); - std::cout << "Execution time: " << _time << "us\n"; - - for (const auto& vector : prob) { - int idx = 0; - for (auto v : vector) { - std::cout << std::setprecision(4) << v << ", " << std::flush; - if (++idx > 9) { - std::cout << "\n====\n"; - break; - } - } - } + auto firstProb = doInference(*context, img.data, 1); + printFirstOutputs("lenet", firstProb[0].data(), firstProb[0].size()); + std::cout << "prediction result:\n"; + int _top = 0; + for (auto& [idx, logits] : topk(firstProb[0], 3)) { + std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits << ", label: " << idx << "\n"; + } - if (i == 99) { - std::cout << "prediction result:\n"; - int _top = 0; - for (auto& [idx, logits] : topk(prob[0], 3)) { - std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits << ", label: " << idx - << "\n"; - } - } + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int32_t i = 0; i < kBenchmarkRuns; ++i) { + auto _start = std::chrono::steady_clock::now(); + (void)doInference(*context, img.data, 1); + auto _end = std::chrono::steady_clock::now(); + auto _time = std::chrono::duration_cast(_end - _start).count(); + latencies.push_back(static_cast(_time) / 1000.0); } + printBenchmark("lenet", latencies); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete context; delete engine; delete runtime; diff --git a/lenet/macros.h b/lenet/macros.h index daae7bfb..6d851752 100644 --- a/lenet/macros.h +++ b/lenet/macros.h @@ -16,14 +16,19 @@ #endif #endif // API_EXPORTS -#define TRT_VERSION \ - ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) \ + ((NV_TENSORRT_MAJOR > (major)) || (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR > (minor)) || \ + (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR == (minor) && NV_TENSORRT_PATCH >= (patch))) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) -#if TRT_VERSION < 7220 +#if TRT_VERSION_LT(7, 2, 2) #error "TensorRT >= 7.2.2 is required for this demo." #endif -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const #else diff --git a/lenet/utils.h b/lenet/utils.h index 2dcf2436..e866807a 100644 --- a/lenet/utils.h +++ b/lenet/utils.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -27,7 +28,7 @@ enum : std::uint32_t { WORKSPACE_SIZE = 16 << 20 }; } while (0) static void checkTrtEnv(int device = 0) { -#if TRT_VERSION < 8000 +#if TRT_VERSION_LT(8, 0, 0) CHECK(cudaGetDevice(&device)); cudaDeviceProp prop{}; CHECK(cudaGetDeviceProperties(&prop, device)); @@ -98,7 +99,7 @@ static std::vector> topk(const std::vector& v, int6 static size_t getSize(DataType dt) { switch (dt) { -#if TRT_VERSION >= 8510 +#if TRT_VERSION_GE(8, 5, 1) case DataType::kUINT8: #endif case DataType::kINT8: @@ -115,3 +116,41 @@ static size_t getSize(DataType dt) { } } } + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} diff --git a/lprnet/FindTensorRT.cmake b/lprnet/FindTensorRT.cmake index af75bfe2..61203d17 100644 --- a/lprnet/FindTensorRT.cmake +++ b/lprnet/FindTensorRT.cmake @@ -73,8 +73,11 @@ if(WIN32) endif() if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) - set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 - nvinfer_dispatch_10 nvinfer_lean_10) + set(_trt_lib_suffix "_${TRT_MAJOR_VERSION}") + set(_modules nvinfer${_trt_lib_suffix} nvinfer_plugin${_trt_lib_suffix} + nvinfer_vc_plugin${_trt_lib_suffix} + nvinfer_dispatch${_trt_lib_suffix} + nvinfer_lean${_trt_lib_suffix}) message(DEBUG "Using ${_modules}") else() set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch @@ -139,7 +142,8 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") unset(TRT_MAJOR_VERSION) -unset(_modules) +unset(_modules) +unset(_trt_lib_suffix) unset(_trt_include_candidates) unset(_trt_library_candidates) unset(_trt_arch) diff --git a/lprnet/lprnet.cpp b/lprnet/lprnet.cpp index 44de873b..28990a63 100644 --- a/lprnet/lprnet.cpp +++ b/lprnet/lprnet.cpp @@ -10,6 +10,7 @@ #include #include #include + #include "logging.h" #include "utils.h" #ifdef _WIN32 @@ -34,7 +35,7 @@ static constexpr const int32_t INPUT_H = 24; static constexpr const int32_t INPUT_W = 94; static constexpr const std::array NAMES = {"data", "prob"}; static constexpr const std::array SIZES = {3 * INPUT_H * INPUT_W, 18 * 68}; -static constexpr const bool TRT_PREPROCESS = TRT_VERSION >= 8510 ? true : false; +static constexpr const bool TRT_PREPROCESS = TRT_VERSION_GE(8, 5, 1) ? true : false; static constexpr const std::array mean = {0.5f, 0.5f, 0.5f}; static constexpr const std::array stdv = {1.f, 1.f, 1.f}; @@ -115,9 +116,9 @@ ICudaEngine* createEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IBuil const int nc = 68; WeightMap w = loadWeights(WTS_PATH); -#if TRT_VERSION >= 11200 +#if TRT_VERSION_GE(10, 12, 0) auto flag = 1U << static_cast(NDCF::kSTRONGLY_TYPED); -#elif TRT_VERSION >= 10000 +#elif TRT_VERSION_GE(10, 0, 0) auto flag = 0U; #else auto flag = 1U << static_cast(NDCF::kEXPLICIT_BATCH); @@ -251,7 +252,7 @@ ICudaEngine* createEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IBuil network->markOutput(*logits->getOutput(0)); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); IHostMemory* mem = builder->buildSerializedNetwork(*network, *config); ICudaEngine* engine = runtime->deserializeCudaEngine(mem->data(), mem->size()); @@ -281,7 +282,7 @@ void APIToModel(int32_t N, IRuntime* runtime, IHostMemory** modelStream) { (*modelStream) = engine->serialize(); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete engine; delete config; delete builder; @@ -298,7 +299,7 @@ auto doInference(IExecutionContext& context, void* input, int64_t batchSize) -> CHECK(cudaStreamCreate(&stream)); std::vector buffers; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const int32_t nIO = engine.getNbIOTensors(); #else const int32_t nIO = engine.getNbBindings(); @@ -307,7 +308,7 @@ auto doInference(IExecutionContext& context, void* input, int64_t batchSize) -> buffers.resize(nIO); for (auto i = 0; i < nIO; ++i) { std::size_t size = 0; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* tensor_name = engine.getIOTensorName(i); auto s = getSize(engine.getTensorDataType(tensor_name)); size = s * batchSize * SIZES[i]; @@ -331,7 +332,7 @@ auto doInference(IExecutionContext& context, void* input, int64_t batchSize) -> #endif } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) if (!context.enqueueV3(stream)) { std::cerr << "enqueueV3 failed\n"; std::abort(); @@ -395,7 +396,7 @@ int main(int argc, char** argv) { const auto* data_ptr = reinterpret_cast(modelStream->data()); auto data_size = static_cast(modelStream->size()); p.write(data_ptr, data_size); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete modelStream; #else modelStream->destroy(); @@ -428,7 +429,7 @@ int main(int argc, char** argv) { input = data.data(); } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size); #else ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size, nullptr); @@ -437,44 +438,35 @@ int main(int argc, char** argv) { IExecutionContext* context = engine->createExecutionContext(); assert(context != nullptr); - for (int32_t i = 0; i < 100; ++i) { - auto _start = std::chrono::system_clock::now(); - auto prob = doInference(*context, input, 1); - auto _end = std::chrono::system_clock::now(); - auto _time = std::chrono::duration_cast(_end - _start).count(); - std::cout << "Execution time: " << _time << "us\n"; - - for (const auto& vector : prob) { - int idx = 0; - for (auto v : vector) { - std::cout << std::setprecision(4) << v << ", " << std::flush; - if (++idx > 20) { - std::cout << "\n====\n"; - break; - } - } - } - - if (i == 99) { - int prev = 67; - std::string str; - for (int t = 0; t < 18; ++t) { - std::array scores{}; - for (int c = 0; c < 68; ++c) { - scores[c] = prob[0][t + 18 * c]; - } - int best = - static_cast(std::distance(scores.begin(), std::max_element(scores.begin(), scores.end()))); - if (best != prev && best != 67) - str += alphabet[best]; - prev = best; - } - std::cout << "result: " << str << "\n"; + auto firstProb = doInference(*context, input, 1); + printFirstOutputs("lprnet", firstProb[0].data(), firstProb[0].size()); + int prev = 67; + std::string str; + for (int t = 0; t < 18; ++t) { + std::array scores{}; + for (int c = 0; c < 68; ++c) { + scores[c] = firstProb[0][t + 18 * c]; } + int best = static_cast(std::distance(scores.begin(), std::max_element(scores.begin(), scores.end()))); + if (best != prev && best != 67) + str += alphabet[best]; + prev = best; + } + std::cout << "result: " << str << "\n"; + + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int32_t i = 0; i < kBenchmarkRuns; ++i) { + auto _start = std::chrono::steady_clock::now(); + (void)doInference(*context, input, 1); + auto _end = std::chrono::steady_clock::now(); + auto _time = std::chrono::duration_cast(_end - _start).count(); + latencies.push_back(static_cast(_time) / 1000.0); } + printBenchmark("lprnet", latencies); delete[] trtModelStream; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete context; delete engine; delete runtime; diff --git a/lprnet/macros.h b/lprnet/macros.h index daae7bfb..6d851752 100644 --- a/lprnet/macros.h +++ b/lprnet/macros.h @@ -16,14 +16,19 @@ #endif #endif // API_EXPORTS -#define TRT_VERSION \ - ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) \ + ((NV_TENSORRT_MAJOR > (major)) || (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR > (minor)) || \ + (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR == (minor) && NV_TENSORRT_PATCH >= (patch))) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) -#if TRT_VERSION < 7220 +#if TRT_VERSION_LT(7, 2, 2) #error "TensorRT >= 7.2.2 is required for this demo." #endif -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const #else diff --git a/lprnet/utils.h b/lprnet/utils.h index a2cd4ee3..784646e3 100644 --- a/lprnet/utils.h +++ b/lprnet/utils.h @@ -2,7 +2,10 @@ #include #include #include +#include +#include #include +#include #include #include #include @@ -23,7 +26,7 @@ using namespace nvinfer1; } while (0) static inline void checkTrtEnv(int device = 0) { -#if TRT_VERSION < 8000 +#if TRT_VERSION_LT(8, 0, 0) CHECK(cudaGetDevice(&device)); cudaDeviceProp prop{}; CHECK(cudaGetDeviceProperties(&prop, device)); @@ -179,7 +182,7 @@ static inline ILayer* addTransformLayer(INetworkDefinition* network, ITensor& in ITensor* in = &input; if (input.getType() != DataType::kFLOAT) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* cast = network->addCast(input, DataType::kFLOAT); assert(cast); cast->setName("Cast to FP32"); @@ -224,7 +227,7 @@ static inline ILayer* addTransformLayer(INetworkDefinition* network, ITensor& in auto* trans = network->addScale(*data, ScaleMode::kCHANNEL, shift, scale, empty); assert(trans); trans->setName("mean & std"); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) trans->setChannelAxis(1); #endif return trans; @@ -232,7 +235,7 @@ static inline ILayer* addTransformLayer(INetworkDefinition* network, ITensor& in static inline size_t getSize(DataType dt) { switch (dt) { -#if TRT_VERSION >= 8510 +#if TRT_VERSION_GE(8, 5, 1) case DataType::kUINT8: #endif case DataType::kINT8: @@ -249,3 +252,41 @@ static inline size_t getSize(DataType dt) { } } } + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} diff --git a/mlp/FindTensorRT.cmake b/mlp/FindTensorRT.cmake index af75bfe2..61203d17 100644 --- a/mlp/FindTensorRT.cmake +++ b/mlp/FindTensorRT.cmake @@ -73,8 +73,11 @@ if(WIN32) endif() if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) - set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 - nvinfer_dispatch_10 nvinfer_lean_10) + set(_trt_lib_suffix "_${TRT_MAJOR_VERSION}") + set(_modules nvinfer${_trt_lib_suffix} nvinfer_plugin${_trt_lib_suffix} + nvinfer_vc_plugin${_trt_lib_suffix} + nvinfer_dispatch${_trt_lib_suffix} + nvinfer_lean${_trt_lib_suffix}) message(DEBUG "Using ${_modules}") else() set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch @@ -139,7 +142,8 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") unset(TRT_MAJOR_VERSION) -unset(_modules) +unset(_modules) +unset(_trt_lib_suffix) unset(_trt_include_candidates) unset(_trt_library_candidates) unset(_trt_arch) diff --git a/mlp/macros.h b/mlp/macros.h index daae7bfb..6d851752 100644 --- a/mlp/macros.h +++ b/mlp/macros.h @@ -16,14 +16,19 @@ #endif #endif // API_EXPORTS -#define TRT_VERSION \ - ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) \ + ((NV_TENSORRT_MAJOR > (major)) || (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR > (minor)) || \ + (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR == (minor) && NV_TENSORRT_PATCH >= (patch))) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) -#if TRT_VERSION < 7220 +#if TRT_VERSION_LT(7, 2, 2) #error "TensorRT >= 7.2.2 is required for this demo." #endif -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const #else diff --git a/mlp/mlp.cpp b/mlp/mlp.cpp index 562bb257..4bd8c9cd 100644 --- a/mlp/mlp.cpp +++ b/mlp/mlp.cpp @@ -4,6 +4,7 @@ #include #include #include + #include "logging.h" #include "utils.h" @@ -35,7 +36,7 @@ ICudaEngine* createMLPEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IB std::map weightMap = loadWeights(WTS_PATH); // Create an empty network -#if TRT_VERSION >= 10000 +#if TRT_VERSION_GE(10, 0, 0) auto* network = builder->createNetworkV2(0); #else auto* network = builder->createNetworkV2(1u << static_cast(NetworkDefinitionCreationFlag::kEXPLICIT_BATCH)); @@ -62,7 +63,7 @@ ICudaEngine* createMLPEngine(int32_t N, IRuntime* runtime, IBuilder* builder, IB // mark the output network->markOutput(*output); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) IHostMemory* serialized_mem = builder->buildSerializedNetwork(*network, *config); ICudaEngine* engine = runtime->deserializeCudaEngine(serialized_mem->data(), serialized_mem->size()); delete network; @@ -101,7 +102,7 @@ void APIToModel(int32_t maxBatchSize, IRuntime* runtime, IHostMemory** modelStre // serialize the engine into binary stream (*modelStream) = engine->serialize(); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete engine; delete config; delete builder; @@ -124,7 +125,7 @@ void doInference(IExecutionContext& ctx, void* input, float* output, int64_t bat // Get engine from the ctx const ICudaEngine& engine = ctx.getEngine(); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) int32_t nIO = engine.getNbIOTensors(); const int inputIndex = 0; const int outputIndex = engine.getNbIOTensors() - 1; @@ -143,7 +144,7 @@ void doInference(IExecutionContext& ctx, void* input, float* output, int64_t bat std::vector buffers(nIO, nullptr); size_t inputSize = 0; size_t outputSize = batchSize * OUTPUT_SIZE * sizeof(float); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* input_name = engine.getIOTensorName(inputIndex); inputSize = batchSize * INPUT_SIZE * getSize(engine.getTensorDataType(input_name)); #else @@ -154,12 +155,9 @@ void doInference(IExecutionContext& ctx, void* input, float* output, int64_t bat CHECK(cudaMemcpyAsync(buffers[inputIndex], input, inputSize, cudaMemcpyHostToDevice, stream)); // execute inference using ctx provided by engine -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) for (int32_t i = 0; i < engine.getNbIOTensors(); i++) { auto const name = engine.getIOTensorName(i); - auto dims = ctx.getTensorShape(name); - auto total = std::accumulate(dims.d, dims.d + dims.nbDims, 1ll, std::multiplies<>()); - std::cout << name << "\t" << total << "\n"; if (!ctx.setTensorAddress(name, buffers[i])) { std::cerr << "setTensorAddress failed\n"; std::abort(); @@ -216,7 +214,7 @@ int main(int argc, char** argv) { auto data_size = static_cast(modelStream->size()); p.write(data_ptr, data_size); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete modelStream; #else modelStream->destroy(); @@ -237,7 +235,7 @@ int main(int argc, char** argv) { } } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size); #else ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size, nullptr); @@ -251,16 +249,21 @@ int main(int argc, char** argv) { std::array output = {-1.f}; std::array input = {12.0f}; - for (int i = 0; i < 100; i++) { - auto start = std::chrono::high_resolution_clock::now(); + doInference(*ctx, input.data(), output.data()); + printFirstOutputs("mlp", output.data(), output.size()); + + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int i = 0; i < kBenchmarkRuns; i++) { + auto start = std::chrono::steady_clock::now(); doInference(*ctx, input.data(), output.data()); - auto end = std::chrono::high_resolution_clock::now(); + auto end = std::chrono::steady_clock::now(); auto time = std::chrono::duration_cast(end - start).count(); - std::cout << "Execution time: " << time << "us\n" - << "output: " << output[0] << "\n"; + latencies.push_back(static_cast(time) / 1000.0); } + printBenchmark("mlp", latencies); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete ctx; delete engine; delete runtime; diff --git a/mlp/utils.h b/mlp/utils.h index 5579b214..eb70a85e 100644 --- a/mlp/utils.h +++ b/mlp/utils.h @@ -1,11 +1,17 @@ #pragma once #include +#include #include +#include +#include #include +#include #include #include +#include #include #include +#include #include "macros.h" using namespace nvinfer1; @@ -22,7 +28,7 @@ constexpr const std::size_t WORKSPACE_SIZE = 16 << 20; } while (0) static void checkTrtEnv(int device = 0) { -#if TRT_VERSION < 8000 +#if TRT_VERSION_LT(8, 0, 0) CHECK(cudaGetDevice(&device)); cudaDeviceProp prop{}; CHECK(cudaGetDeviceProperties(&prop, device)); @@ -76,7 +82,7 @@ static auto loadWeights(const std::string& file) { static size_t getSize(DataType dt) { switch (dt) { -#if TRT_VERSION >= 8510 +#if TRT_VERSION_GE(8, 5, 1) case DataType::kUINT8: #endif case DataType::kINT8: @@ -93,3 +99,41 @@ static size_t getSize(DataType dt) { } } } + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} diff --git a/mnasnet/FindTensorRT.cmake b/mnasnet/FindTensorRT.cmake index af75bfe2..61203d17 100644 --- a/mnasnet/FindTensorRT.cmake +++ b/mnasnet/FindTensorRT.cmake @@ -73,8 +73,11 @@ if(WIN32) endif() if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) - set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 - nvinfer_dispatch_10 nvinfer_lean_10) + set(_trt_lib_suffix "_${TRT_MAJOR_VERSION}") + set(_modules nvinfer${_trt_lib_suffix} nvinfer_plugin${_trt_lib_suffix} + nvinfer_vc_plugin${_trt_lib_suffix} + nvinfer_dispatch${_trt_lib_suffix} + nvinfer_lean${_trt_lib_suffix}) message(DEBUG "Using ${_modules}") else() set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch @@ -139,7 +142,8 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") unset(TRT_MAJOR_VERSION) -unset(_modules) +unset(_modules) +unset(_trt_lib_suffix) unset(_trt_include_candidates) unset(_trt_library_candidates) unset(_trt_arch) diff --git a/mnasnet/macros.h b/mnasnet/macros.h index daae7bfb..6d851752 100644 --- a/mnasnet/macros.h +++ b/mnasnet/macros.h @@ -16,14 +16,19 @@ #endif #endif // API_EXPORTS -#define TRT_VERSION \ - ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) \ + ((NV_TENSORRT_MAJOR > (major)) || (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR > (minor)) || \ + (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR == (minor) && NV_TENSORRT_PATCH >= (patch))) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) -#if TRT_VERSION < 7220 +#if TRT_VERSION_LT(7, 2, 2) #error "TensorRT >= 7.2.2 is required for this demo." #endif -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const #else diff --git a/mnasnet/mnasnet.cpp b/mnasnet/mnasnet.cpp index 61babc2d..486b32bc 100644 --- a/mnasnet/mnasnet.cpp +++ b/mnasnet/mnasnet.cpp @@ -22,7 +22,7 @@ static constexpr const std::array SIZES = {3 * INPUT_H * INPUT_W, static const std::string WTS_PATH = "../models/mnasnet0_5.wts"; static const std::string ENGINE_PATH = "../models/mnasnet0_5.engine"; static constexpr const char* LABELS_PATH = "../assets/imagenet1000_clsidx_to_labels.txt"; -static constexpr const bool TRT_PREPROCESS = TRT_VERSION >= 8510 ? true : false; +static constexpr const bool TRT_PREPROCESS = TRT_VERSION_GE(8, 5, 1) ? true : false; static constexpr const std::array mean = {0.485f, 0.456f, 0.406f}; static constexpr const std::array stdv = {0.229f, 0.224f, 0.225f}; @@ -159,9 +159,9 @@ ICudaEngine* createEngine(unsigned int maxBatchSize, IRuntime* runtime, IBuilder DataType dt) { auto weightMap = loadWeights(WTS_PATH); -#if TRT_VERSION >= 11200 +#if TRT_VERSION_GE(10, 12, 0) auto flag = 1U << static_cast(NDCF::kSTRONGLY_TYPED); -#elif TRT_VERSION >= 10000 +#elif TRT_VERSION_GE(10, 0, 0) auto flag = 0U; #else auto flag = 1U << static_cast(NDCF::kEXPLICIT_BATCH); @@ -216,7 +216,7 @@ ICudaEngine* createEngine(unsigned int maxBatchSize, IRuntime* runtime, IBuilder network->markOutput(*fc1->getOutput(0)); // Build engine -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); auto* _serialized = builder->buildSerializedNetwork(*network, *config); auto* engine = runtime->deserializeCudaEngine(_serialized->data(), _serialized->size()); @@ -251,7 +251,7 @@ void APIToModel(unsigned int maxBatchSize, IRuntime* runtime, IHostMemory** mode (*modelStream) = engine->serialize(); // Close everything down -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete engine; delete config; delete builder; @@ -268,7 +268,7 @@ std::vector> do_inference(IExecutionContext& context, void* i CHECK(cudaStreamCreate(&stream)); std::vector buffers; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const int32_t nIO = engine.getNbIOTensors(); #else const int32_t nIO = engine.getNbBindings(); @@ -277,7 +277,7 @@ std::vector> do_inference(IExecutionContext& context, void* i buffers.resize(nIO); for (auto i = 0; i < nIO; ++i) { std::size_t size = 0; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* tensor_name = engine.getIOTensorName(i); auto s = getSize(engine.getTensorDataType(tensor_name)); size = s * batch_size * SIZES[i]; @@ -301,7 +301,7 @@ std::vector> do_inference(IExecutionContext& context, void* i #endif } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) if (!context.enqueueV3(stream)) { std::cerr << "enqueueV3 failed\n"; std::abort(); @@ -362,7 +362,7 @@ int main(int argc, char** argv) { const auto* data_ptr = reinterpret_cast(modelStream->data()); auto data_size = static_cast(modelStream->size()); p.write(data_ptr, data_size); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete modelStream; #else modelStream->destroy(); @@ -383,7 +383,7 @@ int main(int argc, char** argv) { return -1; } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* engine = runtime->deserializeCudaEngine(trt_model_stream, size); #else auto* engine = runtime->deserializeCudaEngine(trt_model_stream, size, nullptr); @@ -406,34 +406,26 @@ int main(int argc, char** argv) { input = flat_img.data(); } - for (int32_t i = 0; i < 100; ++i) { - auto _start = std::chrono::system_clock::now(); - auto prob = do_inference(*context, input, 1); - auto _end = std::chrono::system_clock::now(); - auto _time = std::chrono::duration_cast(_end - _start).count(); - std::cout << "Execution time: " << _time << "ms\n"; - - for (const auto& vector : prob) { - int idx = 0; - for (auto v : vector) { - std::cout << std::setprecision(4) << v << ", " << std::flush; - if (++idx > 20) { - std::cout << "\n====\n"; - break; - } - } - } + auto firstProb = do_inference(*context, input, 1); + printFirstOutputs("mnasnet", firstProb[0].data(), firstProb[0].size()); + std::cout << "prediction result:\n"; + auto labels = loadImagenetLabelMap(LABELS_PATH); + int _top = 0; + for (auto& [idx, logits] : topk(firstProb[0], 3)) { + std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits << ", label: " << labels[idx] + << "\n"; + } - if (i == 99) { - std::cout << "prediction result:\n"; - auto labels = loadImagenetLabelMap(LABELS_PATH); - int _top = 0; - for (auto& [idx, logits] : topk(prob[0], 3)) { - std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits - << ", label: " << labels[idx] << "\n"; - } - } + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int32_t i = 0; i < kBenchmarkRuns; ++i) { + auto _start = std::chrono::steady_clock::now(); + (void)do_inference(*context, input, 1); + auto _end = std::chrono::steady_clock::now(); + auto _time = std::chrono::duration_cast(_end - _start).count(); + latencies.push_back(static_cast(_time) / 1000.0); } + printBenchmark("mnasnet", latencies); delete[] trt_model_stream; return 0; diff --git a/mnasnet/utils.h b/mnasnet/utils.h index 75ff771a..df8e9085 100644 --- a/mnasnet/utils.h +++ b/mnasnet/utils.h @@ -2,7 +2,10 @@ #include #include #include +#include +#include #include +#include #include #include #include @@ -25,7 +28,7 @@ constexpr const std::size_t WORKSPACE_SIZE = 16 << 20; } while (0) static void checkTrtEnv(int device = 0) { -#if TRT_VERSION < 8000 +#if TRT_VERSION_LT(8, 0, 0) CHECK(cudaGetDevice(&device)); cudaDeviceProp prop{}; CHECK(cudaGetDeviceProperties(&prop, device)); @@ -181,7 +184,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo ITensor* in = &input; if (input.getType() != DataType::kFLOAT) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* cast = network->addCast(input, DataType::kFLOAT); assert(cast); cast->setName("Cast to FP32"); @@ -226,7 +229,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo auto* trans = network->addScale(*data, ScaleMode::kCHANNEL, shift, scale, empty); assert(trans); trans->setName("mean & std"); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) trans->setChannelAxis(1); #endif return trans; @@ -234,7 +237,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo static size_t getSize(DataType dt) { switch (dt) { -#if TRT_VERSION >= 8510 +#if TRT_VERSION_GE(8, 5, 1) case DataType::kUINT8: #endif case DataType::kINT8: @@ -251,3 +254,41 @@ static size_t getSize(DataType dt) { } } } + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} diff --git a/shufflenetv2/FindTensorRT.cmake b/shufflenetv2/FindTensorRT.cmake index af75bfe2..61203d17 100644 --- a/shufflenetv2/FindTensorRT.cmake +++ b/shufflenetv2/FindTensorRT.cmake @@ -73,8 +73,11 @@ if(WIN32) endif() if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) - set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 - nvinfer_dispatch_10 nvinfer_lean_10) + set(_trt_lib_suffix "_${TRT_MAJOR_VERSION}") + set(_modules nvinfer${_trt_lib_suffix} nvinfer_plugin${_trt_lib_suffix} + nvinfer_vc_plugin${_trt_lib_suffix} + nvinfer_dispatch${_trt_lib_suffix} + nvinfer_lean${_trt_lib_suffix}) message(DEBUG "Using ${_modules}") else() set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch @@ -139,7 +142,8 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") unset(TRT_MAJOR_VERSION) -unset(_modules) +unset(_modules) +unset(_trt_lib_suffix) unset(_trt_include_candidates) unset(_trt_library_candidates) unset(_trt_arch) diff --git a/shufflenetv2/macros.h b/shufflenetv2/macros.h index daae7bfb..6d851752 100644 --- a/shufflenetv2/macros.h +++ b/shufflenetv2/macros.h @@ -16,14 +16,19 @@ #endif #endif // API_EXPORTS -#define TRT_VERSION \ - ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) \ + ((NV_TENSORRT_MAJOR > (major)) || (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR > (minor)) || \ + (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR == (minor) && NV_TENSORRT_PATCH >= (patch))) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) -#if TRT_VERSION < 7220 +#if TRT_VERSION_LT(7, 2, 2) #error "TensorRT >= 7.2.2 is required for this demo." #endif -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const #else diff --git a/shufflenetv2/shufflenetv2.cpp b/shufflenetv2/shufflenetv2.cpp index 3e94f7c2..1773526a 100644 --- a/shufflenetv2/shufflenetv2.cpp +++ b/shufflenetv2/shufflenetv2.cpp @@ -14,6 +14,7 @@ #include #include #include + #include "logging.h" #include "utils.h" @@ -31,7 +32,7 @@ static constexpr const int32_t INPUT_H = 224; static constexpr const int32_t INPUT_W = 224; static constexpr const std::array SIZES = {3 * INPUT_H * INPUT_W, 1000}; static constexpr const std::array NAMES = {"data", "logits"}; -static constexpr const bool TRT_PREPROCESS = TRT_VERSION >= 8510; +static constexpr const bool TRT_PREPROCESS = TRT_VERSION_GE(8, 5, 1); static constexpr const std::array mean = {0.485f, 0.456f, 0.406f}; static constexpr const std::array stdv = {0.229f, 0.224f, 0.225f}; static constexpr const char* LABELS_PATH = "assets/imagenet1000_clsidx_to_labels.txt"; @@ -158,7 +159,7 @@ static auto addInvertedResidual(INetworkDefinition* network, WeightMap& weight_m 1, true, 5); std::array cat_tensors = {branch1_output, branch2->getOutput(0)}; - auto* cat = network->addConcatenation(cat_tensors.data(), static_cast(cat_tensors.size())); + auto* cat = network->addConcatenation(cat_tensors.data(), toI32(cat_tensors.size())); assert(cat); cat->setName((lname + ".cat").c_str()); cat->setAxis(1); @@ -181,9 +182,9 @@ static auto createEngine(int32_t batch_size, IRuntime* runtime, IBuilder* builde const ShuffleNetV2Variant& variant) -> ICudaEngine* { WeightMap weight_map = loadWeights(variant.wts_path); -#if TRT_VERSION >= 11200 +#if TRT_VERSION_GE(10, 12, 0) auto flag = 1U << static_cast(NDCF::kSTRONGLY_TYPED); -#elif TRT_VERSION >= 10000 +#elif TRT_VERSION_GE(10, 0, 0) auto flag = 0U; #else auto flag = 1U << static_cast(NDCF::kEXPLICIT_BATCH); @@ -235,7 +236,7 @@ static auto createEngine(int32_t batch_size, IRuntime* runtime, IBuilder* builde fc->getOutput(0)->setName(NAMES[1]); network->markOutput(*fc->getOutput(0)); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); IHostMemory* mem = builder->buildSerializedNetwork(*network, *config); assert(mem); @@ -265,7 +266,7 @@ static void APIToModel(int32_t batch_size, IRuntime* runtime, IHostMemory** mode (*model_stream) = engine->serialize(); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete engine; delete config; delete builder; @@ -283,7 +284,7 @@ static auto doInference(IExecutionContext& context, void* input, CHECK(cudaStreamCreate(&stream)); std::vector buffers; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const int32_t nIO = engine.getNbIOTensors(); #else const int32_t nIO = engine.getNbBindings(); @@ -292,7 +293,7 @@ static auto doInference(IExecutionContext& context, void* input, buffers.resize(nIO); for (auto i = 0; i < nIO; ++i) { std::size_t size = 0; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* tensor_name = engine.getIOTensorName(i); const std::string name = tensor_name; auto element_size = getSize(engine.getTensorDataType(tensor_name)); @@ -317,7 +318,7 @@ static auto doInference(IExecutionContext& context, void* input, #endif } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) if (!context.enqueueV3(stream)) { std::cerr << "enqueueV3 failed\n"; std::abort(); @@ -331,7 +332,7 @@ static auto doInference(IExecutionContext& context, void* input, std::vector> prob; for (int i = 0; i < nIO; ++i) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const std::string name = engine.getIOTensorName(i); if (name == NAMES[0]) { continue; @@ -391,7 +392,7 @@ auto main(int argc, char** argv) -> int { const auto* data_ptr = reinterpret_cast(model_stream->data()); const auto data_size = static_cast(model_stream->size()); plan.write(data_ptr, data_size); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete model_stream; delete runtime; #else @@ -415,7 +416,7 @@ auto main(int argc, char** argv) -> int { return -1; } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) ICudaEngine* engine = runtime->deserializeCudaEngine(trt_model_stream, size); #else ICudaEngine* engine = runtime->deserializeCudaEngine(trt_model_stream, size, nullptr); @@ -436,28 +437,31 @@ auto main(int argc, char** argv) -> int { input = flat_img.data(); } - for (int i = 0; i < 100; ++i) { - auto start = std::chrono::system_clock::now(); - auto prob = doInference(*context, input, N); - auto end = std::chrono::system_clock::now(); + auto firstProb = doInference(*context, input, N); + printFirstOutputs("shufflenetv2", firstProb[0].data(), firstProb[0].size()); + std::cout << "prediction result:\n"; + auto labels = loadImagenetLabelMap(LABELS_PATH); + if (labels.empty()) { + std::cerr << "failed to load labels from " << LABELS_PATH << "\n"; + std::abort(); + } + int top = 0; + for (auto& [idx, logits] : topk(firstProb[0], 3)) { + std::cout << "Top: " << top++ << " idx: " << idx << ", logits: " << std::setprecision(4) << logits + << ", label: " << labels[idx] << "\n"; + } + + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int i = 0; i < kBenchmarkRuns; ++i) { + auto start = std::chrono::steady_clock::now(); + (void)doInference(*context, input, N); + auto end = std::chrono::steady_clock::now(); auto period = std::chrono::duration_cast(end - start); - std::cout << period.count() << "us\n"; - - if (i == 99) { - std::cout << "prediction result:\n"; - auto labels = loadImagenetLabelMap(LABELS_PATH); - if (labels.empty()) { - std::cerr << "failed to load labels from " << LABELS_PATH << "\n"; - std::abort(); - } - int top = 0; - for (auto& [idx, logits] : topk(prob[0], 3)) { - std::cout << "Top: " << top++ << " idx: " << idx << ", logits: " << std::setprecision(4) << logits - << ", label: " << labels[idx] << "\n"; - } - } + latencies.push_back(static_cast(period.count()) / 1000.0); } -#if TRT_VERSION >= 8000 + printBenchmark("shufflenetv2", latencies, N); +#if TRT_VERSION_GE(8, 0, 0) delete context; delete engine; delete runtime; diff --git a/shufflenetv2/utils.h b/shufflenetv2/utils.h index 74a1da19..4f111957 100644 --- a/shufflenetv2/utils.h +++ b/shufflenetv2/utils.h @@ -2,13 +2,19 @@ #include #include #include +#include +#include #include #include +#include #include +#include #include #include #include #include +#include +#include #include #include "macros.h" @@ -24,7 +30,7 @@ using namespace nvinfer1; } while (0) static void checkTrtEnv(int device = 0) { -#if TRT_VERSION < 8000 +#if TRT_VERSION_LT(8, 0, 0) CHECK(cudaGetDevice(&device)); cudaDeviceProp prop{}; CHECK(cudaGetDeviceProperties(&prop, device)); @@ -36,6 +42,17 @@ static void checkTrtEnv(int device = 0) { #endif } +template +static auto toI32(T value) -> int32_t { + static_assert(std::is_integral::value, "toI32 requires an integral type."); + static_assert(sizeof(T) > sizeof(int32_t), "toI32 is only for values wider than int32_t."); + if constexpr (std::is_signed::value) { + assert(value >= static_cast(std::numeric_limits::min())); + } + assert(value <= static_cast(std::numeric_limits::max())); + return static_cast(value); +} + /** * @brief TensorRT weight files have a simple space delimited format: * [type] [size] @@ -184,7 +201,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo ITensor* in = &input; if (input.getType() != DataType::kFLOAT) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* cast = network->addCast(input, DataType::kFLOAT); assert(cast); cast->setName("Cast to FP32"); @@ -229,7 +246,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo auto* trans = network->addScale(*data, ScaleMode::kCHANNEL, shift, scale, empty); assert(trans); trans->setName("mean & std"); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) trans->setChannelAxis(1); #endif return trans; @@ -237,7 +254,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo static size_t getSize(DataType dt) { switch (dt) { -#if TRT_VERSION >= 8510 +#if TRT_VERSION_GE(8, 5, 1) case DataType::kUINT8: #endif case DataType::kINT8: @@ -254,3 +271,41 @@ static size_t getSize(DataType dt) { } } } + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} diff --git a/squeezenet/FindTensorRT.cmake b/squeezenet/FindTensorRT.cmake index 338f592b..114bea76 100644 --- a/squeezenet/FindTensorRT.cmake +++ b/squeezenet/FindTensorRT.cmake @@ -84,8 +84,11 @@ if(WIN32) endif() if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) - set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 - nvinfer_dispatch_10 nvinfer_lean_10) + set(_trt_lib_suffix "_${TRT_MAJOR_VERSION}") + set(_modules nvinfer${_trt_lib_suffix} nvinfer_plugin${_trt_lib_suffix} + nvinfer_vc_plugin${_trt_lib_suffix} + nvinfer_dispatch${_trt_lib_suffix} + nvinfer_lean${_trt_lib_suffix}) message(DEBUG "Using ${_modules}") else() set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch @@ -150,7 +153,8 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") unset(TRT_MAJOR_VERSION) -unset(_modules) +unset(_modules) +unset(_trt_lib_suffix) unset(_trt_include_candidates) unset(_trt_library_candidates) unset(_trt_arch) diff --git a/squeezenet/macros.h b/squeezenet/macros.h index daae7bfb..6d851752 100644 --- a/squeezenet/macros.h +++ b/squeezenet/macros.h @@ -16,14 +16,19 @@ #endif #endif // API_EXPORTS -#define TRT_VERSION \ - ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) \ + ((NV_TENSORRT_MAJOR > (major)) || (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR > (minor)) || \ + (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR == (minor) && NV_TENSORRT_PATCH >= (patch))) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) -#if TRT_VERSION < 7220 +#if TRT_VERSION_LT(7, 2, 2) #error "TensorRT >= 7.2.2 is required for this demo." #endif -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const #else diff --git a/squeezenet/squeezenet.cpp b/squeezenet/squeezenet.cpp index 1ae46501..30daf500 100644 --- a/squeezenet/squeezenet.cpp +++ b/squeezenet/squeezenet.cpp @@ -23,7 +23,7 @@ static constexpr const int32_t INPUT_H = 224; static constexpr const int32_t INPUT_W = 224; static constexpr const std::array SIZES = {3 * INPUT_H * INPUT_W, N * 1000}; static constexpr const std::array NAMES = {"data", "prob"}; -static constexpr const bool TRT_PREPROCESS = TRT_VERSION >= 8510 ? true : false; +static constexpr const bool TRT_PREPROCESS = TRT_VERSION_GE(8, 5, 1) ? true : false; static constexpr const std::array mean = {0.485f, 0.456f, 0.406f}; static constexpr const std::array stdv = {0.229f, 0.224f, 0.225f}; @@ -71,9 +71,9 @@ static auto createEngine(int32_t batch_size, IRuntime* runtime, IBuilder* builde DataType dt) -> ICudaEngine* { auto weightMap = loadWeights(WTS_PATH); -#if TRT_VERSION >= 11200 +#if TRT_VERSION_GE(10, 12, 0) auto flag = 1U << static_cast(NDCF::kSTRONGLY_TYPED); -#elif TRT_VERSION >= 10000 +#elif TRT_VERSION_GE(10, 0, 0) auto flag = 0U; #else auto flag = 1U << static_cast(NDCF::kEXPLICIT_BATCH); @@ -138,7 +138,7 @@ static auto createEngine(int32_t batch_size, IRuntime* runtime, IBuilder* builde network->markOutput(*pool4->getOutput(0)); // Build engine -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); IHostMemory* mem = builder->buildSerializedNetwork(*network, *config); auto* engine = runtime->deserializeCudaEngine(mem->data(), mem->size()); @@ -161,7 +161,7 @@ static auto createEngine(int32_t batch_size, IRuntime* runtime, IBuilder* builde } static void destroyRuntime(IRuntime* runtime) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete runtime; #else runtime->destroy(); @@ -169,7 +169,7 @@ static void destroyRuntime(IRuntime* runtime) { } static void destroyHostMemory(IHostMemory* memory) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete memory; #else memory->destroy(); @@ -185,7 +185,7 @@ static void APIToModel(int32_t batch_size, IRuntime* runtime, IHostMemory** mode (*modelStream) = engine->serialize(); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete engine; delete config; delete builder; @@ -203,7 +203,7 @@ static auto doInference(IExecutionContext& context, void* input, CHECK(cudaStreamCreate(&stream)); std::vector buffers; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const int32_t nIO = engine.getNbIOTensors(); #else const int32_t nIO = engine.getNbBindings(); @@ -212,7 +212,7 @@ static auto doInference(IExecutionContext& context, void* input, buffers.resize(nIO); for (auto i = 0; i < nIO; ++i) { std::size_t size = 0; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const auto* tensor_name = engine.getIOTensorName(i); auto s = getSize(engine.getTensorDataType(tensor_name)); size = s * batch_size * SIZES[i]; @@ -236,7 +236,7 @@ static auto doInference(IExecutionContext& context, void* input, #endif } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) if (!context.enqueueV3(stream)) { std::cerr << "enqueueV3 failed\n"; std::abort(); @@ -323,7 +323,7 @@ int main(int argc, char** argv) { return -1; } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* engine = runtime->deserializeCudaEngine(trtModelStream, size); #else auto* engine = runtime->deserializeCudaEngine(trtModelStream, size, nullptr); @@ -349,37 +349,29 @@ int main(int argc, char** argv) { } assert(input); - for (int32_t i = 0; i < 100; ++i) { - auto _start = std::chrono::system_clock::now(); - auto prob = doInference(*context, input, N); - auto _end = std::chrono::system_clock::now(); - auto _time = std::chrono::duration_cast(_end - _start).count(); - std::cout << "Execution time: " << _time << "us\n"; - - for (const auto& vector : prob) { - int idx = 0; - for (auto v : vector) { - std::cout << std::setprecision(4) << v << ", " << std::flush; - if (++idx > 20) { - std::cout << "\n====\n"; - break; - } - } - } + auto firstProb = doInference(*context, input, N); + printFirstOutputs("squeezenet", firstProb[0].data(), firstProb[0].size()); + std::cout << "prediction result:\n"; + auto labels = loadImagenetLabelMap(LABELS_PATH); + int _top = 0; + for (const auto& [idx, logits] : topk(firstProb[0], 3)) { + std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits << ", label: " << labels[idx] + << "\n"; + } - if (i == 99) { - std::cout << "prediction result:\n"; - auto labels = loadImagenetLabelMap(LABELS_PATH); - int _top = 0; - for (const auto& [idx, logits] : topk(prob[0], 3)) { - std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits - << ", label: " << labels[idx] << "\n"; - } - } + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int32_t i = 0; i < kBenchmarkRuns; ++i) { + auto _start = std::chrono::steady_clock::now(); + (void)doInference(*context, input, N); + auto _end = std::chrono::steady_clock::now(); + auto _time = std::chrono::duration_cast(_end - _start).count(); + latencies.push_back(static_cast(_time) / 1000.0); } + printBenchmark("squeezenet", latencies, N); delete[] trtModelStream; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete context; delete engine; #else diff --git a/squeezenet/utils.h b/squeezenet/utils.h index d44db18e..bdfd66ea 100644 --- a/squeezenet/utils.h +++ b/squeezenet/utils.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -28,7 +29,7 @@ using namespace nvinfer1; } while (0) static void checkTrtEnv(int device = 0) { -#if TRT_VERSION < 8000 +#if TRT_VERSION_LT(8, 0, 0) CHECK(cudaGetDevice(&device)); cudaDeviceProp prop{}; CHECK(cudaGetDeviceProperties(&prop, device)); @@ -184,7 +185,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo ITensor* in = &input; if (input.getType() != DataType::kFLOAT) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* cast = network->addCast(input, DataType::kFLOAT); assert(cast); cast->setName("Cast to FP32"); @@ -229,7 +230,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo auto* trans = network->addScale(*data, ScaleMode::kCHANNEL, shift, scale, empty); assert(trans); trans->setName("mean & std"); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) trans->setChannelAxis(1); #endif return trans; @@ -237,7 +238,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo static std::size_t getSize(DataType dt) { switch (dt) { -#if TRT_VERSION >= 8510 +#if TRT_VERSION_GE(8, 5, 1) case DataType::kUINT8: #endif case DataType::kINT8: @@ -254,3 +255,41 @@ static std::size_t getSize(DataType dt) { } } } + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} diff --git a/vgg/FindTensorRT.cmake b/vgg/FindTensorRT.cmake index af75bfe2..61203d17 100644 --- a/vgg/FindTensorRT.cmake +++ b/vgg/FindTensorRT.cmake @@ -73,8 +73,11 @@ if(WIN32) endif() if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) - set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 - nvinfer_dispatch_10 nvinfer_lean_10) + set(_trt_lib_suffix "_${TRT_MAJOR_VERSION}") + set(_modules nvinfer${_trt_lib_suffix} nvinfer_plugin${_trt_lib_suffix} + nvinfer_vc_plugin${_trt_lib_suffix} + nvinfer_dispatch${_trt_lib_suffix} + nvinfer_lean${_trt_lib_suffix}) message(DEBUG "Using ${_modules}") else() set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch @@ -139,7 +142,8 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") unset(TRT_MAJOR_VERSION) -unset(_modules) +unset(_modules) +unset(_trt_lib_suffix) unset(_trt_include_candidates) unset(_trt_library_candidates) unset(_trt_arch) diff --git a/vgg/macros.h b/vgg/macros.h index daae7bfb..6d851752 100644 --- a/vgg/macros.h +++ b/vgg/macros.h @@ -16,14 +16,19 @@ #endif #endif // API_EXPORTS -#define TRT_VERSION \ - ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) \ + ((NV_TENSORRT_MAJOR > (major)) || (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR > (minor)) || \ + (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR == (minor) && NV_TENSORRT_PATCH >= (patch))) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) -#if TRT_VERSION < 7220 +#if TRT_VERSION_LT(7, 2, 2) #error "TensorRT >= 7.2.2 is required for this demo." #endif -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const #else diff --git a/vgg/utils.h b/vgg/utils.h index b6bff804..b9abe1eb 100644 --- a/vgg/utils.h +++ b/vgg/utils.h @@ -3,10 +3,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -31,7 +33,7 @@ constexpr const std::size_t WORKSPACE_SIZE = 16 << 20; } while (0) static void checkTrtEnv(int device = 0) { -#if TRT_VERSION < 8000 +#if TRT_VERSION_LT(8, 0, 0) CHECK(cudaGetDevice(&device)); cudaDeviceProp prop{}; CHECK(cudaGetDeviceProperties(&prop, device)); @@ -196,7 +198,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo ITensor* in = &input; if (input.getType() != DataType::kFLOAT) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* cast = network->addCast(input, DataType::kFLOAT); assert(cast); cast->setName("Cast to FP32"); @@ -241,7 +243,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo auto* trans = network->addScale(*data, ScaleMode::kCHANNEL, shift, scale, empty); assert(trans); trans->setName("mean & std"); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) trans->setChannelAxis(1); #endif return trans; @@ -249,7 +251,7 @@ static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bo static size_t getSize(DataType dt) { switch (dt) { -#if TRT_VERSION >= 8510 +#if TRT_VERSION_GE(8, 5, 1) case DataType::kUINT8: #endif case DataType::kINT8: @@ -266,3 +268,41 @@ static size_t getSize(DataType dt) { } } } + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} diff --git a/vgg/vgg.cc b/vgg/vgg.cc index fa3b9386..1256275c 100644 --- a/vgg/vgg.cc +++ b/vgg/vgg.cc @@ -40,7 +40,7 @@ static constexpr const int32_t INPUT_W = 224; static constexpr const std::array SIZES = {3 * INPUT_H * INPUT_W, 1000}; static constexpr const std::array NAMES = {"data", "prob"}; static constexpr const char* LABELS_PATH = "assets/imagenet1000_clsidx_to_labels.txt"; -static constexpr const bool TRT_PREPROCESS = TRT_VERSION >= 8510; +static constexpr const bool TRT_PREPROCESS = TRT_VERSION_GE(8, 5, 1); static constexpr const std::array mean = {0.485f, 0.456f, 0.406f}; static constexpr const std::array stdv = {0.229f, 0.224f, 0.225f}; @@ -155,7 +155,7 @@ static auto make_layers(INetworkDefinition* net, WeightMap& w, ITensor& input, c auto create_engine(const VggVariant& variant, int32_t batch_size, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, DataType dt) -> ICudaEngine* { auto w = loadWeights(variant.wts_path); -#if TRT_VERSION >= 10000 +#if TRT_VERSION_GE(10, 0, 0) auto* net = builder->createNetworkV2(1U << static_cast(NetworkDefinitionCreationFlag::kSTRONGLY_TYPED)); #else auto* net = builder->createNetworkV2(1u << static_cast(NetworkDefinitionCreationFlag::kEXPLICIT_BATCH)); @@ -203,7 +203,7 @@ auto create_engine(const VggVariant& variant, int32_t batch_size, IRuntime* runt _fc3_1->getOutput(0)->setName(NAMES[1]); net->markOutput(*_fc3_1->getOutput(0)); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); auto* _serialized = builder->buildSerializedNetwork(*net, *config); auto* _engine = runtime->deserializeCudaEngine(_serialized->data(), _serialized->size()); @@ -231,7 +231,7 @@ void APIToModel(const VggVariant& variant, int32_t batch_size, IRuntime* runtime (*model_stream) = engine->serialize(); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete engine; delete config; delete builder; @@ -248,7 +248,7 @@ auto doInference(IExecutionContext& context, void* input, std::size_t batch_size CHECK(cudaStreamCreate(&stream)); std::vector buffers; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const int32_t nIO = engine.getNbIOTensors(); #else const int32_t nIO = engine.getNbBindings(); @@ -257,7 +257,7 @@ auto doInference(IExecutionContext& context, void* input, std::size_t batch_size buffers.resize(nIO); for (auto i = 0; i < nIO; ++i) { std::size_t size = 0; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* tensor_name = engine.getIOTensorName(i); const std::string name = tensor_name; auto s = getSize(engine.getTensorDataType(tensor_name)); @@ -282,7 +282,7 @@ auto doInference(IExecutionContext& context, void* input, std::size_t batch_size #endif } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) if (!context.enqueueV3(stream)) { std::cerr << "enqueueV3 failed\n"; std::abort(); @@ -296,7 +296,7 @@ auto doInference(IExecutionContext& context, void* input, std::size_t batch_size std::vector> prob; for (int i = 0; i < nIO; ++i) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const std::string name = engine.getIOTensorName(i); if (name == NAMES[0]) { continue; @@ -356,7 +356,7 @@ auto main(int argc, char** argv) -> int { const auto* data_ptr = reinterpret_cast(mem->data()); auto data_size = static_cast(mem->size()); _plan.write(data_ptr, data_size); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete mem; delete runtime; #else @@ -382,7 +382,7 @@ auto main(int argc, char** argv) -> int { return 1; } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* engine = runtime->deserializeCudaEngine(trt_model_stream, size); #else auto* engine = runtime->deserializeCudaEngine(trt_model_stream, size, nullptr); @@ -407,37 +407,29 @@ auto main(int argc, char** argv) -> int { } assert(input); - for (int32_t i = 0; i < 100; ++i) { - auto _start = std::chrono::system_clock::now(); - auto prob = doInference(*context, input, 1); - auto _end = std::chrono::system_clock::now(); - auto _time = std::chrono::duration_cast(_end - _start).count(); - std::cout << "Execution time: " << _time << "ms" << '\n'; - - for (const auto& vector : prob) { - int idx = 0; - for (auto v : vector) { - std::cout << std::setprecision(4) << v << ", " << std::flush; - if (++idx > 20) { - std::cout << "\n====" << '\n'; - break; - } - } - } + auto firstProb = doInference(*context, input, 1); + printFirstOutputs("vgg", firstProb[0].data(), firstProb[0].size()); + std::cout << "prediction result:\n"; + auto labels = loadImagenetLabelMap(LABELS_PATH); + int _top = 0; + for (auto& [idx, logits] : topk(firstProb[0], 3)) { + std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits << ", label: " << labels[idx] + << '\n'; + } - if (i == 99) { - std::cout << "prediction result:\n"; - auto labels = loadImagenetLabelMap(LABELS_PATH); - int _top = 0; - for (auto& [idx, logits] : topk(prob[0], 3)) { - std::cout << "Top: " << _top++ << " idx: " << idx << ", logits: " << logits - << ", label: " << labels[idx] << '\n'; - } - } + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int32_t i = 0; i < kBenchmarkRuns; ++i) { + auto _start = std::chrono::steady_clock::now(); + (void)doInference(*context, input, 1); + auto _end = std::chrono::steady_clock::now(); + auto _time = std::chrono::duration_cast(_end - _start).count(); + latencies.push_back(static_cast(_time) / 1000.0); } + printBenchmark("vgg", latencies); delete[] trt_model_stream; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete context; delete engine; delete runtime; From 2dae0e3bf0974f0bd8ef21a1da649e7c9afda719 Mon Sep 17 00:00:00 2001 From: zjq Date: Sun, 14 Jun 2026 16:08:29 +0800 Subject: [PATCH 6/7] refactor mobilenet_v2, mobilenet_v3_small, resnet, csrnet series to support latest TensorRT 11 SDK. --- csrnet/FindTensorRT.cmake | 10 +- csrnet/csrnet.cpp | 75 ++--- csrnet/macros.h | 11 +- csrnet/utils.h | 46 ++- mobilenet/CMakeLists.txt | 41 +++ mobilenet/FindTensorRT.cmake | 90 +++++ mobilenet/gen_wts.py | 87 +++++ mobilenet/logging.h | 456 +++++++++++++++++++++++++ mobilenet/macros.h | 35 ++ mobilenet/mobilenet.cpp | 634 +++++++++++++++++++++++++++++++++++ mobilenet/utils.h | 311 +++++++++++++++++ resnet/CMakeLists.txt | 83 ++--- resnet/FindTensorRT.cmake | 149 ++++++++ resnet/gen_wts.py | 107 ++++++ resnet/logging.h | 271 +++++++-------- resnet/macros.h | 35 ++ resnet/resnet.cpp | 514 ++++++++++++++++++++++++++++ resnet/utils.h | 311 +++++++++++++++++ 18 files changed, 3019 insertions(+), 247 deletions(-) create mode 100644 mobilenet/CMakeLists.txt create mode 100644 mobilenet/FindTensorRT.cmake create mode 100644 mobilenet/gen_wts.py create mode 100644 mobilenet/logging.h create mode 100644 mobilenet/macros.h create mode 100644 mobilenet/mobilenet.cpp create mode 100644 mobilenet/utils.h create mode 100644 resnet/FindTensorRT.cmake create mode 100644 resnet/gen_wts.py create mode 100644 resnet/macros.h create mode 100644 resnet/resnet.cpp create mode 100644 resnet/utils.h diff --git a/csrnet/FindTensorRT.cmake b/csrnet/FindTensorRT.cmake index af75bfe2..61203d17 100644 --- a/csrnet/FindTensorRT.cmake +++ b/csrnet/FindTensorRT.cmake @@ -73,8 +73,11 @@ if(WIN32) endif() if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) - set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 - nvinfer_dispatch_10 nvinfer_lean_10) + set(_trt_lib_suffix "_${TRT_MAJOR_VERSION}") + set(_modules nvinfer${_trt_lib_suffix} nvinfer_plugin${_trt_lib_suffix} + nvinfer_vc_plugin${_trt_lib_suffix} + nvinfer_dispatch${_trt_lib_suffix} + nvinfer_lean${_trt_lib_suffix}) message(DEBUG "Using ${_modules}") else() set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch @@ -139,7 +142,8 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") unset(TRT_MAJOR_VERSION) -unset(_modules) +unset(_modules) +unset(_trt_lib_suffix) unset(_trt_include_candidates) unset(_trt_library_candidates) unset(_trt_arch) diff --git a/csrnet/csrnet.cpp b/csrnet/csrnet.cpp index 9c62d7ee..93f42b65 100644 --- a/csrnet/csrnet.cpp +++ b/csrnet/csrnet.cpp @@ -13,6 +13,7 @@ #include #include #include + #include "NvInfer.h" #include "utils.h" @@ -120,7 +121,7 @@ auto createEngine(int32_t maxBatchSize, IRuntime* runtime, IBuilder* builder, IB DataType dt) -> ICudaEngine* { WeightMap w = loadWeights(WTS_PATH); -#if TRT_VERSION >= 10000 +#if TRT_VERSION_GE(10, 0, 0) auto* network = builder->createNetworkV2(0); #else auto* network = builder->createNetworkV2(1u << static_cast(NetworkDefinitionCreationFlag::kEXPLICIT_BATCH)); @@ -145,7 +146,7 @@ auto createEngine(int32_t maxBatchSize, IRuntime* runtime, IBuilder* builder, IB profile->setDimensions(NAMES[0], O::kMAX, Dims4(maxBatchSize, 3, MAX_INPUT_SIZE, MAX_INPUT_SIZE)); config->addOptimizationProfile(profile); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); auto* serialized = builder->buildSerializedNetwork(*network, *config); auto* engine = runtime->deserializeCudaEngine(serialized->data(), serialized->size()); @@ -175,7 +176,7 @@ void APIToModel(int32_t batch_size, IRuntime* runtime, IHostMemory** model_strea (*model_stream) = engine->serialize(); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete engine; delete config; delete builder; @@ -192,7 +193,7 @@ std::vector doInference(IExecutionContext& context, void* input, in CHECK(cudaStreamCreate(&stream)); std::vector buffers; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const int32_t nIO = engine.getNbIOTensors(); #else const int32_t nIO = engine.getNbBindings(); @@ -203,7 +204,7 @@ std::vector doInference(IExecutionContext& context, void* input, in std::size_t out_size; context.setOptimizationProfileAsync(0, stream); for (auto i = 0; i < nIO; ++i) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const auto* tensor_name = engine.getIOTensorName(i); auto s = getSize(engine.getTensorDataType(tensor_name)); if (i == 0) { @@ -220,7 +221,7 @@ std::vector doInference(IExecutionContext& context, void* input, in out_shape = context.getTensorShape(tensor_name); out_size = std::accumulate(out_shape.d, out_shape.d + out_shape.nbDims, 1ULL, std::multiplies<>()); CHECK(cudaMalloc(&buffers[i], s * out_size)); -#if TRT_VERSION >= 10000 +#if TRT_VERSION_GE(10, 0, 0) if (!context.setOutputTensorAddress(tensor_name, buffers[i])) { std::cerr << "setOutputTensorAddress failed\n"; std::abort(); @@ -250,7 +251,7 @@ std::vector doInference(IExecutionContext& context, void* input, in #endif } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) if (!context.enqueueV3(stream)) { std::cerr << "enqueueV3 failed\n"; std::abort(); @@ -307,7 +308,7 @@ static auto mainImpl(int argc, char** argv) -> int { const auto* data_ptr = reinterpret_cast(modelStream->data()); auto data_size = static_cast(modelStream->size()); p.write(data_ptr, data_size); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete modelStream; #else modelStream->destroy(); @@ -329,7 +330,7 @@ static auto mainImpl(int argc, char** argv) -> int { return -1; } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size); #else ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size, nullptr); @@ -358,38 +359,34 @@ static auto mainImpl(int argc, char** argv) -> int { } } - // run inference - for (int32_t i = 0; i < 100; ++i) { - auto _start = std::chrono::system_clock::now(); - auto output = doInference(*context, data.data(), 1); - auto _end = std::chrono::system_clock::now(); + auto output = doInference(*context, data.data(), 1); + const auto* data_ptr = reinterpret_cast(output[0].data); + + DummyTensor& t = output[0]; + const auto out_h = t.dims.d[2]; + const auto out_w = t.dims.d[3]; + auto stride = static_cast(out_h) * out_w; + printFirstOutputs("csrnet", data_ptr, static_cast(stride)); + float num = std::accumulate(data_ptr, data_ptr + stride, 0.0f); + cv::Mat density((int)out_h, (int)out_w, CV_32FC1, t.data); + cv::Mat scaled, heatmap, save; + cv::normalize(density, scaled, 0, 255, cv::NORM_MINMAX, CV_8UC1); + cv::applyColorMap(scaled, heatmap, cv::COLORMAP_JET); + cv::resize(heatmap, heatmap, img.size(), 0., 0., cv::INTER_LINEAR); + cv::addWeighted(img, 0.7, heatmap, 0.3, 0, save); + cv::imwrite("../assets/csrnet_output_tensorrt.jpg", save); + std::cout << "approximate people num: " << std::ceil(num) << "\nsave to ../assets/csrnet_output_tensorrt.jpg\n"; + + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int32_t i = 0; i < kBenchmarkRuns; ++i) { + auto _start = std::chrono::steady_clock::now(); + (void)doInference(*context, data.data(), 1); + auto _end = std::chrono::steady_clock::now(); auto _time = std::chrono::duration_cast(_end - _start).count(); - std::cout << "Execution time: " << _time << "us\n"; - - const auto* data_ptr = reinterpret_cast(output[0].data); - for (int j = 0; j < 10; ++j) { - std::cout << data_ptr[j] << " " << std::flush; - } - std::cout << "\n====\n"; - - if (i == 99) { - // write output to jpg - DummyTensor& t = output[0]; - const auto out_h = t.dims.d[2]; - const auto out_w = t.dims.d[3]; - auto stride = static_cast(out_h) * out_w; - float num = std::accumulate(data_ptr, data_ptr + stride, 0.0f); - cv::Mat density((int)out_h, (int)out_w, CV_32FC1, t.data); - cv::Mat scaled, heatmap, save; - cv::normalize(density, scaled, 0, 255, cv::NORM_MINMAX, CV_8UC1); - cv::applyColorMap(scaled, heatmap, cv::COLORMAP_JET); - cv::resize(heatmap, heatmap, img.size(), 0., 0., cv::INTER_LINEAR); - cv::addWeighted(img, 0.7, heatmap, 0.3, 0, save); - cv::imwrite("../assets/csrnet_output_tensorrt.jpg", save); - std::cout << "approximate people num: " << std::ceil(num) - << "\nsave to ../assets/csrnet_output_tensorrt.jpg\n"; - } + latencies.push_back(static_cast(_time) / 1000.0); } + printBenchmark("csrnet", latencies); return 0; } diff --git a/csrnet/macros.h b/csrnet/macros.h index ccc3b106..b8155b4c 100644 --- a/csrnet/macros.h +++ b/csrnet/macros.h @@ -16,10 +16,15 @@ #endif #endif // API_EXPORTS -#define TRT_VERSION \ - ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) \ + ((NV_TENSORRT_MAJOR > (major)) || (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR > (minor)) || \ + (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR == (minor) && NV_TENSORRT_PATCH >= (patch))) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const #else diff --git a/csrnet/utils.h b/csrnet/utils.h index a5225dad..5e55857f 100644 --- a/csrnet/utils.h +++ b/csrnet/utils.h @@ -2,13 +2,17 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include #include +#include #include "macros.h" using namespace nvinfer1; @@ -25,7 +29,7 @@ constexpr const std::size_t WORKSPACE_SIZE = 16 << 20; } while (0) static void checkTrtEnv(int device = 0) { -#if TRT_VERSION < 8000 +#if TRT_VERSION_LT(8, 0, 0) CHECK(cudaGetDevice(&device)); cudaDeviceProp prop{}; CHECK(cudaGetDeviceProperties(&prop, device)); @@ -79,7 +83,7 @@ static auto loadWeights(const std::string& file) { static size_t getSize(DataType dt) { switch (dt) { -#if TRT_VERSION >= 8510 +#if TRT_VERSION_GE(8, 5, 1) case DataType::kUINT8: #endif case DataType::kINT8: @@ -155,3 +159,41 @@ class DummyTensor final { Dims dims; void* data; }; + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} diff --git a/mobilenet/CMakeLists.txt b/mobilenet/CMakeLists.txt new file mode 100644 index 00000000..0670cea2 --- /dev/null +++ b/mobilenet/CMakeLists.txt @@ -0,0 +1,41 @@ +cmake_minimum_required(VERSION 3.14) + +project( + mobilenet + VERSION 0.1 + LANGUAGES C CXX CUDA) + +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES + 60 + 70 + 72 + 75 + 80 + 86 + 89) +endif() + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CUDA_STANDARD 17) +set(CMAKE_CUDA_STANDARD_REQUIRED ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_INCLUDE_CURRENT_DIR TRUE) + +option(CUDA_USE_STATIC_CUDA_RUNTIME "Use static cudaruntime library" OFF) + +find_package(Threads REQUIRED) +find_package(CUDAToolkit REQUIRED) +find_package(OpenCV REQUIRED) + +if(NOT TARGET TensorRT::TensorRT) + include(FindTensorRT.cmake) +endif() + +add_executable(${PROJECT_NAME} ${PROJECT_NAME}.cpp) + +target_include_directories(${PROJECT_NAME} PRIVATE ${OpenCV_INCLUDE_DIRS}) + +target_link_libraries(${PROJECT_NAME} PRIVATE Threads::Threads CUDA::cudart + TensorRT::TensorRT ${OpenCV_LIBS}) diff --git a/mobilenet/FindTensorRT.cmake b/mobilenet/FindTensorRT.cmake new file mode 100644 index 00000000..c877d7df --- /dev/null +++ b/mobilenet/FindTensorRT.cmake @@ -0,0 +1,90 @@ +cmake_minimum_required(VERSION 3.17.0) + +set(TRT_VERSION + $ENV{TRT_VERSION} + CACHE STRING + "TensorRT version, e.g. \"8.6.1.6\" or \"8.6.1.6+cuda12.0.1.011\"") + +function(_guess_path var_name) + set(_result "") + + foreach(path_entry IN LISTS ARGN) + if(EXISTS "${path_entry}") + list(APPEND _result "${path_entry}") + endif() + endforeach() + + set(${var_name} + "${_result}" + PARENT_SCOPE) +endfunction() + +# find TensorRT include folder +if(NOT DEFINED TensorRT_INCLUDE_DIR) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + _guess_path(TensorRT_INCLUDE_DIR + "/usr/local/cuda/targets/aarch64-linux/include") + else() + _guess_path( + TensorRT_INCLUDE_DIR + "/usr/local/tensorrt/targets/x86_64-linux-gnu/include" + "/usr/include/x86_64-linux-gnu") + endif() + message(STATUS "TensorRT includes: ${TensorRT_INCLUDE_DIR}") +endif() + +# find TensorRT library folder +if(NOT TensorRT_LIBRARY_DIR) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + _guess_path(TensorRT_LIBRARY_DIR "/usr/lib/aarch64-linux-gnu/tegra") + else() + _guess_path(TensorRT_LIBRARY_DIR "/usr/lib/x86_64-linux-gnu" + "/usr/local/tensorrt/targets/x86_64-linux-gnu/lib") + endif() + message(STATUS "TensorRT libraries: ${TensorRT_LIBRARY_DIR}") +endif() + +set(TensorRT_LIBRARIES) + +message(STATUS "Found TensorRT lib: ${TensorRT_LIBRARIES}") + +# process for different TensorRT version +if(DEFINED TRT_VERSION AND NOT TRT_VERSION STREQUAL "") + string(REGEX MATCH "([0-9]+)" _match ${TRT_VERSION}) + set(TRT_MAJOR_VERSION "${_match}") + set(_modules nvinfer nvinfer_plugin) + unset(_match) + + if(TRT_MAJOR_VERSION GREATER_EQUAL 8) + list(APPEND _modules nvinfer_vc_plugin nvinfer_dispatch nvinfer_lean) + endif() +else() + message(FATAL_ERROR "Please set a environment variable \"TRT_VERSION\"") +endif() + +# find and add all modules of TensorRT into list +foreach(lib IN LISTS _modules) + find_library( + TensorRT_${lib}_LIBRARY + NAMES ${lib} + HINTS ${TensorRT_LIBRARY_DIR}) + list(APPEND TensorRT_LIBRARIES ${TensorRT_${lib}_LIBRARY}) +endforeach() + +# make the "TensorRT target" +add_library(TensorRT IMPORTED INTERFACE) +add_library(TensorRT::TensorRT ALIAS TensorRT) +target_link_libraries(TensorRT INTERFACE ${TensorRT_LIBRARIES}) + +set_target_properties( + TensorRT + PROPERTIES C_STANDARD 17 + CXX_STANDARD 17 + POSITION_INDEPENDENT_CODE ON + SKIP_BUILD_RPATH TRUE + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "$ORIGIN" + INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") + +unset(TRT_MAJOR_VERSION) +unset(_modules) diff --git a/mobilenet/gen_wts.py b/mobilenet/gen_wts.py new file mode 100644 index 00000000..1c4fce37 --- /dev/null +++ b/mobilenet/gen_wts.py @@ -0,0 +1,87 @@ +import argparse +import struct +from pathlib import Path + +import cv2 +import numpy as np +import torch +from torchvision.models import MobileNet_V2_Weights, MobileNet_V3_Small_Weights +from torchvision.models.mobilenet import mobilenet_v2, mobilenet_v3_small + +SUPPORTED_MODELS = ("mobilenet_v2", "mobilenet_v3_small") + + +def read_imagenet_labels(path: Path) -> dict[int, str]: + clsid2label: dict[int, str] = {} + with path.open("r", encoding="utf-8") as f: + for line in f.readlines(): + k, v = line.split(": ") + clsid2label.setdefault(int(k), v[1:-3]) + return clsid2label + + +def preprocess(img: np.ndarray) -> torch.Tensor: + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0 + img = cv2.resize(img, (224, 224), interpolation=cv2.INTER_LINEAR) + mean = np.array([0.485, 0.456, 0.406], dtype=np.float32) + std = np.array([0.229, 0.224, 0.225], dtype=np.float32) + img = (img - mean) / std + img = img.transpose(2, 0, 1)[None, ...] + return torch.from_numpy(img) + + +def create_model(name: str) -> torch.nn.Module: + if name == "mobilenet_v2": + return mobilenet_v2(weights=MobileNet_V2_Weights.DEFAULT) + if name == "mobilenet_v3_small": + return mobilenet_v3_small(weights=MobileNet_V3_Small_Weights.DEFAULT) + raise ValueError(f"unsupported model: {name}") + + +def export_model(name: str, output_dir: Path, labels: dict[int, str], image: torch.Tensor) -> None: + print(f"Now dealing with model: {name}") + model = create_model(name).eval() + with torch.inference_mode(): + output = model(image) + for i, batch in enumerate(torch.topk(output, k=3).indices): + for j, idx in enumerate(batch): + print(f"\tBatch: {i}, Top: {j}, logits: {output[i][idx]:.4f}, label: {labels[int(idx)]}") + print(f"{'=' * 32}") + + output_dir.mkdir(parents=True, exist_ok=True) + with (output_dir / f"{name}.wts").open("w", encoding="utf-8") as f: + f.write(f"{len(model.state_dict().keys())}\n") + for key, value in model.state_dict().items(): + values = value.reshape(-1).cpu().numpy() + f.write(f"{key} {len(values)} ") + print(key, value.shape) + for item in values: + f.write(" ") + f.write(struct.pack(">f", float(item)).hex()) + f.write("\n") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Export torchvision MobileNet weights to tensorrtx .wts format.") + parser.add_argument("--model", choices=SUPPORTED_MODELS, action="append", help="model to export; repeatable") + parser.add_argument("--output-dir", type=Path, default=Path(__file__).resolve().parents[1] / "models") + parser.add_argument("--image", type=Path, default=Path(__file__).resolve().parents[1] / "assets" / "cats.jpg") + parser.add_argument( + "--labels", + type=Path, + default=Path(__file__).resolve().parents[1] / "assets" / "imagenet1000_clsidx_to_labels.txt", + ) + args = parser.parse_args() + + labels = read_imagenet_labels(args.labels) + img = cv2.imread(str(args.image), cv2.IMREAD_COLOR) + if img is None: + raise FileNotFoundError(args.image) + image = preprocess(img) + + for name in args.model or SUPPORTED_MODELS: + export_model(name, args.output_dir, labels, image) + + +if __name__ == "__main__": + main() diff --git a/mobilenet/logging.h b/mobilenet/logging.h new file mode 100644 index 00000000..f7f955a5 --- /dev/null +++ b/mobilenet/logging.h @@ -0,0 +1,456 @@ +/* + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TENSORRT_LOGGING_H +#define TENSORRT_LOGGING_H + +#include +#include +#include +#include +#include +#include +#include +#include "NvInferRuntime.h" +#include "macros.h" + +using Severity = nvinfer1::ILogger::Severity; + +class LogStreamConsumerBuffer : public std::stringbuf { + public: + LogStreamConsumerBuffer(std::ostream& stream, const std::string& prefix, bool shouldLog) + : mOutput(stream), mPrefix(prefix), mShouldLog(shouldLog) {} + + LogStreamConsumerBuffer(LogStreamConsumerBuffer&& other) : mOutput(other.mOutput) {} + + ~LogStreamConsumerBuffer() { + // std::streambuf::pbase() gives a pointer to the beginning of the buffered part of the output sequence + // std::streambuf::pptr() gives a pointer to the current position of the output sequence + // if the pointer to the beginning is not equal to the pointer to the current position, + // call putOutput() to log the output to the stream + if (pbase() != pptr()) { + putOutput(); + } + } + + // synchronizes the stream buffer and returns 0 on success + // synchronizing the stream buffer consists of inserting the buffer contents into the stream, + // resetting the buffer and flushing the stream + virtual int sync() { + putOutput(); + return 0; + } + + void putOutput() { + if (mShouldLog) { + // prepend timestamp + std::time_t timestamp = std::time(nullptr); + tm* tm_local = std::localtime(×tamp); + std::cout << "["; + std::cout << std::setw(2) << std::setfill('0') << 1 + tm_local->tm_mon << "/"; + std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_mday << "/"; + std::cout << std::setw(4) << std::setfill('0') << 1900 + tm_local->tm_year << "-"; + std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_hour << ":"; + std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_min << ":"; + std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_sec << "] "; + // std::stringbuf::str() gets the string contents of the buffer + // insert the buffer contents pre-appended by the appropriate prefix into the stream + mOutput << mPrefix << str(); + // set the buffer to empty + str(""); + // flush the stream + mOutput.flush(); + } + } + + void setShouldLog(bool shouldLog) { mShouldLog = shouldLog; } + + private: + std::ostream& mOutput; + std::string mPrefix; + bool mShouldLog; +}; + +//! +//! \class LogStreamConsumerBase +//! \brief Convenience object used to initialize LogStreamConsumerBuffer before std::ostream in LogStreamConsumer +//! +class LogStreamConsumerBase { + public: + LogStreamConsumerBase(std::ostream& stream, const std::string& prefix, bool shouldLog) + : mBuffer(stream, prefix, shouldLog) {} + + protected: + LogStreamConsumerBuffer mBuffer; +}; + +//! +//! \class LogStreamConsumer +//! \brief Convenience object used to facilitate use of C++ stream syntax when logging messages. +//! Order of base classes is LogStreamConsumerBase and then std::ostream. +//! This is because the LogStreamConsumerBase class is used to initialize the LogStreamConsumerBuffer member field +//! in LogStreamConsumer and then the address of the buffer is passed to std::ostream. +//! This is necessary to prevent the address of an uninitialized buffer from being passed to std::ostream. +//! Please do not change the order of the parent classes. +//! +class LogStreamConsumer : protected LogStreamConsumerBase, public std::ostream { + public: + //! \brief Creates a LogStreamConsumer which logs messages with level severity. + //! Reportable severity determines if the messages are severe enough to be logged. + LogStreamConsumer(Severity reportableSeverity, Severity severity) + : LogStreamConsumerBase(severityOstream(severity), severityPrefix(severity), severity <= reportableSeverity), + std::ostream(&mBuffer) // links the stream buffer with the stream + , + mShouldLog(severity <= reportableSeverity), + mSeverity(severity) {} + + LogStreamConsumer(LogStreamConsumer&& other) + : LogStreamConsumerBase(severityOstream(other.mSeverity), severityPrefix(other.mSeverity), other.mShouldLog), + std::ostream(&mBuffer) // links the stream buffer with the stream + , + mShouldLog(other.mShouldLog), + mSeverity(other.mSeverity) {} + + void setReportableSeverity(Severity reportableSeverity) { + mShouldLog = mSeverity <= reportableSeverity; + mBuffer.setShouldLog(mShouldLog); + } + + private: + static std::ostream& severityOstream(Severity severity) { + return severity >= Severity::kINFO ? std::cout : std::cerr; + } + + static std::string severityPrefix(Severity severity) { + switch (severity) { + case Severity::kINTERNAL_ERROR: + return "[F] "; + case Severity::kERROR: + return "[E] "; + case Severity::kWARNING: + return "[W] "; + case Severity::kINFO: + return "[I] "; + case Severity::kVERBOSE: + return "[V] "; + default: + assert(0); + return ""; + } + } + + bool mShouldLog; + Severity mSeverity; +}; + +//! \class Logger +//! +//! \brief Class which manages logging of TensorRT tools and samples +//! +//! \details This class provides a common interface for TensorRT tools and samples to log information to the console, +//! and supports logging two types of messages: +//! +//! - Debugging messages with an associated severity (info, warning, error, or internal error/fatal) +//! - Test pass/fail messages +//! +//! The advantage of having all samples use this class for logging as opposed to emitting directly to stdout/stderr is +//! that the logic for controlling the verbosity and formatting of sample output is centralized in one location. +//! +//! In the future, this class could be extended to support dumping test results to a file in some standard format +//! (for example, JUnit XML), and providing additional metadata (e.g. timing the duration of a test run). +//! +//! TODO: For backwards compatibility with existing samples, this class inherits directly from the nvinfer1::ILogger +//! interface, which is problematic since there isn't a clean separation between messages coming from the TensorRT +//! library and messages coming from the sample. +//! +//! In the future (once all samples are updated to use Logger::getTRTLogger() to access the ILogger) we can refactor the +//! class to eliminate the inheritance and instead make the nvinfer1::ILogger implementation a member of the Logger +//! object. + +class Logger : public nvinfer1::ILogger { + public: + Logger(Severity severity = Severity::kWARNING) : mReportableSeverity(severity) {} + + //! + //! \enum TestResult + //! \brief Represents the state of a given test + //! + enum class TestResult { + kRUNNING, //!< The test is running + kPASSED, //!< The test passed + kFAILED, //!< The test failed + kWAIVED //!< The test was waived + }; + + //! + //! \brief Forward-compatible method for retrieving the nvinfer::ILogger associated with this Logger + //! \return The nvinfer1::ILogger associated with this Logger + //! + //! TODO Once all samples are updated to use this method to register the logger with TensorRT, + //! we can eliminate the inheritance of Logger from ILogger + //! + nvinfer1::ILogger& getTRTLogger() { return *this; } + + //! + //! \brief Implementation of the nvinfer1::ILogger::log() virtual method + //! + //! Note samples should not be calling this function directly; it will eventually go away once we eliminate the + //! inheritance from nvinfer1::ILogger + //! + void log(Severity severity, const char* msg) TRT_NOEXCEPT override { + LogStreamConsumer(mReportableSeverity, severity) << "[TRT] " << std::string(msg) << std::endl; + } + + //! + //! \brief Method for controlling the verbosity of logging output + //! + //! \param severity The logger will only emit messages that have severity of this level or higher. + //! + void setReportableSeverity(Severity severity) { mReportableSeverity = severity; } + + //! + //! \brief Opaque handle that holds logging information for a particular test + //! + //! This object is an opaque handle to information used by the Logger to print test results. + //! The sample must call Logger::defineTest() in order to obtain a TestAtom that can be used + //! with Logger::reportTest{Start,End}(). + //! + class TestAtom { + public: + TestAtom(TestAtom&&) = default; + + private: + friend class Logger; + + TestAtom(bool started, const std::string& name, const std::string& cmdline) + : mStarted(started), mName(name), mCmdline(cmdline) {} + + bool mStarted; + std::string mName; + std::string mCmdline; + }; + + //! + //! \brief Define a test for logging + //! + //! \param[in] name The name of the test. This should be a string starting with + //! "TensorRT" and containing dot-separated strings containing + //! the characters [A-Za-z0-9_]. + //! For example, "TensorRT.sample_googlenet" + //! \param[in] cmdline The command line used to reproduce the test + // + //! \return a TestAtom that can be used in Logger::reportTest{Start,End}(). + //! + static TestAtom defineTest(const std::string& name, const std::string& cmdline) { + return TestAtom(false, name, cmdline); + } + + //! + //! \brief A convenience overloaded version of defineTest() that accepts an array of command-line arguments + //! as input + //! + //! \param[in] name The name of the test + //! \param[in] argc The number of command-line arguments + //! \param[in] argv The array of command-line arguments (given as C strings) + //! + //! \return a TestAtom that can be used in Logger::reportTest{Start,End}(). + static TestAtom defineTest(const std::string& name, int argc, char const* const* argv) { + auto cmdline = genCmdlineString(argc, argv); + return defineTest(name, cmdline); + } + + //! + //! \brief Report that a test has started. + //! + //! \pre reportTestStart() has not been called yet for the given testAtom + //! + //! \param[in] testAtom The handle to the test that has started + //! + static void reportTestStart(TestAtom& testAtom) { + reportTestResult(testAtom, TestResult::kRUNNING); + assert(!testAtom.mStarted); + testAtom.mStarted = true; + } + + //! + //! \brief Report that a test has ended. + //! + //! \pre reportTestStart() has been called for the given testAtom + //! + //! \param[in] testAtom The handle to the test that has ended + //! \param[in] result The result of the test. Should be one of TestResult::kPASSED, + //! TestResult::kFAILED, TestResult::kWAIVED + //! + static void reportTestEnd(const TestAtom& testAtom, TestResult result) { + assert(result != TestResult::kRUNNING); + assert(testAtom.mStarted); + reportTestResult(testAtom, result); + } + + static int reportPass(const TestAtom& testAtom) { + reportTestEnd(testAtom, TestResult::kPASSED); + return EXIT_SUCCESS; + } + + static int reportFail(const TestAtom& testAtom) { + reportTestEnd(testAtom, TestResult::kFAILED); + return EXIT_FAILURE; + } + + static int reportWaive(const TestAtom& testAtom) { + reportTestEnd(testAtom, TestResult::kWAIVED); + return EXIT_SUCCESS; + } + + static int reportTest(const TestAtom& testAtom, bool pass) { + return pass ? reportPass(testAtom) : reportFail(testAtom); + } + + Severity getReportableSeverity() const { return mReportableSeverity; } + + private: + //! + //! \brief returns an appropriate string for prefixing a log message with the given severity + //! + static const char* severityPrefix(Severity severity) { + switch (severity) { + case Severity::kINTERNAL_ERROR: + return "[F] "; + case Severity::kERROR: + return "[E] "; + case Severity::kWARNING: + return "[W] "; + case Severity::kINFO: + return "[I] "; + case Severity::kVERBOSE: + return "[V] "; + default: + assert(0); + return ""; + } + } + + //! + //! \brief returns an appropriate string for prefixing a test result message with the given result + //! + static const char* testResultString(TestResult result) { + switch (result) { + case TestResult::kRUNNING: + return "RUNNING"; + case TestResult::kPASSED: + return "PASSED"; + case TestResult::kFAILED: + return "FAILED"; + case TestResult::kWAIVED: + return "WAIVED"; + default: + assert(0); + return ""; + } + } + + //! + //! \brief returns an appropriate output stream (cout or cerr) to use with the given severity + //! + static std::ostream& severityOstream(Severity severity) { + return severity >= Severity::kINFO ? std::cout : std::cerr; + } + + //! + //! \brief method that implements logging test results + //! + static void reportTestResult(const TestAtom& testAtom, TestResult result) { + severityOstream(Severity::kINFO) << "&&&& " << testResultString(result) << " " << testAtom.mName << " # " + << testAtom.mCmdline << std::endl; + } + + //! + //! \brief generate a command line string from the given (argc, argv) values + //! + static std::string genCmdlineString(int argc, char const* const* argv) { + std::stringstream ss; + for (int i = 0; i < argc; i++) { + if (i > 0) + ss << " "; + ss << argv[i]; + } + return ss.str(); + } + + Severity mReportableSeverity; +}; + +namespace { + +//! +//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kVERBOSE +//! +//! Example usage: +//! +//! LOG_VERBOSE(logger) << "hello world" << std::endl; +//! +inline LogStreamConsumer LOG_VERBOSE(const Logger& logger) { + return LogStreamConsumer(logger.getReportableSeverity(), Severity::kVERBOSE); +} + +//! +//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kINFO +//! +//! Example usage: +//! +//! LOG_INFO(logger) << "hello world" << std::endl; +//! +inline LogStreamConsumer LOG_INFO(const Logger& logger) { + return LogStreamConsumer(logger.getReportableSeverity(), Severity::kINFO); +} + +//! +//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kWARNING +//! +//! Example usage: +//! +//! LOG_WARN(logger) << "hello world" << std::endl; +//! +inline LogStreamConsumer LOG_WARN(const Logger& logger) { + return LogStreamConsumer(logger.getReportableSeverity(), Severity::kWARNING); +} + +//! +//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kERROR +//! +//! Example usage: +//! +//! LOG_ERROR(logger) << "hello world" << std::endl; +//! +inline LogStreamConsumer LOG_ERROR(const Logger& logger) { + return LogStreamConsumer(logger.getReportableSeverity(), Severity::kERROR); +} + +//! +//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kINTERNAL_ERROR +// ("fatal" severity) +//! +//! Example usage: +//! +//! LOG_FATAL(logger) << "hello world" << std::endl; +//! +inline LogStreamConsumer LOG_FATAL(const Logger& logger) { + return LogStreamConsumer(logger.getReportableSeverity(), Severity::kINTERNAL_ERROR); +} + +} // anonymous namespace + +#endif // TENSORRT_LOGGING_H diff --git a/mobilenet/macros.h b/mobilenet/macros.h new file mode 100644 index 00000000..4f442e6a --- /dev/null +++ b/mobilenet/macros.h @@ -0,0 +1,35 @@ +#pragma once +#include + +#ifdef API_EXPORTS +#if defined(_MSC_VER) +#define API __declspec(dllexport) +#else +#define API __attribute__((visibility("default"))) +#endif +#else + +#if defined(_MSC_VER) +#define API __declspec(dllimport) +#else +#define API +#endif +#endif // API_EXPORTS + +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) (TRT_VERSION >= TRT_VERSION_ENCODE((major), (minor), (patch), 0)) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) + +#if TRT_VERSION_LT(7, 2, 2) +#error "TensorRT >= 7.2.2 is required for this demo." +#endif + +#if TRT_VERSION_GE(8, 0, 0) +#define TRT_NOEXCEPT noexcept +#define TRT_CONST_ENQUEUE const +#else +#define TRT_NOEXCEPT +#define TRT_CONST_ENQUEUE +#endif diff --git a/mobilenet/mobilenet.cpp b/mobilenet/mobilenet.cpp new file mode 100644 index 00000000..a5708d22 --- /dev/null +++ b/mobilenet/mobilenet.cpp @@ -0,0 +1,634 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "logging.h" +#include "utils.h" + +struct MobileNetVariant { + std::string name; + std::string wts_path; + std::string engine_path; + bool is_v3_small; +}; + +struct V2BlockParams { + int32_t index; + int32_t in_channels; + int32_t out_channels; + int32_t stride; + int32_t expansion; +}; + +struct V3BlockParams { + int32_t index; + int32_t in_channels; + int32_t out_channels; + int32_t hidden_channels; + int32_t kernel_size; + int32_t stride; + bool use_se; + bool use_hs; +}; + +static constexpr const std::size_t WORKSPACE_SIZE = 16 << 20; +static constexpr const int64_t N = 1; +static constexpr const int32_t INPUT_H = 224; +static constexpr const int32_t INPUT_W = 224; +static constexpr const std::array SIZES = {3 * INPUT_H * INPUT_W, 1000}; +static constexpr const std::array NAMES = {"data", "logits"}; +static constexpr const bool TRT_PREPROCESS = TRT_VERSION_GE(8, 5, 1); +static constexpr const std::array mean = {0.485f, 0.456f, 0.406f}; +static constexpr const std::array stdv = {0.229f, 0.224f, 0.225f}; +static constexpr const char* LABELS_PATH = "assets/imagenet1000_clsidx_to_labels.txt"; + +using namespace nvinfer1; +using WeightMap = std::map; +using M = MatrixOperation; +using E = ElementWiseOperation; +using NDCF = NetworkDefinitionCreationFlag; + +static Logger gLogger; + +static auto getVariantConfig(const std::string& name) -> MobileNetVariant { + if (name == "mobilenet_v2" || name == "v2") { + return {"mobilenet_v2", "models/mobilenet_v2.wts", "models/mobilenet_v2.engine", false}; + } + if (name == "mobilenet_v3_small" || name == "v3_small") { + return {"mobilenet_v3_small", "models/mobilenet_v3_small.wts", "models/mobilenet_v3_small.engine", true}; + } + std::cerr << "Unsupported MobileNet variant: " << name << "\n"; + std::cerr << "Choose one of: mobilenet_v2, mobilenet_v3_small\n"; + std::abort(); +} + +static auto emptyWeights() -> Weights { + return Weights{DataType::kFLOAT, nullptr, 0ll}; +} + +static auto getWeight(const WeightMap& weight_map, const std::string& key) -> Weights { + const auto iter = weight_map.find(key); + if (iter == weight_map.end()) { + std::cerr << "Missing MobileNet weight: " << key << "\n"; + std::abort(); + } + return iter->second; +} + +static auto addBatchNorm2d(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, const std::string& lname, + float eps = 1e-5f) -> ILayer* { + const auto* gamma = static_cast(getWeight(weight_map, lname + ".weight").values); + const auto* beta = static_cast(getWeight(weight_map, lname + ".bias").values); + const auto* bn_mean = static_cast(getWeight(weight_map, lname + ".running_mean").values); + const auto* var = static_cast(getWeight(weight_map, lname + ".running_var").values); + const auto len = getWeight(weight_map, lname + ".running_var").count; + + auto* scale_values = static_cast(std::malloc(sizeof(float) * static_cast(len))); + auto* shift_values = static_cast(std::malloc(sizeof(float) * static_cast(len))); + if (scale_values == nullptr || shift_values == nullptr) { + std::cerr << "batchnorm weight allocation failed\n"; + std::abort(); + } + for (int64_t i = 0; i < len; ++i) { + scale_values[i] = gamma[i] / std::sqrt(var[i] + eps); + shift_values[i] = beta[i] - bn_mean[i] * gamma[i] / std::sqrt(var[i] + eps); + } + + Weights scale{DataType::kFLOAT, scale_values, len}; + Weights shift{DataType::kFLOAT, shift_values, len}; + static const Weights power{DataType::kFLOAT, nullptr, 0ll}; + + weight_map[lname + ".scale"] = scale; + weight_map[lname + ".shift"] = shift; + weight_map[lname + ".power"] = power; + auto* scale_layer = network->addScale(input, ScaleMode::kCHANNEL, shift, scale, power); + assert(scale_layer); + return scale_layer; +} + +static auto addHardSigmoid(INetworkDefinition* network, ITensor& input, const std::string& name) -> IActivationLayer* { + auto* hsigmoid = network->addActivation(input, ActivationType::kHARD_SIGMOID); + assert(hsigmoid); + hsigmoid->setAlpha(1.0f / 6.0f); + hsigmoid->setBeta(0.5f); + hsigmoid->setName(name.c_str()); + return hsigmoid; +} + +static auto addHardSwish(INetworkDefinition* network, ITensor& input, const std::string& name) -> ILayer* { + auto* hsigmoid = addHardSigmoid(network, input, name + ".hsigmoid"); + auto* hswish = network->addElementWise(input, *hsigmoid->getOutput(0), E::kPROD); + assert(hswish); + hswish->setName(name.c_str()); + return hswish; +} + +static auto addRelu6(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, + const std::string& name) -> ILayer* { + auto* relu = network->addActivation(input, ActivationType::kRELU); + assert(relu); + relu->setName((name + ".relu").c_str()); + + auto* six = static_cast(std::malloc(sizeof(float))); + if (six == nullptr) { + std::cerr << "relu6 constant allocation failed\n"; + std::abort(); + } + *six = 6.0f; + const std::string key = name + ".relu6.max"; + weight_map[key] = Weights{DataType::kFLOAT, six, 1ll}; + auto* c = network->addConstant(Dims4{1, 1, 1, 1}, weight_map[key]); + assert(c); + auto* clipped = network->addElementWise(*relu->getOutput(0), *c->getOutput(0), E::kMIN); + assert(clipped); + clipped->setName(name.c_str()); + return clipped; +} + +static auto addActivation(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, const std::string& name, + bool hard_swish, bool relu6) -> ILayer* { + if (hard_swish) { + return addHardSwish(network, input, name); + } + if (relu6) { + return addRelu6(network, weight_map, input, name); + } + auto* relu = network->addActivation(input, ActivationType::kRELU); + assert(relu); + relu->setName(name.c_str()); + return relu; +} + +static auto addConvBnAct(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, + const std::string& conv_key, const std::string& bn_key, int32_t out_channels, + int32_t kernel_size, int32_t stride, int32_t padding, int32_t groups, bool with_act, + bool hard_swish = false, bool relu6 = false, float eps = 1e-5f) -> ILayer* { + auto* conv = network->addConvolutionNd(input, out_channels, DimsHW{kernel_size, kernel_size}, + getWeight(weight_map, conv_key), emptyWeights()); + assert(conv); + conv->setStrideNd(DimsHW{stride, stride}); + conv->setPaddingNd(DimsHW{padding, padding}); + conv->setNbGroups(groups); + conv->setName(conv_key.c_str()); + + auto* bn = addBatchNorm2d(network, weight_map, *conv->getOutput(0), bn_key, eps); + bn->setName((bn_key + ".bn").c_str()); + if (!with_act) { + return bn; + } + return addActivation(network, weight_map, *bn->getOutput(0), bn_key + ".act", hard_swish, relu6); +} + +static auto addLinear(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, const std::string& weight_key, + const std::string& bias_key, int32_t in_features, int32_t out_features, + const std::string& name) -> ILayer* { + auto* fcw = network->addConstant(DimsHW{out_features, in_features}, getWeight(weight_map, weight_key)); + auto* fcb = network->addConstant(DimsHW{1, out_features}, getWeight(weight_map, bias_key)); + assert(fcw); + assert(fcb); + auto* fc_matmul = network->addMatrixMultiply(input, M::kNONE, *fcw->getOutput(0), M::kTRANSPOSE); + assert(fc_matmul); + auto* fc = network->addElementWise(*fc_matmul->getOutput(0), *fcb->getOutput(0), E::kSUM); + assert(fc); + fc->setName(name.c_str()); + return fc; +} + +static auto addSqueezeExcitation(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, + const std::string& lname, int32_t channels) -> ILayer* { + auto* pool = network->addReduce(input, ReduceOperation::kAVG, 0xc, true); + assert(pool); + pool->setName((lname + ".avgpool").c_str()); + + const auto fc1_weight = getWeight(weight_map, lname + ".fc1.weight"); + const auto squeeze_channels = toI32(fc1_weight.count / channels); + auto* fc1 = network->addConvolutionNd(*pool->getOutput(0), squeeze_channels, DimsHW{1, 1}, fc1_weight, + getWeight(weight_map, lname + ".fc1.bias")); + assert(fc1); + fc1->setName((lname + ".fc1").c_str()); + auto* relu = network->addActivation(*fc1->getOutput(0), ActivationType::kRELU); + assert(relu); + relu->setName((lname + ".relu").c_str()); + auto* fc2 = network->addConvolutionNd(*relu->getOutput(0), channels, DimsHW{1, 1}, + getWeight(weight_map, lname + ".fc2.weight"), + getWeight(weight_map, lname + ".fc2.bias")); + assert(fc2); + fc2->setName((lname + ".fc2").c_str()); + auto* scale = addHardSigmoid(network, *fc2->getOutput(0), lname + ".scale"); + auto* out = network->addElementWise(input, *scale->getOutput(0), E::kPROD); + assert(out); + out->setName((lname + ".mul").c_str()); + return out; +} + +static auto addV2Block(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, + const V2BlockParams& params) -> ILayer* { + const std::string lname = "features." + std::to_string(params.index) + ".conv."; + const int32_t hidden = params.in_channels * params.expansion; + ILayer* layer = nullptr; + if (params.expansion != 1) { + layer = addConvBnAct(network, weight_map, input, lname + "0.0.weight", lname + "0.1", hidden, 1, 1, 0, 1, true, + false, true); + layer = addConvBnAct(network, weight_map, *layer->getOutput(0), lname + "1.0.weight", lname + "1.1", hidden, 3, + params.stride, 1, hidden, true, false, true); + layer = addConvBnAct(network, weight_map, *layer->getOutput(0), lname + "2.weight", lname + "3", + params.out_channels, 1, 1, 0, 1, false); + } else { + layer = addConvBnAct(network, weight_map, input, lname + "0.0.weight", lname + "0.1", hidden, 3, params.stride, + 1, hidden, true, false, true); + layer = addConvBnAct(network, weight_map, *layer->getOutput(0), lname + "1.weight", lname + "2", + params.out_channels, 1, 1, 0, 1, false); + } + if (params.stride == 1 && params.in_channels == params.out_channels) { + auto* ew = network->addElementWise(input, *layer->getOutput(0), E::kSUM); + assert(ew); + ew->setName((lname + "residual").c_str()); + return ew; + } + return layer; +} + +static auto addV3Block(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, + const V3BlockParams& params) -> ILayer* { + const std::string lname = "features." + std::to_string(params.index) + ".block."; + ITensor* tensor = &input; + int32_t depthwise_index = 0; + int32_t project_index = 2; + if (params.in_channels != params.hidden_channels) { + auto* layer = addConvBnAct(network, weight_map, input, lname + "0.0.weight", lname + "0.1", + params.hidden_channels, 1, 1, 0, 1, true, params.use_hs, false, 1e-3f); + tensor = layer->getOutput(0); + depthwise_index = 1; + project_index = params.use_se ? 3 : 2; + } + + auto* layer = addConvBnAct(network, weight_map, *tensor, lname + std::to_string(depthwise_index) + ".0.weight", + lname + std::to_string(depthwise_index) + ".1", params.hidden_channels, + params.kernel_size, params.stride, params.kernel_size / 2, params.hidden_channels, true, + params.use_hs, false, 1e-3f); + if (params.use_se) { + layer = addSqueezeExcitation(network, weight_map, *layer->getOutput(0), + lname + std::to_string(depthwise_index + 1), params.hidden_channels); + } + layer = addConvBnAct(network, weight_map, *layer->getOutput(0), lname + std::to_string(project_index) + ".0.weight", + lname + std::to_string(project_index) + ".1", params.out_channels, 1, 1, 0, 1, false, false, + false, 1e-3f); + if (params.stride == 1 && params.in_channels == params.out_channels) { + auto* ew = network->addElementWise(input, *layer->getOutput(0), E::kSUM); + assert(ew); + ew->setName((lname + "residual").c_str()); + return ew; + } + return layer; +} + +static auto buildMobileNetV2(INetworkDefinition* network, WeightMap& weight_map, ITensor& input) -> ITensor* { + auto* layer = addConvBnAct(network, weight_map, input, "features.0.0.weight", "features.0.1", 32, 3, 2, 1, 1, true, + false, true); + const std::array blocks = {{{1, 32, 16, 1, 1}, + {2, 16, 24, 2, 6}, + {3, 24, 24, 1, 6}, + {4, 24, 32, 2, 6}, + {5, 32, 32, 1, 6}, + {6, 32, 32, 1, 6}, + {7, 32, 64, 2, 6}, + {8, 64, 64, 1, 6}, + {9, 64, 64, 1, 6}, + {10, 64, 64, 1, 6}, + {11, 64, 96, 1, 6}, + {12, 96, 96, 1, 6}, + {13, 96, 96, 1, 6}, + {14, 96, 160, 2, 6}, + {15, 160, 160, 1, 6}, + {16, 160, 160, 1, 6}, + {17, 160, 320, 1, 6}}}; + for (const auto& block : blocks) { + layer = addV2Block(network, weight_map, *layer->getOutput(0), block); + } + layer = addConvBnAct(network, weight_map, *layer->getOutput(0), "features.18.0.weight", "features.18.1", 1280, 1, 1, + 0, 1, true, false, true); + auto* pool = network->addReduce(*layer->getOutput(0), ReduceOperation::kAVG, 0xc, false); + assert(pool); + pool->setName("avgpool"); + return addLinear(network, weight_map, *pool->getOutput(0), "classifier.1.weight", "classifier.1.bias", 1280, 1000, + "classifier.1") + ->getOutput(0); +} + +static auto buildMobileNetV3Small(INetworkDefinition* network, WeightMap& weight_map, ITensor& input) -> ITensor* { + auto* layer = addConvBnAct(network, weight_map, input, "features.0.0.weight", "features.0.1", 16, 3, 2, 1, 1, true, + true, false, 1e-3f); + const std::array blocks = {{{1, 16, 16, 16, 3, 2, true, false}, + {2, 16, 24, 72, 3, 2, false, false}, + {3, 24, 24, 88, 3, 1, false, false}, + {4, 24, 40, 96, 5, 2, true, true}, + {5, 40, 40, 240, 5, 1, true, true}, + {6, 40, 40, 240, 5, 1, true, true}, + {7, 40, 48, 120, 5, 1, true, true}, + {8, 48, 48, 144, 5, 1, true, true}, + {9, 48, 96, 288, 5, 2, true, true}, + {10, 96, 96, 576, 5, 1, true, true}, + {11, 96, 96, 576, 5, 1, true, true}}}; + for (const auto& block : blocks) { + layer = addV3Block(network, weight_map, *layer->getOutput(0), block); + } + layer = addConvBnAct(network, weight_map, *layer->getOutput(0), "features.12.0.weight", "features.12.1", 576, 1, 1, + 0, 1, true, true, false, 1e-3f); + auto* pool = network->addReduce(*layer->getOutput(0), ReduceOperation::kAVG, 0xc, false); + assert(pool); + pool->setName("avgpool"); + layer = addLinear(network, weight_map, *pool->getOutput(0), "classifier.0.weight", "classifier.0.bias", 576, 1024, + "classifier.0"); + layer = addHardSwish(network, *layer->getOutput(0), "classifier.1"); + return addLinear(network, weight_map, *layer->getOutput(0), "classifier.3.weight", "classifier.3.bias", 1024, 1000, + "classifier.3") + ->getOutput(0); +} + +static auto createEngine(int32_t batch_size, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, DataType dt, + const MobileNetVariant& variant) -> ICudaEngine* { + WeightMap weight_map = loadWeights(variant.wts_path); + +#if TRT_VERSION_GE(10, 12, 0) + auto flag = 1U << static_cast(NDCF::kSTRONGLY_TYPED); +#elif TRT_VERSION_GE(10, 0, 0) + auto flag = 0U; +#else + auto flag = 1U << static_cast(NDCF::kEXPLICIT_BATCH); +#endif + auto* network = builder->createNetworkV2(flag); + assert(network); + + ITensor* input = nullptr; + if constexpr (TRT_PREPROCESS) { + dt = DataType::kUINT8; + input = network->addInput(NAMES[0], dt, Dims4{batch_size, INPUT_H, INPUT_W, 3}); + auto* transform = addTransformLayer(network, *input, true, mean, stdv); + input = transform->getOutput(0); + } else { + input = network->addInput(NAMES[0], dt, Dims4{batch_size, 3, INPUT_H, INPUT_W}); + } + assert(input); + + ITensor* logits = variant.is_v3_small ? buildMobileNetV3Small(network, weight_map, *input) + : buildMobileNetV2(network, weight_map, *input); + logits->setName(NAMES[1]); + network->markOutput(*logits); + +#if TRT_VERSION_GE(8, 0, 0) + config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); + IHostMemory* mem = builder->buildSerializedNetwork(*network, *config); + assert(mem); + ICudaEngine* engine = runtime->deserializeCudaEngine(mem->data(), mem->size()); + delete mem; + delete network; +#else + builder->setMaxBatchSize(batch_size); + config->setMaxWorkspaceSize(WORKSPACE_SIZE); + ICudaEngine* engine = builder->buildEngineWithConfig(*network, *config); + network->destroy(); +#endif + std::cout << "build finished\n"; + + for (auto& mem : weight_map) { + std::free(const_cast(mem.second.values)); + } + return engine; +} + +static void APIToModel(int32_t batch_size, IRuntime* runtime, IHostMemory** model_stream, + const MobileNetVariant& variant) { + IBuilder* builder = createInferBuilder(gLogger); + IBuilderConfig* config = builder->createBuilderConfig(); + ICudaEngine* engine = createEngine(batch_size, runtime, builder, config, DataType::kFLOAT, variant); + assert(engine != nullptr); + + (*model_stream) = engine->serialize(); + +#if TRT_VERSION_GE(8, 0, 0) + delete engine; + delete config; + delete builder; +#else + engine->destroy(); + config->destroy(); + builder->destroy(); +#endif +} + +static auto doInference(IExecutionContext& context, void* input, + int64_t batch_size) -> std::vector> { + ICudaEngine const& engine = context.getEngine(); + cudaStream_t stream; + CHECK(cudaStreamCreate(&stream)); + std::vector buffers; + +#if TRT_VERSION_GE(8, 0, 0) + const int32_t nIO = engine.getNbIOTensors(); +#else + const int32_t nIO = engine.getNbBindings(); +#endif + + buffers.resize(nIO); + for (auto i = 0; i < nIO; ++i) { + std::size_t size = 0; +#if TRT_VERSION_GE(8, 0, 0) + auto* tensor_name = engine.getIOTensorName(i); + const std::string name = tensor_name; + auto element_size = getSize(engine.getTensorDataType(tensor_name)); + size = element_size * static_cast(batch_size) * (name == NAMES[0] ? SIZES[0] : SIZES[1]); + CHECK(cudaMalloc(&buffers[i], size)); + if (name == NAMES[0]) { + CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); + } + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } +#else + const int32_t idx = engine.getBindingIndex(NAMES[i]); + auto element_size = getSize(engine.getBindingDataType(idx)); + assert(idx == i); + size = element_size * static_cast(batch_size) * SIZES[i]; + CHECK(cudaMalloc(&buffers[i], size)); + if (i == 0) { + CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); + } +#endif + } + +#if TRT_VERSION_GE(8, 0, 0) + if (!context.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); + } +#else + if (!context.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } +#endif + + std::vector> prob; + for (int i = 0; i < nIO; ++i) { +#if TRT_VERSION_GE(8, 0, 0) + const std::string name = engine.getIOTensorName(i); + if (name == NAMES[0]) { + continue; + } + constexpr auto output_size = SIZES[1]; +#else + if (i == 0) { + continue; + } + const auto output_size = SIZES[i]; +#endif + std::vector tmp(batch_size * output_size, std::nanf("")); + const auto size = static_cast(batch_size) * output_size * sizeof(float); + CHECK(cudaMemcpyAsync(tmp.data(), buffers[i], size, cudaMemcpyDeviceToHost, stream)); + prob.emplace_back(std::move(tmp)); + } + CHECK(cudaStreamSynchronize(stream)); + + for (auto& buffer : buffers) { + CHECK(cudaFree(buffer)); + } + CHECK(cudaStreamDestroy(stream)); + return prob; +} + +auto main(int argc, char** argv) -> int { + checkTrtEnv(); + if (argc < 2 || argc > 3) { + std::cerr << "arguments not right!\n"; + std::cerr << "./mobilenet -s [model] // serialize model to plan file\n"; + std::cerr << "./mobilenet -d [model] // deserialize plan file and run inference\n"; + std::cerr << "model: mobilenet_v2 | mobilenet_v3_small\n"; + return -1; + } + + const auto variant = getVariantConfig(argc == 3 ? argv[2] : "mobilenet_v2"); + std::cout << "Using MobileNet variant: " << variant.name << "\n"; + + IRuntime* runtime = createInferRuntime(gLogger); + assert(runtime != nullptr); + char* trt_model_stream{nullptr}; + std::streamsize size{0}; + + if (std::string(argv[1]) == "-s") { + IHostMemory* model_stream{nullptr}; + APIToModel(N, runtime, &model_stream, variant); + assert(model_stream != nullptr); + + std::ofstream plan(variant.engine_path, std::ios::binary | std::ios::trunc); + if (!plan) { + std::cerr << "could not open plan output file\n"; + return -1; + } + if (model_stream->size() > static_cast(std::numeric_limits::max())) { + std::cerr << "this model is too large to serialize\n"; + return -1; + } + const auto* data_ptr = reinterpret_cast(model_stream->data()); + const auto data_size = static_cast(model_stream->size()); + plan.write(data_ptr, data_size); +#if TRT_VERSION_GE(8, 0, 0) + delete model_stream; + delete runtime; +#else + model_stream->destroy(); + runtime->destroy(); +#endif + return 0; + } + if (std::string(argv[1]) == "-d") { + std::ifstream file(variant.engine_path, std::ios::binary); + if (file.good()) { + file.seekg(0, file.end); + size = file.tellg(); + file.seekg(0, file.beg); + trt_model_stream = new char[size]; + assert(trt_model_stream); + file.read(trt_model_stream, size); + file.close(); + } + } else { + return -1; + } + +#if TRT_VERSION_GE(8, 0, 0) + ICudaEngine* engine = runtime->deserializeCudaEngine(trt_model_stream, size); +#else + ICudaEngine* engine = runtime->deserializeCudaEngine(trt_model_stream, size, nullptr); +#endif + assert(engine != nullptr); + IExecutionContext* context = engine->createExecutionContext(); + assert(context != nullptr); + delete[] trt_model_stream; + + void* input = nullptr; + std::vector flat_img; + cv::Mat img = cv::imread("assets/cats.jpg", cv::IMREAD_COLOR); + if (img.empty()) { + std::cerr << "failed to read image: assets/cats.jpg\n"; + return -1; + } + if constexpr (TRT_PREPROCESS) { + cv::resize(img, img, cv::Size(INPUT_W, INPUT_H), 0, 0, cv::INTER_LINEAR); + input = static_cast(img.data); + } else { + flat_img = preprocess_img(img, true, mean, stdv, N, INPUT_H, INPUT_W); + input = flat_img.data(); + } + + auto firstProb = doInference(*context, input, N); + printFirstOutputs("mobilenet", firstProb[0].data(), firstProb[0].size()); + std::cout << "prediction result:\n"; + auto labels = loadImagenetLabelMap(LABELS_PATH); + if (labels.empty()) { + std::cerr << "failed to load labels from " << LABELS_PATH << "\n"; + std::abort(); + } + int top = 0; + for (auto& [idx, logits] : topk(firstProb[0], 3)) { + std::cout << "Top: " << top++ << " idx: " << idx << ", logits: " << std::setprecision(4) << logits + << ", label: " << labels[idx] << "\n"; + } + + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int i = 0; i < kBenchmarkRuns; ++i) { + auto start = std::chrono::steady_clock::now(); + (void)doInference(*context, input, N); + auto end = std::chrono::steady_clock::now(); + auto period = std::chrono::duration_cast(end - start); + latencies.push_back(static_cast(period.count()) / 1000.0); + } + printBenchmark("mobilenet", latencies, N); +#if TRT_VERSION_GE(8, 0, 0) + delete context; + delete engine; + delete runtime; +#else + context->destroy(); + engine->destroy(); + runtime->destroy(); +#endif + + return 0; +} diff --git a/mobilenet/utils.h b/mobilenet/utils.h new file mode 100644 index 00000000..4f111957 --- /dev/null +++ b/mobilenet/utils.h @@ -0,0 +1,311 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "macros.h" + +using namespace nvinfer1; + +#define CHECK(status) \ + do { \ + auto ret = (status); \ + if (ret != cudaSuccess) { \ + std::cerr << "Cuda failure: " << ret << "\n"; \ + std::abort(); \ + } \ + } while (0) + +static void checkTrtEnv(int device = 0) { +#if TRT_VERSION_LT(8, 0, 0) + CHECK(cudaGetDevice(&device)); + cudaDeviceProp prop{}; + CHECK(cudaGetDeviceProperties(&prop, device)); + const int sm = prop.major * 10 + prop.minor; + if (sm > 86) { + std::cerr << "TensorRT < 8 does not support SM > 86 on this GPU."; + std::abort(); + } +#endif +} + +template +static auto toI32(T value) -> int32_t { + static_assert(std::is_integral::value, "toI32 requires an integral type."); + static_assert(sizeof(T) > sizeof(int32_t), "toI32 is only for values wider than int32_t."); + if constexpr (std::is_signed::value) { + assert(value >= static_cast(std::numeric_limits::min())); + } + assert(value <= static_cast(std::numeric_limits::max())); + return static_cast(value); +} + +/** + * @brief TensorRT weight files have a simple space delimited format: + * [type] [size] + * + * @param file input weight file path + * @return std::map + */ +static std::map loadWeights(const std::string& file) { + std::cout << "Loading weights: " << file << "\n"; + std::map weightMap; + + // Open weights file + std::ifstream input(file); + assert(input.is_open() && "Unable to load weight file."); + + // Read number of weight blobs + int32_t count; + input >> count; + assert(count > 0 && "Invalid weight map file."); + + while (count--) { + nvinfer1::Weights wt{nvinfer1::DataType::kFLOAT, nullptr, 0}; + + // Read name and type of blob + std::string name; + input >> name >> std::dec >> wt.count; + + // Load blob + auto* val = static_cast(std::malloc(sizeof(uint32_t) * static_cast(wt.count))); + if (val == nullptr) { + std::cerr << "weight allocation failed\n"; + std::abort(); + } + input >> std::hex; + for (auto x = 0ll; x < wt.count; ++x) { + input >> val[x]; + } + wt.values = val; + weightMap[name] = wt; + } + + return weightMap; +} + +/** + * @brief a preprocess function aligning with ImageNet preprocess in torchvision, only support 3-channel image + * + * @param img opencv image with BGR layout + * @param bgr2rgb whether to convert BGR to RGB + * @param mean subtract mean + * @param std divide std + * @param n batch size + * @param h resize height + * @param w resize width + * @return std::vector contiguous flatten image data in float32 type + */ +static std::vector preprocess_img(cv::Mat& img, bool bgr2rgb, const std::array& mean, + const std::array& std, int n, int h, int w) { + const auto c = img.channels(); + const auto size = c * h * w; + if (c != 3) { + std::cerr << "this demo only supports 3 channel input image.\n"; + std::abort(); + } + if (bgr2rgb) { + cv::cvtColor(img, img, cv::COLOR_BGR2RGB); + } + cv::resize(img, img, cv::Size(w, h), 0, 0, cv::INTER_LINEAR); + img.convertTo(img, CV_32FC3, 1.f / 255); + img = (img - cv::Scalar(mean[0], mean[1], mean[2])) / cv::Scalar(std[0], std[1], std[2]); + std::vector chw(static_cast(n) * c * h * w, 0.f); + + // fill all batch with the same input image + for (int i = 0; i < n; ++i) { + for (int y = 0; y < h; ++y) { + for (int x = 0; x < w; ++x) { + const cv::Vec3f v = img.at(y, x); + chw[i * size + 0 * h * w + y * w + x] = v[0]; + chw[i * size + 1 * h * w + y * w + x] = v[1]; + chw[i * size + 2 * h * w + y * w + x] = v[2]; + } + } + } + return chw; +} + +static auto topk(const std::vector& v, int k) -> std::vector> { + if (k <= 0) + return {}; + auto stride = std::min(k, static_cast(v.size())); + + std::vector idx(v.size()); + std::iota(idx.begin(), idx.end(), 0); + + std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) { return v[a] > v[b]; }); + + std::vector> out; + out.reserve(stride); + for (auto i = 0; i < stride; ++i) + out.emplace_back(idx[i], v[idx[i]]); + return out; +} + +static std::map loadImagenetLabelMap(const std::string& path) { + std::map labels; + std::ifstream in(path); + if (!in.is_open()) { + return labels; + } + std::string line; + while (std::getline(in, line)) { + auto colon = line.find(':'); + if (colon == std::string::npos) { + continue; + } + auto first_quote = line.find('\'', colon); + if (first_quote == std::string::npos) { + continue; + } + auto second_quote = line.find('\'', first_quote + 1); + if (second_quote == std::string::npos) { + continue; + } + int idx = std::stoi(line.substr(0, colon)); + labels[idx] = line.substr(first_quote + 1, second_quote - first_quote - 1); + } + return labels; +} + +static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bool bgr2rgb, + const std::array& mean, const std::array& std) { + struct ScaleParams { + std::array shift; + std::array scale; + }; + static std::vector> gScaleParams; + auto params = std::make_unique(); + params->shift = {-mean[0] / std[0], -mean[1] / std[1], -mean[2] / std[2]}; + params->scale = {1.f / (std[0] * 255.f), 1.f / (std[1] * 255.f), 1.f / (std[2] * 255.f)}; + + static const Weights empty{DataType::kFLOAT, nullptr, 0ll}; + const Weights shift{DataType::kFLOAT, params->shift.data(), 3ll}; + const Weights scale{DataType::kFLOAT, params->scale.data(), 3ll}; + + gScaleParams.emplace_back(std::move(params)); + + ITensor* in = &input; + if (input.getType() != DataType::kFLOAT) { +#if TRT_VERSION_GE(8, 0, 0) + auto* cast = network->addCast(input, DataType::kFLOAT); + assert(cast); + cast->setName("Cast to FP32"); + in = cast->getOutput(0); +#else + auto* identity = network->addIdentity(input); + assert(identity); + identity->setName("Convert to FP32"); + identity->setOutputType(0, DataType::kFLOAT); + in = identity->getOutput(0); +#endif + } + // Convert from NHWC to NCHW + auto* perm = network->addShuffle(*in); + assert(perm); + perm->setName("NHWC -> NCHW"); + perm->setFirstTranspose(Permutation{0, 3, 1, 2}); + + // Convert from BGR to RGB (optional) + ITensor* data{nullptr}; + if (bgr2rgb) { + auto add_slice = [&](int c, const char* name) -> ITensor* { + auto dims = perm->getOutput(0)->getDimensions(); + Dims4 start = {0, c, 0, 0}, stride = {1, 1, 1, 1}; + Dims4 size = {dims.d[0], 1, dims.d[2], dims.d[3]}; + auto* _slice = network->addSlice(*perm->getOutput(0), start, size, stride); + _slice->setName(name); + assert(_slice && _slice->getNbOutputs() == 1); + return _slice->getOutput(0); + }; + std::array channels = {add_slice(2, "R"), add_slice(1, "G"), add_slice(0, "B")}; + auto* cat = network->addConcatenation(channels.data(), 3); + assert(cat); + cat->setName("RGB"); + cat->setAxis(1); + data = cat->getOutput(0); + } else { + data = perm->getOutput(0); + } + + // Normalize + auto* trans = network->addScale(*data, ScaleMode::kCHANNEL, shift, scale, empty); + assert(trans); + trans->setName("mean & std"); +#if TRT_VERSION_GE(8, 0, 0) + trans->setChannelAxis(1); +#endif + return trans; +} + +static size_t getSize(DataType dt) { + switch (dt) { +#if TRT_VERSION_GE(8, 5, 1) + case DataType::kUINT8: +#endif + case DataType::kINT8: + return sizeof(int8_t); + case DataType::kFLOAT: + return sizeof(float); + case DataType::kHALF: + return sizeof(int16_t); + case DataType::kINT32: + return sizeof(int32_t); + default: { + std::cerr << "Unsupported data type\n"; + std::abort(); + } + } +} + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} diff --git a/resnet/CMakeLists.txt b/resnet/CMakeLists.txt index f7356098..6fa6b3ca 100644 --- a/resnet/CMakeLists.txt +++ b/resnet/CMakeLists.txt @@ -1,41 +1,42 @@ -cmake_minimum_required(VERSION 2.6) - -project(resnet) - -add_definitions(-std=c++11) - -option(CUDA_USE_STATIC_CUDA_RUNTIME OFF) -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_BUILD_TYPE Debug) - -include_directories(${PROJECT_SOURCE_DIR}/include) -# include and link dirs of cuda and tensorrt, you need adapt them if yours are different -# cuda -include_directories(/usr/local/cuda/include) -link_directories(/usr/local/cuda/lib64) -# tensorrt -include_directories(/usr/include/x86_64-linux-gnu/) -link_directories(/usr/lib/x86_64-linux-gnu/) - -add_executable(resnet18 ${PROJECT_SOURCE_DIR}/resnet18.cpp) -target_link_libraries(resnet18 nvinfer) -target_link_libraries(resnet18 cudart) - -add_executable(resnet34 ${PROJECT_SOURCE_DIR}/resnet34.cpp) -target_link_libraries(resnet34 nvinfer) -target_link_libraries(resnet34 cudart) - -add_executable(resnet50 ${PROJECT_SOURCE_DIR}/resnet50.cpp) -target_link_libraries(resnet50 nvinfer) -target_link_libraries(resnet50 cudart) - -add_executable(resnext50 ${PROJECT_SOURCE_DIR}/resnext50_32x4d.cpp) -target_link_libraries(resnext50 nvinfer) -target_link_libraries(resnext50 cudart) - -add_executable(wideresnet50 ${PROJECT_SOURCE_DIR}/wideresnet50.cpp) -target_link_libraries(wideresnet50 nvinfer) -target_link_libraries(wideresnet50 cudart) - -add_definitions(-O2 -pthread) - +cmake_minimum_required(VERSION 3.14) + +project( + resnet + VERSION 0.1 + LANGUAGES C CXX CUDA) + +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES + 60 + 70 + 72 + 75 + 80 + 86 + 89) +endif() + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CUDA_STANDARD 17) +set(CMAKE_CUDA_STANDARD_REQUIRED ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_INCLUDE_CURRENT_DIR TRUE) + +option(CUDA_USE_STATIC_CUDA_RUNTIME "Use static cudaruntime library" OFF) + +find_package(Threads REQUIRED) +find_package(CUDAToolkit REQUIRED) +find_package(OpenCV REQUIRED) + +if(NOT TARGET TensorRT::TensorRT) + include(FindTensorRT.cmake) +else() + message("TensorRT has been found, skipping for ${PROJECT_NAME}") +endif() + +add_executable(${PROJECT_NAME} ${PROJECT_NAME}.cpp) + +target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_LIST_DIR} ${OpenCV_INCLUDE_DIRS}) + +target_link_libraries(${PROJECT_NAME} PRIVATE Threads::Threads CUDA::cudart TensorRT::TensorRT ${OpenCV_LIBS}) diff --git a/resnet/FindTensorRT.cmake b/resnet/FindTensorRT.cmake new file mode 100644 index 00000000..61203d17 --- /dev/null +++ b/resnet/FindTensorRT.cmake @@ -0,0 +1,149 @@ +cmake_minimum_required(VERSION 3.17.0) + +function(_guess_path var_name required_files) + set(_result "") + + foreach(path_entry IN LISTS ARGN) + if(NOT EXISTS "${path_entry}") + message(DEBUG "skip non-existing path '${path_entry}'") + continue() + endif() + + set(_ok TRUE) + foreach(required_file IN LISTS required_files) + if(NOT EXISTS "${path_entry}/${required_file}") + set(_ok FALSE) + message(DEBUG "'${path_entry}' missing '${required_file}'") + break() + endif() + endforeach() + + if(_ok) + list(APPEND _result "${path_entry}") + message(DEBUG "accept '${path_entry}'") + else() + message(DEBUG "reject '${path_entry}'") + endif() + endforeach() + + if(_result STREQUAL "") + message( + FATAL_ERROR + "_guess_path(${var_name}) failed: no valid path found. required_files='${required_files}' candidates='${ARGN}'" + ) + endif() + + set(${var_name} + "${_result}" + PARENT_SCOPE) +endfunction() + +# add library +add_library(TensorRT IMPORTED INTERFACE) +add_library(TensorRT::TensorRT ALIAS TensorRT) + +set(TRT_VERSION + CACHE + STRING + "TensorRT version, e.g. \"8.6.1.6\" or \"8.6.1.6+cuda12.0.1.011\", \"8.6.1.6.Windows10.x86_64.cuda-12.0\" etc" +) + +if(NOT TRT_VERSION STREQUAL "" AND NOT $ENV{TRT_VERSION} STREQUAL "") + message( + WARNING + "TRT_VERSION defined by cmake and environment variable both, using the later one" + ) +endif() + +if(NOT $ENV{TRT_VERSION} STREQUAL "") + set(TRT_VERSION $ENV{TRT_VERSION}) +endif() + +string(REGEX MATCH "([0-9]+)" _match ${TRT_VERSION}) +set(TRT_MAJOR_VERSION "${_match}") +unset(_match) + +if(WIN32) + set(TensorRT_DIR "C:/Program Files/TensorRT-${TRT_VERSION}") + if(NOT EXISTS "${TensorRT_DIR}") + message( + FATAL_ERROR + "TensorRT_DIR=${TensorRT_DIR} does not exist!" + ) + endif() + + if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) + set(_trt_lib_suffix "_${TRT_MAJOR_VERSION}") + set(_modules nvinfer${_trt_lib_suffix} nvinfer_plugin${_trt_lib_suffix} + nvinfer_vc_plugin${_trt_lib_suffix} + nvinfer_dispatch${_trt_lib_suffix} + nvinfer_lean${_trt_lib_suffix}) + message(DEBUG "Using ${_modules}") + else() + set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch + nvinfer_lean) + endif() + + set(TensorRT_LIBRARY_DIR "${TensorRT_DIR}/lib") + set(TensorRT_INCLUDE_DIR "${TensorRT_DIR}/include") +elseif(UNIX) + string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _trt_arch) + set(_trt_include_candidates) + if(_trt_arch MATCHES "^(aarch64|arm64|arch64)$") + set(_trt_include_candidates "/usr/include/aarch64-linux-gnu" "/usr/include" + "/usr/local/cuda/targets/aarch64-linux/include") + set(_trt_library_candidates + "/usr/local/tensorrt/targets/aarch64-linux-gnu/lib" + "/usr/lib/aarch64-linux-gnu" "/usr/lib/aarch64-linux-gnu/tegra" + "/usr/lib") + elseif(_trt_arch MATCHES "^(x86_64|amd64)$") + set(_trt_include_candidates + "/usr/local/tensorrt/targets/x86_64-linux-gnu/include" + "/usr/include/x86_64-linux-gnu" "/usr/include") + set(_trt_library_candidates + "/usr/local/tensorrt/targets/x86_64-linux-gnu/lib" + "/usr/lib/x86_64-linux-gnu" "/usr/lib") + else() + message(FATAL_ERROR "Unknown architecture") + endif() + + set(_modules nvinfer nvinfer_plugin) + if(${TRT_MAJOR_VERSION} GREATER_EQUAL 8) + list(APPEND _modules nvinfer_vc_plugin nvinfer_dispatch nvinfer_lean) + endif() + + _guess_path(TensorRT_LIBRARY_DIR "libnvinfer.so;libnvinfer_plugin.so" + ${_trt_library_candidates}) + message(STATUS "TensorRT libraries: ${TensorRT_LIBRARY_DIR}") + _guess_path(TensorRT_INCLUDE_DIR "NvInfer.h" ${_trt_include_candidates}) + message(STATUS "TensorRT includes: ${TensorRT_INCLUDE_DIR}") +endif() + +foreach(lib IN LISTS _modules) + find_library( + TensorRT_${lib}_LIBRARY + NAMES ${lib} + HINTS ${TensorRT_LIBRARY_DIR}) + list(APPEND TensorRT_LIBRARIES ${TensorRT_${lib}_LIBRARY}) +endforeach() + +target_link_libraries(TensorRT INTERFACE ${TensorRT_LIBRARIES}) + +message(STATUS "Found TensorRT libs: ${TensorRT_LIBRARIES}") + +set_target_properties( + TensorRT + PROPERTIES C_STANDARD 17 + CXX_STANDARD 17 + POSITION_INDEPENDENT_CODE ON + SKIP_BUILD_RPATH TRUE + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "$ORIGIN" + INTERFACE_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}") + +unset(TRT_MAJOR_VERSION) +unset(_modules) +unset(_trt_lib_suffix) +unset(_trt_include_candidates) +unset(_trt_library_candidates) +unset(_trt_arch) diff --git a/resnet/gen_wts.py b/resnet/gen_wts.py new file mode 100644 index 00000000..d9e78703 --- /dev/null +++ b/resnet/gen_wts.py @@ -0,0 +1,107 @@ +import argparse +import struct +from pathlib import Path + +import cv2 +import numpy as np +import torch +from torchvision.models import ( + ResNeXt50_32X4D_Weights, + ResNet18_Weights, + ResNet34_Weights, + ResNet50_Weights, + Wide_ResNet50_2_Weights, + resnet18, + resnet34, + resnet50, + resnext50_32x4d, + wide_resnet50_2, +) + +SUPPORTED_MODELS = ("resnet18", "resnet34", "resnet50", "resnext50_32x4d", "wide_resnet50_2") + + +def read_imagenet_labels(path: Path) -> dict[int, str]: + labels: dict[int, str] = {} + with path.open("r", encoding="utf-8") as file: + for line in file: + key, value = line.split(": ", maxsplit=1) + labels[int(key)] = value.strip()[1:-2] + return labels + + +def preprocess(image_path: Path) -> torch.Tensor: + image = cv2.imread(str(image_path), cv2.IMREAD_COLOR) + if image is None: + raise FileNotFoundError(f"failed to read image: {image_path}") + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0 + image = cv2.resize(image, (224, 224), interpolation=cv2.INTER_LINEAR) + mean = np.array([0.485, 0.456, 0.406], dtype=np.float32) + std = np.array([0.229, 0.224, 0.225], dtype=np.float32) + image = (image - mean) / std + image = image.transpose(2, 0, 1)[None, ...] + return torch.from_numpy(image) + + +def create_model(name: str) -> torch.nn.Module: + if name == "resnet18": + return resnet18(weights=ResNet18_Weights.DEFAULT) + if name == "resnet34": + return resnet34(weights=ResNet34_Weights.DEFAULT) + if name == "resnet50": + return resnet50(weights=ResNet50_Weights.DEFAULT) + if name == "resnext50_32x4d": + return resnext50_32x4d(weights=ResNeXt50_32X4D_Weights.DEFAULT) + if name == "wide_resnet50_2": + return wide_resnet50_2(weights=Wide_ResNet50_2_Weights.DEFAULT) + raise ValueError(f"unsupported model: {name}") + + +def export_model(name: str, output_dir: Path, labels: dict[int, str], image: torch.Tensor) -> None: + print(f"Now dealing with model: {name}") + model = create_model(name).eval() + with torch.inference_mode(): + output = model(image) + for batch_idx, batch in enumerate(torch.topk(output, k=3).indices): + for top_idx, label_idx in enumerate(batch): + print( + f"\tBatch: {batch_idx}, Top: {top_idx}, " + f"logits: {output[batch_idx][label_idx]:.4f}, label: {labels[int(label_idx)]}" + ) + print("=" * 32) + + output_dir.mkdir(parents=True, exist_ok=True) + with (output_dir / f"{name}.wts").open("w", encoding="utf-8") as file: + file.write(f"{len(model.state_dict().keys())}\n") + for key, value in model.state_dict().items(): + values = value.reshape(-1).cpu().numpy() + file.write(f"{key} {len(values)} ") + print(key, value.shape) + for item in values: + file.write(" ") + file.write(struct.pack(">f", float(item)).hex()) + file.write("\n") + + +def parse_args() -> argparse.Namespace: + repo_root = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser(description="Export torchvision ResNet weights to TensorRT .wts files.") + parser.add_argument("--model", choices=SUPPORTED_MODELS, action="append", help="model to export; repeatable") + parser.add_argument("--output-dir", type=Path, default=repo_root / "models") + parser.add_argument("--image", type=Path, default=repo_root / "assets" / "cats.jpg") + parser.add_argument( + "--labels", type=Path, default=repo_root / "assets" / "imagenet1000_clsidx_to_labels.txt" + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + labels = read_imagenet_labels(args.labels) + image = preprocess(args.image) + for name in args.model or SUPPORTED_MODELS: + export_model(name, args.output_dir, labels, image) + + +if __name__ == "__main__": + main() diff --git a/resnet/logging.h b/resnet/logging.h index 602b69fb..f7f955a5 100644 --- a/resnet/logging.h +++ b/resnet/logging.h @@ -17,7 +17,6 @@ #ifndef TENSORRT_LOGGING_H #define TENSORRT_LOGGING_H -#include "NvInferRuntimeCommon.h" #include #include #include @@ -25,32 +24,24 @@ #include #include #include +#include "NvInferRuntime.h" +#include "macros.h" using Severity = nvinfer1::ILogger::Severity; -class LogStreamConsumerBuffer : public std::stringbuf -{ -public: +class LogStreamConsumerBuffer : public std::stringbuf { + public: LogStreamConsumerBuffer(std::ostream& stream, const std::string& prefix, bool shouldLog) - : mOutput(stream) - , mPrefix(prefix) - , mShouldLog(shouldLog) - { - } + : mOutput(stream), mPrefix(prefix), mShouldLog(shouldLog) {} - LogStreamConsumerBuffer(LogStreamConsumerBuffer&& other) - : mOutput(other.mOutput) - { - } + LogStreamConsumerBuffer(LogStreamConsumerBuffer&& other) : mOutput(other.mOutput) {} - ~LogStreamConsumerBuffer() - { + ~LogStreamConsumerBuffer() { // std::streambuf::pbase() gives a pointer to the beginning of the buffered part of the output sequence // std::streambuf::pptr() gives a pointer to the current position of the output sequence // if the pointer to the beginning is not equal to the pointer to the current position, // call putOutput() to log the output to the stream - if (pbase() != pptr()) - { + if (pbase() != pptr()) { putOutput(); } } @@ -58,16 +49,13 @@ class LogStreamConsumerBuffer : public std::stringbuf // synchronizes the stream buffer and returns 0 on success // synchronizing the stream buffer consists of inserting the buffer contents into the stream, // resetting the buffer and flushing the stream - virtual int sync() - { + virtual int sync() { putOutput(); return 0; } - void putOutput() - { - if (mShouldLog) - { + void putOutput() { + if (mShouldLog) { // prepend timestamp std::time_t timestamp = std::time(nullptr); tm* tm_local = std::localtime(×tamp); @@ -88,12 +76,9 @@ class LogStreamConsumerBuffer : public std::stringbuf } } - void setShouldLog(bool shouldLog) - { - mShouldLog = shouldLog; - } + void setShouldLog(bool shouldLog) { mShouldLog = shouldLog; } -private: + private: std::ostream& mOutput; std::string mPrefix; bool mShouldLog; @@ -103,15 +88,12 @@ class LogStreamConsumerBuffer : public std::stringbuf //! \class LogStreamConsumerBase //! \brief Convenience object used to initialize LogStreamConsumerBuffer before std::ostream in LogStreamConsumer //! -class LogStreamConsumerBase -{ -public: +class LogStreamConsumerBase { + public: LogStreamConsumerBase(std::ostream& stream, const std::string& prefix, bool shouldLog) - : mBuffer(stream, prefix, shouldLog) - { - } + : mBuffer(stream, prefix, shouldLog) {} -protected: + protected: LogStreamConsumerBuffer mBuffer; }; @@ -124,49 +106,49 @@ class LogStreamConsumerBase //! This is necessary to prevent the address of an uninitialized buffer from being passed to std::ostream. //! Please do not change the order of the parent classes. //! -class LogStreamConsumer : protected LogStreamConsumerBase, public std::ostream -{ -public: +class LogStreamConsumer : protected LogStreamConsumerBase, public std::ostream { + public: //! \brief Creates a LogStreamConsumer which logs messages with level severity. //! Reportable severity determines if the messages are severe enough to be logged. LogStreamConsumer(Severity reportableSeverity, Severity severity) - : LogStreamConsumerBase(severityOstream(severity), severityPrefix(severity), severity <= reportableSeverity) - , std::ostream(&mBuffer) // links the stream buffer with the stream - , mShouldLog(severity <= reportableSeverity) - , mSeverity(severity) - { - } + : LogStreamConsumerBase(severityOstream(severity), severityPrefix(severity), severity <= reportableSeverity), + std::ostream(&mBuffer) // links the stream buffer with the stream + , + mShouldLog(severity <= reportableSeverity), + mSeverity(severity) {} LogStreamConsumer(LogStreamConsumer&& other) - : LogStreamConsumerBase(severityOstream(other.mSeverity), severityPrefix(other.mSeverity), other.mShouldLog) - , std::ostream(&mBuffer) // links the stream buffer with the stream - , mShouldLog(other.mShouldLog) - , mSeverity(other.mSeverity) - { - } + : LogStreamConsumerBase(severityOstream(other.mSeverity), severityPrefix(other.mSeverity), other.mShouldLog), + std::ostream(&mBuffer) // links the stream buffer with the stream + , + mShouldLog(other.mShouldLog), + mSeverity(other.mSeverity) {} - void setReportableSeverity(Severity reportableSeverity) - { + void setReportableSeverity(Severity reportableSeverity) { mShouldLog = mSeverity <= reportableSeverity; mBuffer.setShouldLog(mShouldLog); } -private: - static std::ostream& severityOstream(Severity severity) - { + private: + static std::ostream& severityOstream(Severity severity) { return severity >= Severity::kINFO ? std::cout : std::cerr; } - static std::string severityPrefix(Severity severity) - { - switch (severity) - { - case Severity::kINTERNAL_ERROR: return "[F] "; - case Severity::kERROR: return "[E] "; - case Severity::kWARNING: return "[W] "; - case Severity::kINFO: return "[I] "; - case Severity::kVERBOSE: return "[V] "; - default: assert(0); return ""; + static std::string severityPrefix(Severity severity) { + switch (severity) { + case Severity::kINTERNAL_ERROR: + return "[F] "; + case Severity::kERROR: + return "[E] "; + case Severity::kWARNING: + return "[W] "; + case Severity::kINFO: + return "[I] "; + case Severity::kVERBOSE: + return "[V] "; + default: + assert(0); + return ""; } } @@ -198,24 +180,19 @@ class LogStreamConsumer : protected LogStreamConsumerBase, public std::ostream //! class to eliminate the inheritance and instead make the nvinfer1::ILogger implementation a member of the Logger //! object. -class Logger : public nvinfer1::ILogger -{ -public: - Logger(Severity severity = Severity::kWARNING) - : mReportableSeverity(severity) - { - } +class Logger : public nvinfer1::ILogger { + public: + Logger(Severity severity = Severity::kWARNING) : mReportableSeverity(severity) {} //! //! \enum TestResult //! \brief Represents the state of a given test //! - enum class TestResult - { - kRUNNING, //!< The test is running - kPASSED, //!< The test passed - kFAILED, //!< The test failed - kWAIVED //!< The test was waived + enum class TestResult { + kRUNNING, //!< The test is running + kPASSED, //!< The test passed + kFAILED, //!< The test failed + kWAIVED //!< The test was waived }; //! @@ -225,10 +202,7 @@ class Logger : public nvinfer1::ILogger //! TODO Once all samples are updated to use this method to register the logger with TensorRT, //! we can eliminate the inheritance of Logger from ILogger //! - nvinfer1::ILogger& getTRTLogger() - { - return *this; - } + nvinfer1::ILogger& getTRTLogger() { return *this; } //! //! \brief Implementation of the nvinfer1::ILogger::log() virtual method @@ -236,8 +210,7 @@ class Logger : public nvinfer1::ILogger //! Note samples should not be calling this function directly; it will eventually go away once we eliminate the //! inheritance from nvinfer1::ILogger //! - void log(Severity severity, const char* msg) override - { + void log(Severity severity, const char* msg) TRT_NOEXCEPT override { LogStreamConsumer(mReportableSeverity, severity) << "[TRT] " << std::string(msg) << std::endl; } @@ -246,10 +219,7 @@ class Logger : public nvinfer1::ILogger //! //! \param severity The logger will only emit messages that have severity of this level or higher. //! - void setReportableSeverity(Severity severity) - { - mReportableSeverity = severity; - } + void setReportableSeverity(Severity severity) { mReportableSeverity = severity; } //! //! \brief Opaque handle that holds logging information for a particular test @@ -258,20 +228,15 @@ class Logger : public nvinfer1::ILogger //! The sample must call Logger::defineTest() in order to obtain a TestAtom that can be used //! with Logger::reportTest{Start,End}(). //! - class TestAtom - { - public: + class TestAtom { + public: TestAtom(TestAtom&&) = default; - private: + private: friend class Logger; TestAtom(bool started, const std::string& name, const std::string& cmdline) - : mStarted(started) - , mName(name) - , mCmdline(cmdline) - { - } + : mStarted(started), mName(name), mCmdline(cmdline) {} bool mStarted; std::string mName; @@ -289,8 +254,7 @@ class Logger : public nvinfer1::ILogger // //! \return a TestAtom that can be used in Logger::reportTest{Start,End}(). //! - static TestAtom defineTest(const std::string& name, const std::string& cmdline) - { + static TestAtom defineTest(const std::string& name, const std::string& cmdline) { return TestAtom(false, name, cmdline); } @@ -303,8 +267,7 @@ class Logger : public nvinfer1::ILogger //! \param[in] argv The array of command-line arguments (given as C strings) //! //! \return a TestAtom that can be used in Logger::reportTest{Start,End}(). - static TestAtom defineTest(const std::string& name, int argc, char const* const* argv) - { + static TestAtom defineTest(const std::string& name, int argc, char const* const* argv) { auto cmdline = genCmdlineString(argc, argv); return defineTest(name, cmdline); } @@ -316,8 +279,7 @@ class Logger : public nvinfer1::ILogger //! //! \param[in] testAtom The handle to the test that has started //! - static void reportTestStart(TestAtom& testAtom) - { + static void reportTestStart(TestAtom& testAtom) { reportTestResult(testAtom, TestResult::kRUNNING); assert(!testAtom.mStarted); testAtom.mStarted = true; @@ -332,86 +294,85 @@ class Logger : public nvinfer1::ILogger //! \param[in] result The result of the test. Should be one of TestResult::kPASSED, //! TestResult::kFAILED, TestResult::kWAIVED //! - static void reportTestEnd(const TestAtom& testAtom, TestResult result) - { + static void reportTestEnd(const TestAtom& testAtom, TestResult result) { assert(result != TestResult::kRUNNING); assert(testAtom.mStarted); reportTestResult(testAtom, result); } - static int reportPass(const TestAtom& testAtom) - { + static int reportPass(const TestAtom& testAtom) { reportTestEnd(testAtom, TestResult::kPASSED); return EXIT_SUCCESS; } - static int reportFail(const TestAtom& testAtom) - { + static int reportFail(const TestAtom& testAtom) { reportTestEnd(testAtom, TestResult::kFAILED); return EXIT_FAILURE; } - static int reportWaive(const TestAtom& testAtom) - { + static int reportWaive(const TestAtom& testAtom) { reportTestEnd(testAtom, TestResult::kWAIVED); return EXIT_SUCCESS; } - static int reportTest(const TestAtom& testAtom, bool pass) - { + static int reportTest(const TestAtom& testAtom, bool pass) { return pass ? reportPass(testAtom) : reportFail(testAtom); } - Severity getReportableSeverity() const - { - return mReportableSeverity; - } + Severity getReportableSeverity() const { return mReportableSeverity; } -private: + private: //! //! \brief returns an appropriate string for prefixing a log message with the given severity //! - static const char* severityPrefix(Severity severity) - { - switch (severity) - { - case Severity::kINTERNAL_ERROR: return "[F] "; - case Severity::kERROR: return "[E] "; - case Severity::kWARNING: return "[W] "; - case Severity::kINFO: return "[I] "; - case Severity::kVERBOSE: return "[V] "; - default: assert(0); return ""; + static const char* severityPrefix(Severity severity) { + switch (severity) { + case Severity::kINTERNAL_ERROR: + return "[F] "; + case Severity::kERROR: + return "[E] "; + case Severity::kWARNING: + return "[W] "; + case Severity::kINFO: + return "[I] "; + case Severity::kVERBOSE: + return "[V] "; + default: + assert(0); + return ""; } } //! //! \brief returns an appropriate string for prefixing a test result message with the given result //! - static const char* testResultString(TestResult result) - { - switch (result) - { - case TestResult::kRUNNING: return "RUNNING"; - case TestResult::kPASSED: return "PASSED"; - case TestResult::kFAILED: return "FAILED"; - case TestResult::kWAIVED: return "WAIVED"; - default: assert(0); return ""; + static const char* testResultString(TestResult result) { + switch (result) { + case TestResult::kRUNNING: + return "RUNNING"; + case TestResult::kPASSED: + return "PASSED"; + case TestResult::kFAILED: + return "FAILED"; + case TestResult::kWAIVED: + return "WAIVED"; + default: + assert(0); + return ""; } } //! //! \brief returns an appropriate output stream (cout or cerr) to use with the given severity //! - static std::ostream& severityOstream(Severity severity) - { + static std::ostream& severityOstream(Severity severity) { return severity >= Severity::kINFO ? std::cout : std::cerr; } //! //! \brief method that implements logging test results //! - static void reportTestResult(const TestAtom& testAtom, TestResult result) - { + static void reportTestResult(const TestAtom& testAtom, TestResult result) { severityOstream(Severity::kINFO) << "&&&& " << testResultString(result) << " " << testAtom.mName << " # " << testAtom.mCmdline << std::endl; } @@ -419,11 +380,9 @@ class Logger : public nvinfer1::ILogger //! //! \brief generate a command line string from the given (argc, argv) values //! - static std::string genCmdlineString(int argc, char const* const* argv) - { + static std::string genCmdlineString(int argc, char const* const* argv) { std::stringstream ss; - for (int i = 0; i < argc; i++) - { + for (int i = 0; i < argc; i++) { if (i > 0) ss << " "; ss << argv[i]; @@ -434,8 +393,7 @@ class Logger : public nvinfer1::ILogger Severity mReportableSeverity; }; -namespace -{ +namespace { //! //! \brief produces a LogStreamConsumer object that can be used to log messages of severity kVERBOSE @@ -444,8 +402,7 @@ namespace //! //! LOG_VERBOSE(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_VERBOSE(const Logger& logger) -{ +inline LogStreamConsumer LOG_VERBOSE(const Logger& logger) { return LogStreamConsumer(logger.getReportableSeverity(), Severity::kVERBOSE); } @@ -456,8 +413,7 @@ inline LogStreamConsumer LOG_VERBOSE(const Logger& logger) //! //! LOG_INFO(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_INFO(const Logger& logger) -{ +inline LogStreamConsumer LOG_INFO(const Logger& logger) { return LogStreamConsumer(logger.getReportableSeverity(), Severity::kINFO); } @@ -468,8 +424,7 @@ inline LogStreamConsumer LOG_INFO(const Logger& logger) //! //! LOG_WARN(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_WARN(const Logger& logger) -{ +inline LogStreamConsumer LOG_WARN(const Logger& logger) { return LogStreamConsumer(logger.getReportableSeverity(), Severity::kWARNING); } @@ -480,8 +435,7 @@ inline LogStreamConsumer LOG_WARN(const Logger& logger) //! //! LOG_ERROR(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_ERROR(const Logger& logger) -{ +inline LogStreamConsumer LOG_ERROR(const Logger& logger) { return LogStreamConsumer(logger.getReportableSeverity(), Severity::kERROR); } @@ -493,11 +447,10 @@ inline LogStreamConsumer LOG_ERROR(const Logger& logger) //! //! LOG_FATAL(logger) << "hello world" << std::endl; //! -inline LogStreamConsumer LOG_FATAL(const Logger& logger) -{ +inline LogStreamConsumer LOG_FATAL(const Logger& logger) { return LogStreamConsumer(logger.getReportableSeverity(), Severity::kINTERNAL_ERROR); } -} // anonymous namespace +} // anonymous namespace -#endif // TENSORRT_LOGGING_H +#endif // TENSORRT_LOGGING_H diff --git a/resnet/macros.h b/resnet/macros.h new file mode 100644 index 00000000..4f442e6a --- /dev/null +++ b/resnet/macros.h @@ -0,0 +1,35 @@ +#pragma once +#include + +#ifdef API_EXPORTS +#if defined(_MSC_VER) +#define API __declspec(dllexport) +#else +#define API __attribute__((visibility("default"))) +#endif +#else + +#if defined(_MSC_VER) +#define API __declspec(dllimport) +#else +#define API +#endif +#endif // API_EXPORTS + +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) (TRT_VERSION >= TRT_VERSION_ENCODE((major), (minor), (patch), 0)) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) + +#if TRT_VERSION_LT(7, 2, 2) +#error "TensorRT >= 7.2.2 is required for this demo." +#endif + +#if TRT_VERSION_GE(8, 0, 0) +#define TRT_NOEXCEPT noexcept +#define TRT_CONST_ENQUEUE const +#else +#define TRT_NOEXCEPT +#define TRT_CONST_ENQUEUE +#endif diff --git a/resnet/resnet.cpp b/resnet/resnet.cpp new file mode 100644 index 00000000..29133fe3 --- /dev/null +++ b/resnet/resnet.cpp @@ -0,0 +1,514 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "logging.h" +#include "utils.h" + +static constexpr int32_t INPUT_H = 224; +static constexpr int32_t INPUT_W = 224; +static constexpr int32_t OUTPUT_SIZE = 1000; +static constexpr int32_t N = 1; +static constexpr std::size_t WORKSPACE_SIZE = 16 << 20; +static constexpr std::array NAMES = {"data", "prob"}; +static constexpr std::array SIZES = {3 * INPUT_H * INPUT_W, OUTPUT_SIZE}; +static constexpr const char* LABELS_PATH = "assets/imagenet1000_clsidx_to_labels.txt"; +static constexpr bool TRT_PREPROCESS = TRT_VERSION_GE(8, 5, 1); +static constexpr std::array mean = {0.485f, 0.456f, 0.406f}; +static constexpr std::array stdv = {0.229f, 0.224f, 0.225f}; + +using namespace nvinfer1; +using WeightMap = std::map; +using M = nvinfer1::MatrixOperation; +using NDCF = nvinfer1::NetworkDefinitionCreationFlag; + +static Logger gLogger; + +struct ResNetVariant { + const char* name; + const char* wts_path; + const char* engine_path; + std::array layers; + bool bottleneck; + int32_t groups; + int32_t width_per_group; +}; + +static auto getVariantConfig(const std::string& name) -> ResNetVariant { + if (name == "resnet18") { + return {"resnet18", "models/resnet18.wts", "models/resnet18.engine", {2, 2, 2, 2}, false, 1, 64}; + } + if (name == "resnet34") { + return {"resnet34", "models/resnet34.wts", "models/resnet34.engine", {3, 4, 6, 3}, false, 1, 64}; + } + if (name == "resnet50") { + return {"resnet50", "models/resnet50.wts", "models/resnet50.engine", {3, 4, 6, 3}, true, 1, 64}; + } + if (name == "resnext50_32x4d") { + return {"resnext50_32x4d", + "models/resnext50_32x4d.wts", + "models/resnext50_32x4d.engine", + {3, 4, 6, 3}, + true, + 32, + 4}; + } + if (name == "wide_resnet50_2") { + return {"wide_resnet50_2", + "models/wide_resnet50_2.wts", + "models/wide_resnet50_2.engine", + {3, 4, 6, 3}, + true, + 1, + 128}; + } + std::cerr << "unsupported resnet variant: " << name << "\n"; + std::cerr << "model: resnet18 | resnet34 | resnet50 | resnext50_32x4d | wide_resnet50_2\n"; + std::abort(); +} + +static auto addBatchNorm2d(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, const std::string& lname, + float eps) -> IScaleLayer* { + const auto* gamma = static_cast(weight_map[lname + ".weight"].values); + const auto* beta = static_cast(weight_map[lname + ".bias"].values); + const auto* mean_ptr = static_cast(weight_map[lname + ".running_mean"].values); + const auto* var = static_cast(weight_map[lname + ".running_var"].values); + const auto len = weight_map[lname + ".running_var"].count; + + auto* scval = static_cast(std::malloc(sizeof(float) * static_cast(len))); + auto* shval = static_cast(std::malloc(sizeof(float) * static_cast(len))); + auto* pval = static_cast(std::malloc(sizeof(float) * static_cast(len))); + for (int64_t i = 0; i < len; ++i) { + scval[i] = gamma[i] / std::sqrt(var[i] + eps); + shval[i] = beta[i] - mean_ptr[i] * gamma[i] / std::sqrt(var[i] + eps); + pval[i] = 1.0f; + } + + Weights scale{DataType::kFLOAT, scval, len}; + Weights shift{DataType::kFLOAT, shval, len}; + Weights power{DataType::kFLOAT, pval, len}; + weight_map[lname + ".scale"] = scale; + weight_map[lname + ".shift"] = shift; + weight_map[lname + ".power"] = power; + + auto* layer = network->addScale(input, ScaleMode::kCHANNEL, shift, scale, power); + assert(layer); +#if TRT_VERSION_GE(8, 0, 0) + layer->setChannelAxis(1); +#endif + return layer; +} + +static auto addConv2d(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, const std::string& name, + int32_t out_channels, int32_t kernel, int32_t stride, int32_t padding, + int32_t groups = 1) -> IConvolutionLayer* { + static constexpr Weights empty{DataType::kFLOAT, nullptr, 0}; + auto* conv = + network->addConvolutionNd(input, out_channels, DimsHW{kernel, kernel}, weight_map[name + ".weight"], empty); + assert(conv); + conv->setStrideNd(DimsHW{stride, stride}); + conv->setPaddingNd(DimsHW{padding, padding}); + conv->setNbGroups(groups); + conv->setName(name.c_str()); + return conv; +} + +static auto addRelu(INetworkDefinition* network, ITensor& input) -> IActivationLayer* { + auto* relu = network->addActivation(input, ActivationType::kRELU); + assert(relu); + return relu; +} + +static auto addBasicBlock(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, int32_t in_channels, + int32_t planes, int32_t stride, const std::string& lname) -> ITensor* { + auto* conv1 = addConv2d(network, weight_map, input, lname + "conv1", planes, 3, stride, 1); + auto* bn1 = addBatchNorm2d(network, weight_map, *conv1->getOutput(0), lname + "bn1", 1e-5f); + auto* relu1 = addRelu(network, *bn1->getOutput(0)); + + auto* conv2 = addConv2d(network, weight_map, *relu1->getOutput(0), lname + "conv2", planes, 3, 1, 1); + auto* bn2 = addBatchNorm2d(network, weight_map, *conv2->getOutput(0), lname + "bn2", 1e-5f); + + ITensor* shortcut = &input; + if (stride != 1 || in_channels != planes) { + auto* conv3 = addConv2d(network, weight_map, input, lname + "downsample.0", planes, 1, stride, 0); + auto* bn3 = addBatchNorm2d(network, weight_map, *conv3->getOutput(0), lname + "downsample.1", 1e-5f); + shortcut = bn3->getOutput(0); + } + + auto* sum = network->addElementWise(*shortcut, *bn2->getOutput(0), ElementWiseOperation::kSUM); + assert(sum); + return addRelu(network, *sum->getOutput(0))->getOutput(0); +} + +static auto addBottleneck(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, int32_t in_channels, + int32_t planes, int32_t stride, int32_t groups, int32_t width_per_group, + const std::string& lname) -> ITensor* { + constexpr int32_t expansion = 4; + const int32_t width = planes * width_per_group / 64 * groups; + const int32_t out_channels = planes * expansion; + + auto* conv1 = addConv2d(network, weight_map, input, lname + "conv1", width, 1, 1, 0); + auto* bn1 = addBatchNorm2d(network, weight_map, *conv1->getOutput(0), lname + "bn1", 1e-5f); + auto* relu1 = addRelu(network, *bn1->getOutput(0)); + + auto* conv2 = addConv2d(network, weight_map, *relu1->getOutput(0), lname + "conv2", width, 3, stride, 1, groups); + auto* bn2 = addBatchNorm2d(network, weight_map, *conv2->getOutput(0), lname + "bn2", 1e-5f); + auto* relu2 = addRelu(network, *bn2->getOutput(0)); + + auto* conv3 = addConv2d(network, weight_map, *relu2->getOutput(0), lname + "conv3", out_channels, 1, 1, 0); + auto* bn3 = addBatchNorm2d(network, weight_map, *conv3->getOutput(0), lname + "bn3", 1e-5f); + + ITensor* shortcut = &input; + if (stride != 1 || in_channels != out_channels) { + auto* conv4 = addConv2d(network, weight_map, input, lname + "downsample.0", out_channels, 1, stride, 0); + auto* bn4 = addBatchNorm2d(network, weight_map, *conv4->getOutput(0), lname + "downsample.1", 1e-5f); + shortcut = bn4->getOutput(0); + } + + auto* sum = network->addElementWise(*shortcut, *bn3->getOutput(0), ElementWiseOperation::kSUM); + assert(sum); + return addRelu(network, *sum->getOutput(0))->getOutput(0); +} + +static auto addLayer(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, int32_t& in_channels, + const ResNetVariant& variant, int32_t layer_index, int32_t planes, int32_t blocks, + int32_t stride) -> ITensor* { + ITensor* x = &input; + for (int32_t i = 0; i < blocks; ++i) { + const int32_t block_stride = i == 0 ? stride : 1; + const std::string lname = "layer" + std::to_string(layer_index) + "." + std::to_string(i) + "."; + if (variant.bottleneck) { + x = addBottleneck(network, weight_map, *x, in_channels, planes, block_stride, variant.groups, + variant.width_per_group, lname); + in_channels = planes * 4; + } else { + x = addBasicBlock(network, weight_map, *x, in_channels, planes, block_stride, lname); + in_channels = planes; + } + } + return x; +} + +static auto addLinear(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, const std::string& lname, + int32_t in_features, int32_t out_features) -> ITensor* { + auto* reshape = network->addShuffle(input); + assert(reshape); + reshape->setReshapeDimensions(Dims2{N, in_features}); + + auto* kernel = network->addConstant(Dims2{out_features, in_features}, weight_map[lname + ".weight"]); + assert(kernel); + auto* matmul = network->addMatrixMultiply(*reshape->getOutput(0), M::kNONE, *kernel->getOutput(0), M::kTRANSPOSE); + assert(matmul); + + auto* bias = network->addConstant(Dims2{1, out_features}, weight_map[lname + ".bias"]); + assert(bias); + auto* sum = network->addElementWise(*matmul->getOutput(0), *bias->getOutput(0), ElementWiseOperation::kSUM); + assert(sum); + return sum->getOutput(0); +} + +static auto buildResNet(INetworkDefinition* network, WeightMap& weight_map, ITensor& input, + const ResNetVariant& variant) -> ITensor* { + auto* conv1 = addConv2d(network, weight_map, input, "conv1", 64, 7, 2, 3); + auto* bn1 = addBatchNorm2d(network, weight_map, *conv1->getOutput(0), "bn1", 1e-5f); + auto* relu1 = addRelu(network, *bn1->getOutput(0)); + + auto* pool1 = network->addPoolingNd(*relu1->getOutput(0), PoolingType::kMAX, DimsHW{3, 3}); + assert(pool1); + pool1->setStrideNd(DimsHW{2, 2}); + pool1->setPaddingNd(DimsHW{1, 1}); + + int32_t in_channels = 64; + ITensor* x = addLayer(network, weight_map, *pool1->getOutput(0), in_channels, variant, 1, 64, variant.layers[0], 1); + x = addLayer(network, weight_map, *x, in_channels, variant, 2, 128, variant.layers[1], 2); + x = addLayer(network, weight_map, *x, in_channels, variant, 3, 256, variant.layers[2], 2); + x = addLayer(network, weight_map, *x, in_channels, variant, 4, 512, variant.layers[3], 2); + + auto* pool2 = network->addPoolingNd(*x, PoolingType::kAVERAGE, DimsHW{7, 7}); + assert(pool2); + pool2->setStrideNd(DimsHW{1, 1}); + return addLinear(network, weight_map, *pool2->getOutput(0), "fc", in_channels, OUTPUT_SIZE); +} + +static auto createEngine(int32_t batch_size, IRuntime* runtime, IBuilder* builder, IBuilderConfig* config, DataType dt, + const ResNetVariant& variant) -> ICudaEngine* { + WeightMap weight_map = loadWeights(variant.wts_path); + +#if TRT_VERSION_GE(10, 12, 0) + auto flag = 1U << static_cast(NDCF::kSTRONGLY_TYPED); +#elif TRT_VERSION_GE(10, 0, 0) + auto flag = 0U; +#else + auto flag = 1U << static_cast(NDCF::kEXPLICIT_BATCH); +#endif + auto* network = builder->createNetworkV2(flag); + assert(network); + + ITensor* input = nullptr; + if constexpr (TRT_PREPROCESS) { + dt = DataType::kUINT8; + input = network->addInput(NAMES[0], dt, Dims4{batch_size, INPUT_H, INPUT_W, 3}); + auto* transform = addTransformLayer(network, *input, true, mean, stdv); + input = transform->getOutput(0); + } else { + input = network->addInput(NAMES[0], dt, Dims4{batch_size, 3, INPUT_H, INPUT_W}); + } + assert(input); + + ITensor* logits = buildResNet(network, weight_map, *input, variant); + logits->setName(NAMES[1]); + network->markOutput(*logits); + +#if TRT_VERSION_GE(8, 0, 0) + config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); + IHostMemory* mem = builder->buildSerializedNetwork(*network, *config); + assert(mem); + ICudaEngine* engine = runtime->deserializeCudaEngine(mem->data(), mem->size()); + delete mem; + delete network; +#else + builder->setMaxBatchSize(batch_size); + config->setMaxWorkspaceSize(WORKSPACE_SIZE); + ICudaEngine* engine = builder->buildEngineWithConfig(*network, *config); + network->destroy(); +#endif + std::cout << "build finished\n"; + + for (auto& mem : weight_map) { + std::free(const_cast(mem.second.values)); + } + return engine; +} + +static void APIToModel(int32_t batch_size, IRuntime* runtime, IHostMemory** model_stream, + const ResNetVariant& variant) { + IBuilder* builder = createInferBuilder(gLogger); + IBuilderConfig* config = builder->createBuilderConfig(); + ICudaEngine* engine = createEngine(batch_size, runtime, builder, config, DataType::kFLOAT, variant); + assert(engine != nullptr); + + (*model_stream) = engine->serialize(); + +#if TRT_VERSION_GE(8, 0, 0) + delete engine; + delete config; + delete builder; +#else + engine->destroy(); + config->destroy(); + builder->destroy(); +#endif +} + +static auto doInference(IExecutionContext& context, void* input, + int64_t batch_size) -> std::vector> { + ICudaEngine const& engine = context.getEngine(); + cudaStream_t stream; + CHECK(cudaStreamCreate(&stream)); + std::vector buffers; + +#if TRT_VERSION_GE(8, 0, 0) + const int32_t nIO = engine.getNbIOTensors(); +#else + const int32_t nIO = engine.getNbBindings(); +#endif + + buffers.resize(nIO); + for (auto i = 0; i < nIO; ++i) { + std::size_t size = 0; +#if TRT_VERSION_GE(8, 0, 0) + auto* tensor_name = engine.getIOTensorName(i); + const std::string name = tensor_name; + auto element_size = getSize(engine.getTensorDataType(tensor_name)); + size = element_size * static_cast(batch_size) * (name == NAMES[0] ? SIZES[0] : SIZES[1]); + CHECK(cudaMalloc(&buffers[i], size)); + if (name == NAMES[0]) { + CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); + } + if (!context.setTensorAddress(tensor_name, buffers[i])) { + std::cerr << "setTensorAddress failed\n"; + std::abort(); + } +#else + const int32_t idx = engine.getBindingIndex(NAMES[i]); + auto element_size = getSize(engine.getBindingDataType(idx)); + assert(idx == i); + size = element_size * static_cast(batch_size) * SIZES[i]; + CHECK(cudaMalloc(&buffers[i], size)); + if (i == 0) { + CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); + } +#endif + } + +#if TRT_VERSION_GE(8, 0, 0) + if (!context.enqueueV3(stream)) { + std::cerr << "enqueueV3 failed\n"; + std::abort(); + } +#else + if (!context.enqueueV2(buffers.data(), stream, nullptr)) { + std::cerr << "enqueueV2 failed\n"; + std::abort(); + } +#endif + + std::vector> prob; + for (int32_t i = 0; i < nIO; ++i) { +#if TRT_VERSION_GE(8, 0, 0) + const std::string name = engine.getIOTensorName(i); + if (name == NAMES[0]) { + continue; + } + constexpr auto output_size = SIZES[1]; +#else + if (i == 0) { + continue; + } + const auto output_size = SIZES[i]; +#endif + std::vector tmp(static_cast(batch_size) * output_size, std::nanf("")); + const auto size = static_cast(batch_size) * output_size * sizeof(float); + CHECK(cudaMemcpyAsync(tmp.data(), buffers[i], size, cudaMemcpyDeviceToHost, stream)); + prob.emplace_back(std::move(tmp)); + } + CHECK(cudaStreamSynchronize(stream)); + + for (auto& buffer : buffers) { + CHECK(cudaFree(buffer)); + } + CHECK(cudaStreamDestroy(stream)); + return prob; +} + +auto main(int argc, char** argv) -> int { + checkTrtEnv(); + if (argc < 2 || argc > 3) { + std::cerr << "arguments not right!\n"; + std::cerr << "./resnet -s [model] // serialize model to plan file\n"; + std::cerr << "./resnet -d [model] // deserialize plan file and run inference\n"; + std::cerr << "model: resnet18 | resnet34 | resnet50 | resnext50_32x4d | wide_resnet50_2\n"; + return -1; + } + + const auto variant = getVariantConfig(argc == 3 ? argv[2] : "resnet18"); + std::cout << "Using ResNet variant: " << variant.name << "\n"; + + IRuntime* runtime = createInferRuntime(gLogger); + assert(runtime != nullptr); + char* trt_model_stream{nullptr}; + std::streamsize size{0}; + + if (std::string(argv[1]) == "-s") { + IHostMemory* model_stream{nullptr}; + APIToModel(N, runtime, &model_stream, variant); + assert(model_stream != nullptr); + + std::ofstream plan(variant.engine_path, std::ios::binary | std::ios::trunc); + if (!plan) { + std::cerr << "could not open plan output file\n"; + return -1; + } + if (model_stream->size() > static_cast(std::numeric_limits::max())) { + std::cerr << "this model is too large to serialize\n"; + return -1; + } + const auto* data_ptr = reinterpret_cast(model_stream->data()); + const auto data_size = static_cast(model_stream->size()); + plan.write(data_ptr, data_size); +#if TRT_VERSION_GE(8, 0, 0) + delete model_stream; + delete runtime; +#else + model_stream->destroy(); + runtime->destroy(); +#endif + return 0; + } + if (std::string(argv[1]) == "-d") { + std::ifstream file(variant.engine_path, std::ios::binary); + if (file.good()) { + file.seekg(0, file.end); + size = file.tellg(); + file.seekg(0, file.beg); + trt_model_stream = new char[size]; + assert(trt_model_stream); + file.read(trt_model_stream, size); + file.close(); + } + } else { + return -1; + } + +#if TRT_VERSION_GE(8, 0, 0) + ICudaEngine* engine = runtime->deserializeCudaEngine(trt_model_stream, size); +#else + ICudaEngine* engine = runtime->deserializeCudaEngine(trt_model_stream, size, nullptr); +#endif + assert(engine != nullptr); + IExecutionContext* context = engine->createExecutionContext(); + assert(context != nullptr); + delete[] trt_model_stream; + + void* input = nullptr; + std::vector flat_img; + cv::Mat img = cv::imread("assets/cats.jpg", cv::IMREAD_COLOR); + if (img.empty()) { + std::cerr << "failed to read image: assets/cats.jpg\n"; + return -1; + } + if constexpr (TRT_PREPROCESS) { + cv::resize(img, img, cv::Size(INPUT_W, INPUT_H), 0, 0, cv::INTER_LINEAR); + input = static_cast(img.data); + } else { + flat_img = preprocess_img(img, true, mean, stdv, N, INPUT_H, INPUT_W); + input = flat_img.data(); + } + + auto first_prob = doInference(*context, input, N); + printFirstOutputs("resnet", first_prob[0].data(), first_prob[0].size()); + std::cout << "prediction result:\n"; + auto labels = loadImagenetLabelMap(LABELS_PATH); + if (labels.empty()) { + std::cerr << "failed to load labels from " << LABELS_PATH << "\n"; + std::abort(); + } + int32_t top = 0; + for (auto& [idx, logits] : topk(first_prob[0], 3)) { + std::cout << "Top: " << top++ << " idx: " << idx << ", logits: " << std::setprecision(4) << logits + << ", label: " << labels[idx] << "\n"; + } + + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int32_t i = 0; i < kBenchmarkRuns; ++i) { + auto start = std::chrono::steady_clock::now(); + (void)doInference(*context, input, N); + auto end = std::chrono::steady_clock::now(); + auto period = std::chrono::duration_cast(end - start); + latencies.push_back(static_cast(period.count()) / 1000.0); + } + printBenchmark("resnet", latencies, N); +#if TRT_VERSION_GE(8, 0, 0) + delete context; + delete engine; + delete runtime; +#else + context->destroy(); + engine->destroy(); + runtime->destroy(); +#endif + + return 0; +} diff --git a/resnet/utils.h b/resnet/utils.h new file mode 100644 index 00000000..4f111957 --- /dev/null +++ b/resnet/utils.h @@ -0,0 +1,311 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "macros.h" + +using namespace nvinfer1; + +#define CHECK(status) \ + do { \ + auto ret = (status); \ + if (ret != cudaSuccess) { \ + std::cerr << "Cuda failure: " << ret << "\n"; \ + std::abort(); \ + } \ + } while (0) + +static void checkTrtEnv(int device = 0) { +#if TRT_VERSION_LT(8, 0, 0) + CHECK(cudaGetDevice(&device)); + cudaDeviceProp prop{}; + CHECK(cudaGetDeviceProperties(&prop, device)); + const int sm = prop.major * 10 + prop.minor; + if (sm > 86) { + std::cerr << "TensorRT < 8 does not support SM > 86 on this GPU."; + std::abort(); + } +#endif +} + +template +static auto toI32(T value) -> int32_t { + static_assert(std::is_integral::value, "toI32 requires an integral type."); + static_assert(sizeof(T) > sizeof(int32_t), "toI32 is only for values wider than int32_t."); + if constexpr (std::is_signed::value) { + assert(value >= static_cast(std::numeric_limits::min())); + } + assert(value <= static_cast(std::numeric_limits::max())); + return static_cast(value); +} + +/** + * @brief TensorRT weight files have a simple space delimited format: + * [type] [size] + * + * @param file input weight file path + * @return std::map + */ +static std::map loadWeights(const std::string& file) { + std::cout << "Loading weights: " << file << "\n"; + std::map weightMap; + + // Open weights file + std::ifstream input(file); + assert(input.is_open() && "Unable to load weight file."); + + // Read number of weight blobs + int32_t count; + input >> count; + assert(count > 0 && "Invalid weight map file."); + + while (count--) { + nvinfer1::Weights wt{nvinfer1::DataType::kFLOAT, nullptr, 0}; + + // Read name and type of blob + std::string name; + input >> name >> std::dec >> wt.count; + + // Load blob + auto* val = static_cast(std::malloc(sizeof(uint32_t) * static_cast(wt.count))); + if (val == nullptr) { + std::cerr << "weight allocation failed\n"; + std::abort(); + } + input >> std::hex; + for (auto x = 0ll; x < wt.count; ++x) { + input >> val[x]; + } + wt.values = val; + weightMap[name] = wt; + } + + return weightMap; +} + +/** + * @brief a preprocess function aligning with ImageNet preprocess in torchvision, only support 3-channel image + * + * @param img opencv image with BGR layout + * @param bgr2rgb whether to convert BGR to RGB + * @param mean subtract mean + * @param std divide std + * @param n batch size + * @param h resize height + * @param w resize width + * @return std::vector contiguous flatten image data in float32 type + */ +static std::vector preprocess_img(cv::Mat& img, bool bgr2rgb, const std::array& mean, + const std::array& std, int n, int h, int w) { + const auto c = img.channels(); + const auto size = c * h * w; + if (c != 3) { + std::cerr << "this demo only supports 3 channel input image.\n"; + std::abort(); + } + if (bgr2rgb) { + cv::cvtColor(img, img, cv::COLOR_BGR2RGB); + } + cv::resize(img, img, cv::Size(w, h), 0, 0, cv::INTER_LINEAR); + img.convertTo(img, CV_32FC3, 1.f / 255); + img = (img - cv::Scalar(mean[0], mean[1], mean[2])) / cv::Scalar(std[0], std[1], std[2]); + std::vector chw(static_cast(n) * c * h * w, 0.f); + + // fill all batch with the same input image + for (int i = 0; i < n; ++i) { + for (int y = 0; y < h; ++y) { + for (int x = 0; x < w; ++x) { + const cv::Vec3f v = img.at(y, x); + chw[i * size + 0 * h * w + y * w + x] = v[0]; + chw[i * size + 1 * h * w + y * w + x] = v[1]; + chw[i * size + 2 * h * w + y * w + x] = v[2]; + } + } + } + return chw; +} + +static auto topk(const std::vector& v, int k) -> std::vector> { + if (k <= 0) + return {}; + auto stride = std::min(k, static_cast(v.size())); + + std::vector idx(v.size()); + std::iota(idx.begin(), idx.end(), 0); + + std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) { return v[a] > v[b]; }); + + std::vector> out; + out.reserve(stride); + for (auto i = 0; i < stride; ++i) + out.emplace_back(idx[i], v[idx[i]]); + return out; +} + +static std::map loadImagenetLabelMap(const std::string& path) { + std::map labels; + std::ifstream in(path); + if (!in.is_open()) { + return labels; + } + std::string line; + while (std::getline(in, line)) { + auto colon = line.find(':'); + if (colon == std::string::npos) { + continue; + } + auto first_quote = line.find('\'', colon); + if (first_quote == std::string::npos) { + continue; + } + auto second_quote = line.find('\'', first_quote + 1); + if (second_quote == std::string::npos) { + continue; + } + int idx = std::stoi(line.substr(0, colon)); + labels[idx] = line.substr(first_quote + 1, second_quote - first_quote - 1); + } + return labels; +} + +static ILayer* addTransformLayer(INetworkDefinition* network, ITensor& input, bool bgr2rgb, + const std::array& mean, const std::array& std) { + struct ScaleParams { + std::array shift; + std::array scale; + }; + static std::vector> gScaleParams; + auto params = std::make_unique(); + params->shift = {-mean[0] / std[0], -mean[1] / std[1], -mean[2] / std[2]}; + params->scale = {1.f / (std[0] * 255.f), 1.f / (std[1] * 255.f), 1.f / (std[2] * 255.f)}; + + static const Weights empty{DataType::kFLOAT, nullptr, 0ll}; + const Weights shift{DataType::kFLOAT, params->shift.data(), 3ll}; + const Weights scale{DataType::kFLOAT, params->scale.data(), 3ll}; + + gScaleParams.emplace_back(std::move(params)); + + ITensor* in = &input; + if (input.getType() != DataType::kFLOAT) { +#if TRT_VERSION_GE(8, 0, 0) + auto* cast = network->addCast(input, DataType::kFLOAT); + assert(cast); + cast->setName("Cast to FP32"); + in = cast->getOutput(0); +#else + auto* identity = network->addIdentity(input); + assert(identity); + identity->setName("Convert to FP32"); + identity->setOutputType(0, DataType::kFLOAT); + in = identity->getOutput(0); +#endif + } + // Convert from NHWC to NCHW + auto* perm = network->addShuffle(*in); + assert(perm); + perm->setName("NHWC -> NCHW"); + perm->setFirstTranspose(Permutation{0, 3, 1, 2}); + + // Convert from BGR to RGB (optional) + ITensor* data{nullptr}; + if (bgr2rgb) { + auto add_slice = [&](int c, const char* name) -> ITensor* { + auto dims = perm->getOutput(0)->getDimensions(); + Dims4 start = {0, c, 0, 0}, stride = {1, 1, 1, 1}; + Dims4 size = {dims.d[0], 1, dims.d[2], dims.d[3]}; + auto* _slice = network->addSlice(*perm->getOutput(0), start, size, stride); + _slice->setName(name); + assert(_slice && _slice->getNbOutputs() == 1); + return _slice->getOutput(0); + }; + std::array channels = {add_slice(2, "R"), add_slice(1, "G"), add_slice(0, "B")}; + auto* cat = network->addConcatenation(channels.data(), 3); + assert(cat); + cat->setName("RGB"); + cat->setAxis(1); + data = cat->getOutput(0); + } else { + data = perm->getOutput(0); + } + + // Normalize + auto* trans = network->addScale(*data, ScaleMode::kCHANNEL, shift, scale, empty); + assert(trans); + trans->setName("mean & std"); +#if TRT_VERSION_GE(8, 0, 0) + trans->setChannelAxis(1); +#endif + return trans; +} + +static size_t getSize(DataType dt) { + switch (dt) { +#if TRT_VERSION_GE(8, 5, 1) + case DataType::kUINT8: +#endif + case DataType::kINT8: + return sizeof(int8_t); + case DataType::kFLOAT: + return sizeof(float); + case DataType::kHALF: + return sizeof(int16_t); + case DataType::kINT32: + return sizeof(int32_t); + default: { + std::cerr << "Unsupported data type\n"; + std::abort(); + } + } +} + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} From 0be91b8ea1a8b28ab7965c5a87b77650176f35af Mon Sep 17 00:00:00 2001 From: zjq Date: Sun, 14 Jun 2026 16:18:43 +0800 Subject: [PATCH 7/7] Fix ViT allocator and TensorRT compatibility --- vit/CMakeLists.txt | 2 +- vit/FindTensorRT.cmake | 8 ++- vit/cuda_allocator.cc | 28 ++++++--- vit/cuda_allocator.h | 4 +- vit/macros.h | 13 ++-- vit/utils.h | 43 ++++++++++++- vit/vit.cc | 140 ++++++++++++++++++++++++----------------- 7 files changed, 161 insertions(+), 77 deletions(-) diff --git a/vit/CMakeLists.txt b/vit/CMakeLists.txt index 2a3676c8..b0ed0da3 100644 --- a/vit/CMakeLists.txt +++ b/vit/CMakeLists.txt @@ -11,7 +11,7 @@ endif() set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CUDA_STANDARD 17) +set(CMAKE_CUDA_STANDARD 20) set(CMAKE_CUDA_STANDARD_REQUIRED ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_INCLUDE_CURRENT_DIR TRUE) diff --git a/vit/FindTensorRT.cmake b/vit/FindTensorRT.cmake index f529daa2..869ad16e 100644 --- a/vit/FindTensorRT.cmake +++ b/vit/FindTensorRT.cmake @@ -70,8 +70,11 @@ if(WIN32) endif() if(${TRT_MAJOR_VERSION} GREATER_EQUAL 10) - set(_modules nvinfer_10 nvinfer_plugin_10 nvinfer_vc_plugin_10 - nvinfer_dispatch_10 nvinfer_lean_10) + set(_trt_lib_suffix "_${TRT_MAJOR_VERSION}") + set(_modules nvinfer${_trt_lib_suffix} nvinfer_plugin${_trt_lib_suffix} + nvinfer_vc_plugin${_trt_lib_suffix} + nvinfer_dispatch${_trt_lib_suffix} + nvinfer_lean${_trt_lib_suffix}) message(DEBUG "Using ${_modules}") else() set(_modules nvinfer nvinfer_plugin nvinfer_vc_plugin nvinfer_dispatch @@ -137,6 +140,7 @@ set_target_properties( unset(TRT_MAJOR_VERSION) unset(_modules) +unset(_trt_lib_suffix) unset(_trt_include_candidates) unset(_trt_library_candidates) unset(_trt_arch) diff --git a/vit/cuda_allocator.cc b/vit/cuda_allocator.cc index 2d46469a..3f06aa08 100644 --- a/vit/cuda_allocator.cc +++ b/vit/cuda_allocator.cc @@ -16,6 +16,7 @@ struct CudaOutputAllocator::Allocation { void* ptr{nullptr}; std::size_t size{0}; OutputAllocKind kind{OutputAllocKind::kCudaMallocManaged}; + cudaStream_t stream{}; CUmemGenericAllocationHandle handle{}; CUdeviceptr addr{}; std::size_t mapped_size{0}; @@ -61,7 +62,7 @@ CudaOutputAllocator::~CudaOutputAllocator() { } } -#if TRT_VERSION < 10000 +#if TRT_VERSION_LT(10, 0, 0) // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) void* CudaOutputAllocator::reallocateOutput(const char* tensorName, void* currentMemory, uint64_t size, uint64_t alignment) TRT_NOEXCEPT { @@ -80,7 +81,7 @@ void* CudaOutputAllocator::reallocateOutput(const char* tensorName, void* curren return currentMemory; } - Allocation fresh = allocate(static_cast(size)); + Allocation fresh = allocate(static_cast(size), stream_); if (!fresh.ptr) { return nullptr; } @@ -98,10 +99,12 @@ void* CudaOutputAllocator::reallocateOutputAsync(const char* tensorName, void* c if (stream == nullptr) { stream = stream_; } - stream_ = stream; std::lock_guard lock(mutex_); auto& alloc = allocations_[tensorName]; if (alloc.ptr && size <= alloc.size) { + if (alloc.kind == OutputAllocKind::kCudaMallocAsync) { + alloc.stream = stream; + } return alloc.ptr; } if (alloc.ptr) { @@ -110,7 +113,7 @@ void* CudaOutputAllocator::reallocateOutputAsync(const char* tensorName, void* c return currentMemory; } - Allocation fresh = allocate(static_cast(size)); + Allocation fresh = allocate(static_cast(size), stream); if (!fresh.ptr) { return nullptr; } @@ -121,19 +124,26 @@ void* CudaOutputAllocator::reallocateOutputAsync(const char* tensorName, void* c void CudaOutputAllocator::notifyShape(const char* /*tensorName*/, nvinfer1::Dims const& /*dims*/) TRT_NOEXCEPT {} -CudaOutputAllocator::Allocation CudaOutputAllocator::allocate(std::size_t size) { +CudaOutputAllocator::Allocation CudaOutputAllocator::allocate(std::size_t size, cudaStream_t stream) { Allocation alloc{}; if (size == 0) { return alloc; } + if (stream == nullptr) { + stream = stream_; + } + if (cudaSetDevice(device_) != cudaSuccess) { + return alloc; + } if (kind_ == OutputAllocKind::kCudaMallocAsync) { void* ptr = nullptr; - if (cudaMallocAsync(&ptr, size, stream_) != cudaSuccess) { + if (cudaMallocAsync(&ptr, size, stream) != cudaSuccess) { return alloc; } alloc.ptr = ptr; alloc.size = size; alloc.kind = OutputAllocKind::kCudaMallocAsync; + alloc.stream = stream; return alloc; } if (kind_ == OutputAllocKind::kCudaMallocManaged) { @@ -147,9 +157,6 @@ CudaOutputAllocator::Allocation CudaOutputAllocator::allocate(std::size_t size) return alloc; } - if (cudaSetDevice(device_) != cudaSuccess) { - return alloc; - } if (cuInit(0) != CUDA_SUCCESS) { return alloc; } @@ -205,8 +212,9 @@ void CudaOutputAllocator::release(const std::string& /*tensorName*/, Allocation& if (!alloc.ptr) { return; } + CHECK(cudaSetDevice(device_)); if (alloc.kind == OutputAllocKind::kCudaMallocAsync) { - cudaFreeAsync(alloc.ptr, stream_); + cudaFreeAsync(alloc.ptr, alloc.stream); } else if (alloc.kind == OutputAllocKind::kCudaMallocManaged) { cudaFree(alloc.ptr); } else if (alloc.kind == OutputAllocKind::kCuMem) { diff --git a/vit/cuda_allocator.h b/vit/cuda_allocator.h index dcd6c09e..c5e4bc2e 100644 --- a/vit/cuda_allocator.h +++ b/vit/cuda_allocator.h @@ -17,7 +17,7 @@ class CudaOutputAllocator final : public nvinfer1::IOutputAllocator { explicit CudaOutputAllocator(cudaStream_t stream, OutputAllocKind kind, int device = 0); ~CudaOutputAllocator() override; -#if TRT_VERSION < 10000 +#if TRT_VERSION_LT(10, 0, 0) void* reallocateOutput(const char* tensorName, void* currentMemory, uint64_t size, uint64_t alignment) TRT_NOEXCEPT override; #else @@ -32,7 +32,7 @@ class CudaOutputAllocator final : public nvinfer1::IOutputAllocator { private: struct Allocation; - Allocation allocate(std::size_t size); + Allocation allocate(std::size_t size, cudaStream_t stream); void release(const std::string& tensorName, Allocation& alloc); cudaStream_t stream_{}; diff --git a/vit/macros.h b/vit/macros.h index daae7bfb..6d851752 100644 --- a/vit/macros.h +++ b/vit/macros.h @@ -16,14 +16,19 @@ #endif #endif // API_EXPORTS -#define TRT_VERSION \ - ((NV_TENSORRT_MAJOR * 1000) + (NV_TENSORRT_MINOR * 100) + (NV_TENSORRT_PATCH * 10) + NV_TENSORRT_BUILD) +#define TRT_VERSION_ENCODE(major, minor, patch, build) \ + (((major) * 1000000) + ((minor) * 10000) + ((patch) * 100) + (build)) +#define TRT_VERSION TRT_VERSION_ENCODE(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH, NV_TENSORRT_BUILD) +#define TRT_VERSION_GE(major, minor, patch) \ + ((NV_TENSORRT_MAJOR > (major)) || (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR > (minor)) || \ + (NV_TENSORRT_MAJOR == (major) && NV_TENSORRT_MINOR == (minor) && NV_TENSORRT_PATCH >= (patch))) +#define TRT_VERSION_LT(major, minor, patch) (!TRT_VERSION_GE((major), (minor), (patch))) -#if TRT_VERSION < 7220 +#if TRT_VERSION_LT(7, 2, 2) #error "TensorRT >= 7.2.2 is required for this demo." #endif -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) #define TRT_NOEXCEPT noexcept #define TRT_CONST_ENQUEUE const #else diff --git a/vit/utils.h b/vit/utils.h index 6a86347c..92ba2a87 100644 --- a/vit/utils.h +++ b/vit/utils.h @@ -4,7 +4,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -25,7 +27,7 @@ namespace { } while (0) static void checkTrtEnv(int device = 0) { -#if TRT_VERSION < 8000 +#if TRT_VERSION_LT(8, 0, 0) CHECK(cudaGetDevice(&device)); cudaDeviceProp prop{}; CHECK(cudaGetDeviceProperties(&prop, device)); @@ -162,4 +164,43 @@ static auto loadImagenetLabelMap(const std::string& path) { } return labels; } + +static constexpr int32_t kBenchmarkRuns = 200; +static constexpr std::size_t kMaxFirstOutputs = 10; + +inline auto percentile(const std::vector& sorted, double percent) -> double { + assert(!sorted.empty()); + const double rank = percent / 100.0 * static_cast(sorted.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, sorted.size() - 1); + if (lower == upper) { + return sorted[lower]; + } + const double weight = rank - static_cast(lower); + return sorted[lower] * (1.0 - weight) + sorted[upper] * weight; +} + +inline void printBenchmark(const std::string& tag, const std::vector& latenciesMs, int64_t batchSize = 1) { + assert(!latenciesMs.empty()); + auto sorted = latenciesMs; + std::sort(sorted.begin(), sorted.end()); + const double avg = + std::accumulate(latenciesMs.begin(), latenciesMs.end(), 0.0) / static_cast(latenciesMs.size()); + std::cout << "[" << tag << "] benchmark_runs=" << latenciesMs.size() << " batch=" << batchSize << " AVG=" << avg + << "ms P50=" << percentile(sorted, 50.0) << "ms P90=" << percentile(sorted, 90.0) + << "ms P95=" << percentile(sorted, 95.0) << "ms P99=" << percentile(sorted, 99.0) << "ms\n"; +} + +inline void printFirstOutputs(const std::string& tag, const float* values, std::size_t count) { + const auto limit = std::min(count, kMaxFirstOutputs); + std::cout << "[" << tag << "] first_outputs="; + for (std::size_t i = 0; i < limit; ++i) { + if (i > 0) { + std::cout << ", "; + } + std::cout << std::setprecision(4) << values[i]; + } + std::cout << '\n'; +} + } // namespace diff --git a/vit/vit.cc b/vit/vit.cc index 762fc098..808bcb61 100644 --- a/vit/vit.cc +++ b/vit/vit.cc @@ -6,18 +6,19 @@ #include #include #include +#include #include #include #include #include #include -#include #include -#include #include #include #include +#include #include + #include "cuda_allocator.h" #include "logging.h" #include "macros.h" @@ -37,6 +38,12 @@ static constexpr int64_t BUILD_MIN_BATCH = 1; static constexpr int64_t BUILD_OPT_BATCH = 1; static constexpr int64_t BUILD_MAX_BATCH = 2; +template +requires(sizeof(T) > sizeof(int32_t)) static auto toI32(T value) -> int32_t { + assert(std::in_range(value)); + return static_cast(value); +} + // ViT model variant table. // All variants share the same architecture; only sizes differ. // `model_type` is the canonical name (e.g. "ViT-B/16"); aliases without slash @@ -155,10 +162,10 @@ static auto bytesPerElement(DataType t) -> std::size_t { return 8; case DataType::kINT32: return 4; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) case DataType::kBOOL: #endif -#if TRT_VERSION >= 8500 +#if TRT_VERSION_GE(8, 5, 0) case DataType::kUINT8: #endif case DataType::kINT8: @@ -241,7 +248,8 @@ static auto addShapeConstant(INetworkDefinition* net, std::initializer_list(data.size()); - auto* c = net->addConstant(dims, Weights{DataType::kINT64, data.data(), static_cast(data.size())}); + auto* c = net->addConstant( + dims, Weights{.type = DataType::kINT64, .values = data.data(), .count = static_cast(data.size())}); return c->getOutput(0); } @@ -260,7 +268,7 @@ static auto addBatchedConstantSlice(INetworkDefinition* net, ITensor& input, ITe auto* batch = net->addSlice(*input_shape->getOutput(0), batch_start, batch_size, batch_stride); auto* size_tail = addShapeConstant(net, {dim1, dim2}, storage); const std::array size_inputs = {batch->getOutput(0), size_tail}; - auto* size = net->addConcatenation(size_inputs.data(), static_cast(size_inputs.size())); + auto* size = net->addConcatenation(size_inputs.data(), toI32(size_inputs.size())); size->setAxis(0); auto* start = addShapeConstant(net, {0, 0, 0}, storage); @@ -275,7 +283,7 @@ static auto addBatchedConstantSlice(INetworkDefinition* net, ITensor& input, ITe } static auto addGeLU(INetworkDefinition* net, ITensor& input) -> ILayer* { -#if TRT_VERSION < 10000 +#if TRT_VERSION_LT(10, 0, 0) // tanh approximation: 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))) const auto inputDims = input.getDimensions(); @@ -289,10 +297,11 @@ static auto addGeLU(INetworkDefinition* net, ITensor& input) -> ILayer* { static float _one = 1.0f; static float _sqrt_2_div_pi = std::sqrt(2.0f / M_PI); static float _coeff = 0.044715f; - auto* _w_half = net->addConstant(scalarDims, Weights{DataType::kFLOAT, &_half, 1}); - auto* _w_one = net->addConstant(scalarDims, Weights{DataType::kFLOAT, &_one, 1}); - auto* _w_sqrt_2_div_pi = net->addConstant(scalarDims, Weights{DataType::kFLOAT, &_sqrt_2_div_pi, 1}); - auto* _w_coeff = net->addConstant(scalarDims, Weights{DataType::kFLOAT, &_coeff, 1}); + auto* _w_half = net->addConstant(scalarDims, Weights{.type = DataType::kFLOAT, .values = &_half, .count = 1}); + auto* _w_one = net->addConstant(scalarDims, Weights{.type = DataType::kFLOAT, .values = &_one, .count = 1}); + auto* _w_sqrt_2_div_pi = + net->addConstant(scalarDims, Weights{.type = DataType::kFLOAT, .values = &_sqrt_2_div_pi, .count = 1}); + auto* _w_coeff = net->addConstant(scalarDims, Weights{.type = DataType::kFLOAT, .values = &_coeff, .count = 1}); auto* _x2 = net->addElementWise(input, input, E::kPROD); auto* x3_0 = net->addElementWise(*_x2->getOutput(0), input, E::kPROD); @@ -312,7 +321,7 @@ static auto addGeLU(INetworkDefinition* net, ITensor& input) -> ILayer* { static auto addLinearNorm(INetworkDefinition* net, ITensor& input, ITensor& scale, ITensor& bias, uint32_t axesMask) noexcept -> ILayer* { -#if TRT_VERSION >= 11500 +#if TRT_VERSION_GE(10, 15, 0) auto* ln = net->addNormalizationV2(input, scale, bias, axesMask); #else auto* ln = net->addNormalization(input, scale, bias, axesMask); @@ -378,12 +387,12 @@ auto ViTLayer(INetworkDefinition* net, WeightMap& w, ITensor& input, const ViTPa auto* qk_scale_w = net->addConstant(Dims4{1, 1, 1, 1}, w.at(attn_name + ".scale")); // 2.3 QKV attention output and reshape -#if TRT_VERSION >= 11400 && TRT_VERSION < 11500 +#if TRT_VERSION_GE(10, 14, 0) && TRT_VERSION_LT(10, 15, 0) gLogger.log(Severity::kWARNING, "IAttention is available in TensorRT 10.14.1 SDK but have bugs, use 10.15.1+ to enable native fused " "kernel"); #endif -#if TRT_VERSION >= 11500 +#if TRT_VERSION_GE(10, 15, 0) using ANO = AttentionNormalizationOp; auto* q_scaled = net->addElementWise(*q_s->getOutput(0), *qk_scale_w->getOutput(0), E::kPROD)->getOutput(0); auto* attn = net->addAttention(*q_scaled, *k_s->getOutput(0), *v_s->getOutput(0), ANO::kSOFTMAX, false); @@ -458,7 +467,7 @@ auto createEngine(const ViTConfig& cfg, IRuntime* runtime, IBuilder* builder, IB RepeatedWeightsStorage repeated_storage; -#if TRT_VERSION >= 10000 +#if TRT_VERSION_GE(10, 0, 0) auto* net = builder->createNetworkV2(1U << static_cast(NDCF::kSTRONGLY_TYPED)); #else auto* net = builder->createNetworkV2(1U << static_cast(NDCF::kEXPLICIT_BATCH)); @@ -526,10 +535,10 @@ auto createEngine(const ViTConfig& cfg, IRuntime* runtime, IBuilder* builder, IB cls_1->getOutput(0)->setName(NAMES[1]); net->markOutput(*cls_1->getOutput(0)); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) config->setMemoryPoolLimit(MemoryPoolType::kWORKSPACE, WORKSPACE_SIZE); config->setBuilderOptimizationLevel(5); -#if TRT_VERSION < 10000 +#if TRT_VERSION_LT(10, 0, 0) // Strongly-typed networks (TRT 10+) take their precision from the // tensor types declared on inputs/weights and reject BuilderFlag::kFP16. // Pre-TRT-10 networks are weakly typed, so explicitly request FP16 @@ -580,11 +589,11 @@ std::vector> doInference(IExecutionContext& context, const vo cudaStream_t stream; CHECK(cudaStreamCreate(&stream)); std::vector buffers; -#if TRT_VERSION >= 10000 +#if TRT_VERSION_GE(10, 0, 0) auto allocator = CudaOutputAllocator::Create(stream); #endif -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) const int32_t nIO = engine.getNbIOTensors(); #else const int32_t nIO = engine.getNbBindings(); @@ -597,7 +606,7 @@ std::vector> doInference(IExecutionContext& context, const vo return i == 0 ? in_per_sample : out_per_sample; }; -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) if (engine.getTensorShape(NAMES[0]).d[0] == -1) { Dims in_dims{.nbDims = 4, .d = {static_cast(batchSize), 3, cfg.img_size, cfg.img_size}}; if (!context.setInputShape(NAMES[0], in_dims)) { @@ -610,11 +619,11 @@ std::vector> doInference(IExecutionContext& context, const vo buffers.resize(nIO, nullptr); for (auto i = 0; i < nIO; ++i) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* tensor_name = engine.getIOTensorName(i); const auto dtype = engine.getTensorDataType(tensor_name); std::size_t size = batchSize * sizeOf(i) * bytesPerElement(dtype); -#if TRT_VERSION >= 10000 +#if TRT_VERSION_GE(10, 0, 0) if (engine.getTensorIOMode(tensor_name) == TensorIOMode::kINPUT) { CHECK(cudaMalloc(&buffers[i], size)); CHECK(cudaMemcpyAsync(buffers[i], input, size, cudaMemcpyHostToDevice, stream)); @@ -648,7 +657,7 @@ std::vector> doInference(IExecutionContext& context, const vo #endif } -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) if (!context.enqueueV3(stream)) { std::cerr << "enqueueV3 failed\n"; std::abort(); @@ -662,14 +671,14 @@ std::vector> doInference(IExecutionContext& context, const vo std::vector> prob; for (int i = 0; i < nIO; ++i) { -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) auto* tensor_name = engine.getIOTensorName(i); if (engine.getTensorIOMode(tensor_name) != TensorIOMode::kOUTPUT) continue; const auto dtype = engine.getTensorDataType(tensor_name); std::size_t count = batchSize * out_per_sample; std::size_t size = count * bytesPerElement(dtype); -#if TRT_VERSION >= 10000 +#if TRT_VERSION_GE(10, 0, 0) void* out_ptr = allocator->getBuffer(tensor_name); #else void* out_ptr = buffers[i]; @@ -703,7 +712,7 @@ std::vector> doInference(IExecutionContext& context, const vo CHECK(cudaFree(buffer)); } } -#if TRT_VERSION >= 10000 +#if TRT_VERSION_GE(10, 0, 0) allocator.reset(); #endif CHECK(cudaStreamDestroy(stream)); @@ -719,7 +728,7 @@ void APIToModel(const ViTConfig& cfg, IRuntime* runtime, IHostMemory** modelStre (*modelStream) = engine->serialize(); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete engine; delete config; delete builder; @@ -798,10 +807,11 @@ static std::vector collectImages(const std::string& path) { auto main(int argc, char** argv) -> int { std::cout << "TensorRT version: " << TRT_VERSION << "\n"; - if (argc < 2 || (std::string(argv[1]) == "-s" && argc != 5) || (std::string(argv[1]) == "-d" && argc != 4)) { + if (argc < 2 || (std::string(argv[1]) == "-s" && argc != 5) || + (std::string(argv[1]) == "-d" && argc != 4 && argc != 5)) { std::cerr << "usage:\n" << " ./vit -s \n" - << " ./vit -d \n" + << " ./vit -d [--profile]\n" << " model_type: ViT-B/16 | ViT-B/32 | ViT-L/16 | ViT-L/32 | ViT-H/14\n" << " (aliases: b16, b32, l16, l32, h14 also accepted)\n" << " build precision is configured in code via BUILD_PRECISION (see top of vit.cc).\n"; @@ -840,7 +850,7 @@ auto main(int argc, char** argv) -> int { const auto* data_ptr = reinterpret_cast(modelStream->data()); auto data_size = static_cast(modelStream->size()); p.write(data_ptr, data_size); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) delete modelStream; #else modelStream->destroy(); @@ -849,6 +859,14 @@ auto main(int argc, char** argv) -> int { } else if (mode == "-d") { std::string engine_path = argv[2]; std::string image_dir = argv[3]; + bool enable_profile = false; + if (argc == 5) { + if (std::string(argv[4]) != "--profile") { + std::cerr << "unknown option: " << argv[4] << "\n"; + return 1; + } + enable_profile = true; + } std::ifstream file(engine_path, std::ios::binary); if (!file.good()) { @@ -866,14 +884,20 @@ auto main(int argc, char** argv) -> int { file.read(trtModelStream.data(), size); file.close(); -#if TRT_VERSION >= 8000 +#if TRT_VERSION_GE(8, 0, 0) ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream.data(), size); #else ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream.data(), size, nullptr); #endif assert(engine != nullptr); + trtModelStream.clear(); + trtModelStream.shrink_to_fit(); auto* context = engine->createExecutionContext(); assert(context != nullptr); + Profiler profiler("VisionTransformerProfiler"); + if (enable_profile) { + context->setProfiler(&profiler); + } ViTConfig cfg = configFromEngine(*engine); std::cout << "[engine] img=" << cfg.img_size << " classes=" << cfg.num_classes @@ -896,8 +920,7 @@ auto main(int argc, char** argv) -> int { std::cerr << "cannot read image: " << images[i] << "\n"; return -1; } - auto one = preprocess_img(img, false, mean, stdv, 1, static_cast(cfg.img_size), - static_cast(cfg.img_size)); + auto one = preprocess_img(img, false, mean, stdv, 1, toI32(cfg.img_size), toI32(cfg.img_size)); input_buf.insert(input_buf.end(), one.begin(), one.end()); } @@ -918,37 +941,40 @@ auto main(int argc, char** argv) -> int { return -1; } - Profiler profiler("VisionTransformerProfiler"); - for (int i = 0; i < 5; ++i) { (void)doInference(*context, input_ptr, infer_batch, cfg); } - context->setProfiler(&profiler); - for (int i = 0; i < 20; ++i) { - auto start = std::chrono::system_clock::now(); - auto prob = doInference(*context, input_ptr, infer_batch, cfg); - auto end = std::chrono::system_clock::now(); - auto period = std::chrono::duration_cast(end - start); - std::cout << period.count() << "us\n"; - - if (i == 19) { - auto labels = (cfg.num_classes == 1000) ? loadLabels() : std::map{}; - for (int64_t b = 0; b < infer_batch; ++b) { - std::cout << "[sample " << b << "] " << images[b] << "\n"; - int _top = 0; - std::vector sample(prob[0].begin() + b * cfg.num_classes, - prob[0].begin() + (b + 1) * cfg.num_classes); - for (auto& [idx, logits] : topk(sample, 3)) { - std::cout << " Top: " << _top++ << " idx: " << idx << ", logits: " << logits; - if (!labels.empty()) - std::cout << ", label: " << labels[idx]; - std::cout << "\n"; - } + auto firstProb = doInference(*context, input_ptr, infer_batch, cfg); + auto labels = (cfg.num_classes == 1000) ? loadLabels() : std::map{}; + for (int64_t b = 0; b < infer_batch; ++b) { + std::cout << "[sample " << b << "] " << images[b] << "\n"; + std::vector sample(firstProb[0].begin() + b * cfg.num_classes, + firstProb[0].begin() + (b + 1) * cfg.num_classes); + printFirstOutputs("vit", sample.data(), sample.size()); + int _top = 0; + for (auto& [idx, logits] : topk(sample, 3)) { + std::cout << " Top: " << _top++ << " idx: " << idx << ", logits: " << logits; + if (!labels.empty()) { + std::cout << ", label: " << labels[idx]; } - std::cout << profiler << "\n"; + std::cout << "\n"; } } + + std::vector latencies; + latencies.reserve(kBenchmarkRuns); + for (int i = 0; i < kBenchmarkRuns; ++i) { + auto start = std::chrono::steady_clock::now(); + (void)doInference(*context, input_ptr, infer_batch, cfg); + auto end = std::chrono::steady_clock::now(); + auto period = std::chrono::duration_cast(end - start); + latencies.push_back(static_cast(period.count()) / 1000.0); + } + printBenchmark("vit", latencies, infer_batch); + if (enable_profile) { + std::cout << profiler; + } return 0; } return 0;