{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# TVM" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "from matplotlib import pyplot as plt\n", "import torch\n", "\n", "from mod import load_mod\n", "\n", "plt.ion()\n", "# 载入自定义模块\n", "load_mod()\n", "\n", "from xinet import CV" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Files already downloaded and verified\n", "Files already downloaded and verified\n" ] } ], "source": [ "import torch\n", "from torchvision.models import quantization as models\n", "from torch import nn\n", "\n", "def create_combined_model(model_fe):\n", " # 步骤1:分离特征提取器\n", " model_fe_features = nn.Sequential(\n", " model_fe.quant, # 量化 input\n", " model_fe.conv1,\n", " model_fe.bn1,\n", " model_fe.relu,\n", " model_fe.maxpool,\n", " model_fe.layer1,\n", " model_fe.layer2,\n", " model_fe.layer3,\n", " model_fe.layer4,\n", " model_fe.avgpool,\n", " model_fe.dequant, # 反量化 output\n", " )\n", "\n", " # 步骤2:创建一个新的“头”\n", " new_head = nn.Sequential(\n", " nn.Dropout(p=0.5),\n", " nn.Linear(num_ftrs, 10),\n", " )\n", "\n", " # 步骤3:合并,不要忘记量化 stubs\n", " new_model = nn.Sequential(\n", " model_fe_features,\n", " nn.Flatten(1),\n", " new_head,\n", " )\n", " return new_model\n", "\n", "batch_size = 128\n", "train_iter, test_iter = CV.load_data_cifar10(batch_size=batch_size)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import time\n", "import torch\n", "\n", "import sys\n", "TVM_HOME = '/media/pc/data/4tb/xinet/tvm'\n", "sys.path.extend([f'{TVM_HOME}/python', f'{TVM_HOME}/vta/python'])\n", "\n", "import tvm\n", "from tvm import relay\n", "\n", "\n", "def find_topk(array, k, axis=-1, largest=True, sorted=True):\n", " if axis is None:\n", " axis_size = array.size\n", " else:\n", " axis_size = array.shape[axis]\n", " assert 1 <= k <= axis_size\n", "\n", " array = np.asanyarray(array)\n", " if largest:\n", " index_array = np.argpartition(array, axis_size-k, axis=axis)\n", " topk_indices = np.take(index_array, -np.arange(k)-1, axis=axis)\n", " else:\n", " index_array = np.argpartition(array, k-1, axis=axis)\n", " topk_indices = np.take(index_array, np.arange(k), axis=axis)\n", " topk_values = np.take_along_axis(array, topk_indices, axis=axis)\n", " if sorted:\n", " sorted_indices_in_topk = np.argsort(topk_values, axis=axis)\n", " if largest:\n", " sorted_indices_in_topk = np.flip(sorted_indices_in_topk, axis=axis)\n", " sorted_topk_values = np.take_along_axis(\n", " topk_values, sorted_indices_in_topk, axis=axis)\n", " sorted_topk_indices = np.take_along_axis(\n", " topk_indices, sorted_indices_in_topk, axis=axis)\n", " return sorted_topk_values, sorted_topk_indices\n", " return topk_values, topk_indices\n", "\n", "\n", "def calibrate_dataset(val_loader, calibration_samples, batch_size):\n", " for i, (input, _) in enumerate(val_loader):\n", " if i * batch_size >= calibration_samples:\n", " break\n", " yield {\"input\": input.numpy()}\n", "\n", "\n", "def quantize(mod, params, data_aware, val_loader, calibration_samples=500, batch_size=1):\n", " if data_aware:\n", " print(\"tvm calibration quantize begin---------------------------->>\")\n", " with relay.quantize.qconfig(calibrate_mode=\"kl_divergence\", weight_scale=\"max\", skip_conv_layers=[0], skip_dense_layer=True):\n", " mod = relay.quantize.quantize(\n", " mod, params, dataset=calibrate_dataset(val_loader, calibration_samples, batch_size))\n", " #print(mod)\n", " print(\"tvm calibration quantize end---------------------------->>\")\n", " else:\n", " print(\"tvm global scale quantize begin---------------------------->>\")\n", " with relay.quantize.qconfig(calibrate_mode=\"global_scale\", global_scale=8.0):\n", " mod = relay.quantize.quantize(mod, params)\n", " print(\"tvm global scale quantize end---------------------------->>\")\n", " return mod\n", "\n", "\n", "def run_tvm_model(mod, params, target=\"llvm\"):\n", " with tvm.transform.PassContext(opt_level=3):\n", " lib = relay.build(mod, target=target, params=params)\n", " runtime = tvm.contrib.graph_executor.GraphModule(\n", " lib[\"default\"](tvm.device(target, 0)))\n", " return runtime\n", "\n", "\n", "def tvm_model(model, batch_size):\n", " input_shape = (batch_size, 3, 32, 32)\n", " shape_list = [(\"input\", input_shape)]\n", " input_data = torch.randn(input_shape)\n", " scripted_model = torch.jit.trace(model, input_data).eval()\n", " mod, params = relay.frontend.from_pytorch(scripted_model, shape_list)\n", " return mod, params\n", "\n", "\n", "def tvm_test(model, val_loader,\n", " batch_size, data_aware,\n", " calibration_samples=500,\n", " print_freq=100,\n", " pre_quantization=False):\n", " mod, params = tvm_model(model, batch_size)\n", " if not pre_quantization:\n", " mod = quantize(mod, params, data_aware, val_loader,\n", " calibration_samples=calibration_samples,\n", " batch_size=batch_size)\n", " runtime = run_tvm_model(mod, params)\n", " #print(runtime.benchmark(dev, number=1, repeat=100))\n", "\n", " test_nums = len(val_loader)\n", " top1_correct = 0\n", " top5_correct = 0\n", " print('llvm inference-------------->>')\n", " for i, (input, label) in enumerate(val_loader, 1):\n", " runtime.set_input('input', input)\n", " runtime.run()\n", " output = runtime.get_output(0).asnumpy()\n", "\n", " # find topk index\n", " _, preds = find_topk(output, 5)\n", " print(preds, label)\n", " if label.item() == preds[0][0]:\n", " top1_correct += 1\n", "\n", " if label.item() in preds[0]:\n", " top5_correct += 1\n", "\n", " if i % print_freq == 0:\n", " print('Test: [{}/{}] \\t'\n", " 'Acc@1 {:.4f} \\t'\n", " 'Acc@5 {:.4f}'.format(\n", " i, test_nums, top1_correct / i, top5_correct / i))\n", "\n", " top1 = top1_correct / test_nums\n", " top5 = top5_correct / test_nums\n", " print(' * Acc@1 {:.4f} Acc@5 {:.4f}'\n", " .format(top1, top5))\n", "\n", " time_start = time.time()\n", " repeat = 100\n", " for i, (input, label) in enumerate(val_loader, 1):\n", " for r in range(repeat):\n", " runtime.set_input('input', input)\n", " runtime.run()\n", " output = runtime.get_output(0).asnumpy()\n", " time_end = time.time()\n", " print(\"平均推理时间:\", (time_end - time_start)/repeat)\n", " exit()\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "ename": "ValueError", "evalue": "Expected more than 1 value per channel when training, got input size torch.Size([1, 512, 1, 1])", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", "\u001b[1;32m/media/pc/data/4tb/xinet/web/quantization/docs/study/transfer-learning/tvm.ipynb Cell 5'\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[39mfor\u001b[39;00m param \u001b[39min\u001b[39;00m model_ft\u001b[39m.\u001b[39mparameters():\n\u001b[1;32m 12\u001b[0m param\u001b[39m.\u001b[39mrequires_grad \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[0;32m---> 15\u001b[0m tvm_test(model_ft, test_iter,\n\u001b[1;32m 16\u001b[0m batch_size\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m, data_aware\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[1;32m 17\u001b[0m calibration_samples\u001b[39m=\u001b[39;49m\u001b[39m500\u001b[39;49m,\n\u001b[1;32m 18\u001b[0m print_freq\u001b[39m=\u001b[39;49m\u001b[39m100\u001b[39;49m,\n\u001b[1;32m 19\u001b[0m pre_quantization\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\n", "\u001b[1;32m/media/pc/data/4tb/xinet/web/quantization/docs/study/transfer-learning/tvm.ipynb Cell 4'\u001b[0m in \u001b[0;36mtvm_test\u001b[0;34m(model, val_loader, batch_size, data_aware, calibration_samples, print_freq, pre_quantization)\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mtvm_test\u001b[39m(model, val_loader,\n\u001b[1;32m 81\u001b[0m batch_size, data_aware,\n\u001b[1;32m 82\u001b[0m calibration_samples\u001b[39m=\u001b[39m\u001b[39m500\u001b[39m,\n\u001b[1;32m 83\u001b[0m print_freq\u001b[39m=\u001b[39m\u001b[39m100\u001b[39m,\n\u001b[1;32m 84\u001b[0m pre_quantization\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m):\n\u001b[0;32m---> 85\u001b[0m mod, params \u001b[39m=\u001b[39m tvm_model(model, batch_size)\n\u001b[1;32m 86\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m pre_quantization:\n\u001b[1;32m 87\u001b[0m mod \u001b[39m=\u001b[39m quantize(mod, params, data_aware, val_loader,\n\u001b[1;32m 88\u001b[0m calibration_samples\u001b[39m=\u001b[39mcalibration_samples,\n\u001b[1;32m 89\u001b[0m batch_size\u001b[39m=\u001b[39mbatch_size)\n", "\u001b[1;32m/media/pc/data/4tb/xinet/web/quantization/docs/study/transfer-learning/tvm.ipynb Cell 4'\u001b[0m in \u001b[0;36mtvm_model\u001b[0;34m(model, batch_size)\u001b[0m\n\u001b[1;32m 73\u001b[0m shape_list \u001b[39m=\u001b[39m [(\u001b[39m\"\u001b[39m\u001b[39minput\u001b[39m\u001b[39m\"\u001b[39m, input_shape)]\n\u001b[1;32m 74\u001b[0m input_data \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mrandn(input_shape)\n\u001b[0;32m---> 75\u001b[0m scripted_model \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39;49mjit\u001b[39m.\u001b[39;49mtrace(model, input_data)\u001b[39m.\u001b[39meval()\n\u001b[1;32m 76\u001b[0m mod, params \u001b[39m=\u001b[39m relay\u001b[39m.\u001b[39mfrontend\u001b[39m.\u001b[39mfrom_pytorch(scripted_model, shape_list)\n\u001b[1;32m 77\u001b[0m \u001b[39mreturn\u001b[39;00m mod, params\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/jit/_trace.py:733\u001b[0m, in \u001b[0;36mtrace\u001b[0;34m(func, example_inputs, optimize, check_trace, check_inputs, check_tolerance, strict, _force_outplace, _module_class, _compilation_unit)\u001b[0m\n\u001b[1;32m 730\u001b[0m \u001b[39mreturn\u001b[39;00m func\n\u001b[1;32m 732\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(func, torch\u001b[39m.\u001b[39mnn\u001b[39m.\u001b[39mModule):\n\u001b[0;32m--> 733\u001b[0m \u001b[39mreturn\u001b[39;00m trace_module(\n\u001b[1;32m 734\u001b[0m func,\n\u001b[1;32m 735\u001b[0m {\u001b[39m\"\u001b[39;49m\u001b[39mforward\u001b[39;49m\u001b[39m\"\u001b[39;49m: example_inputs},\n\u001b[1;32m 736\u001b[0m \u001b[39mNone\u001b[39;49;00m,\n\u001b[1;32m 737\u001b[0m check_trace,\n\u001b[1;32m 738\u001b[0m wrap_check_inputs(check_inputs),\n\u001b[1;32m 739\u001b[0m check_tolerance,\n\u001b[1;32m 740\u001b[0m strict,\n\u001b[1;32m 741\u001b[0m _force_outplace,\n\u001b[1;32m 742\u001b[0m _module_class,\n\u001b[1;32m 743\u001b[0m )\n\u001b[1;32m 745\u001b[0m \u001b[39mif\u001b[39;00m (\n\u001b[1;32m 746\u001b[0m \u001b[39mhasattr\u001b[39m(func, \u001b[39m\"\u001b[39m\u001b[39m__self__\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 747\u001b[0m \u001b[39mand\u001b[39;00m \u001b[39misinstance\u001b[39m(func\u001b[39m.\u001b[39m\u001b[39m__self__\u001b[39m, torch\u001b[39m.\u001b[39mnn\u001b[39m.\u001b[39mModule)\n\u001b[1;32m 748\u001b[0m \u001b[39mand\u001b[39;00m func\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mforward\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 749\u001b[0m ):\n\u001b[1;32m 750\u001b[0m \u001b[39mreturn\u001b[39;00m trace_module(\n\u001b[1;32m 751\u001b[0m func\u001b[39m.\u001b[39m\u001b[39m__self__\u001b[39m,\n\u001b[1;32m 752\u001b[0m {\u001b[39m\"\u001b[39m\u001b[39mforward\u001b[39m\u001b[39m\"\u001b[39m: example_inputs},\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 759\u001b[0m _module_class,\n\u001b[1;32m 760\u001b[0m )\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/jit/_trace.py:934\u001b[0m, in \u001b[0;36mtrace_module\u001b[0;34m(mod, inputs, optimize, check_trace, check_inputs, check_tolerance, strict, _force_outplace, _module_class, _compilation_unit)\u001b[0m\n\u001b[1;32m 932\u001b[0m func \u001b[39m=\u001b[39m mod \u001b[39mif\u001b[39;00m method_name \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mforward\u001b[39m\u001b[39m\"\u001b[39m \u001b[39melse\u001b[39;00m \u001b[39mgetattr\u001b[39m(mod, method_name)\n\u001b[1;32m 933\u001b[0m example_inputs \u001b[39m=\u001b[39m make_tuple(example_inputs)\n\u001b[0;32m--> 934\u001b[0m module\u001b[39m.\u001b[39;49m_c\u001b[39m.\u001b[39;49m_create_method_from_trace(\n\u001b[1;32m 935\u001b[0m method_name,\n\u001b[1;32m 936\u001b[0m func,\n\u001b[1;32m 937\u001b[0m example_inputs,\n\u001b[1;32m 938\u001b[0m var_lookup_fn,\n\u001b[1;32m 939\u001b[0m strict,\n\u001b[1;32m 940\u001b[0m _force_outplace,\n\u001b[1;32m 941\u001b[0m )\n\u001b[1;32m 942\u001b[0m check_trace_method \u001b[39m=\u001b[39m module\u001b[39m.\u001b[39m_c\u001b[39m.\u001b[39m_get_method(method_name)\n\u001b[1;32m 944\u001b[0m \u001b[39m# Check the trace against new traces created from user-specified inputs\u001b[39;00m\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:887\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 884\u001b[0m \u001b[39minput\u001b[39m \u001b[39m=\u001b[39m bw_hook\u001b[39m.\u001b[39msetup_input_hook(\u001b[39minput\u001b[39m)\n\u001b[1;32m 886\u001b[0m \u001b[39mif\u001b[39;00m torch\u001b[39m.\u001b[39m_C\u001b[39m.\u001b[39m_get_tracing_state():\n\u001b[0;32m--> 887\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_slow_forward(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 888\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 889\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mforward(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:860\u001b[0m, in \u001b[0;36mModule._slow_forward\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 858\u001b[0m recording_scopes \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m 859\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 860\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mforward(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 861\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 862\u001b[0m \u001b[39mif\u001b[39;00m recording_scopes:\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/container.py:119\u001b[0m, in \u001b[0;36mSequential.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m):\n\u001b[1;32m 118\u001b[0m \u001b[39mfor\u001b[39;00m module \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m:\n\u001b[0;32m--> 119\u001b[0m \u001b[39minput\u001b[39m \u001b[39m=\u001b[39m module(\u001b[39minput\u001b[39;49m)\n\u001b[1;32m 120\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39minput\u001b[39m\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:887\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 884\u001b[0m \u001b[39minput\u001b[39m \u001b[39m=\u001b[39m bw_hook\u001b[39m.\u001b[39msetup_input_hook(\u001b[39minput\u001b[39m)\n\u001b[1;32m 886\u001b[0m \u001b[39mif\u001b[39;00m torch\u001b[39m.\u001b[39m_C\u001b[39m.\u001b[39m_get_tracing_state():\n\u001b[0;32m--> 887\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_slow_forward(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 888\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 889\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mforward(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:860\u001b[0m, in \u001b[0;36mModule._slow_forward\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 858\u001b[0m recording_scopes \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m 859\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 860\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mforward(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 861\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 862\u001b[0m \u001b[39mif\u001b[39;00m recording_scopes:\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/container.py:119\u001b[0m, in \u001b[0;36mSequential.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m):\n\u001b[1;32m 118\u001b[0m \u001b[39mfor\u001b[39;00m module \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m:\n\u001b[0;32m--> 119\u001b[0m \u001b[39minput\u001b[39m \u001b[39m=\u001b[39m module(\u001b[39minput\u001b[39;49m)\n\u001b[1;32m 120\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39minput\u001b[39m\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:887\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 884\u001b[0m \u001b[39minput\u001b[39m \u001b[39m=\u001b[39m bw_hook\u001b[39m.\u001b[39msetup_input_hook(\u001b[39minput\u001b[39m)\n\u001b[1;32m 886\u001b[0m \u001b[39mif\u001b[39;00m torch\u001b[39m.\u001b[39m_C\u001b[39m.\u001b[39m_get_tracing_state():\n\u001b[0;32m--> 887\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_slow_forward(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 888\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 889\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mforward(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:860\u001b[0m, in \u001b[0;36mModule._slow_forward\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 858\u001b[0m recording_scopes \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m 859\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 860\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mforward(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 861\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 862\u001b[0m \u001b[39mif\u001b[39;00m recording_scopes:\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/container.py:119\u001b[0m, in \u001b[0;36mSequential.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m):\n\u001b[1;32m 118\u001b[0m \u001b[39mfor\u001b[39;00m module \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m:\n\u001b[0;32m--> 119\u001b[0m \u001b[39minput\u001b[39m \u001b[39m=\u001b[39m module(\u001b[39minput\u001b[39;49m)\n\u001b[1;32m 120\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39minput\u001b[39m\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:887\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 884\u001b[0m \u001b[39minput\u001b[39m \u001b[39m=\u001b[39m bw_hook\u001b[39m.\u001b[39msetup_input_hook(\u001b[39minput\u001b[39m)\n\u001b[1;32m 886\u001b[0m \u001b[39mif\u001b[39;00m torch\u001b[39m.\u001b[39m_C\u001b[39m.\u001b[39m_get_tracing_state():\n\u001b[0;32m--> 887\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_slow_forward(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 888\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 889\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mforward(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:860\u001b[0m, in \u001b[0;36mModule._slow_forward\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 858\u001b[0m recording_scopes \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m 859\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 860\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mforward(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 861\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 862\u001b[0m \u001b[39mif\u001b[39;00m recording_scopes:\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torchvision/models/quantization/resnet.py:32\u001b[0m, in \u001b[0;36mQuantizableBasicBlock.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 29\u001b[0m identity \u001b[39m=\u001b[39m x\n\u001b[1;32m 31\u001b[0m out \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mconv1(x)\n\u001b[0;32m---> 32\u001b[0m out \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbn1(out)\n\u001b[1;32m 33\u001b[0m out \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrelu(out)\n\u001b[1;32m 35\u001b[0m out \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mconv2(out)\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:887\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 884\u001b[0m \u001b[39minput\u001b[39m \u001b[39m=\u001b[39m bw_hook\u001b[39m.\u001b[39msetup_input_hook(\u001b[39minput\u001b[39m)\n\u001b[1;32m 886\u001b[0m \u001b[39mif\u001b[39;00m torch\u001b[39m.\u001b[39m_C\u001b[39m.\u001b[39m_get_tracing_state():\n\u001b[0;32m--> 887\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_slow_forward(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 888\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 889\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mforward(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:860\u001b[0m, in \u001b[0;36mModule._slow_forward\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 858\u001b[0m recording_scopes \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m 859\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 860\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mforward(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 861\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 862\u001b[0m \u001b[39mif\u001b[39;00m recording_scopes:\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/modules/batchnorm.py:135\u001b[0m, in \u001b[0;36m_BatchNorm.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 133\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrunning_mean \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mor\u001b[39;00m \u001b[39misinstance\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrunning_mean, torch\u001b[39m.\u001b[39mTensor)\n\u001b[1;32m 134\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrunning_var \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mor\u001b[39;00m \u001b[39misinstance\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrunning_var, torch\u001b[39m.\u001b[39mTensor)\n\u001b[0;32m--> 135\u001b[0m \u001b[39mreturn\u001b[39;00m F\u001b[39m.\u001b[39;49mbatch_norm(\n\u001b[1;32m 136\u001b[0m \u001b[39minput\u001b[39;49m,\n\u001b[1;32m 137\u001b[0m \u001b[39m# If buffers are not to be tracked, ensure that they won't be updated\u001b[39;49;00m\n\u001b[1;32m 138\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mrunning_mean \u001b[39mif\u001b[39;49;00m \u001b[39mnot\u001b[39;49;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtraining \u001b[39mor\u001b[39;49;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtrack_running_stats \u001b[39melse\u001b[39;49;00m \u001b[39mNone\u001b[39;49;00m,\n\u001b[1;32m 139\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mrunning_var \u001b[39mif\u001b[39;49;00m \u001b[39mnot\u001b[39;49;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtraining \u001b[39mor\u001b[39;49;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtrack_running_stats \u001b[39melse\u001b[39;49;00m \u001b[39mNone\u001b[39;49;00m,\n\u001b[1;32m 140\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mweight, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbias, bn_training, exponential_average_factor, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49meps)\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/functional.py:2147\u001b[0m, in \u001b[0;36mbatch_norm\u001b[0;34m(input, running_mean, running_var, weight, bias, training, momentum, eps)\u001b[0m\n\u001b[1;32m 2134\u001b[0m \u001b[39mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 2135\u001b[0m batch_norm,\n\u001b[1;32m 2136\u001b[0m (\u001b[39minput\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 2144\u001b[0m eps\u001b[39m=\u001b[39meps,\n\u001b[1;32m 2145\u001b[0m )\n\u001b[1;32m 2146\u001b[0m \u001b[39mif\u001b[39;00m training:\n\u001b[0;32m-> 2147\u001b[0m _verify_batch_size(\u001b[39minput\u001b[39;49m\u001b[39m.\u001b[39;49msize())\n\u001b[1;32m 2149\u001b[0m \u001b[39mreturn\u001b[39;00m torch\u001b[39m.\u001b[39mbatch_norm(\n\u001b[1;32m 2150\u001b[0m \u001b[39minput\u001b[39m, weight, bias, running_mean, running_var, training, momentum, eps, torch\u001b[39m.\u001b[39mbackends\u001b[39m.\u001b[39mcudnn\u001b[39m.\u001b[39menabled\n\u001b[1;32m 2151\u001b[0m )\n", "File \u001b[0;32m~/.local/lib/python3.8/site-packages/torch/nn/functional.py:2114\u001b[0m, in \u001b[0;36m_verify_batch_size\u001b[0;34m(size)\u001b[0m\n\u001b[1;32m 2112\u001b[0m size_prods \u001b[39m*\u001b[39m\u001b[39m=\u001b[39m size[i \u001b[39m+\u001b[39m \u001b[39m2\u001b[39m]\n\u001b[1;32m 2113\u001b[0m \u001b[39mif\u001b[39;00m size_prods \u001b[39m==\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[0;32m-> 2114\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mExpected more than 1 value per channel when training, got input size \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39mformat(size))\n", "\u001b[0;31mValueError\u001b[0m: Expected more than 1 value per channel when training, got input size torch.Size([1, 512, 1, 1])" ] } ], "source": [ "# 注意 `quantize=False`\n", "model = models.resnet18(pretrained=True, progress=True, quantize=False)\n", "num_ftrs = model.fc.in_features\n", "\n", "# Step 1\n", "model.train()\n", "# model.fuse_model()\n", "# Step 2\n", "model_ft = create_combined_model(model)\n", "\n", "for param in model_ft.parameters():\n", " param.requires_grad = True\n", "\n", "\n", "tvm_test(model_ft, test_iter,\n", " batch_size=1, data_aware=True,\n", " calibration_samples=500,\n", " print_freq=100,\n", " pre_quantization=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "interpreter": { "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" }, "kernelspec": { "display_name": "Python 3.8.10 64-bit", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }