mirror of
https://github.com/labmlai/annotated_deep_learning_paper_implementations.git
synced 2025-08-14 17:41:37 +08:00
notebook
This commit is contained in:
Varuna Jayasiri
684
labml_nn/normalization/weight_standardization/experiment.ipynb
Normal file
684
labml_nn/normalization/weight_standardization/experiment.ipynb
Normal file
@ -0,0 +1,684 @@
|
||||
{
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "Weight Standardization & Batch-Channel Normalization",
|
||||
"provenance": [],
|
||||
"collapsed_sections": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"accelerator": "GPU",
|
||||
"widgets": {
|
||||
"application/vnd.jupyter.widget-state+json": {
|
||||
"1dacd9c82caf40c8b2def0c5fe4f7643": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "HBoxModel",
|
||||
"state": {
|
||||
"_view_name": "HBoxView",
|
||||
"_dom_classes": [],
|
||||
"_model_name": "HBoxModel",
|
||||
"_view_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_view_count": null,
|
||||
"_view_module_version": "1.5.0",
|
||||
"box_style": "",
|
||||
"layout": "IPY_MODEL_67e37901c44b4bde9062014542cae018",
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"children": [
|
||||
"IPY_MODEL_9d57be286b7244ca9a5f74cff6c3cf4e",
|
||||
"IPY_MODEL_9e511aaba1204ea9985fb6f55e024039"
|
||||
]
|
||||
}
|
||||
},
|
||||
"67e37901c44b4bde9062014542cae018": {
|
||||
"model_module": "@jupyter-widgets/base",
|
||||
"model_name": "LayoutModel",
|
||||
"state": {
|
||||
"_view_name": "LayoutView",
|
||||
"grid_template_rows": null,
|
||||
"right": null,
|
||||
"justify_content": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"overflow": null,
|
||||
"_model_module_version": "1.2.0",
|
||||
"_view_count": null,
|
||||
"flex_flow": null,
|
||||
"width": null,
|
||||
"min_width": null,
|
||||
"border": null,
|
||||
"align_items": null,
|
||||
"bottom": null,
|
||||
"_model_module": "@jupyter-widgets/base",
|
||||
"top": null,
|
||||
"grid_column": null,
|
||||
"overflow_y": null,
|
||||
"overflow_x": null,
|
||||
"grid_auto_flow": null,
|
||||
"grid_area": null,
|
||||
"grid_template_columns": null,
|
||||
"flex": null,
|
||||
"_model_name": "LayoutModel",
|
||||
"justify_items": null,
|
||||
"grid_row": null,
|
||||
"max_height": null,
|
||||
"align_content": null,
|
||||
"visibility": null,
|
||||
"align_self": null,
|
||||
"height": null,
|
||||
"min_height": null,
|
||||
"padding": null,
|
||||
"grid_auto_rows": null,
|
||||
"grid_gap": null,
|
||||
"max_width": null,
|
||||
"order": null,
|
||||
"_view_module_version": "1.2.0",
|
||||
"grid_template_areas": null,
|
||||
"object_position": null,
|
||||
"object_fit": null,
|
||||
"grid_auto_columns": null,
|
||||
"margin": null,
|
||||
"display": null,
|
||||
"left": null
|
||||
}
|
||||
},
|
||||
"9d57be286b7244ca9a5f74cff6c3cf4e": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "FloatProgressModel",
|
||||
"state": {
|
||||
"_view_name": "ProgressView",
|
||||
"style": "IPY_MODEL_61849c45ed0c4436995bb81d7efdac7e",
|
||||
"_dom_classes": [],
|
||||
"description": "",
|
||||
"_model_name": "FloatProgressModel",
|
||||
"bar_style": "success",
|
||||
"max": 170498071,
|
||||
"_view_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"value": 170498071,
|
||||
"_view_count": null,
|
||||
"_view_module_version": "1.5.0",
|
||||
"orientation": "horizontal",
|
||||
"min": 0,
|
||||
"description_tooltip": null,
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"layout": "IPY_MODEL_7ed3376588734d64a229fd8dd6bfe06e"
|
||||
}
|
||||
},
|
||||
"9e511aaba1204ea9985fb6f55e024039": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "HTMLModel",
|
||||
"state": {
|
||||
"_view_name": "HTMLView",
|
||||
"style": "IPY_MODEL_afd6865a483f4273addb343059632961",
|
||||
"_dom_classes": [],
|
||||
"description": "",
|
||||
"_model_name": "HTMLModel",
|
||||
"placeholder": "",
|
||||
"_view_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"value": " 170499072/? [00:04<00:00, 42242006.52it/s]",
|
||||
"_view_count": null,
|
||||
"_view_module_version": "1.5.0",
|
||||
"description_tooltip": null,
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"layout": "IPY_MODEL_7b765f7623474999a8c9815e02c1cc6b"
|
||||
}
|
||||
},
|
||||
"61849c45ed0c4436995bb81d7efdac7e": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "ProgressStyleModel",
|
||||
"state": {
|
||||
"_view_name": "StyleView",
|
||||
"_model_name": "ProgressStyleModel",
|
||||
"description_width": "initial",
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_view_count": null,
|
||||
"_view_module_version": "1.2.0",
|
||||
"bar_color": null,
|
||||
"_model_module": "@jupyter-widgets/controls"
|
||||
}
|
||||
},
|
||||
"7ed3376588734d64a229fd8dd6bfe06e": {
|
||||
"model_module": "@jupyter-widgets/base",
|
||||
"model_name": "LayoutModel",
|
||||
"state": {
|
||||
"_view_name": "LayoutView",
|
||||
"grid_template_rows": null,
|
||||
"right": null,
|
||||
"justify_content": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"overflow": null,
|
||||
"_model_module_version": "1.2.0",
|
||||
"_view_count": null,
|
||||
"flex_flow": null,
|
||||
"width": null,
|
||||
"min_width": null,
|
||||
"border": null,
|
||||
"align_items": null,
|
||||
"bottom": null,
|
||||
"_model_module": "@jupyter-widgets/base",
|
||||
"top": null,
|
||||
"grid_column": null,
|
||||
"overflow_y": null,
|
||||
"overflow_x": null,
|
||||
"grid_auto_flow": null,
|
||||
"grid_area": null,
|
||||
"grid_template_columns": null,
|
||||
"flex": null,
|
||||
"_model_name": "LayoutModel",
|
||||
"justify_items": null,
|
||||
"grid_row": null,
|
||||
"max_height": null,
|
||||
"align_content": null,
|
||||
"visibility": null,
|
||||
"align_self": null,
|
||||
"height": null,
|
||||
"min_height": null,
|
||||
"padding": null,
|
||||
"grid_auto_rows": null,
|
||||
"grid_gap": null,
|
||||
"max_width": null,
|
||||
"order": null,
|
||||
"_view_module_version": "1.2.0",
|
||||
"grid_template_areas": null,
|
||||
"object_position": null,
|
||||
"object_fit": null,
|
||||
"grid_auto_columns": null,
|
||||
"margin": null,
|
||||
"display": null,
|
||||
"left": null
|
||||
}
|
||||
},
|
||||
"afd6865a483f4273addb343059632961": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "DescriptionStyleModel",
|
||||
"state": {
|
||||
"_view_name": "StyleView",
|
||||
"_model_name": "DescriptionStyleModel",
|
||||
"description_width": "",
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_view_count": null,
|
||||
"_view_module_version": "1.2.0",
|
||||
"_model_module": "@jupyter-widgets/controls"
|
||||
}
|
||||
},
|
||||
"7b765f7623474999a8c9815e02c1cc6b": {
|
||||
"model_module": "@jupyter-widgets/base",
|
||||
"model_name": "LayoutModel",
|
||||
"state": {
|
||||
"_view_name": "LayoutView",
|
||||
"grid_template_rows": null,
|
||||
"right": null,
|
||||
"justify_content": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"overflow": null,
|
||||
"_model_module_version": "1.2.0",
|
||||
"_view_count": null,
|
||||
"flex_flow": null,
|
||||
"width": null,
|
||||
"min_width": null,
|
||||
"border": null,
|
||||
"align_items": null,
|
||||
"bottom": null,
|
||||
"_model_module": "@jupyter-widgets/base",
|
||||
"top": null,
|
||||
"grid_column": null,
|
||||
"overflow_y": null,
|
||||
"overflow_x": null,
|
||||
"grid_auto_flow": null,
|
||||
"grid_area": null,
|
||||
"grid_template_columns": null,
|
||||
"flex": null,
|
||||
"_model_name": "LayoutModel",
|
||||
"justify_items": null,
|
||||
"grid_row": null,
|
||||
"max_height": null,
|
||||
"align_content": null,
|
||||
"visibility": null,
|
||||
"align_self": null,
|
||||
"height": null,
|
||||
"min_height": null,
|
||||
"padding": null,
|
||||
"grid_auto_rows": null,
|
||||
"grid_gap": null,
|
||||
"max_width": null,
|
||||
"order": null,
|
||||
"_view_module_version": "1.2.0",
|
||||
"grid_template_areas": null,
|
||||
"object_position": null,
|
||||
"object_fit": null,
|
||||
"grid_auto_columns": null,
|
||||
"margin": null,
|
||||
"display": null,
|
||||
"left": null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "AYV_dMVDxyc2"
|
||||
},
|
||||
"source": [
|
||||
"[](https://github.com/lab-ml/nn)\n",
|
||||
"[](https://colab.research.google.com/github/lab-ml/nn/blob/master/labml_nn/normalization/group_norm/experiment.ipynb) \n",
|
||||
"\n",
|
||||
"## Weight Standardization & Batch-Channel Normalization - CIFAR 10\n",
|
||||
"\n",
|
||||
"This is an experiment training a model with Weight Standardization & Batch-Channel Normalization to classify CIFAR-10 dataset."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "AahG_i2y5tY9"
|
||||
},
|
||||
"source": [
|
||||
"Install the `labml-nn` package. Optionally `wandb` package for experiment stats."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"id": "ZCzmCrAIVg0L",
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"outputId": "8332d030-2fab-4a24-876f-152b7f99a226"
|
||||
},
|
||||
"source": [
|
||||
"!pip install labml-nn wandb"
|
||||
],
|
||||
"execution_count": 2,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Collecting labml-nn\n",
|
||||
"\u001b[?25l Downloading https://files.pythonhosted.org/packages/56/22/f649a4fa3c3114e49d568eb2cab2dc836118e3aebb07d781cdae0f975d37/labml_nn-0.4.97-py3-none-any.whl (187kB)\n",
|
||||
"\u001b[K |████████████████████████████████| 194kB 8.4MB/s \n",
|
||||
"\u001b[?25hCollecting wandb\n",
|
||||
"\u001b[?25l Downloading https://files.pythonhosted.org/packages/f6/28/4aefc543967839bdb4e139831b82004279f1c435cede2a9557ccf8369875/wandb-0.10.27-py2.py3-none-any.whl (2.1MB)\n",
|
||||
"\u001b[K |████████████████████████████████| 2.1MB 12.2MB/s \n",
|
||||
"\u001b[?25hCollecting einops\n",
|
||||
" Downloading https://files.pythonhosted.org/packages/5d/a0/9935e030634bf60ecd572c775f64ace82ceddf2f504a5fd3902438f07090/einops-0.3.0-py2.py3-none-any.whl\n",
|
||||
"Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from labml-nn) (1.8.1+cu101)\n",
|
||||
"Collecting labml-helpers>=0.4.76\n",
|
||||
" Downloading https://files.pythonhosted.org/packages/49/df/4d920a4a221acd3cfa384dddb909ed0691b08682c0d8aeaabeee2138624f/labml_helpers-0.4.76-py3-none-any.whl\n",
|
||||
"Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from labml-nn) (1.19.5)\n",
|
||||
"Collecting labml>=0.4.110\n",
|
||||
"\u001b[?25l Downloading https://files.pythonhosted.org/packages/56/7c/513f04703536b50de5f4e0804815325951b76f6a072564b5bb665f9c56ff/labml-0.4.117-py3-none-any.whl (109kB)\n",
|
||||
"\u001b[K |████████████████████████████████| 112kB 27.6MB/s \n",
|
||||
"\u001b[?25hRequirement already satisfied: protobuf>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.12.4)\n",
|
||||
"Collecting sentry-sdk>=0.4.0\n",
|
||||
"\u001b[?25l Downloading https://files.pythonhosted.org/packages/f3/92/5a33be64990ba815364a8f2dd9e6f51de60d23dfddafb4f1fc5577d4dc64/sentry_sdk-1.0.0-py2.py3-none-any.whl (131kB)\n",
|
||||
"\u001b[K |████████████████████████████████| 133kB 29.1MB/s \n",
|
||||
"\u001b[?25hCollecting GitPython>=1.0.0\n",
|
||||
"\u001b[?25l Downloading https://files.pythonhosted.org/packages/a6/99/98019716955ba243657daedd1de8f3a88ca1f5b75057c38e959db22fb87b/GitPython-3.1.14-py3-none-any.whl (159kB)\n",
|
||||
"\u001b[K |████████████████████████████████| 163kB 35.9MB/s \n",
|
||||
"\u001b[?25hRequirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.23.0)\n",
|
||||
"Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.3)\n",
|
||||
"Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.15.0)\n",
|
||||
"Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (5.4.8)\n",
|
||||
"Collecting subprocess32>=3.5.3\n",
|
||||
"\u001b[?25l Downloading https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz (97kB)\n",
|
||||
"\u001b[K |████████████████████████████████| 102kB 12.0MB/s \n",
|
||||
"\u001b[?25hCollecting shortuuid>=0.5.0\n",
|
||||
" Downloading https://files.pythonhosted.org/packages/25/a6/2ecc1daa6a304e7f1b216f0896b26156b78e7c38e1211e9b798b4716c53d/shortuuid-1.0.1-py3-none-any.whl\n",
|
||||
"Collecting docker-pycreds>=0.4.0\n",
|
||||
" Downloading https://files.pythonhosted.org/packages/f5/e8/f6bd1eee09314e7e6dee49cbe2c5e22314ccdb38db16c9fc72d2fa80d054/docker_pycreds-0.4.0-py2.py3-none-any.whl\n",
|
||||
"Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from wandb) (3.13)\n",
|
||||
"Collecting configparser>=3.8.1\n",
|
||||
" Downloading https://files.pythonhosted.org/packages/fd/01/ff260a18caaf4457eb028c96eeb405c4a230ca06c8ec9c1379f813caa52e/configparser-5.0.2-py3-none-any.whl\n",
|
||||
"Collecting pathtools\n",
|
||||
" Downloading https://files.pythonhosted.org/packages/e7/7f/470d6fcdf23f9f3518f6b0b76be9df16dcc8630ad409947f8be2eb0ed13a/pathtools-0.1.2.tar.gz\n",
|
||||
"Requirement already satisfied: Click>=7.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (7.1.2)\n",
|
||||
"Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.8.1)\n",
|
||||
"Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch->labml-nn) (3.7.4.3)\n",
|
||||
"Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.12.0->wandb) (56.0.0)\n",
|
||||
"Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from sentry-sdk>=0.4.0->wandb) (2020.12.5)\n",
|
||||
"Requirement already satisfied: urllib3>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from sentry-sdk>=0.4.0->wandb) (1.24.3)\n",
|
||||
"Collecting gitdb<5,>=4.0.1\n",
|
||||
"\u001b[?25l Downloading https://files.pythonhosted.org/packages/ea/e8/f414d1a4f0bbc668ed441f74f44c116d9816833a48bf81d22b697090dba8/gitdb-4.0.7-py3-none-any.whl (63kB)\n",
|
||||
"\u001b[K |████████████████████████████████| 71kB 10.7MB/s \n",
|
||||
"\u001b[?25hRequirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (3.0.4)\n",
|
||||
"Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2.10)\n",
|
||||
"Collecting smmap<5,>=3.0.1\n",
|
||||
" Downloading https://files.pythonhosted.org/packages/68/ee/d540eb5e5996eb81c26ceffac6ee49041d473bc5125f2aa995cf51ec1cf1/smmap-4.0.0-py2.py3-none-any.whl\n",
|
||||
"Building wheels for collected packages: subprocess32, pathtools\n",
|
||||
" Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
|
||||
" Created wheel for subprocess32: filename=subprocess32-3.5.4-cp37-none-any.whl size=6489 sha256=91ec9a13851693ac0f3ae914c31ec39b7a54da583a7f2cb20cc132dc0db3c648\n",
|
||||
" Stored in directory: /root/.cache/pip/wheels/68/39/1a/5e402bdfdf004af1786c8b853fd92f8c4a04f22aad179654d1\n",
|
||||
" Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
|
||||
" Created wheel for pathtools: filename=pathtools-0.1.2-cp37-none-any.whl size=8786 sha256=fde28c832a62934336b3d858a4aef15d113d96a2d5675823ec4359761a0ac52e\n",
|
||||
" Stored in directory: /root/.cache/pip/wheels/0b/04/79/c3b0c3a0266a3cb4376da31e5bfe8bba0c489246968a68e843\n",
|
||||
"Successfully built subprocess32 pathtools\n",
|
||||
"Installing collected packages: einops, smmap, gitdb, GitPython, labml, labml-helpers, labml-nn, sentry-sdk, subprocess32, shortuuid, docker-pycreds, configparser, pathtools, wandb\n",
|
||||
"Successfully installed GitPython-3.1.14 configparser-5.0.2 docker-pycreds-0.4.0 einops-0.3.0 gitdb-4.0.7 labml-0.4.117 labml-helpers-0.4.76 labml-nn-0.4.97 pathtools-0.1.2 sentry-sdk-1.0.0 shortuuid-1.0.1 smmap-4.0.0 subprocess32-3.5.4 wandb-0.10.27\n"
|
||||
],
|
||||
"name": "stdout"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "SE2VUQ6L5zxI"
|
||||
},
|
||||
"source": [
|
||||
"Imports"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"id": "0hJXx_g0wS2C"
|
||||
},
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"import torch.nn as nn\n",
|
||||
"\n",
|
||||
"from labml import experiment\n",
|
||||
"from labml_nn.normalization.weight_standardization.experiment import CIFAR10Configs as Configs"
|
||||
],
|
||||
"execution_count": 4,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "Lpggo0wM6qb-"
|
||||
},
|
||||
"source": [
|
||||
"Create an experiment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"id": "bFcr9k-l4cAg"
|
||||
},
|
||||
"source": [
|
||||
"experiment.create(name=\"cifar10\", comment=\"WS + BCN\")"
|
||||
],
|
||||
"execution_count": 5,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "-OnHLi626tJt"
|
||||
},
|
||||
"source": [
|
||||
"Initialize configurations"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"id": "Piz0c5f44hRo"
|
||||
},
|
||||
"source": [
|
||||
"conf = Configs()"
|
||||
],
|
||||
"execution_count": 6,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "wwMzCqpD6vkL"
|
||||
},
|
||||
"source": [
|
||||
"Set experiment configurations and assign a configurations dictionary to override configurations"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/",
|
||||
"height": 17
|
||||
},
|
||||
"id": "e6hmQhTw4nks",
|
||||
"outputId": "33a5979f-70eb-4d19-e82a-a6b113316cca"
|
||||
},
|
||||
"source": [
|
||||
"experiment.configs(conf, {\n",
|
||||
" 'optimizer.optimizer': 'Adam',\n",
|
||||
" 'optimizer.learning_rate': 2.5e-4,\n",
|
||||
" 'train_batch_size': 64,\n",
|
||||
"})"
|
||||
],
|
||||
"execution_count": 7,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<pre style=\"overflow-x: scroll;\"></pre>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"tags": []
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "KJZRf8527GxL"
|
||||
},
|
||||
"source": [
|
||||
"Start the experiment and run the training loop."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/",
|
||||
"height": 933,
|
||||
"referenced_widgets": [
|
||||
"1dacd9c82caf40c8b2def0c5fe4f7643",
|
||||
"67e37901c44b4bde9062014542cae018",
|
||||
"9d57be286b7244ca9a5f74cff6c3cf4e",
|
||||
"9e511aaba1204ea9985fb6f55e024039",
|
||||
"61849c45ed0c4436995bb81d7efdac7e",
|
||||
"7ed3376588734d64a229fd8dd6bfe06e",
|
||||
"afd6865a483f4273addb343059632961",
|
||||
"7b765f7623474999a8c9815e02c1cc6b"
|
||||
]
|
||||
},
|
||||
"id": "aIAWo7Fw5DR8",
|
||||
"outputId": "cf3d16aa-a9a1-4fd4-cfb4-23e8ea5ea940"
|
||||
},
|
||||
"source": [
|
||||
"with experiment.start():\n",
|
||||
" conf.run()"
|
||||
],
|
||||
"execution_count": null,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<pre style=\"overflow-x: scroll;\">\n",
|
||||
"<strong><span style=\"text-decoration: underline\">cifar10</span></strong>: <span style=\"color: #208FFB\">f4a783a2a7df11eb921d0242ac1c0002</span>\n",
|
||||
"\t<strong><span style=\"color: #DDB62B\">WS + BCN</span></strong>\n",
|
||||
"\t[dirty]: <strong><span style=\"color: #DDB62B\">\"\"</span></strong>\n",
|
||||
"<span style=\"color: #C5C1B4\"></span>\n",
|
||||
"<span style=\"color: #C5C1B4\">--------------------------------------------------</span><span style=\"color: #DDB62B\"><strong><span style=\"text-decoration: underline\"></span></strong></span>\n",
|
||||
"<span style=\"color: #DDB62B\"><strong><span style=\"text-decoration: underline\">LABML WARNING</span></strong></span>\n",
|
||||
"<span style=\"color: #DDB62B\"><strong><span style=\"text-decoration: underline\"></span></strong></span>LabML App Warning: <span style=\"color: #60C6C8\">empty_token: </span><strong>Please create a valid token at https://app.labml.ai.</strong>\n",
|
||||
"<strong>Click on the experiment link to monitor the experiment and add it to your experiments list.</strong><span style=\"color: #C5C1B4\"></span>\n",
|
||||
"<span style=\"color: #C5C1B4\">--------------------------------------------------</span>\n",
|
||||
"<span style=\"color: #208FFB\">Monitor experiment at </span><a href='https://app.labml.ai/run/f4a783a2a7df11eb921d0242ac1c0002' target='blank'>https://app.labml.ai/run/f4a783a2a7df11eb921d0242ac1c0002</a>\n",
|
||||
"Initialize...\n",
|
||||
" Prepare mode<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t13.37ms</span>\n",
|
||||
" Prepare model...\n",
|
||||
" Prepare device...\n",
|
||||
" Prepare device_info<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t72.67ms</span>\n",
|
||||
" Prepare device<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t84.98ms</span>\n",
|
||||
" Prepare model<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t10,890.59ms</span>\n",
|
||||
"Initialize<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t11,011.70ms</span>\n",
|
||||
"Prepare validator...\n",
|
||||
" Prepare valid_loader...\n",
|
||||
" Prepare valid_dataset...\n",
|
||||
" Prepare dataset_transforms<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t4.17ms</span>\n",
|
||||
" Prepare valid_dataset<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t6,677.35ms</span>\n",
|
||||
" Prepare valid_loader<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t6,792.89ms</span>\n",
|
||||
"Prepare validator<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t6,907.56ms</span>\n",
|
||||
"Prepare trainer...\n",
|
||||
" Prepare train_loader...\n",
|
||||
" Prepare train_dataset<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t1,004.63ms</span>\n",
|
||||
" Prepare train_loader<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t1,117.76ms</span>\n",
|
||||
"Prepare trainer<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t1,156.46ms</span>\n",
|
||||
"Prepare training_loop...\n",
|
||||
" Prepare loop_count<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t42.93ms</span>\n",
|
||||
"Prepare training_loop<span style=\"color: #00A250\">...[DONE]</span><span style=\"color: #208FFB\">\t311.00ms</span>\n",
|
||||
"<strong><span style=\"color: #DDB62B\"> 50,000: </span></strong>Train:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 68,972ms </span>Valid:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 5,060ms </span> loss.train: <span style=\"color: #C5C1B4\">0.811377</span> accuracy.train: <span style=\"color: #C5C1B4\">0.570960</span> loss.valid: <span style=\"color: #C5C1B4\"> 0.81718</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.592600</span> <span style=\"color: #208FFB\">74,141ms</span><span style=\"color: #D160C4\"> 0:01m/ 0:11m </span>\n",
|
||||
"<strong><span style=\"color: #DDB62B\"> 100,000: </span></strong>Train:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 79,315ms </span>Valid:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 5,436ms </span> loss.train: <span style=\"color: #C5C1B4\">0.861817</span> accuracy.train: <span style=\"color: #C5C1B4\">0.764740</span> loss.valid: <span style=\"color: #C5C1B4\">0.729245</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.746400</span> <span style=\"color: #208FFB\">79,902ms</span><span style=\"color: #D160C4\"> 0:02m/ 0:10m </span>\n",
|
||||
"<strong><span style=\"color: #DDB62B\"> 150,000: </span></strong>Train:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 77,688ms </span>Valid:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 5,494ms </span> loss.train: <span style=\"color: #C5C1B4\">0.817582</span> accuracy.train: <span style=\"color: #C5C1B4\">0.825700</span> loss.valid: <span style=\"color: #C5C1B4\">0.578170</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.790600</span> <span style=\"color: #208FFB\">81,153ms</span><span style=\"color: #D160C4\"> 0:04m/ 0:09m </span>\n",
|
||||
"<strong><span style=\"color: #DDB62B\"> 200,000: </span></strong>Train:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 78,789ms </span>Valid:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 5,483ms </span> loss.train: <span style=\"color: #C5C1B4\">0.528081</span> accuracy.train: <span style=\"color: #C5C1B4\">0.864400</span> loss.valid: <span style=\"color: #C5C1B4\">0.540609</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.809200</span> <span style=\"color: #208FFB\">81,860ms</span><span style=\"color: #D160C4\"> 0:05m/ 0:08m </span>\n",
|
||||
"<strong><span style=\"color: #DDB62B\"> 250,000: </span></strong>Train:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 80,515ms </span>Valid:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 5,556ms </span> loss.train: <span style=\"color: #C5C1B4\">0.513213</span> accuracy.train: <span style=\"color: #C5C1B4\">0.892160</span> loss.valid: <span style=\"color: #C5C1B4\">0.562979</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.817000</span> <span style=\"color: #208FFB\">82,512ms</span><span style=\"color: #D160C4\"> 0:06m/ 0:06m </span>\n",
|
||||
"<strong><span style=\"color: #DDB62B\"> 300,000: </span></strong>Train:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 77,969ms </span>Valid:<span style=\"color: #C5C1B4\"> 100%</span><span style=\"color: #208FFB\"> 5,508ms </span> loss.train: <span style=\"color: #C5C1B4\">0.264446</span> accuracy.train: <span style=\"color: #C5C1B4\">0.915060</span> loss.valid: <span style=\"color: #C5C1B4\">0.542280</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.832200</span> <span style=\"color: #208FFB\">82,644ms</span><span style=\"color: #D160C4\"> 0:08m/ 0:05m </span>\n",
|
||||
"<strong><span style=\"color: #DDB62B\"> 313,888: </span></strong>Train:<span style=\"color: #C5C1B4\"> 27%</span><span style=\"color: #208FFB\"> 77,803ms </span>Valid:<span style=\"color: #C5C1B4\"> 20%</span><span style=\"color: #208FFB\"> 5,506ms </span> loss.train: <strong>0.152911</strong> accuracy.train: <strong>0.937572</strong> loss.valid: <span style=\"color: #C5C1B4\">0.572198</span> accuracy.valid: <span style=\"color: #C5C1B4\">0.834961</span> <span style=\"color: #208FFB\">82,644ms</span><span style=\"color: #D160C4\"> 0:08m/ 0:05m </span></pre>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"tags": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"application/javascript": [
|
||||
"\n",
|
||||
" window._wandbApiKey = new Promise((resolve, reject) => {\n",
|
||||
" function loadScript(url) {\n",
|
||||
" return new Promise(function(resolve, reject) {\n",
|
||||
" let newScript = document.createElement(\"script\");\n",
|
||||
" newScript.onerror = reject;\n",
|
||||
" newScript.onload = resolve;\n",
|
||||
" document.body.appendChild(newScript);\n",
|
||||
" newScript.src = url;\n",
|
||||
" });\n",
|
||||
" }\n",
|
||||
" loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
|
||||
" const iframe = document.createElement('iframe')\n",
|
||||
" iframe.style.cssText = \"width:0;height:0;border:none\"\n",
|
||||
" document.body.appendChild(iframe)\n",
|
||||
" const handshake = new Postmate({\n",
|
||||
" container: iframe,\n",
|
||||
" url: 'https://wandb.ai/authorize'\n",
|
||||
" });\n",
|
||||
" const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
|
||||
" handshake.then(function(child) {\n",
|
||||
" child.on('authorize', data => {\n",
|
||||
" clearTimeout(timeout)\n",
|
||||
" resolve(data)\n",
|
||||
" });\n",
|
||||
" });\n",
|
||||
" })\n",
|
||||
" });\n",
|
||||
" "
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.Javascript object>"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"tags": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
|
||||
],
|
||||
"name": "stderr"
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": [
|
||||
"\n",
|
||||
" Tracking run with wandb version 0.10.27<br/>\n",
|
||||
" Syncing run <strong style=\"color:#cdcd00\">icy-gorge-2</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
|
||||
" Project page: <a href=\"https://wandb.ai/vpj/cifar10\" target=\"_blank\">https://wandb.ai/vpj/cifar10</a><br/>\n",
|
||||
" Run page: <a href=\"https://wandb.ai/vpj/cifar10/runs/3flr4k8w\" target=\"_blank\">https://wandb.ai/vpj/cifar10/runs/3flr4k8w</a><br/>\n",
|
||||
" Run data is saved locally in <code>/content/logs/cifar10/f4a783a2a7df11eb921d0242ac1c0002/wandb/run-20210428_051029-3flr4k8w</code><br/><br/>\n",
|
||||
" "
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"tags": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to /content/data/cifar-10-python.tar.gz\n"
|
||||
],
|
||||
"name": "stdout"
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "1dacd9c82caf40c8b2def0c5fe4f7643",
|
||||
"version_minor": 0,
|
||||
"version_major": 2
|
||||
},
|
||||
"text/plain": [
|
||||
"HBox(children=(FloatProgress(value=0.0, max=170498071.0), HTML(value='')))"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"tags": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"Extracting /content/data/cifar-10-python.tar.gz to /content/data\n",
|
||||
"Files already downloaded and verified\n"
|
||||
],
|
||||
"name": "stdout"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"id": "oBXXlP2b7XZO"
|
||||
},
|
||||
"source": [
|
||||
""
|
||||
],
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user