diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..4293a08 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "workbench.editorAssociations": { + "*.copilotmd": "vscode.markdown.preview.editor", + "file:/**/*.csv": "jupyter-data-wrangler" + } +} \ No newline at end of file diff --git a/01_Python_Jupyter/code/00/einfuehrung_jupyter.ipynb b/01_Python_Jupyter/code/00/einfuehrung_jupyter.ipynb index a37b893..c477550 100644 --- a/01_Python_Jupyter/code/00/einfuehrung_jupyter.ipynb +++ b/01_Python_Jupyter/code/00/einfuehrung_jupyter.ipynb @@ -1189,7 +1189,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "dsai", "language": "python", "name": "python3" }, @@ -1203,7 +1203,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.20" + "version": "3.9.23" } }, "nbformat": 4, diff --git a/05_ML/code/unsupervised_2_downprojection.ipynb b/05_ML/code/unsupervised_2_downprojection.ipynb index 1ff5fcf..f78c297 100644 --- a/05_ML/code/unsupervised_2_downprojection.ipynb +++ b/05_ML/code/unsupervised_2_downprojection.ipynb @@ -1524,7 +1524,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.20" + "version": "3.9.23" } }, "nbformat": 4, diff --git a/06_NN/code/nn_10_cnn_2.ipynb b/06_NN/code/nn_10_cnn_2.ipynb new file mode 100644 index 0000000..481b817 --- /dev/null +++ b/06_NN/code/nn_10_cnn_2.ipynb @@ -0,0 +1,126 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "086b9495", + "metadata": {}, + "source": [ + "
\n", + "\n", + "

Neural Networks: Convolutional Neural Networks (2)

\n", + "

DSAI

\n", + "

Jakob Eggl

\n", + "\n", + "
\n", + " \"Logo\"\n", + "
\n", + " © 2025/26 Jakob Eggl. Nutzung oder Verbreitung nur mit ausdrücklicher Genehmigung des Autors.\n", + "
\n", + "
\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "640a7aba", + "metadata": {}, + "source": [ + "Nachdem wir jetzt wissen, wie ein *Convolutional Neuronal Network* (**CNN**) funktioniert, wollen wir nun nochmal die Datasets **MNIST** und **Fashion-MNIST** ausprobieren." + ] + }, + { + "cell_type": "markdown", + "id": "8d8125aa", + "metadata": {}, + "source": [ + "Erstelle somit ein Neuronales Netzwerk, welches auf der CNN Architektur basiert und auf den Datasets **MNIST** und **Fashion-MNIST** trainiert wird." + ] + }, + { + "cell_type": "markdown", + "id": "2b5f313f", + "metadata": {}, + "source": [ + "# Lösung" + ] + }, + { + "cell_type": "markdown", + "id": "6c231b6a", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2a0a6fdf", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "from torch.utils.data import DataLoader, Dataset, random_split\n", + "from torchvision import datasets, transforms\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.metrics import confusion_matrix\n", + "import seaborn as sns" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e942400c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cpu\n" + ] + } + ], + "source": [ + "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('mps') if torch.backends.mps.is_available() else torch.device('cpu')\n", + "print(device)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86c04c79", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dsai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.23" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/06_NN/code/nn_10_cnn_2_solution.ipynb b/06_NN/code/nn_10_cnn_2_solution.ipynb new file mode 100644 index 0000000..74b9256 --- /dev/null +++ b/06_NN/code/nn_10_cnn_2_solution.ipynb @@ -0,0 +1,938 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "086b9495", + "metadata": {}, + "source": [ + "
\n", + "\n", + "

Neural Networks: Convolutional Neural Networks (2)

\n", + "

DSAI

\n", + "

Jakob Eggl

\n", + "\n", + "
\n", + " \"Logo\"\n", + "
\n", + " © 2025/26 Jakob Eggl. Nutzung oder Verbreitung nur mit ausdrücklicher Genehmigung des Autors.\n", + "
\n", + "
\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "640a7aba", + "metadata": {}, + "source": [ + "Nachdem wir jetzt wissen, wie ein *Convolutional Neuronal Network* (**CNN**) funktioniert, wollen wir nun nochmal die Datasets **MNIST** und **Fashion-MNIST** ausprobieren." + ] + }, + { + "cell_type": "markdown", + "id": "7f180ba7", + "metadata": {}, + "source": [ + "Erstelle somit ein Neuronales Netzwerk, welches auf der CNN Architektur basiert und auf den Datasets **MNIST** und **Fashion-MNIST** trainiert wird." + ] + }, + { + "cell_type": "markdown", + "id": "2b5f313f", + "metadata": {}, + "source": [ + "# Lösung" + ] + }, + { + "cell_type": "markdown", + "id": "24949af8", + "metadata": {}, + "source": [ + "Nun können wir testen, wie gut ein CNN bei unseren bisherigen Aufgaben (MNIST und Fashion MNIST performt)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "a5e14abb", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "from torch.utils.data import DataLoader, Dataset, random_split\n", + "from torchvision import datasets, transforms\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.metrics import confusion_matrix\n", + "import seaborn as sns" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "6035d78c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cpu\n" + ] + } + ], + "source": [ + "\n", + "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('mps') if torch.backends.mps.is_available() else torch.device('cpu')\n", + "print(device)" + ] + }, + { + "cell_type": "markdown", + "id": "7c345703", + "metadata": {}, + "source": [ + "## MNIST" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5d56b200", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100.0%\n", + "100.0%\n", + "100.0%\n", + "100.0%\n" + ] + } + ], + "source": [ + "data_path = os.path.join(\"..\", \"..\", \"_data\", \"mnist_data\")\n", + "\n", + "train_dataset = datasets.MNIST(root=data_path, train=True, download=True, transform=transforms.ToTensor()) # ToTensor makes images [0, 1] instead of {1,2,...,255}\n", + "test_dataset = datasets.MNIST(root=data_path, train=False, download=True, transform=transforms.ToTensor())\n", + "\n", + "test_size = len(test_dataset) // 2\n", + "valid_size = len(test_dataset) - test_size\n", + "\n", + "test_dataset, valid_dataset = random_split(test_dataset, [test_size, valid_size])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "05e3dd14", + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 64\n", + "\n", + "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", + "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n", + "valid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "id": "40660012", + "metadata": {}, + "source": [ + "Hier könnte man noch normalisieren und würde wahrscheinlich noch ein bisschen besser abscheiden. Man kann eigentlich direkt die `transform`-Methode vom `nn_8_example_classification_solution` übernehmen." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "536dd4f6", + "metadata": {}, + "outputs": [], + "source": [ + "class CNNMNISTClassifier(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.layers = nn.Sequential(\n", + " nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1), # output: 32 x 28 x 28\n", + " nn.ReLU(),\n", + " nn.MaxPool2d(kernel_size=2, stride=2), # output: 32 x 14 x 14\n", + " nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1), # output: 64 x 14 x 14\n", + " nn.ReLU(),\n", + " nn.MaxPool2d(kernel_size=2, stride=2), # output: 64 x 7 x 7\n", + " nn.Flatten(),\n", + " nn.Linear(64*7*7, 256),\n", + " nn.ReLU(),\n", + " nn.Linear(256, 128),\n", + " nn.ReLU(),\n", + " nn.Linear(128,64),\n", + " nn.ReLU(),\n", + " nn.Linear(64, 10),\n", + " )\n", + " def forward(self, x):\n", + " return self.layers(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f5d7153b", + "metadata": {}, + "outputs": [], + "source": [ + "def evaluate_model(model, data_loader, criterion):\n", + " model.eval()\n", + " loss_total = 0.0\n", + " correct = 0\n", + " total = 0\n", + " \n", + " with torch.no_grad():\n", + " for data, target in data_loader:\n", + " data, target = data.to(device), target.to(device)\n", + " output = model(data)\n", + " loss = criterion(output, target)\n", + " loss_total += loss.item() * data.size(0)\n", + " \n", + " _, predicted = torch.max(output.data, 1)\n", + " total += target.size(0)\n", + " correct += (predicted == target).sum().item()\n", + " \n", + " avg_loss = loss_total / total\n", + " accuracy = 100.0 * correct / total\n", + " return avg_loss, accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "c8297d67", + "metadata": {}, + "outputs": [], + "source": [ + "def train_model(model, train_loader, valid_loader, criterion, optimizer, save_path:str=None,\n", + " epochs=20, validate_at=1, print_at=100, patience=3):\n", + " torch.set_num_threads(8)\n", + " if save_path is None:\n", + " save_path = 'best_model.pt'\n", + "\n", + " best_loss = float(\"inf\")\n", + " patience_counter = 0\n", + "\n", + " for epoch in range(1, epochs+1):\n", + " model.train()\n", + " running_loss = 0.0\n", + "\n", + " for batch_idx, (data, target) in enumerate(train_loader):\n", + " data, target = data.to(device), target.to(device)\n", + "\n", + " optimizer.zero_grad()\n", + " output = model(data)\n", + " loss = criterion(output, target)\n", + " loss.backward()\n", + " optimizer.step()\n", + " \n", + " running_loss += loss.item()\n", + " \n", + " if (batch_idx+1) % print_at == 0:\n", + " print(f\"Epoch [{epoch}/{epochs}], Step [{batch_idx+1}/{len(train_loader)}], Loss: {loss.item():.4f}\")\n", + "\n", + " if epoch % validate_at == 0:\n", + " val_loss, val_acc = evaluate_model(model, valid_loader, criterion)\n", + " print(f\"Epoch [{epoch}/{epochs}] - Validation Loss: {val_loss:.4f}, Validation Accuracy: {val_acc:.2f}%\")\n", + "\n", + " if val_loss < best_loss:\n", + " best_loss = val_loss\n", + " patience_counter = 0\n", + " torch.save(model.state_dict(), save_path)\n", + " print(f\">>> Found a better model and saved it at '{save_path}'\")\n", + " else:\n", + " patience_counter += 1\n", + " print(f\"No Improvement. Early Stopping Counter: {patience_counter}/{patience}\")\n", + " if patience_counter >= patience:\n", + " print(\"Early Stopping triggered.\")\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "cd75f50d", + "metadata": {}, + "outputs": [], + "source": [ + "### HYPERPARAMETER ###\n", + "\n", + "model = CNNMNISTClassifier().to(device)\n", + "criterion = nn.CrossEntropyLoss()\n", + "lr = 0.001\n", + "optimizer = optim.Adam(model.parameters(), lr=lr)\n", + "epochs = 20\n", + "validate_at = 1\n", + "print_at = 200\n", + "early_stopping_patience = 3\n", + "save_path = os.path.join(\"..\", \"models\", \"best_model_cnn_mnist.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "d408a930", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch [1/20], Step [200/938], Loss: 0.7541\n", + "Epoch [1/20], Step [400/938], Loss: 0.4307\n", + "Epoch [1/20], Step [600/938], Loss: 0.4306\n", + "Epoch [1/20], Step [800/938], Loss: 0.3365\n", + "Epoch [1/20] - Validation Loss: 0.3727, Validation Accuracy: 86.10%\n", + ">>> Found a better model and saved it at '../models/best_model_cnn_mnist.pt'\n", + "Epoch [2/20], Step [200/938], Loss: 0.2701\n", + "Epoch [2/20], Step [400/938], Loss: 0.3223\n", + "Epoch [2/20], Step [600/938], Loss: 0.3573\n", + "Epoch [2/20], Step [800/938], Loss: 0.2421\n", + "Epoch [2/20] - Validation Loss: 0.3108, Validation Accuracy: 88.48%\n", + ">>> Found a better model and saved it at '../models/best_model_cnn_mnist.pt'\n", + "Epoch [3/20], Step [200/938], Loss: 0.2437\n", + "Epoch [3/20], Step [400/938], Loss: 0.2717\n", + "Epoch [3/20], Step [600/938], Loss: 0.3106\n", + "Epoch [3/20], Step [800/938], Loss: 0.2901\n", + "Epoch [3/20] - Validation Loss: 0.3020, Validation Accuracy: 88.68%\n", + ">>> Found a better model and saved it at '../models/best_model_cnn_mnist.pt'\n", + "Epoch [4/20], Step [200/938], Loss: 0.2356\n", + "Epoch [4/20], Step [400/938], Loss: 0.3265\n", + "Epoch [4/20], Step [600/938], Loss: 0.2237\n", + "Epoch [4/20], Step [800/938], Loss: 0.2049\n", + "Epoch [4/20] - Validation Loss: 0.2775, Validation Accuracy: 90.18%\n", + ">>> Found a better model and saved it at '../models/best_model_cnn_mnist.pt'\n", + "Epoch [5/20], Step [200/938], Loss: 0.1586\n", + "Epoch [5/20], Step [400/938], Loss: 0.1646\n", + "Epoch [5/20], Step [600/938], Loss: 0.1840\n", + "Epoch [5/20], Step [800/938], Loss: 0.2783\n", + "Epoch [5/20] - Validation Loss: 0.2549, Validation Accuracy: 91.10%\n", + ">>> Found a better model and saved it at '../models/best_model_cnn_mnist.pt'\n", + "Epoch [6/20], Step [200/938], Loss: 0.2141\n", + "Epoch [6/20], Step [400/938], Loss: 0.1137\n", + "Epoch [6/20], Step [600/938], Loss: 0.1712\n", + "Epoch [6/20], Step [800/938], Loss: 0.2002\n", + "Epoch [6/20] - Validation Loss: 0.2805, Validation Accuracy: 90.74%\n", + "No Improvement. Early Stopping Counter: 1/3\n", + "Epoch [7/20], Step [200/938], Loss: 0.1157\n", + "Epoch [7/20], Step [400/938], Loss: 0.1438\n", + "Epoch [7/20], Step [600/938], Loss: 0.1271\n", + "Epoch [7/20], Step [800/938], Loss: 0.1231\n", + "Epoch [7/20] - Validation Loss: 0.2463, Validation Accuracy: 91.92%\n", + ">>> Found a better model and saved it at '../models/best_model_cnn_mnist.pt'\n", + "Epoch [8/20], Step [200/938], Loss: 0.2126\n", + "Epoch [8/20], Step [400/938], Loss: 0.2189\n", + "Epoch [8/20], Step [600/938], Loss: 0.1637\n", + "Epoch [8/20], Step [800/938], Loss: 0.0895\n", + "Epoch [8/20] - Validation Loss: 0.2650, Validation Accuracy: 91.30%\n", + "No Improvement. Early Stopping Counter: 1/3\n", + "Epoch [9/20], Step [200/938], Loss: 0.1041\n", + "Epoch [9/20], Step [400/938], Loss: 0.1339\n", + "Epoch [9/20], Step [600/938], Loss: 0.1669\n", + "Epoch [9/20], Step [800/938], Loss: 0.1946\n", + "Epoch [9/20] - Validation Loss: 0.2602, Validation Accuracy: 91.64%\n", + "No Improvement. Early Stopping Counter: 2/3\n", + "Epoch [10/20], Step [200/938], Loss: 0.0308\n", + "Epoch [10/20], Step [400/938], Loss: 0.1354\n", + "Epoch [10/20], Step [600/938], Loss: 0.1513\n", + "Epoch [10/20], Step [800/938], Loss: 0.0820\n", + "Epoch [10/20] - Validation Loss: 0.2590, Validation Accuracy: 91.76%\n", + "No Improvement. Early Stopping Counter: 3/3\n", + "Early Stopping triggered.\n" + ] + } + ], + "source": [ + "train_model(model, train_loader, valid_loader, criterion, optimizer, epochs=epochs, validate_at=validate_at, print_at=print_at, patience=early_stopping_patience, save_path=save_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "4f97b366", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finaler Test Loss: 0.0260\n", + "Finale Test Accuracy: 98.88%\n" + ] + } + ], + "source": [ + "model.load_state_dict(torch.load(save_path))\n", + "test_loss, test_acc = evaluate_model(model, test_loader, criterion)\n", + "print(f\"Finaler Test Loss: {test_loss:.4f}\")\n", + "print(f\"Finale Test Accuracy: {test_acc:.2f}%\")" + ] + }, + { + "cell_type": "markdown", + "id": "7dde66a7", + "metadata": {}, + "source": [ + "Also funktioniert auch dieses Modell sehr gut.\n", + "\n", + "Probieren wir nun auch das Fashion-MNIST Dataset." + ] + }, + { + "cell_type": "markdown", + "id": "c34d701e", + "metadata": {}, + "source": [ + "## Fashion MNIST" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "d617df78", + "metadata": {}, + "outputs": [], + "source": [ + "torch.cuda.empty_cache() # Clears GPU memory (otherwise it could run out of memory)" + ] + }, + { + "cell_type": "markdown", + "id": "a9c9a4e8", + "metadata": {}, + "source": [ + "Nun testen wir das gleiche Modell (nur anders genannt) für Fashion MNIST." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "c9a9e4ea", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100.0%\n", + "100.0%\n", + "100.0%\n", + "100.0%\n" + ] + } + ], + "source": [ + "data_path = os.path.join(\"..\", \"..\", \"_data\", \"fashion_mnist_data\")\n", + "\n", + "train_dataset = datasets.FashionMNIST(root=data_path, train=True, download=True, transform=transforms.ToTensor()) # ToTensor makes images [0, 1] instead of {1,2,...,255}\n", + "test_dataset = datasets.FashionMNIST(root=data_path, train=False, download=True, transform=transforms.ToTensor())\n", + "\n", + "test_size = len(test_dataset) // 2\n", + "valid_size = len(test_dataset) - test_size\n", + "\n", + "test_dataset, valid_dataset = random_split(test_dataset, [test_size, valid_size])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "1b96937b", + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 64\n", + "\n", + "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", + "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n", + "valid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "id": "4135a271", + "metadata": {}, + "source": [ + "Erneut das gleiche Modell, jedoch anderer Name." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "14eefada", + "metadata": {}, + "outputs": [], + "source": [ + "class CNNFashionMNISTClassifier(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.layers = nn.Sequential(\n", + " nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1), # output: 32 x 28 x 28\n", + " nn.ReLU(),\n", + " nn.MaxPool2d(kernel_size=2, stride=2), # output: 32 x 14 x 14\n", + " nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1), # output: 64 x 14 x 14\n", + " nn.ReLU(),\n", + " nn.MaxPool2d(kernel_size=2, stride=2), # output: 64 x 7 x 7\n", + " nn.Flatten(),\n", + " nn.Linear(64*7*7, 256),\n", + " nn.ReLU(),\n", + " nn.Linear(256, 128),\n", + " nn.ReLU(),\n", + " nn.Linear(128,64),\n", + " nn.ReLU(),\n", + " nn.Linear(64, 10),\n", + " )\n", + " def forward(self, x):\n", + " return self.layers(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "d8af8854", + "metadata": {}, + "outputs": [], + "source": [ + "### HYPERPARAMETER ###\n", + "\n", + "model = CNNFashionMNISTClassifier().to(device)\n", + "criterion = nn.CrossEntropyLoss()\n", + "lr = 0.001\n", + "optimizer = optim.Adam(model.parameters(), lr=lr)\n", + "epochs = 20\n", + "validate_at = 1\n", + "print_at = 200\n", + "early_stopping_patience = 3\n", + "save_path = os.path.join(\"..\", \"models\", \"best_model_cnn_fashion_mnist.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "23f69a13", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch [1/20], Step [200/938], Loss: 0.4498\n", + "Epoch [1/20], Step [400/938], Loss: 0.3739\n", + "Epoch [1/20], Step [600/938], Loss: 0.2920\n", + "Epoch [1/20], Step [800/938], Loss: 0.3433\n", + "Epoch [1/20] - Validation Loss: 0.3789, Validation Accuracy: 85.44%\n", + ">>> Found a better model and saved it at '../models/best_model_cnn_fashion_mnist.pt'\n", + "Epoch [2/20], Step [200/938], Loss: 0.3348\n", + "Epoch [2/20], Step [400/938], Loss: 0.3218\n", + "Epoch [2/20], Step [600/938], Loss: 0.2406\n", + "Epoch [2/20], Step [800/938], Loss: 0.2876\n", + "Epoch [2/20] - Validation Loss: 0.2920, Validation Accuracy: 88.96%\n", + ">>> Found a better model and saved it at '../models/best_model_cnn_fashion_mnist.pt'\n", + "Epoch [3/20], Step [200/938], Loss: 0.2267\n", + "Epoch [3/20], Step [400/938], Loss: 0.1655\n", + "Epoch [3/20], Step [600/938], Loss: 0.2643\n", + "Epoch [3/20], Step [800/938], Loss: 0.1533\n", + "Epoch [3/20] - Validation Loss: 0.2866, Validation Accuracy: 89.46%\n", + ">>> Found a better model and saved it at '../models/best_model_cnn_fashion_mnist.pt'\n", + "Epoch [4/20], Step [200/938], Loss: 0.1869\n", + "Epoch [4/20], Step [400/938], Loss: 0.2363\n", + "Epoch [4/20], Step [600/938], Loss: 0.2081\n", + "Epoch [4/20], Step [800/938], Loss: 0.1707\n", + "Epoch [4/20] - Validation Loss: 0.2570, Validation Accuracy: 90.92%\n", + ">>> Found a better model and saved it at '../models/best_model_cnn_fashion_mnist.pt'\n", + "Epoch [5/20], Step [200/938], Loss: 0.0748\n", + "Epoch [5/20], Step [400/938], Loss: 0.0544\n", + "Epoch [5/20], Step [600/938], Loss: 0.2605\n", + "Epoch [5/20], Step [800/938], Loss: 0.2111\n", + "Epoch [5/20] - Validation Loss: 0.2585, Validation Accuracy: 90.78%\n", + "No Improvement. Early Stopping Counter: 1/3\n", + "Epoch [6/20], Step [200/938], Loss: 0.2887\n", + "Epoch [6/20], Step [400/938], Loss: 0.0977\n", + "Epoch [6/20], Step [600/938], Loss: 0.1891\n", + "Epoch [6/20], Step [800/938], Loss: 0.0624\n", + "Epoch [6/20] - Validation Loss: 0.3020, Validation Accuracy: 89.72%\n", + "No Improvement. Early Stopping Counter: 2/3\n", + "Epoch [7/20], Step [200/938], Loss: 0.1913\n", + "Epoch [7/20], Step [400/938], Loss: 0.1165\n", + "Epoch [7/20], Step [600/938], Loss: 0.0614\n", + "Epoch [7/20], Step [800/938], Loss: 0.1338\n", + "Epoch [7/20] - Validation Loss: 0.2592, Validation Accuracy: 91.74%\n", + "No Improvement. Early Stopping Counter: 3/3\n", + "Early Stopping triggered.\n" + ] + } + ], + "source": [ + "train_model(model, train_loader, valid_loader, criterion, optimizer, epochs=epochs, validate_at=validate_at, print_at=print_at, patience=early_stopping_patience, save_path=save_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "a364d68d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finaler Test Loss: 0.2448\n", + "Finale Test Accuracy: 90.96%\n" + ] + } + ], + "source": [ + "model.load_state_dict(torch.load(save_path))\n", + "test_loss, test_acc = evaluate_model(model, test_loader, criterion)\n", + "print(f\"Finaler Test Loss: {test_loss:.4f}\")\n", + "print(f\"Finale Test Accuracy: {test_acc:.2f}%\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "dc164da3", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArMAAAJLCAYAAAD0Jh5vAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAADQ4UlEQVR4nOzdd1QUVxvA4d/SOyioiGIDLNixY8MWe+9do8ZYY9RYYy8o1hh7xRJ7i1FDTGyJURN7wxZjjWABROltvz/83LguKCCwzOZ9zplz4M6dmffu3VnuvnNnUKnVajVCCCGEEEIokJG+AxBCCCGEECK9ZDArhBBCCCEUSwazQgghhBBCsWQwK4QQQgghFEsGs0IIIYQQQrFkMCuEEEIIIRRLBrNCCCGEEEKxZDArhBBCCCEUSwazQgghhBBCsWQwK9Lt8uXL9O7dm8KFC2NhYYGNjQ1eXl74+fkRGhqaqce+cOECtWvXxt7eHpVKxcKFCzP8GCqVismTJ2f4fj/E398flUqFSqXi2LFjOuvVajXu7u6oVCp8fHzSdYylS5fi7++fpm2OHTuWYkzptW3bNkqWLImlpSUqlYqLFy9m2L7f9Sb+5JZ27dpl6LHe9OHZs2c/WNfHxyfd/ZjWeNLznnqz3axZs1Lc79vtnDx5MiqViufPn2vtf+vWrdSsWZPcuXNjYWFB/vz5adiwIatXrwagV69eKfbP20uvXr0+2N7ffvuNDh06kC9fPszMzLC3t8fb25tly5YRGRmpqVeoUKFU7S+zvHn97t27p1X+9ddfU6BAAUxMTHBwcAAy/32yefPmFD9H9fVZKERqmeg7AKFMq1atYuDAgRQrVoyvvvoKT09P4uPjOXv2LMuXL+fUqVPs2bMn047/6aefEhkZydatW8mRIweFChXK8GOcOnWK/PnzZ/h+U8vW1pY1a9bo/AE7fvw4d+7cwdbWNt37Xrp0KU5OTmn6Q+7l5cWpU6fw9PRM93Hf9uzZM7p3706jRo1YunQp5ubmFC1aNEP2/T4zZ86kTp06WmWOjo6ZftyULF26NMuO9THvqVmzZvHZZ5+RM2fONB937NixzJ49m379+vHVV19ha2vL/fv3OXLkCN9//z19+/ZlwoQJfP7555ptzp8/z6BBg3T6K1euXO891qRJk5g6dSre3t5MmzYNNzc3oqKiOHnyJJMnT+bWrVssWLAgzW3IDE2bNuXUqVPkzZtXU/b9998zY8YMxo8fT+PGjTE3Nwcy/32yefNmrl69yrBhw3TW6fuzUIgPUguRRidPnlQbGxurGzVqpI6JidFZHxsbq/7+++8zNQYTExP1gAEDMvUY+rJu3To1oO7bt6/a0tJSHR4errW+W7du6mrVqqlLliyprl27drqOkZZt4+Li1PHx8ek6zvucOHFCDai3bduWYfuMjIxMcd3Ro0fVgHrHjh0ZdryUvOnDM2fOZPqxUuNj3lOAun79+moTExP18OHDk93v2+2cNGmSGlA/e/ZMrVar1VFRUWpzc3N1jx49ko0tMTEx2fL09Nf27dvVgLpPnz7qpKQknfUvX75U//TTT5rfCxYsqO7Zs2eq958Vpk+frgbUT548ydLjNm3aVF2wYMEsPaYQGUWmGYg0mzlzJiqVipUrV2qyBm8zMzOjRYsWmt+TkpLw8/OjePHimJubkzt3bnr06MGjR4+0tvPx8aFUqVKcOXOGmjVrYmVlRZEiRZg1axZJSUnAv5flEhISWLZsmebSI/x7efNdyV3KO3LkCD4+Pjg6OmJpaUmBAgVo27YtUVFRmjrJXVq7evUqLVu2JEeOHFhYWFCuXDnWr1+vVefN5ewtW7Ywfvx4XFxcsLOzo379+ty8eTN1LzLQuXNnALZs2aIpCw8PZ9euXXz66afJbjNlyhSqVKlCzpw5sbOzw8vLizVr1qBWqzV1ChUqxLVr1zh+/Ljm9XuT2X4T+8aNGxkxYgT58uXD3Nycv/76S2eawfPnz3F1dcXb25v4+HjN/gMDA7G2tqZ79+4ptq1Xr17UqFEDgI4dO+pc3t63bx/VqlXDysoKW1tbGjRowKlTp7T28aa/z58/T7t27ciRIwdubm4ffmFT8OzZMwYOHIinpyc2Njbkzp2bunXr8ttvv+nUXbZsGWXLlsXGxgZbW1uKFy/OuHHjdOq9evWKAQMG4OTkhKOjI23atOHx48dadZK7fBwaGsrAgQM1l8mLFCnC+PHjiY2N1aqnUqkYPHgwGzdupESJElhZWVG2bFn279+fbBvT854CKFasGH369GHJkiXcv38/xXrJiYyMJDY2Viv7+DYjo4z7MzR16lRy5MjBokWLkv0ssLW15ZNPPklx+5iYGEaMGEG5cuWwt7cnZ86cVKtWje+//16n7o4dO6hSpQr29vaaz6q3X8OkpCSmT59OsWLFsLS0xMHBgTJlyvDNN99o6rz72VSoUCG+/vprAPLkyaP1GZTc+yQ2NpapU6dSokQJLCwscHR0pE6dOpw8eVJTZ8mSJdSqVYvcuXNjbW1N6dKl8fPz0zpnfXx8OHDgAPfv39ea0vGGvj8LhfgQGcyKNElMTOTIkSNUqFABV1fXVG0zYMAARo8eTYMGDdi3bx/Tpk0jICAAb29vrXl1AMHBwXTt2pVu3bqxb98+GjduzNixY9m0aRPw72U5gHbt2nHq1CmdQc6H3Lt3j6ZNm2JmZsbatWsJCAhg1qxZWFtbExcXl+J2N2/exNvbm2vXrrFo0SJ2796Np6cnvXr1ws/PT6f+uHHjuH//PqtXr2blypXcvn2b5s2bk5iYmKo47ezsaNeuHWvXrtWUbdmyBSMjIzp27Jhi2/r378/27dvZvXs3bdq0YciQIUybNk1TZ8+ePRQpUoTy5ctrXr93p4SMHTuWBw8esHz5cn744Qdy586tcywnJye2bt3KmTNnGD16NABRUVG0b9+eAgUKsHz58hTbNmHCBJYsWQK8/nJ06tQpzWXUzZs307JlS+zs7NiyZQtr1qwhLCwMHx8fTpw4obOvNm3a4O7uzo4dO957zDeSkpJISEjQWgDNPO9JkyZx4MAB1q1bR5EiRfDx8dGaZ7p161YGDhxI7dq12bNnD3v37uXLL7/Umov5Rt++fTE1NWXz5s34+flx7NgxunXr9t74YmJiqFOnDhs2bGD48OEcOHCAbt264efnR5s2bXTqHzhwgMWLFzN16lR27dpFzpw5ad26NX///bdO3fS8p96YPHkyxsbGTJgw4b313uXk5IS7uztLly5l/vz53LhxQ+vLVUYJCgri6tWrfPLJJ1hZWaVrH7GxsYSGhjJy5Ej27t3Lli1bqFGjBm3atGHDhg2aeqdOnaJjx44UKVKErVu3cuDAASZOnKh5LwH4+fkxefJkOnfuzIEDB9i2bRt9+vThxYsXKR5/z5499OnTB4CAgABOnTpF3759k62bkJBA48aNmTZtGs2aNWPPnj34+/vj7e3NgwcPNPXu3LlDly5d2LhxI/v376dPnz7MmTOH/v37a+osXbqU6tWr4+zsrPlMeN/nalZ/FgrxQfpODQtlCQ4OVgPqTp06par+9evX1YB64MCBWuV//PGHGlCPGzdOU1a7dm01oP7jjz+06np6eqobNmyoVQaoBw0apFX25vLmu95cCr17965arVard+7cqQbUFy9efG/sgHrSpEma3zt16qQ2NzdXP3jwQKte48aN1VZWVuoXL16o1ep/L482adJEq96bS6CnTp1673HfvnT7Zl9Xr15Vq9VqdaVKldS9evVSq9UfniqQmJiojo+PV0+dOlXt6Oioddk1pW3fHK9WrVoprjt69KhW+ezZs9WAes+ePeqePXuqLS0t1ZcvX35vG9/e39uXkRMTE9UuLi7q0qVLa11+fvXqlTp37txqb29vTdmb/p44ceIHj/X28ZJbbt++rVM/ISFBHR8fr65Xr566devWmvLBgwerHRwc3nusN3347vvez89PDaiDgoI0ZbVr19bqi+XLl6sB9fbt27W2ffM6Hzp0SFMGqPPkyaN++fKlpiw4OFhtZGSk9vX11YknPe+pt8+18ePHq42MjNSXLl3S2e8b704zUKvV6j///FNdoEABzetta2urbtasmXrDhg3JTgdQq9M+zeD06dNqQD1mzJhU1VerPzzN4M17oE+fPury5ctryufOnasGNOd8cpo1a6YuV67ce4//7meTWp3866dW675PNmzYoAbUq1ateu8x3vbmM2HDhg1qY2NjdWhoqGbd+6YZ6OuzUIjUksysyFRHjx4F0LnRqHLlypQoUYLDhw9rlTs7O1O5cmWtsjJlyqT50ub7lCtXDjMzMz777DPWr1+fbAYrOUeOHKFevXo6GelevXoRFRWlk8l4e6oFvG4HkKa21K5dGzc3N9auXcuVK1c4c+bMey8HHzlyhPr162Nvb4+xsTGmpqZMnDiRkJAQnj59murjtm3bNtV1v/rqK5o2bUrnzp1Zv3493377LaVLl0719m+7efMmjx8/pnv37lqXn21sbGjbti2nT5/WmgqS1lgBZs+ezZkzZ7SWN326fPlyvLy8sLCwwMTEBFNTUw4fPsz169c121euXJkXL17QuXNnvv/+e52rC29Lz3vgyJEjWFtb6zxh4c059O45U6dOHa0bt/LkyUPu3LlTPEZa31NvGzVqFDlz5tRk4lOrUqVK/PXXXwQEBDBu3DiqVavG4cOH6dGjBy1atMiUTG167dixg+rVq2NjY6N5D6xZs0brPVCpUiUAOnTowPbt2/nnn3909lO5cmUuXbrEwIED+emnn3j58mWGxvnjjz9iYWHxwb67cOECLVq0wNHRUfOZ0KNHDxITE7l161a6jq2Pz0Ih3kcGsyJNnJycsLKy4u7du6mqHxISApDsfDkXFxfN+jeSu6vc3Nyc6OjodESbPDc3N3755Rdy587NoEGDcHNzw83NTWsuW3JCQkJSbMeb9W97ty1v5henpS0qlYrevXuzadMmli9fTtGiRalZs2aydf/880/NfMBVq1bx+++/c+bMGcaPH5/m46Y0vzGlGHv16kVMTAzOzs7vnSv7IR96vyQlJREWFpbuWAGKFClCxYoVtRZzc3Pmz5/PgAEDqFKlCrt27eL06dOcOXOGRo0aab123bt3Z+3atdy/f5+2bduSO3duqlSpws8//6xzrPS8B0JCQnB2dtaZ85k7d25MTEw++pxJy3vqXXZ2dnz99dcEBARovqimlqmpKQ0bNmTGjBn89NNPPHz4EB8fH/bv38+PP/6Ypn0lp0CBAgCp/mxKzu7duzWP9Nq0aROnTp3SDPZjYmI09WrVqsXevXtJSEigR48e5M+fn1KlSmnNRR47dixz587l9OnTNG7cGEdHR+rVq5eqx7WlxrNnz3BxcXnvnOMHDx5Qs2ZN/vnnH7755ht+++03zpw5o5nik97PVX18FgrxPjKYFWlibGxMvXr1OHfunM4NXMl58yEWFBSks+7x48c4OTllWGwWFhYAOjfJJJc5q1mzJj/88APh4eGcPn2aatWqMWzYMLZu3Zri/h0dHVNsB5ChbXlbr169eP78OcuXL6d3794p1tu6dSumpqbs37+fDh064O3tTcWKFdN1zORunklJUFAQgwYNoly5coSEhDBy5Mh0HRM+/H4xMjIiR44c6Y71fTZt2oSPjw/Lli2jadOmVKlShYoVK/Lq1Sudur179+bkyZOEh4dz4MAB1Go1zZo1y5BMk6OjI0+ePNHJVj59+pSEhIQMeZ+l9j2VnAEDBlC4cGFGjx79URlVR0dHzWOgrl69mu79vJE3b15Kly7NoUOHdLL3qbVp0yYKFy7Mtm3baNWqFVWrVqVixYo6nykALVu25PDhw4SHh3Ps2DHy589Ply5dNFlJExMThg8fzvnz5wkNDWXLli08fPiQhg0bpju+t+XKlYvHjx9rbo5Nzt69e4mMjGT37t1069aNGjVqULFiRczMzD7q2Pr6LBQiJTKYFWk2duxY1Go1/fr1S/aGqfj4eH744QcA6tatC6C5geuNM2fOcP36derVq5dhcb25I//y5cta5W9iSY6xsTFVqlTRZCrOnz+fYt169epx5MgRnbvRN2zYgJWVFVWrVk1n5O+XL18+vvrqK5o3b07Pnj1TrKdSqTAxMcHY2FhTFh0dzcaNG3XqZlS2OzExkc6dO6NSqfjxxx/x9fXl22+/Zffu3enaX7FixciXLx+bN2/WGihFRkaya9cuzRMOMoNKpdJ5Osfly5ffeyOMtbU1jRs3Zvz48cTFxXHt2rWPjqNevXpERESwd+9erfI3NyBlxDmT2vdUcszMzJg+fTpnzpxhx44dH6wfHx+vk6l7482l+zcZvY81YcIEwsLCGDp0aLID7YiICA4dOpTi9iqVCjMzM60vSMHBwck+zeANc3NzateuzezZs4HXl/Xf5eDgQLt27Rg0aBChoaE6/yQhPRo3bkxMTMx7//nJm3a8/b5Wq9WsWrVKp25aPhP09VkoRErknyaINKtWrRrLli1j4MCBVKhQgQEDBlCyZEni4+O5cOECK1eupFSpUjRv3pxixYrx2Wef8e2332JkZETjxo25d+8eEyZMwNXVlS+//DLD4mrSpAk5c+akT58+TJ06FRMTE/z9/Xn48KFWveXLl3PkyBGaNm1KgQIFiImJ0dzdXb9+/RT3P2nSJPbv30+dOnWYOHEiOXPm5LvvvuPAgQP4+flhb2+fYW15V3L/feldTZs2Zf78+XTp0oXPPvuMkJAQ5s6dm+zj00qXLs3WrVvZtm0bRYoUwcLCIl3zXCdNmsRvv/3GoUOHcHZ2ZsSIERw/fpw+ffpQvnx5ChcunKb9GRkZ4efnR9euXWnWrBn9+/cnNjaWOXPm8OLFi1S9DunVrFkzpk2bxqRJk6hduzY3b95k6tSpFC5cWOsu9X79+mFpaUn16tXJmzcvwcHB+Pr6Ym9vr5lL+TF69OjBkiVL6NmzJ/fu3aN06dKcOHGCmTNn0qRJk/e+R9PiY17Lzp07M3fu3FRNDwgPD6dQoUK0b9+e+vXr4+rqSkREBMeOHeObb76hRIkSyT6lIT3at2/PhAkTmDZtGjdu3KBPnz6af5rwxx9/sGLFCjp27Jji47maNWvG7t27GThwIO3atePhw4dMmzaNvHnzcvv2bU29iRMn8ujRI+rVq0f+/Pl58eIF33zzDaamptSuXRuA5s2bU6pUKSpWrEiuXLm4f/8+CxcupGDBgnh4eHx0Wzt37sy6dev4/PPPuXnzJnXq1CEpKYk//viDEiVK0KlTJxo0aICZmRmdO3dm1KhRxMTEsGzZMp2pOvD6M2H37t0sW7aMChUqYGRklOKVHX1+FgqRHBnMinTp168flStXZsGCBcyePZvg4GBMTU0pWrQoXbp0YfDgwZq6y5Ytw83NjTVr1rBkyRLs7e1p1KgRvr6+Gfqfl+zs7AgICGDYsGF069YNBwcH+vbtS+PGjbUeb1OuXDkOHTrEpEmTCA4OxsbGhlKlSrFv3773PoOyWLFinDx5knHjxjFo0CCio6MpUaIE69at0+u/xHyjbt26rF27ltmzZ9O8eXPy5ctHv379yJ07t+ZxP29MmTKFoKAg+vXrx6tXryhYsGCas0U///wzvr6+TJgwQStb6O/vT/ny5enYsSMnTpxI8yXNLl26YG1tja+vLx07dsTY2JiqVaty9OhRvL2907SvtBg/fjxRUVGsWbMGPz8/PD09Wb58OXv27NF6NFfNmjXx9/dn+/bthIWF4eTkRI0aNdiwYcMH/ztValhYWHD06FHGjx/PnDlzePbsGfny5WPkyJFMmjTpo/efEVQqFbNnz37v+fKGnZ0dU6ZM4fDhw4wbN44nT56gUqkoXLgww4YNY/To0RmabZ86dSr169fn22+/Zfz48Tx//hxLS0tKlizJ8OHDtR5J9a7evXvz9OlTli9fztq1aylSpAhjxozh0aNHTJkyRVOvSpUqnD17ltGjR/Ps2TMcHByoWLEiR44coWTJksDrG/N27drF6tWrefnyJc7OzjRo0IAJEyZgamr60e00MTHh4MGD+Pr6smXLFhYuXIitrS1ly5alUaNGABQvXpxdu3bx9ddf06ZNGxwdHenSpQvDhw+ncePGWvv74osvuHbtGuPGjSM8PBy1Wp3iNJLs/lko/ntU6ux0G6kQQgghhBBpIHNmhRBCCCGEYslgVgghhBBCKJYMZoUQQgghhGLJYFYIIYQQQiiWDGaFEEIIIYRiyWBWCCGEEEIolgxmhRBCCCGEYsk/TdCjHpsvf7iSAqzsUEbfIXy0JAN53HJETMKHK2Vz1uaG8bFkbKT6cCUF2Hzhgb5D+GhdyhfQdwgZwhA+plSGcVpgocePKcvygz9cKZ2iLyzOtH1nJsnMCiGEEEIIxTKMFIgQQgghxH+BSvKQ75JXRAghhBBCKJZkZoUQQgghlMJQJh5nIBnMCiGEEEIohUwz0CGviBBCCCGEUCzJzAohhBBCKIVMM9AhmVkhhBBCCKFYkpkVQgghhFAKmTOrQ14RIYQQQgihWJKZFUIIIYRQCpkzq0Mys0IIIYQQQrEkMyuEEEIIoRQyZ1aHDGaFEEIIIZRCphnokOG9EEIIIYRQrP/sYNbHx4dhw4a9t45KpWLv3r1ZEo8QQgghxAepjDJvUahsP81A9YF0es+ePfH399cqS0xMxM/Pj/Xr13P//n0sLS0pWrQo/fv3p3fv3qk+dlBQEDly5HhvHX9/f4YNG8aLFy9Svd/0auaZi4qu9uS1Myc+Uc3tZ5FsuxhM8KtYrXouduZ0KJeX4rmtUangn/AYlpx4QEhUPE7WpsxvWSLZ/X/7233OPAzP9HakxbYt3+G/bg3Pnz3Dzd2DUWPG4VWhor7DSpc1q1aw+JsFdOnWg6/GjNN3OCnas3Mre3duIzjoMQCFi7jTq+/nVK1eE4AZk8cTsP97rW08S5Vhhf/mLI81LRISElixdDE/HvyBkOfPcXLKRfOWrenbfwBGRsr6EM/u58XDG5f548AOnty9RcSLUFoPm0zRitU16+Niojm+bTW3zp4kJuIldrnyUPGT1pSv3xyA8GfBLP+ye7L7bjnka4pXqZ0l7UiN7N4XH7Jm1QoO/3KIe3f/xtzCgrLlyjPsy5EUKlxE36Gli9L7Q6RPth/MBgUFaX7etm0bEydO5ObNm5oyS0tLnW0mT57MypUrWbx4MRUrVuTly5ecPXuWsLCwNB3b2dn5vevj4+PTtL+PVTy3Db/cCuFuaBRGKhXtyzozqm5hxuy/SVyiGoDcNmZ83cCN43dC2XMlmKi4JFzszYlLTAIgJCqeIbsDtfbr456TpiVycTnoVZa250MCfjyI3yxfxk+YRLnyXuzcvpWB/fuxZ98B8rq46Du8NLl25Qq7d27Ho2gxfYfyQblzO/P54C/J51oAgID93zN2xBDWfreTwm7uAFTxrsHYidM125iamuol1rTwX7uaXTu2MmXGLNzc3Am8dpXJE8ZhY2tLl2499B1eqinhvIiLjSF3gSKUrvUJe7+ZqrP+8KZlPAi8RPMBY7DPlYe7V85xyH8RNjkc8ajgja1jLgYt3qa1zaWjB/hj/3aKlK2cVc34ICX0xYecO/snHTt3pWSp0iQmJLJ40QIGfNaH3d8fwNLKSt/hpYkh9EeqyJxZHdk+HeHs7KxZ7O3tUalUOmXv+uGHHxg4cCDt27encOHClC1blj59+jB8+HCteklJSYwaNYqcOXPi7OzM5MmTtda/Pc3g3r17qFQqtm/fjo+PDxYWFmzatInevXsTHh6OSqVCpVLp7CMjzT12lxN3w/gnPJaHL2JYdfohTtZmFM757wdOu7LOXHr8im0Xg7kfFsOzyDguPX7Fq9hEANRqCI9J0Foq5rfnjwfhxCYkZVrs6bFx/Tpat21Lm3btKeLmxqix43HO68z2bVv0HVqaREVFMm7MSCZMnoadnZ2+w/mg6rV8qFajFgUKFqJAwUJ8NugLLK2suHblkqaOqakZjk5OmsUumfMwu7l86QK169SjZi0fXPLlp/4njajqXZ3Aa1f1HVqaKOG8cCtbmVrte1OsUs1k1z/+6zqlajaggGdZ7HM5U65uU3IXcCPo71sAGBkZY+OQU2u5dfZ3ilf1wcxCN4GhL0roiw9ZumINLVu1wd3dg2LFizNlui9BQY8JDLym79DSzBD6Q6RPth/MpoezszNHjhzh2bNn7623fv16rK2t+eOPP/Dz82Pq1Kn8/PPP791m9OjRDB06lOvXr1OvXj0WLlyInZ0dQUFBBAUFMXLkyIxsyntZmhoDEBGXAIAKKOtiS/CrWL6qU5jFbTyZ9Ik7XvlTHkAVymFJwZyWHL8TmhUhp1p8XBzXA69RzbuGVnk17+pcunhBT1Glj+/0qdSs5UPVat76DiXNEhMT+eWng8RER1OyTDlN+cVzZ2jeoBad2zRl9vRJhIWG6C/IVCpfvgJ//nGK+/fuAnDr5g0unj9PjZq19BxZ6hnKeZG/aEn+On+KV6HPUavV3A+8SFjwI4qUSf5ycPDdWzy9f4cytRtlcaQpM5S+eFdExOsrdMklirIzQ+2PZMmcWR3ZfppBesyfP5927drh7OxMyZIl8fb2pmXLljRu3FirXpkyZZg0aRIAHh4eLF68mMOHD9OgQYMU9z1s2DDatGmj+f3tbHFW6+Llws2nkfwT/nrOrJ2FCZamxjTzzM3OS8FsuxBEGRdbhtYsiO/hv7n5NFJnH7XdcvBPeAx/PY/K6vDfK+xFGImJiTg6OmqVOzo68fz5+7+kZCcBBw9w43ogm7bu1HcoaXLnr1sM6N2VuLg4LC2tmDHnGwoXcQOgqncN6tT/BGdnF4Ie/8Pq5d/yxed9WL1pO2ZmZnqOPGW9+vQjIuIVbVo0wdjYmMTERAYNHUajJs30HVqqGcp5Ub/HIAJWL2Dp0M4YGRujUhnRqO+X5C9WKtn6l48F4OhSgPxFS2ZxpCkzlL54m1qtZp6fL+W9KuDuUVTf4aSJIfaHSD3FD2ZtbGw0P3fr1o3ly5fj6enJ1atXOXfuHCdOnODXX3+lefPm9OrVi9WrV2vqlylTRmtfefPm5enTp+89XsWK6ZtIHhsbS2ys9o1aifFxGJum749/j4ouuDpYMP3nO5qyN9Nozj8K56ebzwF48CIGdydr6ro76gxmTY1VVC2Ug++vPklXDFnh3RsA1Wr1B28KzC6Cg4KYM2smS1euwdzcXN/hpEmBgoVZu3kXEa9ecuzIz8yYPJ5vV/pTuIgb9T7590thEXcPinmWpH2zBpw6cZzadVP+IqhvhwIOcnD/D8ycPZcibu7cvHmDebNnkitXbpq3bK3v8NJEyecFwNmf9vL4r+u0HT4VO6c8PLxxmZ/9v8XGwZFCpby06sbHxRJ46gjerbrqKdr3U3pfvM13xlRu3bqF/4bsfTPn+xhSf6TI0NqTARQ/mL148aLm57fnIxoZGVGpUiUqVarEl19+yaZNm+jevTvjx4+ncOHCgO5NKyqViqSk988btba2Tlecvr6+TJkyRausTJvPKdt2QJr31b2CC+Xz2THjlzuERf97E9qr2EQSktSaTO0bj1/GUDSXbtyVXO0xN1bx+9203RiXFXI45MDY2Jjnz59rlYeGhuDo6KSnqNLmeuA1QkND6NqxraYsMTGR8+fOsm3Ld/xx/jLGxsZ6jDBlpqam5P//DWDFPUtxI/AaO7ds4qvxk3TqOjnlwjmvC48ePMjqMNNk4bw59OrTj4aNmwLgUbQYwY8fs271SsUMZg3hvIiPi+XX7WtpM2wybuWrAJC7QBGe3r/Dnwd26Axmb/75K/GxsZSqkb2+KBlCX7xt1sxpHD96hLXrN5FHD1caP5ah9cd7KXg6QGZR/Cvi7u6uWXLnzp1iPU9PTwAiI3UvtX8MMzMzEhMTP1hv7NixhIeHay2lWvRJ8/G6V3Shgqs9s478zfNI7acpJCapuRsSRV477Sygs605IZFxOvuq7ZaT8/+81Nwclp2YmplRwrMkp0/+rlV++uRJypYrr6eo0qZy1ars2LOPrTv3aBbPkqVo0rQ5W3fuybYD2eSo1Wri4nXfQwDhL17w9Ekwjk7Z+w9GTEy0ziO4jIyNSFJnrxsf38cQzoukhASSEhPASDu7pDIyRp1MX1w+FoC7VzWs7ByyKMLUMYS+gNfntu+MqRz+5RAr164nX35XfYeULobSHyJ9FJ+ZTU67du2oXr063t7eODs7c/fuXcaOHUvRokUpXrx4hh6rUKFCREREcPjwYcqWLYuVlRVWyTzOxNzcXOdSc1qnGPSs6ELVQjlY+Os9YuKTsLd43X1R8YnE///RXAevP2NQ9QLcfBpJ4JMIyrjYUj6fHb6H72jtK7eNGcVyWzPv2N00xZCVuvfszfgxo/AsVYqyZcuza8c2goKCaN+xk75DSxVraxudeWeWlpbYOzhk6/loK5YspKp3TXLncSYqKpLDP/3IxXNnmLtoOVFRUaxbuYTadRvg6JSL4Mf/sHLpN9g75KBWnfr6Dv29atWuw5qVy3HOmxc3N3du3LjOpg3+tGzV9sMbZyNKOC/iYqIJe/KP5vfwZ8E8uf8XltZ22DnlxrV4GY5tWYWpqTl2Trl5eOMy1078TN2un2vtJyz4Hx7evEL7kTOyugmpooS++JCZ06fw48H9LFy0FGtra838UhsbWywsLPQcXdoYQn+kimRmdRjkYLZhw4Zs2bIFX19fwsPDcXZ2pm7dukyePBkTk4xtsre3N59//jkdO3YkJCSESZMmZdrjueoVfZ35Gl/fTat85amHnPj/VIFzj17if+YfmpXMTbcKLgS9iuXb3+5z65n2DV613HISFhXP1aCITIk1IzRq3ITwF2GsXLaUZ8+e4u5RlCXLV+Likk/foRm0sJAQpk8cS8jzZ1jb2OLmUZS5i5ZTqao3sTEx3PnrNgEHfiDi1UscnXJRvmJlJs+ci1U6p+BklVHjvmbp4kX4Tp9KWGgIuXLlpm27jnw2YKC+Q0sTJZwXwX/fYsvMf5/scuS75QCUqtmApv1H0WLweI5vW8MPy3yJiXiFnVMearbvTbl62jfjXT4egG0OJwqXrpCl8aeWEvriQ3b8/7FVfXtr/5OKKdN9admqTXKbZFuG0B8ifVRqtVqt7yD+q3psvqzvEDLEyg5lPlwpm0sykNMgIiZB3yF8NGtzw/iObWxkGDdpbL6QvedCp0aX8gX0HUKGMISPKUO5d8lCjx9TlnWmZdq+o49OyLR9ZybJVQshhBBCCMUyjBSIEEIIIcR/gcyZ1SGviBBCCCGEUCzJzAohhBBCKIWhTDzOQJKZFUIIIYQQiiWZWSGEEEIIpZA5szpkMCuEEEIIoRQyzUCHDO+FEEIIIYRiSWZWCCGEEEIpZJqBDnlFhBBCCCGEYklmVgghhBBCKWTOrA7JzAohhBBCCMWSzKwQQgghhFLInFkd8ooIIYQQQgjFksysEEIIIYRSyJxZHTKYFUIIIYRQCplmoENeESGEEEIIoViSmRVCCCGEUAqZZqBDBrN6tLJDGX2HkCFy+EzQdwgfLezYNH2HkCEsTI31HcJHM5IP6mylUzlXfYcg/k9ODSGSJ4NZIYQQQgilkDmzOuQVEUIIIYQQiiWZWSGEEEIIpZDMrA55RYQQQgghhGJJZlYIIYQQQinkTkAdMpgVQgghhFAKmWagQ14RIYQQQgihWJKZFUIIIYRQCplmoEMys0IIIYQQQrEkMyuEEEIIoRQyZ1aHvCJCCCGEEEKxJDMrhBBCCKEUMmdWh2RmhRBCCCGEYklmVgghhBBCIVSSmdUhg1khhBBCCIWQwawumWYghBBCCCEUSzKzQgghhBBKIYlZHdkmM6tSqd679OrVS98hKsq2Ld/R+JO6VCpfmk7t23D+3Fl9h5Sikd1qEX1iGnOGNtaURZ+YluzyZefqAOSwtWT+sKZc2vwFIb9M4NauEcz7ogl21ub6akaKlNQXAOfPneHLIQNoXL8WlcqW4NiRX7TWVypbItllo/8aPUWcOmtWraBLx7Z4Vy5PnVrVGDZ0IPfu/q3vsNJFae+pD1mzagXlSxVnzqyZ+g4lzQyhL86dPcOQgZ9T36cGZUsW48jhXz68UTZlCP0h0i7bDGaDgoI0y8KFC7Gzs9Mq++abb7Tqx8fH6ynS94uLi9N3CAT8eBC/Wb70+2wA23buxcurAgP79yPo8WN9h6ajQvF89GlRkct/BWuVF2oxW2v5bOZukpKS2HM8EIC8TrbkdbJl7JIAKvZYTL8Zu2lQ1YPlY1rroxkpUlJfvBEdHU3RYsX4aszXya7/8fCvWsuEKTNQqVTUqf9JFkeaNufO/knHzl3ZsHk7y1euIzEhkQGf9SE6KkrfoaWJEt9T73PtyhV279yOR9Fi+g4lzQylL6KjoyhWrBhjxk/UdygfxVD640M+lPz7mEWpss1g1tnZWbPY29ujUqk0v8fExODg4MD27dvx8fHBwsKCTZs2kZSUxNSpU8mfPz/m5uaUK1eOgIAAzT6PHTuGSqXixYsXmrKLFy+iUqm4d+8eAPfv36d58+bkyJEDa2trSpYsycGDBzX1AwMDadKkCTY2NuTJk4fu3bvz/PlzzXofHx8GDx7M8OHDcXJyokGDBpn+Wn3IxvXraN22LW3ataeImxujxo7HOa8z27dt0XdoWqwtzVg3qR0D/fby4lW01ronoRFaS/MaJTh+/i73HocBEHj3KZ2/3srB329y93EYx8/fZfLKX2hSvRjGxtnmba2Yvnhb9Rq1GDB4GHVTGJw6OeXSWn49doQKlaqQP79rFkeaNktXrKFlqza4u3tQrHhxpkz3JSjoMYGB1/QdWpoo8T2VkqioSMaNGcmEydOws7PTdzhpZih9UaNmbQZ/8SX1G2TvL6QfYij9IdIu+/zVT4XRo0czdOhQrl+/TsOGDfnmm2+YN28ec+fO5fLlyzRs2JAWLVpw+/btVO9z0KBBxMbG8uuvv3LlyhVmz56NjY0N8DpbXLt2bcqVK8fZs2cJCAjgyZMndOjQQWsf69evx8TEhN9//50VK1ZkaJvTKj4ujuuB16jmXUOrvJp3dS5dvKCnqJK3cHgzAk7e4ujZ91/qzZ3DmkbeRVl/4Px769lZW/AyMpbExKSMDDPdlNQX6RUS8pwTvx2nZeu2+g4lzSIiXgFgb2+v50hSz9DeU77Tp1Kzlg9Vq3nrO5Q0M7S+ULr/Un9IZlaXom4AGzZsGG3atNH8PnfuXEaPHk2nTp0AmD17NkePHmXhwoUsWbIkVft88OABbdu2pXTp0gAUKVJEs27ZsmV4eXkxc+a/87jWrl2Lq6srt27domjRogC4u7vj5+f33uPExsYSGxurVaY2NsfcPGPneIa9CCMxMRFHR0etckdHJ54/f5ahx/oY7euVplxRF2r0W/7But0al+dVVCx7/z/FIDk57SwZ28uHNfvOZGSYH0UpffExDuzbi7WVNXXq6f+KRFqo1Wrm+flS3qsC7h5F9R1OqhnSeyrg4AFuXA9k09ad+g4lXQypLwyB9Md/m6IysxUrVtT8/PLlSx4/fkz16tW16lSvXp3r16+nep9Dhw5l+vTpVK9enUmTJnH58mXNunPnznH06FFsbGw0S/HixQG4c+dOsnGlxNfXF3t7e61lzmzfVMeZVu9+w1Kr1dnmW1f+3HbM+aIJn07bSWxcwgfr92jqxbZDl1Osa2tlzp453bl+7ykz1h7N6HA/Wnbui4+1b+9uGjVpluFfyjKb74yp3Lp1i1l+8/UdSroo/T0VHBTEnFkzme47R3HvnXcpvS8MzX+hPyQzq0tRmVlra2udsve9cY2MjDRlb7x741jfvn1p2LAhBw4c4NChQ/j6+jJv3jyGDBlCUlISzZs3Z/bs2TrHzZs373vjetfYsWMZPny4dqzGGf8hnsMhB8bGxlrzegFCQ0NwdHTK8OOlR/li+ciT04aTqz/XlJmYGFOjbEE+b1MF+7pTSEp63WfVyxSkWMFcdJ+0Pdl92ViasW9eDyKi4+g4bgsJ2WSKASijLz7GhfNnuX/vLjMVNiCcNXMax48eYe36TeRxdtZ3OGliKO+p64HXCA0NoWvHf6enJCYmcv7cWbZt+Y4/zl/G2NhYjxF+mKH0haH4L/WHkgedmUVRmdm32dnZ4eLiwokTJ7TKT548SYkSJQDIlSsX8Hru6xsXL17U2Zerqyuff/45u3fvZsSIEaxatQoALy8vrl27RqFChXB3d9daUjOAfZu5uTl2dnZaS2ZkJEzNzCjhWZLTJ3/XKj998iRly5XP8OOlx9Gzd6jQ/Vuq9F6qWc5df8TWQ5ep0nupZiAL0LOZF+du/MOVd552AK8zsvsX9CQuIZF2o79LVZY3KymhLz7G93t2UcKzJEWLFdd3KKmiVqvxnTGVw78cYuXa9eTL5jesJcdQ3lOVq1Zlx559bN25R7N4lixFk6bN2bpzT7YfyILh9IWhkP74b1NUZvZdX331FZMmTcLNzY1y5cqxbt06Ll68yHfffQe8nsvq6urK5MmTmT59Ordv32bevHla+xg2bBiNGzemaNGihIWFceTIEc1geNCgQaxatYrOnTvz1Vdf4eTkxF9//cXWrVtZtWpVtv3A7d6zN+PHjMKzVCnKli3Prh3bCAoKon3HTvoODYCI6DgC7z7VKouMiSf0ZZRWua2VOW3qlGLM4oB3d4GNpRn7F/TE0tyU3lM3Y2dtrnnG7LMXkVoDYn3K7n2RnKioSB4+eKD5/fE/j7h54zr29vY453UBICIigsOHfmLYiFH6CjPNZk6fwo8H97Nw0VKsra018+hsbGyxsLDQc3Spp8T31LusrW105ipbWlpi7+CgqDnMhtAXAFGRkTx465z/59Ejblx/fc7ndXHRY2RpYyj98UGSmNWh6MHs0KFDefnyJSNGjODp06d4enqyb98+PDw8ADA1NWXLli0MGDCAsmXLUqlSJaZPn0779u01+0hMTGTQoEE8evQIOzs7GjVqxIIFCwBwcXHh999/Z/To0TRs2JDY2FgKFixIo0aNNFMYsqNGjZsQ/iKMlcuW8uzZU9w9irJk+UpcXPLpO7Q0aV+/NCoVbP/lss668sVdqFzydWYtcLv29I1i7ebxIPhFVoT4QUrsi+vXrvF5356a3xfMfT3NpmmLVkye9nqe96GAg6hR07BxU73EmB47/v94nr69u2uVT5nuS8tWbZLbJFtS4nvKUBlKX1y7dpW+vXtofp/r9/o8b9GyNdNmztJXWGlmKP0h0k6lfntCqchSMdnrqni65fCZoO8QPlrYsWn6DiFDxCVknznD6WWajZ4T/DEMZVpbkgH8iTAylM4Q2YaFHlOBDl03Zdq+X3zXLdP2nZkM46+GEEIIIYT4T1L0NAMhhBBCiP8SeZqBLsnMCiGEEEIIxZLMrBBCCCGEQkhmVpcMZoUQQgghFEIGs7pkmoEQQgghhFAsycwKIYQQQiiFJGZ1SGZWCCGEEEIolmRmhRBCCCEUQubM6pLMrBBCCCGESDdfX19UKhXDhg3TlKnVaiZPnoyLiwuWlpb4+Phw7do1re1iY2MZMmQITk5OWFtb06JFCx49epTm48tgVgghhBBCIVQqVaYt6XHmzBlWrlxJmTJltMr9/PyYP38+ixcv5syZMzg7O9OgQQNevXqlqTNs2DD27NnD1q1bOXHiBBERETRr1ozExMQ0xSCDWSGEEEIIkWYRERF07dqVVatWkSNHDk25Wq1m4cKFjB8/njZt2lCqVCnWr19PVFQUmzdvBiA8PJw1a9Ywb9486tevT/ny5dm0aRNXrlzhl19+SVMcMpgVQgghhFCIzMzMxsbG8vLlS60lNjY2xVgGDRpE06ZNqV+/vlb53bt3CQ4O5pNPPtGUmZubU7t2bU6ePAnAuXPniI+P16rj4uJCqVKlNHVSSwazQgghhBBKocq8xdfXF3t7e63F19c32TC2bt3K+fPnk10fHBwMQJ48ebTK8+TJo1kXHByMmZmZVkb33TqpJU8zEEIIIYQQjB07luHDh2uVmZub69R7+PAhX3zxBYcOHcLCwiLF/b07D1etVn9wbm5q6rxLMrNCCCGEEAqRmdMMzM3NsbOz01qSG8yeO3eOp0+fUqFCBUxMTDAxMeH48eMsWrQIExMTTUb23Qzr06dPNeucnZ2Ji4sjLCwsxTqpJYNZIYQQQgiRavXq1ePKlStcvHhRs1SsWJGuXbty8eJFihQpgrOzMz///LNmm7i4OI4fP463tzcAFSpUwNTUVKtOUFAQV69e1dRJLZlmoEdqtb4jyBhhx6bpO4SP5jHse32HkCFuLmih7xA+WmRsgr5DyBA2Fobx8fr300h9h/DR3PPY6DsEITJMdvinCba2tpQqVUqrzNraGkdHR035sGHDmDlzJh4eHnh4eDBz5kysrKzo0qULAPb29vTp04cRI0bg6OhIzpw5GTlyJKVLl9a5oexDDOPTVgghhBBCZBujRo0iOjqagQMHEhYWRpUqVTh06BC2traaOgsWLMDExIQOHToQHR1NvXr18Pf3x9jYOE3HUqnVhpIfVJ7oeH1HkDGywZfEjyaZ2ewjKjZtD8vOrgwlM/vXkwh9h/DRJDMrMpo+T++8n+3KtH0HrWybafvOTDJnVgghhBBCKJZhpA6EEEIIIf4DssOc2exGBrNCCCGEEEohY1kdMs1ACCGEEEIolmRmhRBCCCEUQqYZ6JLMrBBCCCGEUCzJzAohhBBCKIRkZnVJZlYIIYQQQiiWZGaFEEIIIRRCMrO6JDMrhBBCCCEUSzKzQgghhBBKIYlZHZKZFUIIIYQQiiWZWSGEEEIIhZA5s7pkMCuEEEIIoRAymNWl6GkGPj4+DBs2TPN7oUKFWLhwod7iEUIIIYQQWUuvg9levXqhUqlQqVSYmppSpEgRRo4cSWRkpD7DUrTtWzfTvnVzqlfxonoVL3p07ciJ347rO6x027blOxp/UpdK5UvTqX0bzp87q++QNLrXKMShsT4EzmlC4Jwm7B1REx/P3Jr1VmbGTGtfmj+nfcLt+c048nVdutcopFnvYGXK1PalOTahHrfmN+X01AZMaVcaW4vsfcFkzaoVlC9VnDmzZuo7lBRtWLuKPt07UL9mJZrWr8mY4UO4f++uVp3QkOdMnzSOFg19qOtdgeGDP+Phg/t6ijhtsvN5kZzoqEjWLp5L/05N6dzIm3GDe/PXjWua9Wq1mm3+K+jbviGdG3kz8cvPeHD3jh4jTj2l9UVKpB3K8WbclBmLUuk9M9uoUSOCgoL4+++/mT59OkuXLmXkyJH6Divd4uLi9Hr8PM7ODP1yJJu37WLztl1UqlyVYUMG8ddft/UaV3oE/HgQv1m+9PtsANt27sXLqwID+/cj6PFjfYcGQNCLaHy/D6TpnOM0nXOck7ees+azKhR1tgVgUttS+HjmZuiGc9SZfpjVR+8wtX1pPintDEAeewvy2Fswfc9VGsw8yvBNF/DxzM2cruX12az3unblCrt3bsejaDF9h/JeF8+foU37zqz038LCpatITEzky0H9iI6OAl4PnsaMGMrjfx4xe/63rNu8E+e8LnwxoI+mTnaV3c+L5CydO41L5/5g6NhpzF+zjbIVqzLlqwGEPHsKwN6t6/lh53f0HTKa2cs24JDTkamjBhIdlb0TG0rsi+RIO4TS6X0wa25ujrOzM66urnTp0oWuXbuyd+9eevXqRatWrbTqDhs2DB8fn1Tv+8GDB7Rs2RIbGxvs7Ozo0KEDT548AeDmzZuoVCpu3Lihtc38+fMpVKgQarUagMDAQJo0aYKNjQ158uShe/fuPH/+XFPfx8eHwYMHM3z4cJycnGjQoEH6XogMUtunLjVr1aZgocIULFSYIV98iZWVFVcuXdRrXOmxcf06WrdtS5t27Sni5saoseNxzuvM9m1b9B0aAL9cfcLRwKfcfRrJ3aeR+P1wnajYBMoXzgFAhcI52fnHQ07fDuFRaDSbf79P4D8vKVPAAYCbQa/ov/oMv1x9wv3nUZy89Ry/H65Tv1QejI2y3zfkqKhIxo0ZyYTJ07Czs9N3OO81f/FKmrZoTRE3dzyKFmfc5Ok8CQ7i5vVAAB4+uM+1K5cYOXYiJUqWpmChwowYM4Ho6Ch+Djio5+jfL7ufF++KjY3h9K9H6NF/KCXLepE3nysde/Unt3M+ftq3E7Vazf5dm2nb9VOq1qpLgcLuDBk9hdiYGH47HKDv8N9LaX2REmmHskhmVpfeB7PvsrS0JD4+/qP3o1aradWqFaGhoRw/fpyff/6ZO3fu0LFjRwCKFStGhQoV+O6777S227x5M126dEGlUhEUFETt2rUpV64cZ8+eJSAggCdPntChQwetbdavX4+JiQm///47K1as+OjYM0piYiIBBw8QHR1FmXLZN9uXnPi4OK4HXqOadw2t8mre1bl08YKeokqZkQpaVMiHpZkx5++GAfDn3yE0KO2Ms70FANU8nCiS24bj15+muB9bCxMiYhJITFJnSdxp4Tt9KjVr+VC1mre+Q0mzyIhXANjZ2QOv318AZmZmmjrGxsaYmphy+eL5rA8wlZR2XgAkJSaSlJSIqZm5VrmZuTk3rl7kSdA/vAgNoWzFqpp1pmZmlCxbgZvXLmV1uKmmxL5IjrRDGIJsNTnvzz//ZPPmzdSrV++j9/XLL79w+fJl7t69i6urKwAbN26kZMmSnDlzhkqVKtG1a1cWL17MtGnTALh16xbnzp1jw4YNACxbtgwvLy9mzvx3buDatWtxdXXl1q1bFC1aFAB3d3f8/Pw+OuaMcvvWTXp07URcXCyWVlbM/2YJbm7u+g4rTcJehJGYmIijo6NWuaOjE8+fP9NTVLqKu9iyd0QtzE2MiIxNpN+qP7kd/HrgNGnHFWZ3KceZGQ2JT0wiKUnNqM0XOfN3aLL7crA25YvGxfju93tZ2ILUCTh4gBvXA9m0dae+Q0kztVrNovl+lCnnRRF3DwAKFiqMc14XVixeyFfjJ2FpacnWTesJCXlOSDZ6f71LKefF2yytrCnmWYadG1eTv0Bh7HPk5MSRn7h9/Sp58xXgRWgIAA45tNtknyMnz54E6SPkVFFiXyRH2qFAyk2gZhq9Z2b379+PjY0NFhYWVKtWjVq1avHtt99+9H6vX7+Oq6urZiAL4OnpiYODA9evXwegU6dO3L9/n9OnTwPw3XffUa5cOTw9PQE4d+4cR48excbGRrMUL14cgDt3/r05oWLFih+MJzY2lpcvX2otsbGxH93O5BQqXJhtu/ay4bttdOjQmYnjR3Pnzl+ZcqzM9u5lD7Vana0uhdx5EkEj32O0nPcbG0/cZUF3Lzz+P2f2U58ieBXKSe/lp2ky+zjT9lxjRsey1CiWS2c/NhYmrP+8KreDXrHg4M2sbsZ7BQcFMWfWTKb7zsHc3PzDG2Qz82dP587tW0yZOUdTZmJqyow5C3nw4B6N63hTr3pFLpw7Q9XqNTEyNtZjtKmT3c+Ldw0dOxW1Wk2/Do3o1LAaB3dvpWa9RhgZ//snSCf8bN6mN5TWFymRdggl03tmtk6dOixbtgxTU1NcXFwwNTUFwMjISDNv9Y20TD9I6Q38dnnevHmpU6cOmzdvpmrVqmzZsoX+/ftr6iYlJdG8eXNmz56ts5+8efNqfra2tv5gPL6+vkyZMkWrbNzXk/h64uTUNinVTE3NKFCgIAAlS5Xm2rUrbN60gQmTpmb4sTJLDoccGBsba81PBggNDcHR0UlPUemKT1Rz7/nrm1QuP3hB2QI5+NSnCFN2XWFUc0/6rfqTI9dez9O+8fglJfPb07+eGydu/pspsDY3YePAaprMbkI2m2JwPfAaoaEhdO3YVlOWmJjI+XNn2bblO/44fxnjbDoAnO83gxO/HmPJqvXkzuOsta54iZKs37KbiFeviE+IJ0eOnPTr0YniniX1FO2HKeW8eJdzPlemLVxFTHQ00VER5HDMxbypY8jt7IJDzteZtLDQEHI4/vtFL/xFGA45cuor5A9Sal+8S9qhPDI416X3zKy1tTXu7u4ULFhQM5AFyJUrF0FB2peYLl68mOr9enp68uDBAx4+fKgpCwwMJDw8nBIlSmjKunbtyrZt2zh16hR37tyhU6dOmnVeXl5cu3aNQoUK4e7urrWkZgD7trFjxxIeHq61fDV6bJr2kV5qtVrvT1lIK1MzM0p4luT0yd+1yk+fPEnZbDz/V6UCcxMjTIyNMDMxIumdL2SJSWqM3vogsrEw4bvB1YhPTOLTFX8Qm5CU1SF/UOWqVdmxZx9bd+7RLJ4lS9GkaXO27tyTLQeyarWaebOnc/zILyxavhaXfPlTrGtja0uOHDl5+OA+N65fo0btulkYadoo9bx4w8LSkhyOuYh49ZKLZ05RqboPefLmwyGnI5fP/aGpFx8fz7VL5yhWsqweo30/pffFG9IO5ZEbwHTpPTObkrp16zJnzhw2bNhAtWrV2LRpE1evXqV8+dS9KevXr0+ZMmXo2rUrCxcuJCEhgYEDB1K7dm2taQFt2rRhwIABDBgwgDp16pAvXz7NukGDBrFq1So6d+7MV199hZOTE3/99Rdbt25l1apVafojbm5urnOJNvrj73PTsWjhfGrUrEUeZ2eiIiMJ+PEgZ8/8yZLlqzP+YJmse8/ejB8zCs9SpShbtjy7dmwjKCiI9h07fXjjLDC6eQmOBj7hcVg0NhYmtKiQn2oeTnRfeoqImARO3X7O161KEhOfyD+h0VR1d6RdZVem7r4KvM7IfjeoGpZmxnyx/hy2FiaaZ8yGRMSSXRK01tY2uHsU1SqztLTE3sFBpzy7mDdrGj8HHGTW/G+xsrLSzIO1sbHF3OL1DXlHfv4Jhxw5yOOcl7//us3Cub7U9KlLlWrV9Rn6B2X38yI5F86cBDW4uBYk+J+HbFjxDflcC1K3UXNUKhXN2nZh13dryZvPlbz5C7Dru7WYW1hQs14jfYf+Xkrsi+RIO4TSZdvBbMOGDZkwYQKjRo0iJiaGTz/9lB49enDlypVUba9Sqdi7dy9DhgyhVq1aGBkZ0ahRI535uHZ2djRv3pwdO3awdu1arXUuLi78/vvvjB49moYNGxIbG0vBggVp1KgRRkZ6T2onKzTkOePHjuL5s6fY2NpStGgxlixfTTXv7P0HOjmNGjch/EUYK5ct5dmzp7h7FGXJ8pW4uOT78MZZwMnWnIU9KpDbzpxXMQlc/+cl3Zee4rcbrwdOg9aeZUxLT77tWQEHKzMehUbht/86G0/cA6B0AXu8Cr++jHpisvYj3apNPMSj0OgsbY8h2bNzGwCDP+ulVT5u0nSatmgNQMjzZ3y7wI/QkOc4OuWiUdMW9O73eVaHmmbZ/bxITlRkBN+tWkzI86fY2NpRtWY9uvQZiInJ66txrTr1JC42lpXfzCLy1Ss8SpRiot8SLK3SdgUsqymxL5Ij7VAWBSdQM41K/e7EVJFlMiMzqw+GcGJ5DPte3yFkiJsLWug7hI8WFZuo7xAyhE02/09uqfXXkwh9h/DR3PPY6DsEYWD0eXq7j/wx0/b919zGmbbvzGQYn7ZCCCGEEP8BSp7bmlmy57VyIYQQQgghUkEys0IIIYQQCiGJWV2SmRVCCCGEEIolmVkhhBBCCIWQObO6ZDArhBBCCKEQMpbVJdMMhBBCCCGEYklmVgghhBBCIYyMJDX7LsnMCiGEEEIIxZLMrBBCCCGEQsicWV2SmRVCCCGEEIolmVkhhBBCCIWQR3PpksysEEIIIYRQLMnMCiGEEEIohCRmdclgVgghhBBCIWSagS6ZZiCEEEIIIRRLMrNCCCGEEAohmVldkpkVQgghhBCKJZlZIYCbC1roO4QM4VhjlL5D+GihJ+boOwTxlnw5LPUdghDiLZKY1SWZWSGEEEIIoViSmRVCCCGEUAiZM6tLMrNCCCGEEEKxJDMrhBBCCKEQkpjVJYNZIYQQQgiFkGkGumSagRBCCCGEUCzJzAohhBBCKIQkZnVJZlYIIYQQQiiWZGaFEEIIIRRC5szqksysEEIIIYRQLMnMCiGEEEIohCRmdUlmVgghhBBCKJZkZoUQQgghFELmzOqSwawQQgghhELIWFaXTDMQQgghhBCKJZlZIYQQQgiFkGkGuhSfme3VqxcqlQqVSoWpqSl58uShQYMGrF27lqSkJH2Hl+XWrFpBl45t8a5cnjq1qjFs6EDu3f1b32Gl27Yt39H4k7pUKl+aTu3bcP7cWX2HlG5rVq2gfKnizJk1U9+hpGhkzzpE/zGHOV+20JRZW5qxYGQr/vphPKHHZ3Jh60j6tammWZ/DzpL5I1pyaftXhByfwa3vxzFveEvsrC300YQUbd+6mfatm1O9ihfVq3jRo2tHTvx2XN9hpYuSzov1a1bSu2sH6lavSOO6NRj15WDu37ubYv1Z0ydRtbwnW7/bkIVRpp+S+uJ9pB1CyRQ/mAVo1KgRQUFB3Lt3jx9//JE6derwxRdf0KxZMxISEpLdJj4+PoujzBrnzv5Jx85d2bB5O8tXriMxIZEBn/UhOipK36GlWcCPB/Gb5Uu/zwawbedevLwqMLB/P4IeP9Z3aGl27coVdu/cjkfRYvoOJUUVSuSnT6uqXL6t/fr6DWtBg6rF6D1pC+U6zeHbrb8xf0RLmtUqCUBeJzvy5rJn7KL9VOwyn35Tt9GgWjGWf91eH81IUR5nZ4Z+OZLN23axedsuKlWuyrAhg/jrr9v6Di1NlHZeXDh/lrYdO7N6wxYWLVtNYmIiXwzoS3S07mfS8aO/cO3KZXLlyq2HSNNOaX2REmmHsqhUmbcolUEMZs3NzXF2diZfvnx4eXkxbtw4vv/+e3788Uf8/f2B12n55cuX07JlS6ytrZk+fToAP/zwAxUqVMDCwoIiRYowZcoUrQHw5MmTKVCgAObm5ri4uDB06FDNuqVLl+Lh4YGFhQV58uShXbt2Wdru5CxdsYaWrdrg7u5BseLFmTLdl6CgxwQGXtN3aGm2cf06WrdtS5t27Sni5saoseNxzuvM9m1b9B1amkRFRTJuzEgmTJ6GnZ2dvsNJlrWlGeumdmHgzJ28eBmtta5K6YJsOniO387/zYOgMNbu/YPLfwXhVSI/AIF/P6HzmA0cPHGdu/+EcPzcHSYvC6BJDU+MjbPPR0xtn7rUrFWbgoUKU7BQYYZ88SVWVlZcuXRR36GlidLOi4VLVtKsRWuKuHngUaw4X0+eQXBwEDcCA7XqPX36hLmzZjBlph/GJsqYAae0vkiJtEMoXfb5S5PB6tatS9myZdm9e7embNKkSbRs2ZIrV67w6aef8tNPP9GtWzeGDh1KYGAgK1aswN/fnxkzZgCwc+dOFixYwIoVK7h9+zZ79+6ldOnSAJw9e5ahQ4cydepUbt68SUBAALVq1dJLW98nIuIVAPb29nqOJG3i4+K4HniNat41tMqreVfn0sULeooqfXynT6VmLR+qVvPWdygpWvhVawJ+v87RM7pZypOX7tKspicuuV4PxGtVcMPD1YlfTt9McX92Nha8jIwhMTF7TvVJTEwk4OABoqOjKFOuvL7DSTVDOC/efCbZvfWZlJSUxJSvx9Ct56cUcfPQV2hpYgh9AdIOJXoztTIzFqVSxtffdCpevDiXL1/W/N6lSxc+/fRTze/du3dnzJgx9OzZE4AiRYowbdo0Ro0axaRJk3jw4AHOzs7Ur18fU1NTChQoQOXKlQF48OAB1tbWNGvWDFtbWwoWLEj58tnrj6JarWaeny/lvSrg7lFU3+GkSdiLMBITE3F0dNQqd3R04vnzZ3qKKu0CDh7gxvVANm3dqe9QUtS+QVnKFctHjd6Lkl0/Yt73LB3Xjjv7JxCfkEhSkpoBM3dw8tK9ZOvntLNi7Kf1WbPndCZGnT63b92kR9dOxMXFYmllxfxvluDm5q7vsFJN6eeFWq3mm3l+lC3vhZv7v4PWjetWY2xsTIfO3fQYXdoovS/ekHYIQ2DQg1m1Wq31TaNixYpa68+dO8eZM2c0mVh4nbGJiYkhKiqK9u3bs3DhQooUKUKjRo1o0qQJzZs3x8TEhAYNGlCwYEHNukaNGtG6dWusrKySjSU2NpbY2FitsiQjc8zNzTOwxdp8Z0zl1q1b+G/YnGnHyGzvflN8t0+zs+CgIObMmsnSlWsytZ8/Rv7c9swZ3pLmQ1cRG5f8/PJBHWtQuVQB2o5Yy4PgF9QoV5hvvmpN8PNXOplcW2tz9iz4lOt3nzBj9c9Z0YQ0KVS4MNt27eXVy5cc/vkQE8ePZrX/JkUNaEG558XcWdP56/ZNVq7bpCm7EXiNbVs2sn7zLkW04V1K7Yt3STuUw8CakyEMdpoBwPXr1ylcuLDmd2tra631SUlJTJkyhYsXL2qWK1eucPv2bSwsLHB1deXmzZssWbIES0tLBg4cSK1atYiPj8fW1pbz58+zZcsW8ubNy8SJEylbtiwvXrxINhZfX1/s7e21ljmzfTOt7bNmTuP40SOsXruePM7OmXaczJLDIQfGxsY8f/5cqzw0NARHRyc9RZU21wOvERoaQteObalYtiQVy5bk3NkzbPluIxXLliQxMVHfIVK+eH7y5LTlpP8XvPp9Fq9+n0WtCm4M7FCdV7/PwsrClCkDGjH6mx84eOI6V/8KYvnOk+z85RLDutbW2peNlTn7FvYlIiqOjqPXk5ANpxiYmppRoEBBSpYqzdAvR1C0WHE2b1LGXfOg7PNi7qzp/Hb8KEtX+ZM7z7+fSRcvnCMsNJRWTepRvWJpqlcsTXDQYxbN96NVk/p6jPj9lNwXb5N2KI9MM9BlsJnZI0eOcOXKFb788ssU63h5eXHz5k3c3VPOylhaWtKiRQtatGjBoEGDKF68OFeuXMHLywsTExPq169P/fr1mTRpEg4ODhw5coQ2bdro7Gfs2LEMHz5cqyzJKOOzdWq1mlkzp3Hk8M+sXreRfPldM/wYWcHUzIwSniU5ffJ36tVvoCk/ffIkPnXr6TGy1KtctSo79uzTKpv09TgKFy5Crz59MTY21lNk/zp69i8qdJ6rVbZyQkdu3n/KvA1HMTY2wszUhKQktVadxCQ1Rkb/fvDZWpvzwzf9iI1LoN3IdSlmebMbtVpNXFycvsNINSWeF2q1mnmzZ3D8yC8sWeWPS778WusbN21BpSrVtMqGDexHo6YtaNaydVaGmiZK7IvkSDuEITCIwWxsbCzBwcEkJiby5MkTAgIC8PX1pVmzZvTo0SPF7SZOnEizZs1wdXWlffv2GBkZcfnyZa5cucL06dPx9/cnMTGRKlWqYGVlxcaNG7G0tKRgwYLs37+fv//+m1q1apEjRw4OHjxIUlISxYol/+glc3PdKQXRmfB0sJnTp/Djwf0sXLQUa2trzVwhGxtbLCyy13M/P6R7z96MHzMKz1KlKFu2PLt2bCMoKIj2HTvpO7RUsba20ZmrbGlpib2DQ7aZwxwRFUvg30+0yiKj4wgNj9KU/3ruDjOHNCM6Np4HQWHU9HKja+MKjP7mB+B1Rnb/on5YmpvRe9IW7KwtNM+YffYiQmcgrC+LFs6nRs1a5HF2JioykoAfD3L2zJ8sWb5a36GlidLOizm+0zj04wH8FizG2tqakP9/Jln//zPJ3sEBewcHrW2MTUxwdHKiYKHCyewx+1BaX6RE2qEsSs6gZhaDGMwGBASQN29eTExMyJEjB2XLlmXRokX07NkTI6OUZ1I0bNiQ/fv3M3XqVPz8/DA1NaV48eL07dsXAAcHB2bNmsXw4cNJTEykdOnS/PDDDzg6OuLg4MDu3buZPHkyMTExeHh4sGXLFkqWLJlVzU7Wjv8/gqRv7+5a5VOm+9KylW7GODtr1LgJ4S/CWLlsKc+ePcXdoyhLlq/ExSWfvkP7T+nx9XdMHdQY/yldyGFnxYPgMCYvD2DV7lMAlC+ej8qlCgIQuHuM1rbFWs3kQVBYlsecnNCQ54wfO4rnz55iY2tL0aLFWLJ8NdW8q+s7tDRR2nmxe8dWAAb266lV/vWUGTRrkX0zr6mhtL5IibRDKJ1KrVZnj7TJf1BmZGb1wRC+JCYZyGngWGOUvkP4aKEn5ug7hAxhCOcFQHSc/ud2fyxLM/1P6RGGxUKPqcDaC37PtH0f/1JZX+7fMOgbwIQQQgghhGEziGkGQgghhBD/BTJnVpdkZoUQQgghhGJJZlYIIYQQQiEkMatLBrNCCCGEEAoh0wx0yTQDIYQQQgihWJKZFUIIIYRQCEnM6pLMrBBCCCGEUCzJzAohhBBCKISRpGZ1SGZWCCGEEEIolgxmhRBCCCEUQqXKvCUtli1bRpkyZbCzs8POzo5q1arx448/atar1WomT56Mi4sLlpaW+Pj4cO3aNa19xMbGMmTIEJycnLC2tqZFixY8evQoza+JDGaFEEIIIUSa5M+fn1mzZnH27FnOnj1L3bp1admypWbA6ufnx/z581m8eDFnzpzB2dmZBg0a8OrVK80+hg0bxp49e9i6dSsnTpwgIiKCZs2akZiYmKZYVGq1Wp2hrROpFh2v7wgyhiFM30kykNPAscYofYfw0UJPzNF3CBnCEM4LgOi4tP1RyY4szYz1HYIwMBZ6vOOo4dI/Mm3fPw2s8lHb58yZkzlz5vDpp5/i4uLCsGHDGD16NPA6C5snTx5mz55N//79CQ8PJ1euXGzcuJGOHTsC8PjxY1xdXTl48CANGzZM9XElMyuEEEIIoRBGqsxbYmNjefnypdYSGxv7wZgSExPZunUrkZGRVKtWjbt37xIcHMwnn3yiqWNubk7t2rU5efIkAOfOnSM+Pl6rjouLC6VKldLUSfVrkqbaQgghhBDCIPn6+mJvb6+1+Pr6plj/ypUr2NjYYG5uzueff86ePXvw9PQkODgYgDx58mjVz5Mnj2ZdcHAwZmZm5MiRI8U6qSWP5hJCCCGEUIjM/He2Y8eOZfjw4Vpl5ubmKdYvVqwYFy9e5MWLF+zatYuePXty/PjxFGNVq9UfjD81dd4lmVkhhBBCCIG5ubnm6QRvlvcNZs3MzHB3d6dixYr4+vpStmxZvvnmG5ydnQF0MqxPnz7VZGudnZ2Ji4sjLCwsxTqpJYNZIYQQQgiFyC6P5kqOWq0mNjaWwoUL4+zszM8//6xZFxcXx/Hjx/H29gagQoUKmJqaatUJCgri6tWrmjqpJdMM9MhQ7nY2BC8iDePREo+PztJ3CB+tzerMu1M3K+3p93F3BWcXN4NefbhSNleuoIO+QxDC4IwbN47GjRvj6urKq1ev2Lp1K8eOHSMgIACVSsWwYcOYOXMmHh4eeHh4MHPmTKysrOjSpQsA9vb29OnThxEjRuDo6EjOnDkZOXIkpUuXpn79+mmKRQazQgghhBAKoSJ7ZMKePHlC9+7dCQoKwt7enjJlyhAQEECDBg0AGDVqFNHR0QwcOJCwsDCqVKnCoUOHsLW11exjwYIFmJiY0KFDB6Kjo6lXrx7+/v4YG6ftcXrynFk9iknQdwTijdCIOH2HkCEM4XmaXdaf1XcIGcJQMrMX77/QdwgfTTKzIqPp8zmzzVacybR97+9fKdP2nZkkMyuEEEIIoRBG2SMxm63IDWBCCCGEEEKxJDMrhBBCCKEQmfmcWaWSwawQQgghhELIWFaXTDMQQgghhBCKJZlZIYQQQgiFMJLUrA7JzAohhBBCCMWSzKwQQgghhEJIYlaXZGaFEEIIIYRiSWZWCCGEEEIh5NFcuiQzK4QQQgghFEsys0IIIYQQCiGJWV0ymBVCCCGEUAh5NJcumWYghBBCCCEUy+AHs8HBwQwZMoQiRYpgbm6Oq6srzZs35/Dhwxl2jEKFCrFw4cIM219G2LblOxp/UpdK5UvTqX0bzp87q++Q0kXJ7fjOfzV1qpRm8fzZmrJZU8dTp0pprWXgp131GKWu9WtW0rtrB+pWr0jjujUY9eVg7t+7m2L9WdMnUbW8J1u/25CFUWprWjI3SzuUZlefiuzqU5H5rT2pWMBeq07XivnY1KM8e/tVYnaLEhTIYam1vnGJXMxuUYJdfSry44AqWJsZZ2UT0iS7nxc3rlxgweQRfNGtKT2bVOHcyeOadQkJCWxbu5jxA7rQr3VtvujWlBVzJxMW8kxrH/HxcWxcNpdBnT6hX+vaLJgyktDnT7K6KR+U3fsitaQdyqHKxEWpDHowe+/ePSpUqMCRI0fw8/PjypUrBAQEUKdOHQYNGqTv8DJNwI8H8ZvlS7/PBrBt5168vCowsH8/gh4/1ndoaaLkdtwIvMr+vTsp4l5UZ13latXZdfCoZpm1YKkeIkzZhfNnaduxM6s3bGHRstUkJibyxYC+REdH6dQ9fvQXrl25TK5cufUQ6b+eR8Sx7vQDhu68ytCdV7n0z0smNiqqGbC2L5eXNmXzsvS3e3yx6yphUfHMbF4cS9N/PwLNTY05+/AFW8//o69mpIoSzovYmGhcC3vQfcBInXVxsTHc/+smLTp/ytRvNzDk61k8+ecBC6do1/1uxQLOnTzGwNHT+XruSmKjo1gweQRJiYlZ1YwPUkJfpIa0QyidQQ9mBw4ciEql4s8//6Rdu3YULVqUkiVLMnz4cE6fPg3AgwcPaNmyJTY2NtjZ2dGhQweePPn32/+dO3do2bIlefLkwcbGhkqVKvHLL79o1vv4+HD//n2+/PJLVCpVtnhkxsb162jdti1t2rWniJsbo8aOxzmvM9u3bdF3aGmi1HZER0UxY+IYRo6bhK2dnc56U1Mzcjo6aRY7e/tk9qI/C5espFmL1hRx88CjWHG+njyD4OAgbgQGatV7+vQJc2fNYMpMP4xN9Dv9/o/7LzjzIJx/wmP4JzyG9X8+IiY+ieJ5bABoVcaZref+4eTdMO6HRjPvyB3MTYzw8XDS7GPv5WB2XAjixpMIfTUjVZRwXpSt5E27np9TsXodnXVW1jaMmvktVWrVJ2/+grgXL023ASO599cNQp4GAxAVGcGvh/bRue8XlCxfmYJuxej/1RQe3rvDtYtnsro5KVJCX6SGtENZ3ow1MmNRKoMdzIaGhhIQEMCgQYOwtrbWWe/g4IBaraZVq1aEhoZy/Phxfv75Z+7cuUPHjh019SIiImjSpAm//PILFy5coGHDhjRv3pwHDx4AsHv3bvLnz8/UqVMJCgoiKCgoy9qYnPi4OK4HXqOadw2t8mre1bl08YKeoko7Jbdj4ZwZVK1ekwqVqyW7/uL5s7RuVJvu7Zoxd+ZkwkJDsjjCtImIeAWgNehOSkpiytdj6NbzU4q4eegrtGQZqaC2e04sTI248SQCZ1tzclqbcf5RuKZOfJKaK49f4elso8dI007J58X7REdGoFKpsLJ53R/3bt8gMSGBUl5VNHVyOOYif8Ei3L5+WV9hajGUvpB2CENgsE8z+Ouvv1Cr1RQvXjzFOr/88guXL1/m7t27uLq6ArBx40ZKlizJmTNnqFSpEmXLlqVs2bKabaZPn86ePXvYt28fgwcPJmfOnBgbG2Nra4uzs3Omt+tDwl6EkZiYiKOjo1a5o6MTz58/S2Gr7Eep7Thy6Edu3wxk+bqtya6vXK0mtes2xDlvXoIe/8PaFYsZPqgvK9Zvw8zMLIuj/TC1Ws038/woW94LN/d/B60b163G2NiYDp276TE6bYVyWjK/TUnMjI2Ijk9kWsAtHoRFU+L/2dmwqHit+i+i48ltk/1e8/dR6nnxPnFxsWxft4SqPg2xtHrdV+FhIZiYmGJtq31lw84hJ+Fh2ePLn6H0hbRDeYyUm0DNNAY7mFWr1cD7/1PG9evXcXV11QxkATw9PXFwcOD69etUqlSJyMhIpkyZwv79+3n8+DEJCQlER0drMrOpFRsbS2xsrHaMxuaYm5unaT+p9W671Wq1Ii8hKKkdT58Es3j+LPwWrcQshX6t26CR5ufCbh4UK1GSTi0/4fTvv1KrTv2sCjXV5s6azl+3b7Jy3SZN2Y3Aa2zbspH1m3dlq7549CKGQduvYGNuQvUiORlR141R31/XrFcns01yZUqgpPPifRISElg262vUajU9B3314Q3UoMpmt6kYSl9IO4SSGew0Aw8PD1QqFdevX0+xTkpv8rfLv/rqK3bt2sWMGTP47bffuHjxIqVLlyYuLi5N8fj6+mJvb6+1zJntm7ZGpUIOhxwYGxvz/PlzrfLQ0BAcHZ1S2Cr7UWI7bt24RlhYKP17daSedznqeZfj0vmz7N7+HfW8y5GYzI0rjk65yOPswj8P7+sh4vebO2s6vx0/ytJV/uTO8+9Vh4sXzhEWGkqrJvWoXrE01SuWJjjoMYvm+9Gqif4G5AlJaoJexnL7WST+fzzk75AoWpbOo8nI5rQy1arvYGnKi+j45HaVbSnxvEhJQkICS3zH8ezJY0bN+FaTlQWwz+FIQkI8ka9eam3zMjwUuxw5szrUZBlKX0g7lEfmzOoy2MFszpw5adiwIUuWLCEyMlJn/YsXL/D09OTBgwc8fPhQUx4YGEh4eDglSpQA4LfffqNXr160bt2a0qVL4+zszL1797T2ZWZmluxA5W1jx44lPDxca/lq9NiPb+g7TM3MKOFZktMnf9cqP33yJGXLlc/w42UWJbbDq2JV1m7ezeqNOzRLsRIlqd+wKas37sDYWPdRT+HhL3j6NJicTrn0EHHy1Go1c2dN5/iRX1i8Yi0u+fJrrW/ctAWbtu9lw9bdmiVXrtx07fEp3yxdpaeodakAU2Mjgl/FEhoZR/n8/875NTFSUdrFlsDg7H2z17uUeF4k581A9snjh4yauRgbO+2bIAt5FMfYxISrF/7UlL0Ifc6j+3/jUaJMVoebLEPpC2mH8qhUmbcolcFOMwBYunQp3t7eVK5cmalTp1KmTBkSEhL4+eefWbZsGYGBgZQpU4auXbuycOFCEhISGDhwILVr16ZixYoAuLu7s3v3bpo3b45KpWLChAkkJSVpHadQoUL8+uuvdOrUCXNzc5ycdL8FmpvrTimIScicdnfv2ZvxY0bhWaoUZcuWZ9eObQQFBdG+Y6fMOWAmUVo7rKytKfzOzVAWlpbY2TtQ2M2D6Kgo/FctpVbd+jg65iI46DGrl32Dvb0DNWvX01PUuub4TuPQjwfwW7AYa2trQv4/38zaxhYLCwvsHRywd3DQ2sbYxARHJycKFiqsh4ihZ5X8nH0QzrOIWKxMjant7khpFzsmHLgBvH5SQUcvFx7//2kHHb1ciE1I4tjtf7M4OSxNyWFliou9BQCFHK2IjkvkaUQsEbHZ53FQSjgvYqKjePL4keb3Z08ec//OLWxs7XBwdGLxzDHc/+smX06eR1JiEi/+fxOkja0dJqamWFnbUOuTFmxd/Q02dvbY2NqxdfUiXAu5UbJcJX01S4cS+iI1pB1C6Qx6MFu4cGHOnz/PjBkzGDFiBEFBQeTKlYsKFSqwbNkyVCoVe/fuZciQIdSqVQsjIyMaNWrEt99+q9nHggUL+PTTT/H29sbJyYnRo0fz8qX2pa+pU6fSv39/3NzciI2N1czX1ZdGjZsQ/iKMlcuW8uzZU9w9irJk+UpcXPLpNa60MpR2vGFkZMTfd25z6McfiHj1EkenXJSrUImJM+ZilcwTN/Rl947XN68N7NdTq/zrKTNo1qK1PkL6oByWpnxV142c1qZExiVyNySKCQducOHR63N1x8UgzEyMGFSzEDbmJtx8GsH4/TeIjv/3i2mTkrnpVunfLPTcVp4AzDtyh19ual+61CclnBd3b19n1piBmt+3rFoIQI36TWnVtS8XTv8GwITB3bW2GzNrKSXKVACgy2fDMDY2ZonvOOLjYvEsW4lhwydilMwVDn1RQl+khrRDWZQ8HSCzqNT6Hnn9h2VWZlakXWhE2uZAZ1eW2fi/VqVWl/WG8R979vSr8uFKCnDx/gt9h/DRyhV00HcIwsBY6DEV2GNz5j2ebkOX7DGNJ60MOjMrhBBCCGFI5NFcugz2BjAhhBBCCGH4JDMrhBBCCKEQMmdWl2RmhRBCCCGEYklmVgghhBBCISQvq0sGs0IIIYQQCmEk0wx0yDQDIYQQQgihWJKZFUIIIYRQCEnM6pLMrBBCCCGEUCzJzAohhBBCKIQ8mkuXZGaFEEIIIYRiSWZWCCGEEEIhJDGrSzKzQgghhBBCsSQzK4QQQgihEPKcWV0ymBVCCCGEUAgZy+qSaQZCCCGEEEKxJDMrhBBCCKEQ8mguXZKZFUIIIYQQiiWZWSGAnDZm+g5B/N+eflX0HUKGyFFpsL5DyBBhZxbrO4SPlqRW6zuEDGEIN/4YSFfolWQhdclrIoQQQgghFEsys0IIIYQQCiFzZnVJZlYIIYQQQiiWZGaFEEIIIRTCSBKzOmQwK4QQQgihEDKY1SXTDIQQQgghhGJJZlYIIYQQQiHkBjBdkpkVQgghhBCKJZlZIYQQQgiFkDmzuiQzK4QQQgghFEsys0IIIYQQCiFTZnVJZlYIIYQQQiiWZGaFEEIIIRTCSFKzOmQwK4QQQgihEHJJXZe8JkIIIYQQQrFkMJuCXr160apVq1TXv3fvHiqViosXL2ZaTEIIIYT4b1OpMm9Rqmw/mH369Cn9+/enQIECmJub4+zsTMOGDTl16pS+Q8vWtm35jsaf1KVS+dJ0at+G8+fO6jukdDGEdhhCG0DakdVGfvoJ0RcWM2dkW01Z7py2rJzSjb8PzSDk5Hy+XzwQtwK5UtzH3sUDiL6wmOY+ZbIi5DRTSl+kxppVKyhfqjhzZs3Udyhpdu7sGYYM/Jz6PjUoW7IYRw7/ou+Q0mzNqhV06dgW78rlqVOrGsOGDuTe3b/1HZbIItl+MNu2bVsuXbrE+vXruXXrFvv27cPHx4fQ0FB9h5ZtBfx4EL9ZvvT7bADbdu7Fy6sCA/v3I+jxY32HliaG0A5DaANIO7JaBc8C9GnjzeVbj7TKty/4jML5nWg/bAVVO8/iQVAoB5cPwcrCTGcfQ7rWQa3OqojTTil9kRrXrlxh987teBQtpu9Q0iU6OopixYoxZvxEfYeSbufO/knHzl3ZsHk7y1euIzEhkQGf9SE6KkrfoWU4I5Uq0xalytaD2RcvXnDixAlmz55NnTp1KFiwIJUrV2bs2LE0bdoUgPnz51O6dGmsra1xdXVl4MCBREREaPbh7++Pg4MDP/30EyVKlMDGxoZGjRoRFBSkqZOYmMjw4cNxcHDA0dGRUaNGoX7nr0BAQAA1atTQ1GnWrBl37tzJmhcijTauX0frtm1p0649RdzcGDV2PM55ndm+bYu+Q0sTQ2iHIbQBpB1ZydrSjHUzezFw2hZevIzWlLsXyE2VMoUZOmMr5wIfcPv+U77w3Ya1pTkdGlfQ2kfpovkY2q0un0/elNXhp5oS+iI1oqIiGTdmJBMmT8POzk7f4aRLjZq1GfzFl9Rv8Im+Q0m3pSvW0LJVG9zdPShWvDhTpvsSFPSYwMBr+g5NZIFsPZi1sbHBxsaGvXv3Ehsbm2wdIyMjFi1axNWrV1m/fj1Hjhxh1KhRWnWioqKYO3cuGzdu5Ndff+XBgweMHDlSs37evHmsXbuWNWvWcOLECUJDQ9mzZ4/WPiIjIxk+fDhnzpzh8OHDGBkZ0bp1a5KSkjK+4R8hPi6O64HXqOZdQ6u8mnd1Ll28oKeo0s4Q2mEIbQBpR1ZbOLYjAb9d5egfN7XKzc1eP3wmJi5BU5aUpCYuPgHvcm6aMksLU9b79uLL2dt5EvIqa4JOI6X0RWr4Tp9KzVo+VK3mre9QxFsiIl6/9+3t7fUcScaTObO6svVg1sTEBH9/f9avX4+DgwPVq1dn3LhxXL58WVNn2LBh1KlTh8KFC1O3bl2mTZvG9u3btfYTHx/P8uXLqVixIl5eXgwePJjDhw9r1i9cuJCxY8fStm1bSpQowfLly3VOgLZt29KmTRs8PDwoV64ca9as4cqVKwQGBmbui5BGYS/CSExMxNHRUavc0dGJ58+f6SmqtDOEdhhCG0DakZXaN6xAueKuTPh2n866m/eCuf84hGlDWuBga4mpiTEjezcgby57nJ3+/bzyG9GW05fusv/YlawMPU2U0BepEXDwADeuBzJk2HB9hyLeolarmefnS3mvCrh7FNV3OCILZOvBLLweRD5+/Jh9+/bRsGFDjh07hpeXF/7+/gAcPXqUBg0akC9fPmxtbenRowchISFERkZq9mFlZYWb27+Zi7x58/L06VMAwsPDCQoKolq1apr1JiYmVKxYUSuOO3fu0KVLF4oUKYKdnR2FCxcG4MGDB6lqR2xsLC9fvtRaUso2ZwTVO1+x1Gq1TpkSGEI7DKENIO3IbPnzODDnq7Z8+vV6Yt/Kvr6RkJBE55GrcS+Ym6Bf5xB6aj41K3gQcOIaif+/QtS0dml8Khflqzk7szr8dMmufZEawUFBzJk1k+m+czA3N9d3OOItvjOmcuvWLWb5zdd3KJnCSJV5i1Ip4p8mWFhY0KBBAxo0aMDEiRPp27cvkyZNok6dOjRp0oTPP/+cadOmkTNnTk6cOEGfPn2Ij4/XbG9qaqq1P5VKpTMn9kOaN2+Oq6srq1atwsXFhaSkJEqVKkVcXFyqtvf19WXKlClaZeMnTOLriZPTFMeH5HDIgbGxMc+fP9cqDw0NwdHRKUOPlZkMoR2G0AaQdmSV8iUKkMfRjpPf/TtNysTEmBpebnzesRb2VYZx4fpDqnaahZ2NBWamJjwPi+DXDSM5F/j6S7VPpaIUye9E8K9ztPa9ZW5ffr9wh4b9vsnSNqUku/dFalwPvEZoaAhdO/77tInExETOnzvLti3f8cf5yxgbG+sxwv+mWTOncfzoEdau30QeZ2d9h5MplHyjVmZRxGD2XZ6enuzdu5ezZ8+SkJDAvHnzMDJ6nWR+d4rBh9jb25M3b15Onz5NrVq1AEhISODcuXN4eXkBEBISwvXr11mxYgU1a9YE4MSJE2k6ztixYxk+XPtSlNo447/Nm5qZUcKzJKdP/k69+g005adPnsSnbr0MP15mMYR2GEIbQNqRVY7+eZMK7WZola2c0o2bd58wz/9nkpL+/QL+MiIGALcCufDyLMCUpfsBmLvuEOv2nNTax7md4xk1bxcHjl/N5BakXnbvi9SoXLUqO/ZoTweZ9PU4ChcuQq8+fWUgm8XUajWzZk7jyOGfWb1uI/nyu+o7JJGFsvVgNiQkhPbt2/Ppp59SpkwZbG1tOXv2LH5+frRs2RI3NzcSEhL49ttvad68Ob///jvLly9P83G++OILZs2ahYeHByVKlGD+/Pm8ePFCsz5Hjhw4OjqycuVK8ubNy4MHDxgzZkyajmFubq5zKSpG90pihujeszfjx4zCs1QpypYtz64d2wgKCqJ9x06Zc8BMYgjtMIQ2gLQjK0RExRJ4J0irLDI6jtDwSE15m/rleRYWwcPgUEp5uDD3q3b8cOwyh0/fAOBJyKtkb/p6GBTG/cchmd+INMjOfZEa1tY2OvMxLS0tsXdwUNw8zajISK0pc/88esSN69dfJ3tcXPQYWerNnD6FHw/uZ+GipVhbW2vmXtvY2GJhYaHn6DKWJGZ1ZevBrI2NDVWqVGHBggXcuXOH+Ph4XF1d6devH+PGjcPS0pL58+cze/Zsxo4dS61atfD19aVHjx5pOs6IESMICgqiV69eGBkZ8emnn9K6dWvCw8OB109M2Lp1K0OHDqVUqVIUK1aMRYsW4ePjkwmt/niNGjch/EUYK5ct5dmzp7h7FGXJ8pW4uOTTd2hpYgjtMIQ2gLQju3DOZcfsEW3I7WhL8POXfLf/D3xXBug7rHRRel8YkmvXrtK3979/N+f6+QLQomVrps2cpa+w0mTH/x/p1rd3d63yKdN9admqjT5CEllIpU7r5FGRYTIrMyuE0L8clQbrO4QMEXZmsb5D+GhJBvJnzhDmShpIV2Bp+uE6mWXG4b8ybd/j67ln2r4zU7Z/moEQQgghhBApydbTDIQQQgghxL9UKD9Dn9EkMyuEEEIIIRRLMrNCCCGEEAqh5H9ukFkkMyuEEEIIIRRLMrNCCCGEEAohmVldMpgVQgghhFAIlQE8oi2jyTQDIYQQQgihWJKZFUIIIYRQCJlmoEsys0IIIYQQQrEkMyuEEEIIoRAyZVaXZGaFEEIIIYRiyWBWCCGEEEIhjFSqTFvSwtfXl0qVKmFra0vu3Llp1aoVN2/e1KqjVquZPHkyLi4uWFpa4uPjw7Vr17TqxMbGMmTIEJycnLC2tqZFixY8evQoba9JmmoLIYQQQoj/vOPHjzNo0CBOnz7Nzz//TEJCAp988gmRkZGaOn5+fsyfP5/Fixdz5swZnJ2dadCgAa9evdLUGTZsGHv27GHr1q2cOHGCiIgImjVrRmJiYqpjUanVanWGtk6kWkyCviMQQmSWHJUG6zuEDBF2ZrG+Q/hoSQbyZy6tmbPsyEC6AktT/R170Ym7mbbvoTUKp3vbZ8+ekTt3bo4fP06tWrVQq9W4uLgwbNgwRo8eDbzOwubJk4fZs2fTv39/wsPDyZUrFxs3bqRjx44APH78GFdXVw4ePEjDhg1TdWzJzAohhBBCKIRKlXlLbGwsL1++1FpiY2NTFVd4eDgAOXPmBODu3bsEBwfzySefaOqYm5tTu3ZtTp48CcC5c+eIj4/XquPi4kKpUqU0dVJDBrNCCCGEEAJfX1/s7e21Fl9f3w9up1arGT58ODVq1KBUqVIABAcHA5AnTx6tunny5NGsCw4OxszMjBw5cqRYJzXk0VxCCCGEEAphROZNNxk7dizDhw/XKjM3N//gdoMHD+by5cucOHFCZ927/35XrVZ/8F/ypqbO22Qwq0eJSYYxecjYAP4dSWx8kr5DyBBqlP+eMjaAeYFgGHNNAZy6+Os7hI/2fHMvfYeQIQxhvqmBnN4Gy9zcPFWD17cNGTKEffv28euvv5I/f35NubOzM/A6+5o3b15N+dOnTzXZWmdnZ+Li4ggLC9PKzj59+hRvb+9UxyDTDIQQQgghFCIz58ymhVqtZvDgwezevZsjR45QuLD2zWOFCxfG2dmZn3/+WVMWFxfH8ePHNQPVChUqYGpqqlUnKCiIq1evpmkwK5lZIYQQQgiRJoMGDWLz5s18//332Nraaua42tvbY2lpiUqlYtiwYcycORMPDw88PDyYOXMmVlZWdOnSRVO3T58+jBgxAkdHR3LmzMnIkSMpXbo09evXT3UsMpgVQgghhFCI7DKzb9myZQD4+Phola9bt45evXoBMGrUKKKjoxk4cCBhYWFUqVKFQ4cOYWtrq6m/YMECTExM6NChA9HR0dSrVw9/f3+MjY1THYs8Z1aPIuMM46WXObPZh8yZzT5MTQxjFpfMmc0+DOGvtYGc3ljoMRW4/NS9TNv359UKZdq+M5NkZoUQQgghFMIQ/nlGRpPBrBBCCCGEQshYVpdhXAcTQgghhBD/SZKZFUIIIYRQCJlmoEsys0IIIYQQQrEkMyuEEEIIoRCSmNUlmVkhhBBCCKFYkpkVQgghhFAIyULqktdECCGEEEIolmRmhRBCCCEUQiWTZnXIYFYIIYQQQiFkKKtLphkIIYQQQgjF+k8PZlUqFXv37k1x/bFjx1CpVLx48SLLYhJCCCGESImRSpVpi1IZ9GD26dOn9O/fnwIFCmBubo6zszMNGzbk1KlTqdre29uboKAg7O3t31uvV69etGrVKgMizhiRkRHMmT2TJp/UpVrFsvTq1olrV6/oO6w0OXf2DEMGfk59nxqULVmMI4d/0XdIqXL+3BmGDx1Akwa1qFyuBMeOaMcdFRXJHN9pNPvEh5pVytGhdVN2bt+ip2iT579mJb26dKCOd0Ua1anBV8MGc//eXa06arWaVcsW07RBbWpVKc+APj35+6/beoo4eefPneHLIQNoVL8WFcvq9sXkCWOpWLaE1tKrW0c9RZs227Z8R+NP6lKpfGk6tW/D+XNn9R1Sika0Kk3E9l7M7lkZABNjFVO7VuCPuS15sqErt5d3YOWgGjjnsNTablG/alxe1IZnm7pxb3Untn5Vl6Iu7/8s1gcl9UVy1qxaQZeObfGuXJ46taoxbOhA7t39W99hpZvS+0Okj0EPZtu2bculS5dYv349t27dYt++ffj4+BAaGpqq7c3MzHB2dk5xsnViYiJJSUkZGXKGmDppAn+cOsm0mbPZtnsfVb2rM6Bfb54+eaLv0FItOjqKYsWKMWb8RH2HkiYx0dF4FC3GV2O+Tnb9gjmzOHXyBFNm+LFt9wE6d+3JvNkzOH70cBZHmrIL587SrmNn1mzYwqLlq0lMTGTogL5ER0dp6mz0X8PmTesZOeZr1n23nZxOTgwZ0JfIyEg9Rq4tOjoaj2LFGJVCXwB4V69JwOFfNcs3S1ZkYYTpE/DjQfxm+dLvswFs27kXL68KDOzfj6DHj/Udmg4vN0d61y/KlXv/fuZamZlQrrAjs3ddosboH+gy7yjuee3ZPqqe1rYX/g5hwLLfqfDlXlrOOIRKBd9/3SBbZY+U1BcpOXf2Tzp27sqGzdtZvnIdiQmJDPisD9FRUR/eOJsxhP5IDVUmLkplsIPZFy9ecOLECWbPnk2dOnUoWLAglStXZuzYsTRt2lRT7/nz57Ru3RorKys8PDzYt2+fZt270wz8/f1xcHBg//79eHp6Ym5uTu/evVm/fj3ff/89KpUKlUrFsWPHsri1/4qJieHIL4f4YvhIKlSsRIECBfl84BBc8uVnx7bslQF8nxo1azP4iy+p3+ATfYeSJt41ajFg8DDq1Es+7iuXL9K0eUsqVKqMS758tG7XAY+ixbgeeDWLI03ZN0tX0qxla4q4e1C0WHEmTJlBcFAQNwIDgddZ2a3fbaB33/7UqdcAN3cPJk3zJSY6hp9+3K/n6P9VvUYtBg4eRt36Kb+HTM3McHLKpVns7R2yLsB02rh+Ha3btqVNu/YUcXNj1NjxOOd1Zns2O7+tzU1YM6QWg1ec5EVknKb8ZXQ8LaYfYvepe9wOesmZ288Yue40Xm5O5He01tRbd/gWv19/woNnEVy6G8rUrRdwdbKhYG4bfTQnWUrpi/dZumINLVu1wd3dg2LFizNlui9BQY8JDLym79DSzBD6Q6SPwQ5mbWxssLGxYe/evcTGxqZYb8qUKXTo0IHLly/TpEkTunbt+t7MbVRUFL6+vqxevZpr166xaNEiOnToQKNGjQgKCiIoKAhvb+/MaFKqJCYmkJiYiJmZuVa5ubk5Fy+c01NU4o2y5Svw67GjPH3yBLVazdkzf/Dg/j2qetfQd2gpioh4BYDd/6fbPP7nESHPn1Ol2r/vczMzM8pXrMiVixf1EWK6nTv7Jw18qtOmeSOmT5lAaEiIvkN6r/i4OK4HXqPaO++Xat7VuXTxgp6iSt78vlX56cIjjl0J+mBdOyszkpLUhEfFJbveytyE7nXcufvkFY+eZ4/sv5L6Ii3enO8fml6X3RhqfyRHpcq8RakM9tFcJiYm+Pv7069fP5YvX46Xlxe1a9emU6dOlClTRlOvV69edO7cGYCZM2fy7bff8ueff9KoUaNk9xsfH8/SpUspW7aspszS0pLY2FicnZ1TjCc2NlZnUJ2gMsPc3DyFLdLH2tqGMmXLsXrFUooUKUJORycCDh7g6pXLFChYMEOPJdJu5OhxzJgykWYNfTA2McFIpWL8pGmUK19B36ElS61W8808P8qW98LN3QOAkOfPAciZ00mrbs6cTgQHKedynnf1mtRv0BDnvC48/ucfli9dxOf9erFp6y7MzMz0HV6ywl6EkZiYiKOjo1a5o6MTz58/01NUutp5F6ZcYUdqjf1wpt7c1JipXSqw/fe/eRUdr7Wu3yfFmNatIjYWptx89IIW0w8Rn5g9pnYppS/SQq1WM8/Pl/JeFXD3KKrvcNLEEPtDpJ7BZmbh9ZzZx48fs2/fPho2bMixY8fw8vLC399fU+ftga21tTW2trY8ffo0xX2amZlpbZNavr6+2Nvbay1z/XzTvJ/UmObrh1qtpmG92lStUIatmzfSqEkzjIyMM+V4IvW2bd7E1SuXmPfNUjZs3skXI0bjN3Mqf54+qe/QkjXHdzp/3brJtFlzddbpzCVXqxX1MO9PGjWhRi0f3D2KUsunDouWrODB/fuc+PWYvkP7oHdfZ3U2eu3zOVrh16syfb79ldj4xPfWNTFW4T+sNkYqFV+uPq2zfttvf1N91D4aTvqRv4JfsuHL2pibZq/PsezcF2nlO2Mqt27dYpbffH2Hkm6G1B8peTOlMTMWpTLYzOwbFhYWNGjQgAYNGjBx4kT69u3LpEmT6NWrFwCmpqZa9VUq1Xtv6rK0tExXh48dO5bhw4drlSWoMif74+pagNX+m4iOiiIiMoJcuXIzeuSX5MuXP1OOJ1InJiaGpd8uxG/+ImrU8gHAo2gxbt28zqYN66hcVX/TU5Izd9Z0fjt+lBVrN5Anz79XHRydXmdkQ0Ke4ZQrl6Y8NCyEnDkddfajFE65cpPXJS8PHtzXdygpyuGQA2NjY57/Pzv+RmhoCI6OTilslbXKF3Eit4MlJ2Y115SZGBtRvUQe+jcqTs4uG0lSqzExVrHxSx8K5bKh6dSfdLKy8Hp+7cvoeO4Ev+LPW894tK4zLSoXYMfvd3XqZjUl9EVazJo5jeNHj7B2/SbyvOcqY3ZlaP3xPgadhUyn/9xr4unpmeF3XJuZmZGY+P4MhLm5OXZ2dlpLRk8xeJellRW5cuXmZXg4p06eoHadupl6PPF+CQkJJCTEY2SkfdoZGxmjzkZPxVCr1czxnc6xw7+wZOVaXN75EuSSLz+OTk78+dYj7uLj47hw9iyly5XL4mgzzosXYTwJDtYaoGc3pmZmlPAsyemTv2uVnz55krLlyuspKm3Hrjym8oi9eI/ap1nO/fWcbSf+xnvUPq2BrJuzHc2n/URoRMr3NbxNpVJhZpI9MrNK6IvUUKvV+M6YyuFfDrFy7Xry5XfVd0jpYij9IdLHYDOzISEhtG/fnk8//ZQyZcpga2vL2bNn8fPzo2XLlhl6rEKFCvHTTz9x8+ZNHB0dsbe318n4ZqWTv/+GWg2FChXm4YP7LJw/h0KFCtOiVRu9xZRWUZGRPHjwQPP7P48eceP6dezt7cnr4qLHyN4vKiqSR2/F/fifR9y6cR07e3uc87rgVaESixbMwdzcAmcXFy6cPcPB/d/zxYjReoxa25yZ0/jpxwPMWbgYa2trQv4/38zaxhYLCwtUKhWduvbAf81KXAsWxLVAQfxXr8TC0oKGjZvpOfp/RUVF8vDt99A/j7h54/V7yM7enpXLllC3fgOcnHLz+PE/LP12AQ4OOahTt4Eeo/6w7j17M37MKDxLlaJs2fLs2rGNoKAg2nfspO/QAIiISSDw4QutsqjYBEJfxRL48AXGRio2Da9DucKOtJv9C0ZGRuS2f/2M2bCIWOITkyiU24a23oU5fOkxz1/G4JLTii9blSY6LoFDFx7poVXJy+59kRozp0/hx4P7WbhoKdbW1pr5pTb/P9+VxBD6IzWUPB0gsxjsYNbGxoYqVaqwYMEC7ty5Q3x8PK6urvTr149x48Zl6LH69evHsWPHqFixIhERERw9ehQfH58MPUZaRLyKYPE383nyJBh7ewfq1m/AoKFf6nWAnVbXrl2lb+8emt/fzC9u0bI102bO0ldYH3T92jUG9Oup+X3hvNkANG3eiknTfJk+ex5LFy1g4rivePkyHOe8Lnw+eBht22efD9tdO7YCMKBvT63yCVNm0KxlawC69+pDbEwMfjOn8urlS0qWLsOiZauxtrbW2Z++BF67xudvtWHB3Nd90axFK8aMn8Rft29x4IfvefXqFU65nKhYqQoz/eZnqzYkp1HjJoS/CGPlsqU8e/YUd4+iLFm+EheXfPoOLVXyOVrTrFIBAE7P0U4sNJ4cwG+BwcTEJ+JdPA+DmnjiYGPG0xcx/H49mPpfH+TZyxh9hJ0spfcFoHlkY9/e3bXKp0z3paWCEiBgGP0h0kelVqvV+g7ivyoyzjBeemMj5X9LjI3PPpf5P4Ya5b+njA0k62BqYhizuJy6+Os7hI/2fHMvfYeQIQzhr7WBnN5Y6DEVuONi5j01pn257Hvl830M49NWCCGEEEL8JxnsNAMhhBBCCEMjc2Z1SWZWCCGEEEIolmRmhRBCCCEUQrKQumQwK4QQQgihEDLNQJcM8IUQQgghhGJJZlYIIYQQQiEkL6tLMrNCCCGEEEKxJDMrhBBCCKEQMmVWl2RmhRBCCCGEYklmVgghhBBCIYxk1qwOycwKIYQQQgjFksysEEIIIYRCyJxZXTKYFUIIIYRQCJVMM9Ah0wyEEEIIIYRiSWZWCCGEEEIhZJqBLsnMCiGEEEIIxZLMrB4ZG8nXq+zC3FS+14mMFRweo+8QMsTzzb30HcJHy9Fsvr5DyBAh+77UdwgZQP7ufSx5NJcu+QsuhBBCCCEUSzKzQgghhBAKIXNmdUlmVgghhBBCKJZkZoUQQgghFEIys7pkMCuEEEIIoRDyTxN0yTQDIYQQQgihWJKZFUIIIYRQCHmqpy7JzAohhBBCCMWSzKwQQgghhELInFldkpkVQgghhBCKJZlZIYQQQgiFkEdz6ZLMrBBCCCGEUCzJzAohhBBCKITMmdUlmVkhhBBCCKFYkpkVQgghhFAIec6sLhnMCiGEEEIohEwz0PWfn2Zw7949VCoVFy9e1HcoQgghhBAijfQ6mH369Cn9+/enQIECmJub4+zsTMOGDTl16pQ+wzII27Z8R+NP6lKpfGk6tW/D+XNn9R1SuhhCOwyhDSDt0IeNq5fR0Lus1tKpWV3N+uioKBbPm0nXlg1o7lOZvp1b8cPu7XqMOG2U0hcjO1YiOmA4c/r7aMpWjmhIdMBwreX4gs5a233auDQ/+bXnya5BRAcMx97aPIsjT52nT54wfsxX+NSoQrVK5ejYrhWB167qO6w0WbNqBV06tsW7cnnq1KrGsKEDuXf3b32HlSlUqsxblEqvg9m2bdty6dIl1q9fz61bt9i3bx8+Pj6EhobqM6yPFh8fr9fjB/x4EL9ZvvT7bADbdu7Fy6sCA/v3I+jxY73GlVaG0A5DaANIO/SpYGE3tvxwWLMs37hTs275N3M4e/okoybNZNWWPbTp2I2lC2Zx8tejeow4dZTSFxWK5qFP4zJc/vuZzrqfztylUOflmqXVhD1a663MTfj57D3mbPszq8JNs5fh4fTq0RkTExMWL1vFrr37GT5yNLZ2dvoOLU3Onf2Tjp27smHzdpavXEdiQiIDPutDdFSUvkMTWUBvg9kXL15w4sQJZs+eTZ06dShYsCCVK1dm7NixNG3aFACVSsXq1atp3bo1VlZWeHh4sG/fPq39BAYG0qRJE2xsbMiTJw/du3fn+fPnmvUBAQHUqFEDBwcHHB0dadasGXfu3EkxrqSkJPr160fRokW5f/8+AD/88AMVKlTAwsKCIkWKMGXKFBISEjTbqFQqli9fTsuWLbG2tmb69OkZ+VKl2cb162jdti1t2rWniJsbo8aOxzmvM9u3bdFrXGllCO0whDaAtEOfjE1MyOnopFkccuTUrLt+9RINmjSnrFclnPPmo0mrdhRxL8rtG9f0GHHqKKEvrC1MWTeqCQO/+ZkXETE66+PiE3kSFqVZwt6ps3jvBeZuP8MfN4KyKuQ0W7d2Nc7OeZky3ZdSpcvgki8/VapWw9W1gL5DS5OlK9bQslUb3N09KFa8OFOm+xIU9JjAwOx/LqSVKhMXpdLbYNbGxgYbGxv27t1LbGxsivWmTJlChw4duHz5Mk2aNKFr166azG1QUBC1a9emXLlynD17loCAAJ48eUKHDh0020dGRjJ8+HDOnDnD4cOHMTIyonXr1iQlJekcKy4ujg4dOnD27FlOnDhBwYIF+emnn+jWrRtDhw4lMDCQFStW4O/vz4wZM7S2nTRpEi1btuTKlSt8+umnGfQqpV18XBzXA69RzbuGVnk17+pcunhBT1GlnSG0wxDaANIOffvn4X06t6hPj7aNmTlhFEH/PNKsK1m2PKd/O87zZ09Qq9VcPPcn/zy8T4Uq3nqM+MOU0hcLB9Ul4M+/OXrhQbLra5bJz/2tn3N5dW+WfNGAXPaWWRzhxzt+7AienqX4avgX1K3tTaf2rdm9UzlTVVISEfEKAHt7ez1HIrKC3p5mYGJigr+/P/369WP58uV4eXlRu3ZtOnXqRJkyZTT1evXqRef/tXfncTVn/x/AX7d9VSlaSCRtslX2skZFWccyY8s2xpZthvomKWSbkjEz1qhsZQxmZGTLNkJEikqWKERU0p66n98fft1x3aKb6nM/t/dzHvcx7rmfbq/j43ZP73s+53z7YR6Sv78/tmzZgtjYWDg5OWHr1q2wtraGv7+/4Pjdu3fD0NAQqampMDU1xejRo4W+b3BwMJo3b46kpCRYWVkJ2gsKCjB06FAUFxfjwoULghfAmjVr4OHhgSlTpgAAjI2NsWrVKixduhQ+Pj6Cr//uu+9YHcRWyn2bi4qKCmhrawu1a2vr4M0b0Y/JJJU09EMa+gBQP9hk3r4DfvJeg5atjJCbk42DITuxaNZk7Nh/BE00NDFnkQeC1vliwvDBkJWVg4wMDws9fGDVyZrt6J/FhXMxpq8ZOpvows59f5WPn76RhiOXU5H+6h1a62lgxeReOLl+DHrN34+y9xUNnLb2nj/LwB+HDmLiZDdMnzkLdxMTsGHdGsgrKMB12Ai249UKwzAI2LAWXaxtYNLOlO04dU6Gy5Nb6wmrS3ONHj0aQ4cOxeXLl3H16lVERUVhw4YN2LVrF9zc3ABAaGCrqqoKdXV1ZGVlAQDi4uJw/vx5qKmpiTz3o0ePYGpqikePHsHb2xvXrl3DmzdvBBXZ9PR0ocHst99+i5YtW+LcuXNQUVERtMfFxeHGjRtCldiKigqUlJSgqKhIcKytre1n+1paWipSgWZkFaGoWD8XBPA++cfOMIxIGxdIQz+koQ8A9YMNXXv+V7ls07YdLK06wm2MC8788zdGfzsZx/44gJR7CfDdsBnN9QyQGB+HXwP80VSnGay79mAxec1I6rloqaOGjT/0g+v//kRpNQPTw5dSBX9OepqNWw9e4X7oDDh3a4O/rjxsqKhfjc9nYNm+PeYvWAwAMLewxKNHD/FHxEHODmbXrvFDamoqQsIOsB2FNBDW15lVUlLCoEGDMGjQIKxYsQIzZsyAj4+PYDArLy8vdDyPxxMMSPl8PlxdXbF+/XqR59XX1wcAuLq6wtDQEDt37oSBgQH4fD6srKxQVlYmdPyQIUOwb98+XLt2DQMG/He1MJ/Ph6+vL0aNGlVl9kqqqqqf7efatWvh6+sr1Obl7YPlK1Z+9uvEpaWpBVlZWaF5wwCQk5MNbW2dOv1e9Uka+iENfQCoH5JESVkFrdu2w/Nn6SgtLUHItl+wYu0mdO/dBwBgbGKKxw/u4/CBUIkezEr6uejSThe6WqqI+XWioE1OVgZ2Vi3xw7DO0HDdDD6fEfqalzmFSM96BxMDrYaO+1V0mjWDcVsTobY2xm1x7uxplhJ9nXX+q3DxfDR2h+6Drp4e23HqBfu/7kkeiVtn1tLSEoWFhTU61traGvfu3UPr1q1hYmIidFNVVUV2djaSk5OxfPlyDBw4EBYWFsjNza3yuWbPno1169Zh2LBhuHjxotD3uH//vsjzm5iYQEam5n99np6eyMvLE7r9tMyzxl9fU/IKCrCwbI9rMVeE2q/FxKBT5y51/v3qizT0Qxr6AFA/JElZWRkynjxGU20dlJeXo7y8XOTnkIyMDJgqrgmQJJJ+Ls7Hp8NmVii6z9kruMWlvkT4+WR0n7NXZCALAE3VldCymToycwpYSFx7nTt3wdMnaUJt6U+eQF/fgKVEtcMwDNau8cO5s6exY3coWrQ0ZDtS/aErwESwVpnNzs7GmDFjMG3aNHTs2BHq6uq4efMmNmzYgOHDh9foOebOnYudO3fi22+/xU8//QQdHR08fPgQ4eHh2LlzJ7S0tKCtrY0dO3ZAX18f6enp8PDwqPb55s+fj4qKCri4uODkyZOws7PDihUr4OLiAkNDQ4wZMwYyMjJISEhAYmKiWKsWKCqKTikoKa/m4K80acpUeHkshaWVFTp16oI//4hAZmYmxowbXz/fsJ5IQz+koQ8A9YMtO7YEoIddXzTX1cPb3BwcCNmJosJCDHIeBlVVNXTsYoudvwZCQVERunr6SLgdh7MnI/G9+49sR/8iST4XBcXvkfQ0W6itsOQ9ct6VIOlpNlSV5LF8Yk8cu/IAmTmFMNJtAj83O2TnFePvmP+mGOhqqUBXSxVtDTQBAFatdZBfXIaMrHyRlQ/YMnGyG9wmfYvgndswyNEZ9xIT8Oefh+C9wo/taGLxX+2Lk/9EIuiX36GqqiqYe62mpi70KSqRTqwNZtXU1NC9e3ds2rQJjx49wvv372FoaIiZM2fif//7X42ew8DAAFeuXMGyZcvg6OiI0tJSGBkZwcnJCTIyMuDxeAgPD4e7uzusrKxgZmaGX375Bf369av2ORcuXAg+n48hQ4YgKioKjo6OiIyMhJ+fHzZs2AB5eXmYm5tjxowZdfQ3UfecnIcg720udmz9Ha9fZ8GknSl+27YDBgYt2I4mFmnohzT0AaB+sOVN1ius9fHAu7e50NDUgrlVRwTt3Avd/6+aefqtx+6tm7F+pSfy371Dcz19uM2aB5eRY1hO/mVcOxcfq+AzaN9GB985WEJTVREvcwpxMSEDk/wjUVD83zrjM4Z2wvKJPQX3zwaMAwDMDIjCvjNJDZ67Ku2tOiAgaAu2BAVix7bf0aJFS/y01BNDXFzZjiaWP/5/SbcZUycJtfuuXovhI0SnCXIZbWcriscwjOjnJaRB1FdllhDCvpd5klF5+1p6Gtyvamm5BLIdoU5k/72I7QhfTRIu8KsLyvJfPqa+XH+UV2/P3b0tN5cyY/0CMEIIIYQQUjNS8vtAnZK4C8AIIYQQQgipKarMEkIIIYRwBBVmRVFllhBCCCGEcBZVZgkhhBBCuIJKsyJoMEsIIYQQwhG0NJcommZACCGEEEI4iyqzhBBCCCEcQUtziaLKLCGEEEII4SyqzBJCCCGEcAQVZkVRZZYQQgghhHAWVWYJIYQQQriCSrMiqDJLCCGEEELEcunSJbi6usLAwAA8Hg/Hjh0TepxhGKxcuRIGBgZQVlZGv379cO/ePaFjSktLMX/+fOjo6EBVVRXDhg3Ds2fPxM5Cg1lCCCGEEI7g1eN/4igsLESnTp3w66+/Vvn4hg0bEBgYiF9//RU3btyAnp4eBg0ahPz8fMExCxcuxNGjRxEeHo5///0XBQUFcHFxQUVFhVhZaJoBIYQQQghHSMrSXM7OznB2dq7yMYZhEBQUBC8vL4waNQoAEBoaCl1dXRw4cACzZs1CXl4egoODsXfvXjg4OAAA9u3bB0NDQ5w9exaOjo41zkKVWUIIIYQQUmfS0tLw8uVLDB48WNCmqKiIvn37IiYmBgAQFxeH9+/fCx1jYGAAKysrwTE1RZVZQgghhBCOqM/CbGlpKUpLS4XaFBUVoaioKNbzvHz5EgCgq6sr1K6rq4unT58KjlFQUICWlpbIMZVfX1NUmSWEEEIIIVi7di00NDSEbmvXrq318/E+mRPBMIxI26dqcsynqDJLiBTh8xm2I3w1GRkJmRD2lfQ0lNiOUCf4DPf/TeVGLmY7Qp3QsvdgO8JXy728ju0I3FePPyI9PT2xeLHw60XcqiwA6OnpAfhQfdXX1xe0Z2VlCaq1enp6KCsrQ25urlB1NisrC7169RLr+1FllhBCCCGEQFFREU2aNBG61WYw26ZNG+jp6eHMmTOCtrKyMly8eFEwULWxsYG8vLzQMZmZmbh7967Yg1mqzBJCCCGEcIS4S2jVl4KCAjx8+FBwPy0tDfHx8WjatClatWqFhQsXwt/fH+3atUO7du3g7+8PFRUVfPfddwAADQ0NTJ8+HUuWLIG2tjaaNm2KH3/8ER06dBCsblBTNJglhBBCCCFiuXnzJvr37y+4Xzk9YcqUKQgJCcHSpUtRXFyMOXPmIDc3F927d8fp06ehrq4u+JpNmzZBTk4OY8eORXFxMQYOHIiQkBDIysqKlYXHMFIwIYqjSsrZTkCkDc2ZJXVNGubMykjKwpxfiebMSg4lFkuBic8K6u25O7RUq7fnrk9UmSWEEEII4Qjp+NWsbtEFYIQQQgghhLOoMksIIYQQwhVUmhVBlVlCCCGEEMJZVJklhBBCCOEISVmaS5JQZZYQQgghhHAWVWYJIYQQQjhCSlaaq1NUmSWEEEIIIZxFlVlCCCGEEI6gwqwoGswSQgghhHAFjWZF0DQDQgghhBDCWVSZJYQQQgjhCFqaS1Sjq8y6ubmBx+MJbtra2nByckJCQgLb0epUxMH9cB48AF27dMD4MaNwK+4m25Fqhev9iLt5A/Pn/ACHfnbo1N4M0efOsh1JbEMcB6BLB3OR29rVfmxHE4s0nItKXH9dfCp453Z0sTLHxnX+bEcRG5fOxY+T+6H46jpsXOgiaCu+uq7K26IJfQAArfS0qj1m1IAObHWlWlw6H6TuNLrBLAA4OTkhMzMTmZmZOHfuHOTk5ODi4vLlL+SIqJP/YMO6tZj5/WxEHD4Ga2sbzJk1E5kvXrAdTSzS0I/i4iKYmZnBw2sF21Fqbd/Bwzhz/rLgtnXHbgDAIEdHlpOJRxrOBSAdr4uP3UtMxJHDh9DO1IztKGLj0rmwsWiJ6cO7IeFBplB766GrhW7fr/4DfD4fR8/fBQA8y3orcozfzjMoKCrFqav32ehKtbh0Pr4Gj1d/N65qlINZRUVF6OnpQU9PD507d8ayZcuQkZGB169fAwCWLVsGU1NTqKiowNjYGN7e3nj//r3Qc6xevRrNmzeHuro6ZsyYAQ8PD3Tu3JmF3ojaG7oHI0ePxqhvxsC4bVss9fSCnr4eDkUcZDuaWKShH3b2fTFvwSI4DBrMdpRaa9q0KXR0mgluly9dgKFhK9jYdmM7mlik4VwA0vG6qFRUVIj/efwI75Wr0KRJE7bjiI0r50JVWQF7Vo7DnHVH8Da/WOixVzkFQjdXe0tcvPUYT17kAAD4fEbkmGF92+PwuQQUFpex0Z1qceV8kLrXKAezHysoKMD+/fthYmICbW1tAIC6ujpCQkKQlJSEzZs3Y+fOndi0aZPga/bv3481a9Zg/fr1iIuLQ6tWrbB161a2uiDkfVkZkpPuoWcvO6H2nr164078bZZSiU9a+iFt3r8vwz+Rf2P4yFHgcfnXeI6SttfF2tV+sO/TDz169mI7iti4dC6CfhyOqJj7OH/j4WePa66lBqfe5gg9fqPaY7qYtUBnU4PPHsMGLp2Pr8WrxxtXNcoLwCIjI6GmpgYAKCwshL6+PiIjIyEj82Fsv3z5csGxrVu3xpIlSxAREYGlS5cCALZs2YLp06dj6tSpAIAVK1bg9OnTKCgoaOCeiMp9m4uKigrBwLyStrYO3rx5zVIq8UlLP6TN+XPnkJ+fD9fhI9mO0ihJ0+si6p8TSElOwr7ww2xHqRWunIsxDh3R2awF7Kb9+sVjJw6xRn5RKY5duFftMVNcbZGc9grXEtPrMuZX48r5IPWjUVZm+/fvj/j4eMTHx+P69esYPHgwnJ2d8fTpUwDA4cOHYWdnBz09PaipqcHb2xvp6f+9cO/fv49u3YQ/Yv30/qdKS0vx7t07oVtpaWndd+7/fVo1YxiGk5U0aemHtDh29DB629mjeXNdtqM0alx/XbzMzMTGdf5YvXYjFBUV2Y7zVST5XLRsroGNi1wxbWUESsvKv3j8ZFdbRJyKr/ZYJUU5jBvcGaHHJfeiKkk+H3WGSrMiGmVlVlVVFSYmJoL7NjY20NDQwM6dO+Hi4oLx48fD19cXjo6O0NDQQHh4OAICAoSeo6oXzOesXbsWvr6+Qm1e3j5YvmLl13XmE1qaWpCVlcWbN2+E2nNysqGtrVOn36s+SUs/pMmLF89x/dpV/LxpC9tRGi1peV0kJ91DTk42JowbLWirqKjArbibiDi4H9dvJUBWVpbFhF/GhXPRxbwFdJuqI2bPPEGbnJws7Dq3xg+je0Kj73Lw+R/eu3p3ag0zo+aYtLz6+aUj+3eAipI89p+8Ve/ZxcWF81FXaGkuUY1yMPspHo8HGRkZFBcX48qVKzAyMoKXl5fg8cqKbSUzMzPExsZi0qRJgrabNz//m6qnpycWL14s1MbI1n1FQl5BARaW7XEt5goGOgwStF+LiUG/AQPr/PvVF2nphzT5+9gRNG2qDfs+fdmO0mhJy+uiW48e+OPo30JtPsv/hzZtjOE2fYbED2QBbpyL8zcfwmbCJqG2HV7f4P7T1wjYd1EwkAWAKa5dEZf8DIkPMz99GgE31644cTkZb94W1lvm2uLC+SD1p1EOZktLS/Hy5UsAQG5uLn799VcUFBTA1dUVeXl5SE9PR3h4OLp27YoTJ07g6NGjQl8/f/58zJw5E7a2tujVqxciIiKQkJAAY2Pjar+noqKiyMdpJV/+1KdWJk2ZCi+PpbC0skKnTl3w5x8RyMzMxJhx4+vnG9YTaehHUWGh0BSV58+eISU5GRoaGtA3MGAxmXj4fD7+OnYULsNGQE6Omz82pOVcSMPrQlVVDSbtTIXalJWVoaGpKdIuyST9XBQUlSHp8SuhtsKS98h5VyTUrq6iiFEDOsBjy4lqn8u4pTbsOrfGiCUh9RX3q0n6+agr0jZroi5w813pK0VFRUFfXx/Ah5ULzM3N8ccff6Bfv34AgEWLFmHevHkoLS3F0KFD4e3tjZUrVwq+fsKECXj8+DF+/PFHlJSUYOzYsXBzc0NsbCwLvRHl5DwEeW9zsWPr73j9Ogsm7Uzx27YdMDBowXY0sUhDP+7du4sZUycL7v+8YS0AYNjwkVjlv46tWGK7fi0GLzNfYMTIUWxHqTVpORfS8LqQFtJyLsYM6gQeDzh0Or7aY6a42OLF63c4e/1BwwUTk7ScDyI+HvOlyZ6kRgYNGgQ9PT3s3bu3xl9TX5VZ0nh9/LEhV8nIUNlBkvCl4C1CRkpKWVr2HmxH+Gq5l7nzi+PnKLFYCnyUVfzlg2qpbXPlenvu+tQoK7Nfq6ioCNu2bYOjoyNkZWVx8OBBnD17FmfOnGE7GiGEEEJIo0KD2Vrg8Xj4559/sHr1apSWlsLMzAx//vknHBwc2I5GCCGEEGkmHR801CkazNaCsrIyzp49y3YMQgghhJBGjwazhBBCCCEcQevMiqLBLCGEEEIIR0jJ9Yx1qlFuZ0sIIYQQQqQDVWYJIYQQQjiCCrOiqDJLCCGEEEI4iyqzhBBCCCFcQaVZEVSZJYQQQgghnEWVWUIIIYQQjqCluURRZZYQQgghhHAWVWYJIYQQQjiC1pkVRYNZQgghhBCOoLGsKJpmQAghhBBCOIsqs4QQQgghHEHTDERRZZYQQgghhHAWVWYJIYQQQjiDSrOf4jEMw7AdorEqKWc7ASGkvpRXSMePVjlZ7r9xFkjJD1s1Je7Xn7RcAtmOUCeKoxaz9r2f5ZbV23O31FKot+euT9x/ZRBCCCGENBI0Z1YUzZklhBBCCCGcRZVZQgghhBCOoMKsKKrMEkIIIYQQzqLKLCGEEEIIR9CcWVE0mCWEEEII4QgeTTQQQdMMCCGEEEIIZ1FllhBCCCGEK6gwK4Iqs4QQQgghhLOoMksIIYQQwhFUmBVFlVlCCCGEEMJZVJklhBBCCOEIWppLFFVmCSGEEEIIZ1FllhBCCCGEI2idWVE0mCWEEEII4Qoay4pgZZrBypUr0blz52ofDwkJgaamZoPlIYQQQggh3FSrwWxMTAxkZWXh5ORU13nqzMqVK8Hj8QQ3DQ0N2Nvb4+LFi3X+vXg8Ho4dO1bnz/s1Ig7uh/PgAejapQPGjxmFW3E32Y5UK9LQD2noA0D9YMOtmzewcN4PcBxoD5uO5jgffVbocYZhsP33LXAcaI9eXTvh+2mT8OjhA5bSio9L5+LoH+GYPG4kBvXphkF9uuF7t+9w9cplweMMwyB4+28Y5tgP/XtZY973bnj86CGLiWsu7uYNzJ/zAxz62aFTezNEnzv75S9i0Y/juqI4ajE2zuonaNuxxBHFUYuFbhc3fSv0ddOcO+DUhjF49edcFEcthoaqYgMnrxu8erxxVa0Gs7t378b8+fPx77//Ij09va4z1Zn27dsjMzMTmZmZuHr1Ktq1awcXFxfk5eWxHa1eRZ38BxvWrcXM72cj4vAxWFvbYM6smch88YLtaGKRhn5IQx8A6gdbiouLYWpmjmWe3lU+HrpnF/bvDcEyT2+EHfgD2jrNMGfWNBQWFjRwUvFx7Vw009XFD/MXIXjvIQTvPQSbrt3hsXieYMC6PzQY4ftDsXiZF4LDItBUWwcL58xAYWEhy8m/rLi4CGZmZvDwWsF2lC+yMdXFdOeOSHj8WuSxUzfS0PrbbYLbCO+jQo+rKMrhzM0n2BgR21BxSQMRezBbWFiIQ4cOYfbs2XBxcUFISIjQ4xcuXACPx8O5c+dga2sLFRUV9OrVC/fv36/2OdPS0mBiYoLZs2eDz+dXeczx48dhY2MDJSUlGBsbw9fXF+Xl5Z/NKicnBz09Pejp6cHS0hK+vr4oKChAamqq4Jj09HQMHz4campqaNKkCcaOHYtXr14JPc/WrVvRtm1bKCgowMzMDHv37hU81rp1awDAyJEjwePxBPfZtDd0D0aOHo1R34yBcdu2WOrpBT19PRyKOMh2NLFIQz+koQ8A9YMtve37YM78hRjgMFjkMYZhcGBfGKbN/AEDHAbDpJ0pfFevQ0lJCaL+iWQhrXi4di7s+vRHL7s+aGXUGq2MWmPW3AVQVlHBvcQ7YBgGhw7sxZRp36PfgEEwNmmH5b7+KC0pwZmoE2xH/yI7+76Yt2ARHAaJ/juTJKpK8tizdAjmbD6DtwUlIo+Xva/Aq9wiwS33k2N+PXYbPx+6gespmQ0VuV7wePV34yqxB7MREREwMzODmZkZJk6ciD179oBhGJHjvLy8EBAQgJs3b0JOTg7Tpk2r8vnu3r2L3r17Y8yYMdi6dStkZEQjnTp1ChMnToS7uzuSkpKwfft2hISEYM2aNTXOXVpaKpiLa2ZmBuDDm8GIESOQk5ODixcv4syZM3j06BHGjRsn+LqjR49iwYIFWLJkCe7evYtZs2Zh6tSpOH/+PADgxo0bAIA9e/YgMzNTcJ8t78vKkJx0Dz172Qm19+zVG3fib7OUSnzS0A9p6ANA/ZBUz58/Q/ab1+jRs7egTUFBATY2XSW+P1w/FxUVFTh76h+UFBfDqmMnvHj+DNnZb9Cth/C56Gxji8Q7kt8frgiaOwBRsY9x/nbVnwjbd2yJp+E/IGHXVPy2YBCaaSg3cELCFrFXMwgODsbEiRMBAE5OTigoKMC5c+fg4OAgdNyaNWvQt29fAICHhweGDh2KkpISKCkpCY65evUqXFxc4OnpiR9//LHa77lmzRp4eHhgypQpAABjY2OsWrUKS5cuhY+PT7Vfl5iYCDU1NQBAUVER1NXVERERgSZNmgAAzp49i4SEBKSlpcHQ0BAAsHfvXrRv3x43btxA165d8fPPP8PNzQ1z5swBACxevBjXrl3Dzz//jP79+6NZs2YAAE1NTejp6dX8L7Ke5L7NRUVFBbS1tYXatbV18OaN6Mcykkoa+iENfQCoH5Iq+/8zf9qfptrayMyUzI/qK3H1XDx6kIpZU79DWVkZlJVV4P/zL2hjbCIYsGp9ei6aauOlhJ8LrhjT1wydTXRh576/ysdP30jDkcupSH/1Dq31NLBici+cXD8GvebvR9n7igZOW79oaS5RYlVm79+/j9jYWIwfPx7Ah4/xx40bh927d4sc27FjR8Gf9fX1AQBZWVmCtvT0dDg4OGD58uWfHcgCQFxcHPz8/KCmpia4zZw5E5mZmSgqKqr268zMzBAfH4/4+HjExcVh9uzZGDNmDG7e/HCRQXJyMgwNDQUDWQCwtLSEpqYmkpOTBcf07t1b6Hl79+4teLymSktL8e7dO6FbaWmpWM8hDt4nnxcwDCPSxgXS0A9p6ANA/ZBYIv3hzpsd185Fq9atEXLwT2wPOYAR34zDGp//Ie3xfxd5ffr3Lun94YqWOmrY+EM/TNvwD0qrGZgevpSKqNg0JD3Nxj/XH2OE91G0a6EF525tGjgtYYNYldng4GCUl5ejRYsWgjaGYSAvL4/c3FxoaWkJ2uXl5QV/rnwxfzwftlmzZjAwMEB4eDimT58uqJZWhc/nw9fXF6NGjRJ57ONK76cUFBRgYmIiuN+lSxccO3YMQUFB2LdvX7U/aD5tr4sfuGvXroWvr69Qm5e3D5avWCnW83yJlqYWZGVl8ebNG6H2nJxsaGvr1On3qk/S0A9p6ANA/ZBU2jofPhXKfvMGzZo1F7Tn5mSj6ScVQknD1XMhL6+AloZGAAALSyukJN3FHwf3YcKU6QCAnOw30Pn/T+sAIDc3B1pNJftccEGXdrrQ1VJFzK8TBW1ysjKws2qJH4Z1hobrZvD5wtMdX+YUIj3rHUwMtD59Os6j349E1bgyW15ejrCwMAQEBAiqnfHx8bhz5w6MjIywf3/Vpf/qKCsrIzIyEkpKSnB0dER+fn61x1pbW+P+/fswMTERuVU1x/ZzZGVlUVxcDOBDFTY9PR0ZGRmCx5OSkpCXlwcLCwsAgIWFBf7991+h54iJiRE8DnwYuFdUfP5jDE9PT+Tl5QndflrmKVb2mpBXUICFZXtci7ki1H4tJgadOnep8+9XX6ShH9LQB4D6IalatGgJbZ1muH41RtD2/n0Z4uJuSHx/pOVcMAyDsrIyGLRoCW1tHdy4Lnwu4uNuokMn7vRHUp2PT4fNrFB0n7NXcItLfYnw88noPmevyEAWAJqqK6FlM3Vk5kj+yh7k69W4MhsZGYnc3FxMnz4dGhoaQo998803CA4Oxrx588T65qqqqjhx4gScnZ3h7OyMqKgowRzXj61YsQIuLi4wNDTEmDFjICMjg4SEBCQmJmL16tXVPn95eTlevnwJAMjPz0dERASSkpKwbNkyAICDgwM6duyICRMmICgoCOXl5ZgzZw769u0LW1tbAMBPP/2EsWPHwtraGgMHDsTx48dx5MgRnD373zp8rVu3xrlz59C7d28oKioKVagrKSoqQlFReE27ks8vxlBrk6ZMhZfHUlhaWaFTpy74848IZGZmYsy48fXzDeuJNPRDGvoAUD/YUlRUiIyPlj988fwZ7qcko4mGBvT1DfDdxMnYHbwdhkZGaNXKCLt3bYeSkhKchriwmLpmuHYutv0ahB697aGrq4eiwkKcPX0St+NuIGDLdvB4PIz9bhLCdu9ES0MjGLYyQtjuHVBUUsIgp6FsR/+iosJCoWU2nz97hpTkZGhoaEDfwIDFZB8UFL9H0tNsobbCkvfIeVeCpKfZUFWSx/KJPXHsygNk5hTCSLcJ/NzskJ1XjL9j/psGoqulAl0tVbQ10AQAWLXWQX5xGTKy8kVWPiDcUuPBbHBwMBwcHEQGsgAwevRo+Pv749atW2IHUFNTw8mTJ+Ho6IghQ4bg5MmTIsc4OjoiMjISfn5+2LBhA+Tl5WFubo4ZM2Z89rnv3bsnmK+roqKCtm3bYuvWrZg8eTKA/zY7mD9/Pvr06QMZGRk4OTlhy5YtgucYMWIENm/ejI0bN8Ld3R1t2rTBnj170K9fP8ExAQEBWLx4MXbu3IkWLVrgyZMnYv891CUn5yHIe5uLHVt/x+vXWTBpZ4rftu2AgUGLL3+xBJGGfkhDHwDqB1uS7t3FrOlTBPcDN64DALgMGwHf1eswZeoMlJaUYN0aP+S/y4NVh474bVswVFVFiwKShmvnIjcnG6u8PZD95jVU1dRh0s4UAVu2o1uPXgCACVOmo7S0FAHrViE//x0srToi6LedUFVVZTn5l927dxczpk4W3P95w1oAwLDhI7HKfx1bsWqsgs+gfRsdfOdgCU1VRbzMKcTFhAxM8o9EQfF7wXEzhnbC8ok9BffPBnxYuWhmQBT2nUlq8Ny1RdMMRPGYqtbVIg2iviqzhBD2lVdIx49WOVnuv3MWSMkPWzUlsRcgkjhaLoFsR6gTxVGLWfveb4vrb3UGTWXZenvu+sT9VwYhhBBCSCPBldVKGlKttrMlhBBCCCFEElBllhBCCCGEI2jOrCiqzBJCCCGEEM6iyiwhhBBCCEdQYVYUDWYJIYQQQriCRrMiaJoBIYQQQgjhLKrMEkIIIYRwBC3NJYoqs4QQQgghhLOoMksIIYQQwhG0NJcoqswSQgghhBDOososIYQQQghHUGFWFFVmCSGEEEIIZ1FllhBCCCGEK6g0K4Iqs4QQQgghHMGrx/9q4/fff0ebNm2gpKQEGxsbXL58uY57/GU0mCWEEEIIIWKLiIjAwoUL4eXlhdu3b8Pe3h7Ozs5IT09v0Bw0mCWEEEII4Qger/5u4goMDMT06dMxY8YMWFhYICgoCIaGhti6dWvdd/wzaDBLCCGEEEJQWlqKd+/eCd1KS0urPLasrAxxcXEYPHiwUPvgwYMRExPTEHH/wxCpVVJSwvj4+DAlJSVsR6k1aegDw1A/JIk09IFhpKMf0tAHhqF+SBJp6AObfHx8GABCNx8fnyqPff78OQOAuXLlilD7mjVrGFNT0wZI+x8ewzBMww6fSUN59+4dNDQ0kJeXhyZNmrAdp1akoQ8A9UOSSEMfAOnohzT0AaB+SBJp6AObSktLRSqxioqKUFRUFDn2xYsXaNGiBWJiYtCzZ09B+5o1a7B3716kpKTUe95KtDQXIYQQQgipduBaFR0dHcjKyuLly5dC7VlZWdDV1a2PeNWiObOEEEIIIUQsCgoKsLGxwZkzZ4Taz5w5g169ejVoFqrMEkIIIYQQsS1evBiTJk2Cra0tevbsiR07diA9PR0//PBDg+agwawUU1RUhI+PT40/MpBE0tAHgPohSaShD4B09EMa+gBQPySJNPSBS8aNG4fs7Gz4+fkhMzMTVlZW+Oeff2BkZNSgOegCMEIIIYQQwlk0Z5YQQgghhHAWDWYJIYQQQghn0WCWEEIIIYRwFg1mCSGEEEIIZ9FgVor4+fmhqKhIpL24uBh+fn4sJBLP+/fv0b9/f6SmprIdhRBCCAcYGxsjOztbpP3t27cwNjZmIRFhAw1mpYivry8KCgpE2ouKiuDr68tCIvHIy8vj7t274PF4bEchUkZWVhZZWVki7dnZ2ZCVlWUhUeN16dIllJeXi7SXl5fj0qVLLCQSX3l5OXx9fZGRkcF2lEbvyZMnqKioEGkvLS3F8+fPWUhE2EDrzEoRhmGqHAjeuXMHTZs2ZSGR+CZPnozg4GCsW7eO7Shf5f379xg8eDC2b98OU1NTtuPUqXfv3iE6OhpmZmawsLBgO06NVLcCYWlpKRQUFBo4jXgSEhJqfGzHjh3rMUnd6N+/PzIzM9G8eXOh9ry8PPTv37/KgYmkkZOTw8aNGzFlyhS2ozRaf//9t+DPp06dgoaGhuB+RUUFzp07h9atW7OQjLCBBrNSQEtLCzweDzweD6ampkID2oqKChQUFDT4bhy1VVZWhl27duHMmTOwtbWFqqqq0OOBgYEsJROPNFWZx44diz59+mDevHkoLi6Gra0tnjx5AoZhEB4ejtGjR7MdsVq//PILAIDH42HXrl1QU1MTPFZRUYFLly7B3NycrXg10rlzZ/B4vGoH5JWP8Xg8TgwEq/ulOzs7W+T1LskcHBxw4cIFuLm5sR3lq3Tp0qXK88Hj8aCkpAQTExO4ubmhf//+LKSr3ogRIwB8yPnpLxXy8vJo3bo1AgICWEhG2ECDWSkQFBQEhmEwbdo0+Pr6Cv2GqqCggNatW6Nnz54sJqy5u3fvwtraGgBE5s5ybWAoLVXmS5cuwcvLCwBw9OhRMAyDt2/fIjQ0FKtXr5boweymTZsAfBhAbdu2TWhKQeVrY9u2bWzFq5G0tDS2I9SJUaNGAfjwOnZzcxPaoamiogIJCQkNvp/713B2doanpyfu3r0LGxsbkYH4sGHDWEomHicnJ2zduhUdOnRAt27dwDAMbt68iYSEBLi5uSEpKQkODg44cuQIhg8fznZcAT6fDwBo06YNbty4AR0dHZYTETbRDmBSory8HPv27YODgwNatmzJdhwCYP78+QgLC4OJiQmnq8zKyspITU2FoaEhJk+eDAMDA6xbtw7p6emwtLSscp62pOnfvz+OHDkCLS0ttqM0WlOnTgUAhIaGYuzYsVBWVhY8VvmLxcyZMzkzKJGRqf6SE65UyQFg5syZaNWqFby9vYXaV69ejadPn2Lnzp3w8fHBiRMncPPmTZZSEvJ5NJiVIioqKkhOTm7wPZHrw8OHD/Ho0SP06dMHysrK1X40Kck+97Ecj8dDdHR0A6apPVNTU6xevRpDhw5FmzZtEB4ejgEDBuDOnTsYOHAg3rx5w3bERicpKQnp6ekoKysTapf0aiDDMJg6dSq2bNkCdXV1tuMQABoaGoiLi4OJiYlQ+8OHD2FjY4O8vDykpKSga9euyM/PZynl5128eBE///wzkpOTwePxYGFhgZ9++gn29vZsRyMNhKYZSJHu3bvj9u3bnB7MZmdnY+zYsTh//jx4PB4ePHgAY2NjzJgxA5qampyaA3X+/Hm2I9SJhQsXYsKECVBTU4ORkRH69esH4MP0gw4dOrAbTgzPnj3D33//XeUgkCtV8sePH2PkyJFITEwUmkdb+YuepFcDGYbBgQMH4OXlJVWD2ZKSEigpKbEdo1aUlJQQExMjMpiNiYkR9InP5wtNC5Ek+/btw9SpUzFq1Ci4u7uDYRjExMRg4MCBCAkJwXfffcd2RNIQGCI1Dh06xBgbGzNbtmxhYmJimDt37gjduGDSpEmMo6Mjk5GRwaipqTGPHj1iGIZhTp06xVhaWrKcrnYePHjAREVFMUVFRQzDMAyfz2c5kfhu3LjBHDlyhMnPzxe0RUZGMv/++y+LqWru7NmzjIqKCtO+fXtGTk6O6dy5M6OpqcloaGgw/fv3Zztejbm4uDDDhw9nsrKyGDU1NSYpKYm5fPky061bN+bSpUtsx6sRS0tL5urVq2zH+Grl5eWMn58fY2BgwMjKygp+Vi1fvpzZtWsXy+lqbtWqVYyysjLj7u7O7N27l9m3bx/j7u7OqKioMKtXr2YYhmECAwMZBwcHlpNWzdzcnAkMDBRpDwgIYMzNzVlIRNhAg1kpwuPxRG4yMjKC/3OBrq4uEx8fzzAMIzSYffz4MaOqqspmNLG9efOGGTBggODvv7Iv06ZNYxYvXsxyutorLy9nbt++zeTk5LAdpca6du3KeHt7Mwzz37+r/Px8ZtiwYczvv//Ocrqa09bWFvxi2qRJEyYlJYVhGIY5d+4c07lzZzaj1VhkZCRjZ2fHJCYmsh3lq/j6+jLGxsbMvn37GGVlZcHrOyIigunRowfL6cSzb98+pkePHoyWlhajpaXF9OjRg9m/f7/g8aKiIqa4uJjFhNVTUFBgHjx4INL+4MEDRlFRkYVEhA20aYIUSUtLE7k9fvxY8H8uKCwshIqKikj7mzdvJPZjruosWrQI8vLySE9PF+rTuHHjEBUVxWIy8SxcuBDBwcEAPnyM3bdvX1hbW8PQ0BAXLlxgN1wNJScnC5bvkZOTQ3FxMdTU1ODn54f169eznK7mKioqBMuL6ejo4MWLFwAAIyMj3L9/n81oNTZx4kTExsaiU6dOUFZWRtOmTYVuXBEWFoYdO3ZgwoQJQqtkdOzYESkpKSwmE9+ECRNw9epV5OTkICcnB1evXhX6eF5ZWVlip1EYGhri3LlzIu3nzp2DoaEhC4kIG2jOrBTh8lzZSn369EFYWBhWrVoF4MNcQD6fj40bN0rcOodfcvr0aZw6dUpkdYl27drh6dOnLKUS3+HDhzFx4kQAwPHjx5GWloaUlBSEhYXBy8sLV65cYTnhl6mqqqK0tBQAYGBggEePHqF9+/YAwKkL2KysrJCQkABjY2N0794dGzZsgIKCAnbs2MGZrTuDgoLYjlAnnj9/LjLPFPgwv/T9+/csJGqclixZAnd3d8THx6NXr17g8Xj4999/ERISgs2bN7MdjzQQGsxy3N9//w1nZ2fIy8sL7YhSFUm/0hkANm7ciH79+uHmzZsoKyvD0qVLce/ePeTk5HBi0PQxaakyv3nzBnp6egCAf/75B2PGjIGpqSmmT58u2JRA0vXo0QNXrlyBpaUlhg4diiVLliAxMRFHjhxBjx492I5XY8uXL0dhYSGAD0snubi4wN7eHtra2oiIiGA5Xc1Iy65Z7du3x+XLl0WKCH/88Qe6dOnCUirxVVRUYNOmTTh06FCVF0fm5OSwlKxmZs+eDT09PQQEBODQoUMAAAsLC0REREjUurikftFgluNGjBiBly9fonnz5oIdUarClXUPLS0tkZCQgK1bt0JWVhaFhYUYNWoU5s6dC319fbbjiUVaqsy6urpISkqCvr4+oqKi8PvvvwMAioqKhD5elWSBgYGC9XBXrlyJgoICREREwMTERLCxAhc4OjoK/mxsbIykpCTk5OQIdgGUVO/evUOTJk0Ef/6cyuMknY+PDyZNmoTnz5+Dz+fjyJEjuH//PsLCwhAZGcl2vBrz9fXFrl27sHjxYnh7e8PLywtPnjzBsWPHsGLFCrbj1cjIkSMxcuRItmMQFtE6s4TUk6SkJPTr1w82NjaIjo7GsGHDhKrMbdu2ZTtijaxcuRJBQUHQ19dHUVERUlNToaioiN27d2Pnzp24evUq2xGJhJOVlUVmZiaaN28OGRmZKgfeDIe25K106tQp+Pv7Iy4uDnw+H9bW1lixYgUGDx7MdrQaa9u2LX755RcMHToU6urqiI+PF7Rdu3YNBw4cYDtijcTFxQnWmbW0tORUdZx8PRrMEokSFRUFNTU12NnZAQB+++037Ny5E5aWlvjtt984t4PTy5cvsXXrVqE3Oy5WmQ8fPoyMjAyMGTNGMAc4NDQUmpqanPooj4tveJXbwNbEkSNH6jFJ7V28eBG9e/eGnJwcLl68+Nlj+/bt20Cpvk5GRka1Fxhdu3aNM9NXVFVVkZycjFatWkFfXx8nTpyAtbU1Hj9+jC5duiAvL4/tiJ+VlZWF8ePH48KFC9DU1ATDMMjLy0P//v0RHh6OZs2asR2RNAAazEqZc+fO4dy5c8jKyhLsXV1p9+7dLKWquQ4dOmD9+vUYMmQIEhMTYWtriyVLliA6OhoWFhbYs2cP2xEbNa4uDs/lN7zKbWCBD9XLo0ePQkNDA7a2tgA+DNDfvn2LUaNG0eujAZmbm+PKlSvQ1tYWar9y5QqGDh2Kt2/fshNMTGZmZggLC0P37t1hb2+PoUOHwsPDAxEREZg/fz6ysrLYjvhZ48aNw6NHj7B3715YWFgA+PCp2JQpU2BiYoKDBw+ynJA0BJozK0V8fX3h5+cHW1tb6OvrS/QcuuqkpaXB0tISAPDnn3/C1dUV/v7+uHXrFoYMGcJyOvG0adMGEydOxMSJE2FmZsZ2nFqrqKiAv78/tm3bhlevXiE1NRXGxsbw9vZG69atMX36dLYjftH8+fPx7t073Lt3T+QNz93dXaLf8D4eoC5btgxjx47Ftm3bBPOVKyoqMGfOHM7MNQWAt2/fIjY2tspfuidPnsxSKvHY29tj8ODBuHDhgmA3s0uXLsHV1RUrV65kN5wYRo4ciXPnzqF79+5YsGABvv32WwQHByM9PR2LFi1iO94XRUVF4ezZs4LXNQDBJ3lcmu5BvhJbC9ySuqenp8eEhYWxHeOraGlpMffu3WMYhmF69+7NbN++nWEYhklLS2OUlZXZjCa2gIAAxtbWluHxeIy1tTWzadMm5sWLF2zHEps0LA7fpEkTJjY2VqT9+vXrjIaGRsMHqiUdHR3BRgkfS0lJYZo2bcpCIvH9/fffjLq6OiMjI8NoaGgwmpqagpuWlhbb8WqMz+czo0ePZuzt7Zni4mImOjqaUVNTY4KCgtiO9lWuXbvGBAQEMH/99RfbUWpETU2NuX37tkj7rVu3GHV19YYPRFhBmyZIkbKyMvTq1YvtGF/Fzs4OixcvxqpVqxAbG4uhQ4cCAFJTU0XWa5V0ixcvxo0bN5CSkgIXFxds3boVrVq1wuDBgxEWFsZ2vBqThsXh+Xw+5OXlRdrl5eVFKoOSrLy8HMnJySLtycnJnOnHkiVLMG3aNOTn5+Pt27fIzc0V3CR9GaiP8Xg8HDx4EEpKShg4cCCGDRuGtWvXYsGCBWxHE0t2drbgzxkZGThx4gQyMzOhqanJXigxDBgwAAsWLBBsIAJ8WAN40aJFGDhwIIvJSEOiObNSZNmyZVBTU4O3tzfbUWotPT0dc+bMQUZGBtzd3QUfYS9atAgVFRWcWde0OteuXcPs2bORkJDAmau2lZWVkZKSAiMjI6irq+POnTuCZaG6desmWPJKkg0fPhxv377FwYMHYWBgAODDG96ECROgpaWFo0ePspywZhYvXoyQkBD873//E1xgdO3aNaxbtw6TJ09GYGAgywm/TFVVFYmJiZzZ5OFjCQkJIm35+fn49ttvMXToUMyePVvQ3rFjx4aMJrbExES4uroiIyMD7dq1Q3h4OJycnFBYWAgZGRkUFhbi8OHDn13yURJkZGRg+PDhuHv3LgwNDcHj8ZCeno4OHTrgr7/+4lwRhNQODWY5bvHixYI/8/l8hIaGomPHjujYsaNIJYoLb3TSKjY2FgcOHEBERATy8vLg6urKmUXubW1tsXDhQkycOFFoMOvr64uzZ8/i8uXLbEf8Iml5w+Pz+fj555+xefNmZGZmAgD09fWxYMECLFmyhBPr/o4aNQrjx4/H2LFj2Y4itsplxT5+2/z4fuWfubDEmLOzM+Tk5LBs2TLs27cPkZGRGDx4MHbt2gXgwzzzuLg4XLt2jeWkNXPmzBmkpKSAYRhYWlrCwcGB7UikAdFgluNquvg+j8dDdHR0Paf5eunp6Z99vFWrVg2U5OulpqZi//79OHDgAJ48eYL+/ftjwoQJGDVqlOCCES44fvw4Jk2aBE9PT/j5+cHX11docfhBgwaxHbHGpOkNr3LzAS5c+PXx7oSvX7+Gn58fpk6dig4dOoj80i3JOxWKsw21pG8vrqOjg+joaHTs2BEFBQVo0qQJYmNjBatkpKSkoEePHpxZlYE0bjSYJRKlugXVK0l6teNjMjIysLW1xXfffYfx48cLtoTlIq4uDh8dHY158+bh2rVrIoO+vLw89OrVC9u2bYO9vT1LCRsHGZmaXZ7BhYqmtJCRkRHsHglA6FMXAHj16hUMDAwk+nzk5+cjNTUVZmZmUFNTw61btxAUFITi4mKMGDECEyZMYDsiaSC0NJcUe/r0KQoLC2Fubl7jNxO23b59W+j++/fvcfv2bQQGBmLNmjUspaqdlJQUmJqash3jq5SXl2PNmjWYNm3aFxe7l0RBQUGYOXNmldVLDQ0NzJo1C4GBgZwZzL569Qo//vijYC3pT2sRkjrw4MrFabWRlJSE9PR0lJWVCbVLcoW50qeFAy4t53jp0iW4uLigoKAAWlpaOHjwIL755hu0aNECsrKyOHLkCIqKijBz5ky2o5IGQJVZKRAaGorc3FwsXLhQ0Pb9998jODgYwIdFsU+dOlXtbjVccOLECWzcuBEXLlxgO4rYPt51ysLCAtbW1mxHEouamhru3r2L1q1bsx1FbEZGRoiKihJag/JjKSkpGDx48Bent0gKZ2dnpKenY968eVWuJS3Ju7Fdv34dOTk5cHZ2FrSFhYXBx8cHhYWFGDFiBLZs2QJFRUUWU9bc48ePMXLkSCQmJorMmwUk9xeLSjIyMnB2dhb8fR8/fhwDBgyAqqoqAKC0tBRRUVES248+ffqgXbt28PX1xZ49exAYGIjZs2fD398fALB69WocPnwY8fHx7AYlDYIGs1KgZ8+e+P777wU7BUVFRcHV1RUhISGwsLDAvHnzYGlpKZjYz0UPHjxA586dUVhYyHaUGuPyrlMfGzFiBEaMGAE3Nze2o4hNSUkJd+/ehYmJSZWPP3z4EB06dEBxcXEDJ6sddXV1XL58GZ07d2Y7iticnJzQv39/LFu2DMCHq+mtra3h5uYGCwsLbNy4EbNmzeLMhgOurq6QlZXFzp07YWxsjNjYWGRnZ2PJkiX4+eefJb7a//HOcp8jqbvKaWpq4tq1azA3N0dZWRmUlZVx69YtdOrUCcCH13aXLl2Qn5/PclLSEGiagRRITU0VTNoHgL/++gvDhg0TzBfy9/ev8Q8utlVe1FKJYRhkZmZi5cqVaNeuHUupaofLu059zNnZGZ6enrh79y5sbGwElZtKkvxxaosWLZCYmFjtYDYhIQH6+voNnKr2DA0NRaYWcMWdO3ewevVqwf3w8HB0794dO3fuBPChbz4+PpwZzF69ehXR0dFo1qwZZGRkICMjAzs7O6xduxbu7u4iU6YkjaQOUmvq3bt3aNq0KQBAQUEBKioqQhfWqquro6ioiK14pIHRYFYKFBcXC80JjImJwbRp0wT3jY2N8fLlSzaiiU1TU1Pko1OGYWBoaIjw8HCWUtWOtGyzWLl2ZlVLu0n6BTtDhgzBihUr4OzsDCUlJaHHiouL4ePjAxcXF5bSiS8oKAgeHh7Yvn0756Z95ObmQldXV3D/4sWLcHJyEtzv2rUrMjIy2IhWKxUVFVBTUwPwYWWAFy9ewMzMDEZGRrh//z7L6aQfj8cTeq/49D5pXGgwKwWMjIwQFxcHIyMjvHnzBvfu3YOdnZ3g8ZcvX0JDQ4PFhDV3/vx5ofsyMjJo1qwZTExMICfHrX+u0rLrFJeyfmr58uU4cuQITE1NMW/ePJiZmYHH4yE5ORm//fYbKioq4OXlxXbMGhs3bhyKiorQtm1bqKioiPz7kuQdtHR1dZGWlgZDQ0OUlZXh1q1b8PX1FTyen59f5etFUllZWSEhIQHGxsbo3r07NmzYAAUFBezYsYOTG0JwDcMwGDhwoOB9oaioCK6urlBQUADw4eJV0nhwa3RAqjR58mTMnTsX9+7dQ3R0NMzNzWFjYyN4PCYmBlZWViwmrLm+ffuyHaHOVG6z+OmuU1zaZpHP5yMkJARHjhzBkydPwOPxYGxsjNGjR2PSpEkSXwnR1dVFTEwMZs+eDU9PT6GLdBwdHfH7778LVQslXVBQENsRas3JyQkeHh5Yv349jh07BhUVFaF5pQkJCWjbti2LCcWzfPlywRz+1atXw8XFBfb29tDW1ubMhihc5uPjI3S/qosfR48e3VBxCMvoAjApwOfz4ePjg8jISOjp6SEwMFDoo+0xY8bAyclJsDWspHv06BGCgoKEVgBYsGABp97oAO7vOsUwDFxdXfHPP/+gU6dOMDc3B8MwSE5ORmJiIoYNG4Zjx46xHbPGcnNz8fDhQzAMg3bt2kFLS4vtSI3K69evMWrUKFy5cgVqamoIDQ3FyJEjBY8PHDgQPXr04NwSfB/LycmBlpaWxP+SR4i0ocEskSinTp3CsGHD0LlzZ/Tu3RsMwyAmJgZ37tzB8ePHObXbVCWu7jq1Z88eLFiwAH/99ZfITnPR0dEYMWIEfv31V0yePJmlhI1bcXEx3r9/L9TGhd3A8vLyoKamJrL1bk5ODtTU1AQfE3PFw4cP8ejRI/Tp0wfKysqC7WwJIQ2HBrNSas6cOfDz84OOjg7bUcTSpUsXODo6Yt26dULtHh4eOH36NG7dusVSssZn8ODBGDBgADw8PKp83N/fHxcvXsSpU6caOFnjVVhYiGXLluHQoUPIzs4WeVySL8aTNtnZ2Rg7dizOnz8PHo+HBw8ewNjYGNOnT4empiYCAgLYjkhIo0GDWSnVpEkTxMfHc+5CBCUlJSQmJoosw5WamoqOHTuipKSEpWQ188svv9T4WHd393pM8vX09PQQFRVV7Zqmt2/fhrOzM2dWypAGc+fOxfnz5+Hn54fJkyfjt99+w/Pnz7F9+3asW7eOtu9sQJMnT0ZWVhZ27doFCwsLwVawp0+fxqJFi3Dv3j22IxLSaNAFYFKKq7+jNGvWDPHx8SKD2fj4eMEe4pJs06ZNNTqOx+NJ/GA2JyfnsxdH6erqIjc3twETkePHjyMsLAz9+vXDtGnTYG9vDxMTExgZGWH//v00mG1Ap0+fxqlTp0Tmvrdr1w5Pnz5lKRUhjRMNZqXAtGnTsHnzZqEFo7lq5syZ+P777/H48WP06tULPB4P//77L9avX48lS5awHe+L0tLS2I5QZyoqKj67HJqsrCwtf9PAcnJy0KZNGwAfPn2pXIrLzs5OsB4waRiFhYVQUVERaX/z5g1ntuSVNiUlJSLrSZPGgQazUiA0NBTr1q0TGsxydQs/b29vqKurIyAgAJ6engAAAwMDrFy5UuIrmdKGYRi4ublV+8ZcWlrawImIsbExnjx5AiMjI1haWuLQoUPo1q0bjh8/Dk1NTbbjNSp9+vRBWFgYVq1aBeDDpy18Ph8bN24UuWCS1B8+n481a9Zg27ZtePXqFVJTU2FsbAxvb2+0bt2aM6v4kK9Dc2algIyMDF6+fMmJj+E/p7y8HPv374ejoyP09PQEA3IuVZwXL15c42Or2lFLknB973ZptGnTJsjKysLd3R3nz5/H0KFDUVFRgfLycgQGBmLBggVsR2w0kpKS0K9fP9jY2CA6OhrDhg3DvXv3kJOTgytXrnBuKUGu8vPzQ2hoKPz8/DBz5kzcvXsXxsbGOHToEDZt2oSrV6+yHZE0ABrMSgEZGRm8evUKzZo1YzvKV1NRUUFycjKMjIzYjlIrNa3I8Hg8REdH13MaIu3S09Nx8+ZNtG3bFp06dWI7TqPz8uVLbN26FXFxceDz+bC2tsbcuXOhr6/PdrRGw8TEBNu3b8fAgQOhrq4uuBAvJSUFPXv2pHn9jQRNM5ASpqamX1zbUJK3uqzUvXt33L59m7OD2U+34yWkLly/fh05OTlwdnYWtIWFhcHHxweFhYUYMWIEtmzZQnM1G5ienp7Qlryk4T1//hwmJiYi7Xw+X2QdZiK9aDArJXx9faGhocF2jK82Z84cLFmyBM+ePYONjQ1UVVWFHu/YsSNLyQhhz8qVK9GvXz/BYDYxMRHTp0+Hm5sbLC0tsWHDBsHcctJw3r59i9jYWGRlZYHP5ws9RpuJNIz27dvj8uXLIgWQP/74A126dGEpFWloNM1ACkjDnNlp06YhKCioyotYeDyeYFcdLi0K379//89Wy2maAakpfX19HD9+HLa2tgAALy8vXLx4Ef/++y+AD2/cPj4+SEpKYjNmo3L8+HFMmDABhYWFUFdXF3qt83g8TnwSJg2OHz+OSZMmwdPTE35+fvD19cX9+/cRFhaGyMhITu4aScRHg1kpICsri8zMTE4PZiv7UFxc/NnjuDT9YNGiRUL3379/j/j4eNy9exdTpkzB5s2bWUpGuEZJSQkPHjyAoaEhgA9LcTk5OWH58uUAgCdPnqBDhw6cXcWEi0xNTTFkyBD4+/tXuUQXaTinTp2Cv7+/0NzlFStWYPDgwWxHIw2EphlIAWn4faSyD1warH5JdRsorFy5EgUFBQ2chnCZrq4u0tLSYGhoiLKyMty6dUtormZ+fj7k5eVZTNj4PH/+HO7u7jSQlQCOjo5wdHRkOwZhkQzbAcjX4/P5nK7KVvrSBWzSYuLEidi9ezfbMQiHODk5wcPDA5cvX4anpydUVFRgb28veDwhIYGWgmpgjo6OuHnzJtsxCCGgyiyRINKyIsOXXL16lXapIWJZvXo1Ro0ahb59+0JNTQ2hoaFQUFAQPL579276SLWBDR06FD/99BOSkpLQoUMHkcr4sGHDWEom/bS0tGpc/JCG9wzyZTRnlkgEGRkZBAUFfXFFhilTpjRQoq83cuRIoR+4DMMgMzMTN2/ehLe3N3x8fFhMR7goLy8PampqkJWVFWrPycmBmpqa0ACX1C8Zmeo/2OTaxapcExoaWuNjufSeQWqPBrNEIkjDigyfmjp1qmAlBuBDH5s1a4YBAwZQFY0QQupZcXExlJWV2Y5BGgDNmSUSQZrmyxYVFWHu3Lk4deoUIiMjUVJSgo0bNyI4OBjr1q2jgSwhHHb9+nWcPHlSqC0sLAxt2rRB8+bN8f3336O0tJSldI3P3Llzq2wvLCwU2mSESDcazBKJIE0fEPj4+CAkJAQuLi749ttvcfbsWcyePZvtWISQOrBy5UokJCQI7lduYOHg4AAPDw8cP34ca9euZTFh43L69GnBEnWVCgsL4eTkRFM9GhGaZkBIHWvbti3WrFmD8ePHAwBiY2PRu3dvlJSUiMx1JIRwC21gIVnS0tJgZ2eHH3/8EYsWLUJ+fj4cHR0hJyeHkydPiuwiSaQTrWZASB3LyMgQWjapW7dukJOTw4sXLwSL3hNCuCk3Nxe6urqC+xcvXoSTk5PgfteuXZGRkcFGtEapTZs2OHXqFPr16wcZGRmEh4dDUVERJ06coIFsI0LTDAipYxUVFSJXlcvJyaG8vJylRISQulK5gQUAwQYWPXv2FDxOG1g0PCsrK0RGRsLLywsqKipUkW2EqDJLSB1jGAZubm5QVFQUtJWUlOCHH34Q+gF75MgRNuIRQr5C5QYW69evx7Fjx2gDCxZ06dKlyouGFRUV8eLFC/Tu3VvQduvWrYaMRlhCg1lC6lhV6xpOnDiRhSSEkLpGG1iwb8SIEWxHIBKGLgAjhBBCxEQbWBAiOWgwSwghhBBOKysrQ1ZWFvh8vlB7q1atWEpEGhJNMyCEEEIIJ6WmpmL69OmIiYkRamcYhrYVbkRoMEsIIYQQTpo6dSrk5OQQGRkJfX19qdpNktQcTTMghBBCCCepqqoiLi4O5ubmbEchLKJ1ZgkhhBDCSZaWlnjz5g3bMQjLaDBLCCGEEE5av349li5digsXLiA7Oxvv3r0TupHGgaYZEEIIIYSTZGQ+1OQ+nStLF4A1LnQBGCGEEEI46fz582xHIBKAKrOEEEIIkTrx8fHo3Lkz2zFIA6A5s4QQQgiRCnl5efj9999hbW0NGxsbtuOQBkKDWUIIIYRwWnR0NCZOnAh9fX1s2bIFQ4YMwc2bN9mORRoIzZklhBBCCOc8e/YMISEh2L17NwoLCzF27Fi8f/8ef/75JywtLdmORxoQVWYJIYQQwilDhgyBpaUlkpKSsGXLFrx48QJbtmxhOxZhCVVmCSGEEMIpp0+fhru7O2bPno127dqxHYewjCqzhBBCCOGUy5cvIz8/H7a2tujevTt+/fVXvH79mu1YhCW0NBchhBBCOKmoqAjh4eHYvXs3YmNjUVFRgcDAQEybNg3q6upsxyMNhAazhBBCCOG8+/fvIzg4GHv37sXbt28xaNAg/P3332zHIg2ABrOEEEIIkRoVFRU4fvw4du/eTYPZRoIGs4QQQgghhLPoAjBCCCGEEMJZNJglhBBCCCGcRYNZQgghhBDCWTSYJYQQQgghnEWDWUIIIYQQwlk0mCWEEEIIIZxFg1lCCCGEEMJZNJglhBBCCCGc9X82XSMpEn8S1wAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Confusion Matrix of FashionMNIST model\n", + "\n", + "label_dict = {\n", + " 0: \"T-Shirt\",\n", + " 1: \"Trouser\",\n", + " 2: \"Pullover\",\n", + " 3: \"Dress\",\n", + " 4: \"Coat\",\n", + " 5: \"Sandal\",\n", + " 6: \"Shirt\",\n", + " 7: \"Sneaker\",\n", + " 8: \"Bag\",\n", + " 9: \"Ankle Boot\"\n", + "}\n", + "\n", + "test_data = test_loader.dataset.dataset.data[test_loader.dataset.indices]\n", + "test_targets = test_loader.dataset.dataset.targets[test_loader.dataset.indices]\n", + "\n", + "pred = model(test_data.unsqueeze(1).float().to(device))\n", + "cm = confusion_matrix(test_targets.cpu(), pred.argmax(dim=1).cpu())\n", + "\n", + "plt.figure(figsize=(8, 6))\n", + "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=label_dict.values(), yticklabels=label_dict.values())\n", + "plt.title(\"Confusion Matrix for FashionMNIST Classification\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "f6023126", + "metadata": {}, + "source": [ + "Wir wollen das noch weiter verbessern und verwenden ein noch größeres Modell." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "1e023b36", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "\n", + "class SigmaCNNFashionMNISTClassifier(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.conv_layers = nn.Sequential(\n", + " nn.Conv2d(1, 32, kernel_size=3, padding=1), # -> 32x28x28\n", + " nn.BatchNorm2d(32),\n", + " nn.ReLU(),\n", + " \n", + " nn.Conv2d(32, 64, kernel_size=3, padding=1), # -> 64x28x28\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.MaxPool2d(2,2), # -> 64x14x14\n", + " nn.Dropout(0.25),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, padding=1), # -> 128x14x14\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " \n", + " nn.Conv2d(128, 128, kernel_size=3, padding=1), # -> 128x14x14\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.MaxPool2d(2,2), # -> 128x7x7\n", + " nn.Dropout(0.25),\n", + " )\n", + " \n", + " self.fc_layers = nn.Sequential(\n", + " nn.Flatten(),\n", + " nn.Linear(128*7*7, 512),\n", + " nn.ReLU(),\n", + " nn.Dropout(0.5),\n", + " nn.Linear(512, 10)\n", + " )\n", + " \n", + " def forward(self, x):\n", + " x = self.conv_layers(x)\n", + " x = self.fc_layers(x)\n", + " return x\n" + ] + }, + { + "cell_type": "markdown", + "id": "3befeeae", + "metadata": {}, + "source": [ + "**Hinweis:** Wie bereits vorher einmal angekündigt, verwenden wir hier auch \"neue\" Layers, welche wir jetzt nicht im Detail besprechen, bei Interesse sind diese aber leicht selber zu lernen." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "7804ef2d", + "metadata": {}, + "outputs": [], + "source": [ + "### HYPERPARAMETER ###\n", + "\n", + "model = SigmaCNNFashionMNISTClassifier().to(device)\n", + "criterion = nn.CrossEntropyLoss()\n", + "lr = 0.001\n", + "optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay = 1e-4)\n", + "epochs = 20\n", + "validate_at = 1\n", + "print_at = 200\n", + "early_stopping_patience = 3\n", + "save_path = os.path.join(\"..\", \"models\", \"best_model_sophisticated_cnn_fashion_mnist.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "3f762a6d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch [1/20], Step [200/938], Loss: 0.6775\n", + "Epoch [1/20], Step [400/938], Loss: 0.3635\n", + "Epoch [1/20], Step [600/938], Loss: 0.4439\n", + "Epoch [1/20], Step [800/938], Loss: 0.5197\n", + "Epoch [1/20] - Validation Loss: 0.3771, Validation Accuracy: 86.14%\n", + ">>> Found a better model and saved it at '../models/best_model_sophisticated_cnn_fashion_mnist.pt'\n", + "Epoch [2/20], Step [200/938], Loss: 0.2703\n", + "Epoch [2/20], Step [400/938], Loss: 0.2146\n", + "Epoch [2/20], Step [600/938], Loss: 0.5231\n", + "Epoch [2/20], Step [800/938], Loss: 0.3303\n", + "Epoch [2/20] - Validation Loss: 0.2523, Validation Accuracy: 90.44%\n", + ">>> Found a better model and saved it at '../models/best_model_sophisticated_cnn_fashion_mnist.pt'\n", + "Epoch [3/20], Step [200/938], Loss: 0.3770\n", + "Epoch [3/20], Step [400/938], Loss: 0.1671\n", + "Epoch [3/20], Step [600/938], Loss: 0.3214\n", + "Epoch [3/20], Step [800/938], Loss: 0.3550\n", + "Epoch [3/20] - Validation Loss: 0.2688, Validation Accuracy: 89.86%\n", + "No Improvement. Early Stopping Counter: 1/3\n", + "Epoch [4/20], Step [200/938], Loss: 0.2788\n", + "Epoch [4/20], Step [400/938], Loss: 0.2245\n", + "Epoch [4/20], Step [600/938], Loss: 0.2170\n", + "Epoch [4/20], Step [800/938], Loss: 0.2386\n", + "Epoch [4/20] - Validation Loss: 0.2620, Validation Accuracy: 90.18%\n", + "No Improvement. Early Stopping Counter: 2/3\n", + "Epoch [5/20], Step [200/938], Loss: 0.3376\n", + "Epoch [5/20], Step [400/938], Loss: 0.1734\n", + "Epoch [5/20], Step [600/938], Loss: 0.1676\n", + "Epoch [5/20], Step [800/938], Loss: 0.3558\n", + "Epoch [5/20] - Validation Loss: 0.2116, Validation Accuracy: 92.12%\n", + ">>> Found a better model and saved it at '../models/best_model_sophisticated_cnn_fashion_mnist.pt'\n", + "Epoch [6/20], Step [200/938], Loss: 0.1225\n", + "Epoch [6/20], Step [400/938], Loss: 0.1172\n", + "Epoch [6/20], Step [600/938], Loss: 0.2205\n", + "Epoch [6/20], Step [800/938], Loss: 0.2128\n", + "Epoch [6/20] - Validation Loss: 0.2128, Validation Accuracy: 92.24%\n", + "No Improvement. Early Stopping Counter: 1/3\n", + "Epoch [7/20], Step [200/938], Loss: 0.1310\n", + "Epoch [7/20], Step [400/938], Loss: 0.1170\n", + "Epoch [7/20], Step [600/938], Loss: 0.2907\n", + "Epoch [7/20], Step [800/938], Loss: 0.3590\n", + "Epoch [7/20] - Validation Loss: 0.2218, Validation Accuracy: 91.98%\n", + "No Improvement. Early Stopping Counter: 2/3\n", + "Epoch [8/20], Step [200/938], Loss: 0.2786\n", + "Epoch [8/20], Step [400/938], Loss: 0.3544\n", + "Epoch [8/20], Step [600/938], Loss: 0.0602\n", + "Epoch [8/20], Step [800/938], Loss: 0.1725\n", + "Epoch [8/20] - Validation Loss: 0.2126, Validation Accuracy: 92.56%\n", + "No Improvement. Early Stopping Counter: 3/3\n", + "Early Stopping triggered.\n" + ] + } + ], + "source": [ + "train_model(model, train_loader, valid_loader, criterion, optimizer, epochs=epochs, validate_at=validate_at, print_at=print_at, patience=early_stopping_patience, save_path=save_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "a5258817", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Finaler Test Loss: 0.2012\n", + "Finale Test Accuracy: 92.88%\n" + ] + } + ], + "source": [ + "model.load_state_dict(torch.load(save_path))\n", + "test_loss, test_acc = evaluate_model(model, test_loader, criterion)\n", + "print(f\"Finaler Test Loss: {test_loss:.4f}\")\n", + "print(f\"Finale Test Accuracy: {test_acc:.2f}%\")" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "d11f050c", + "metadata": {}, + "outputs": [], + "source": [ + "torch.cuda.empty_cache()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "bc3edbfa", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArMAAAJLCAYAAAD0Jh5vAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAADPGklEQVR4nOzdd3QUVRvA4d+m95ACKRBCC72GHlrovXeQDiKhGIpU6SUQighSld5ViiBFpCoCGkCkgyAQkVASQoD0Mt8ffKwsm0ACCZtZ3+ecOQfu3Jl5796dzd137sxqFEVREEIIIYQQQoVMDB2AEEIIIYQQb0sGs0IIIYQQQrVkMCuEEEIIIVRLBrNCCCGEEEK1ZDArhBBCCCFUSwazQgghhBBCtWQwK4QQQgghVEsGs0IIIYQQQrVkMCuEEEIIIVRLBrP/EefOnaNXr17kz58fKysr7Ozs8PX1JTg4mEePHmXpsX///Xdq1aqFo6MjGo2G+fPnZ/oxNBoNkyZNyvT9vsnq1avRaDRoNBqOHDmit15RFAoVKoRGo8Hf3/+tjrF48WJWr16doW2OHDmSZkxva8uWLZQoUQJra2s0Gg1nz57NtH2n5a+//mLQoEEULlwYa2trbGxsKFGiBJ9++in//POPtl7Pnj3RaDSUKFGC5ORkvf1oNBoGDRqk/f+tW7e0/bZ582a9+pMmTUKj0RAeHv7a+F7u/1eXESNGvEPL9aU3JoB8+fLRs2fPTD1+WvGYmJjw119/6a2Pjo7GwcEBjUajE8vbvPY9e/bEzs5Op15iYiLLli2jYsWKODs7Y2Njg7e3Ny1btmT79u0A+Pv7p9k/Ly/p+ezYtWsXzZs3x83NDQsLC5ydnalbty4bNmwgMTFRW89Qn0UvvHj9XpaQkMBHH32Eh4cHpqamlC1bFsj690lan10v3gMZ/VwTIi1mhg5AZL0vv/ySgIAAihQpwieffELx4sVJTEzk1KlTLF26lBMnTmg//LNC7969iY6OZvPmzTg5OZEvX75MP8aJEyfIkydPpu83vezt7VmxYoXegPXo0aPcuHEDe3v7t9734sWLcXV1zdAfHV9fX06cOEHx4sXf+rgve/jwId26daNRo0YsXrwYS0tLChcunCn7Tsv3339Pp06dcHV1ZdCgQZQrVw6NRsP58+dZuXIlu3fv5vfff9fZ5tKlS6xevZo+ffqk+zjjxo2jbdu2mJubv3Wsq1atomjRojplnp6eb72/d7V9+3YcHBzey7Hs7OxYtWoVU6dO1Sn/5ptvSExMfO3r+i6vfbdu3di2bRuBgYFMnjwZS0tL/vrrL/bt28cPP/xA69atWbx4MU+ePNFus3v3bqZNm6bXX6/77FAUhd69e7N69WqaNGnCvHnz8PLyIioqisOHDxMQEEB4eDgff/xxhtuQFfr27UujRo10ypYsWcKyZctYuHAh5cuX134xyOr3SVqfXR4eHpw4cYKCBQtm2bHFf4wijNrx48cVU1NTpVGjRkpcXJze+vj4eOW7777L0hjMzMyUAQMGZOkxDGXVqlUKoPTt21extrZWoqKidNZ/8MEHStWqVZUSJUootWrVeqtjZGTbhIQEJTEx8a2O8zrHjh1TAGXLli2Zts/o6Og01/3111+Kra2tUq5cOeXx48d661NSUpStW7dq/9+jRw/F1tZWqVGjhpI7d24lJiZGpz6gDBw4UPv/mzdvKoDSuHFjBVAWLFigU3/ixIkKoDx8+PC1bXjR/yEhIa+tlxnSG9P78iKevn37Kl5eXkpycrLO+urVqyudO3dWbG1tlR49emjL3+a1f9G/L/z1118KoEyYMCHV2F6N5YW36a9Zs2YpgDJ58uRU14eFhSk///yz9v+AMnHixHTv/3148fn0vr3L554QGSHTDIzcjBkz0Gg0LF++HEtLS731FhYWtGjRQvv/lJQUgoODKVq0KJaWluTKlYvu3btz584dne38/f0pWbIkISEh1KhRAxsbGwoUKMDMmTNJSUkB/r0Em5SUxJIlS7SX9CD1S2Evb3Pr1i1t2aFDh/D398fFxQVra2vy5s1L27ZtiYmJ0dZJ7dLehQsXaNmyJU5OTlhZWVG2bFnWrFmjU+fF5fhNmzYxbtw4PD09cXBwoF69ely9ejV9LzLQuXNnADZt2qQti4qKYuvWrfTu3TvVbSZPnkzlypVxdnbGwcEBX19fVqxYgaIo2jr58uXj4sWLHD16VPv6vchsv4h93bp1DB8+nNy5c2Npacn169f1phmEh4fj5eWFn5+fziXRS5cuYWtrS7du3dJsW8+ePalevToAHTt21JsysXPnTqpWrYqNjQ329vbUr1+fEydO6OzjRX+fOXOGdu3a4eTk9NqszLx584iOjmbx4sU4OjrqrddoNLRp00avfNasWfzzzz98/vnnae77ZXXq1KFhw4ZMnTqVp0+fpmubjLh+/Tq9evXCx8cHGxsbcufOTfPmzTl//rxOvZSUFKZNm0aRIkWwtrYmR44clC5dOtV23L9/n86dO+Po6Iibmxu9e/cmKipKp05ql49DQ0P54IMPyJUrF5aWlhQrVoy5c+dqz1f49/LvnDlzmDdvHvnz58fOzo6qVaty8uTJVNvYu3dv/v77b3788Udt2bVr1zh27Fia7314t9c+IiICeJ7hS42JSeb8aUtMTGTWrFkULVqU8ePHp1rH3d1de36k5uHDhwQEBFC8eHHs7OzIlSsXderU4eeff9aru2TJEsqUKYOdnR329vYULVqUsWPHatfHxMQwYsQI7XQxZ2dnKlSooPO58+pnq0aj4auvviI2Nlb7GfLi8n5q75PHjx8zfPhwChQooP0b0KRJE65cuaKt866fXWlNMzh27Bh169bF3t4eGxsb/Pz82L17t06dF38jDh8+zIABA3B1dcXFxYU2bdpw9+7dNPtBGDcZzBqx5ORkDh06RPny5fHy8krXNgMGDGDUqFHUr1+fnTt3MnXqVPbt24efn5/eXL179+7RtWtXPvjgA3bu3Enjxo0ZM2YM69evB6Bp06baQU27du04ceKE3iDnTW7dukXTpk2xsLBg5cqV7Nu3j5kzZ2Jra0tCQkKa2129ehU/Pz8uXrzIggUL2LZtG8WLF6dnz54EBwfr1R87diy3b9/mq6++Yvny5fz55580b9481fmXqXFwcKBdu3asXLlSW7Zp0yZMTEzo2LFjmm3r378/X3/9Ndu2baNNmzYMHjxY53Lt9u3bKVCgAOXKldO+fq9OCRkzZgyhoaEsXbqUXbt2kStXLr1jubq6snnzZkJCQhg1ahTw/A9j+/btyZs3L0uXLk2zbePHj2fRokXA8y9HJ06cYPHixQBs3LiRli1b4uDgwKZNm1ixYgWRkZH4+/tz7NgxvX21adOGQoUK8c0337z2mPv378fNzY0qVaqkWSc1VatWpXXr1syaNSvdc8FnzZpFeHg4s2fPztCxXpacnExSUpLOAnD37l1cXFyYOXMm+/btY9GiRZiZmVG5cmWdL0vBwcFMmjSJzp07s3v3brZs2UKfPn14/Pix3rHatm1L4cKF2bp1K6NHj2bjxo0MHTr0tfE9fPgQPz8/9u/fz9SpU9m5cyf16tVjxIgROnOJX1i0aBE//vgj8+fPZ8OGDURHR9OkSRO9QTOAj48PNWrU0Hnvr1y5knz58lG3bt3XxvW2r32xYsXIkSMHkydPZvny5TpffjPTqVOnePToES1btkz1y3d6vHgfTpw4kd27d7Nq1SoKFCiAv7+/zpz2zZs3ExAQQK1atdi+fTs7duxg6NChREdHa+sMGzaMJUuWMGTIEPbt28e6deto3769dnCfmhMnTtCkSROsra21nyFNmzZNte7Tp0+pXr06y5Yto1evXuzatYulS5dSuHBhwsLCtPUy67PrZUePHqVOnTpERUWxYsUKNm3ahL29Pc2bN2fLli169fv27Yu5uTkbN24kODiYI0eO8MEHH6S5f2HkDJ0aFlnn3r17CqB06tQpXfUvX76sAEpAQIBO+a+//qoAytixY7VltWrVUgDl119/1albvHhxpWHDhjplvHKJV1H+vZT4qheXAW/evKkoiqJ8++23CqCcPXv2tbHzyqW9Tp06KZaWlkpoaKhOvcaNGys2NjbaS9eHDx9WAKVJkyY69b7++msFUE6cOPHa47582fLFvi5cuKAoiqJUrFhR6dmzp6Iob77clpycrCQmJipTpkxRXFxclJSUFO26tLZ9cbyaNWumue7w4cM65S8umW7fvl3p0aOHYm1trZw7d+61bXx5f998841OzJ6enkqpUqV0Lus+ffpUyZUrl+Ln56cte9HfaV0WfpWVlZVSpUqVdNVVFN3L0FeuXFFMTU2V4cOHa9e/+h58cal79uzZiqIoSteuXRVbW1slLCxMJ970TjNIbUltukdSUpKSkJCg+Pj4KEOHDtWWN2vWTClbtuxrj/UipuDgYJ3ygIAAxcrKSuc94+3trXNpf/To0amerwMGDFA0Go1y9epVndelVKlSSlJSkrbeb7/9pgDKpk2b9OJ5+PChsmrVKsXS0lKJiIhQkpKSFA8PD2XSpEmKoihpTjPIyGv/6jQDRVGU3bt3K66urtrX28XFRWnfvr2yc+fONF/DjE4z2Lx5swIoS5cuTVd9RXnzNIOkpCQlMTFRqVu3rtK6dWtt+aBBg5QcOXK8dt8lS5ZUWrVq9do6qX22pvb6KYr++2TKlCkKoPz444+vPcbL3uaz68V7YNWqVdqyKlWqKLly5VKePn2qLUtKSlJKliyp5MmTR7vfF3346t+p4OBgBdC+j8R/i2Rmhdbhw4cB9C47VapUiWLFinHw4EGdcnd3dypVqqRTVrp0aW7fvp1pMZUtWxYLCws+/PBD1qxZk+pd06k5dOgQdevW1ctI9+zZk5iYGL0M8ctTLeB5O4AMtaVWrVoULFiQlStXcv78eUJCQl57mfXQoUPUq1cPR0dHTE1NMTc3Z8KECURERPDgwYN0H7dt27bprvvJJ5/QtGlTOnfuzJo1a1i4cCGlSpVK9/Yvu3r1Knfv3qVbt246l3Xt7Oxo27YtJ0+e1JkKktFY31aRIkXo06cPX3zxBaGhoenaZtq0aSQmJjJ58uS3OubatWsJCQnRWczMzEhKSmLGjBkUL14cCwsLzMzMsLCw4M8//+Ty5cva7StVqsQff/xBQEAAP/zwg85NS69K7b0aFxf32vfMoUOHKF68uN752rNnTxRF4dChQzrlTZs2xdTUVOcYkPb50L59eywsLNiwYQN79uzh3r176b5h8W1f+yZNmhAaGsr27dsZMWIEJUqUYMeOHbRo0SLVbLMhLV26FF9fX6ysrDAzM8Pc3JyDBw/qvQceP35M586d+e6771J9akWlSpXYu3cvo0eP5siRI8TGxmZqnHv37qVw4cLUq1fvtfUy67PrhejoaH799VfatWun89QKU1NTunXrxp07d/SmfWXGZ7YwHjKYNWKurq7Y2Nhw8+bNdNV/3Tw0T09PvUtZLi4uevUsLS0z9QO2YMGCHDhwgFy5cjFw4EAKFixIwYIF3zgnMiIiIs12vFj/slfb8mJ+cUbaotFo6NWrF+vXr9demqtRo0aqdX/77TcaNGgAPH/axC+//EJISAjjxo3L8HHTmjeYVow9e/YkLi4Od3f3186VfZM3vV9SUlKIjIx8q1jz5s2b7vdtaiZNmoSpqWma8xxflS9fPgICAvjqq6/4888/M3y8YsWKUaFCBZ0Fnl8WHj9+PK1atWLXrl38+uuvhISEUKZMGZ0+HjNmDHPmzOHkyZM0btwYFxcX6taty6lTp/SO9Tbv1aw+H2xtbenYsSMrV65kxYoV1KtXD29v7zTjedm7vPbW1ta0atWK2bNnc/ToUa5fv07x4sVZtGgRFy9ezNC+UpM3b16Ad3ovzps3jwEDBlC5cmW2bt3KyZMnCQkJoVGjRjqvZ7du3Vi5ciW3b9+mbdu25MqVi8qVK+vMRV6wYAGjRo1ix44d1K5dG2dnZ1q1avVW79nUPHz48I1PhcnMz64XIiMjURTlvX9mC+Mhg1kjZmpqSt26dTl9+rTeDVypefHh8PLcqBfu3r2Lq6trpsVmZWUFQHx8vE55atmIGjVqsGvXLqKiojh58iRVq1YlMDAw1WdUvuDi4pJmO4BMbcvLevbsSXh4OEuXLqVXr15p1tu8eTPm5uZ8//33dOjQAT8/P+0AKKMyMpcvLCyMgQMHUrZsWSIiIt7pWahver+YmJjg5OT0VrE2bNiQ+/fvp3nT0Zt4eHgQGBjI+vXrOXfuXLq2+fTTT7GxsdG54eZdrV+/nu7duzNjxgwaNmxIpUqVqFChgt773MzMjGHDhnHmzBkePXrEpk2b+Pvvv2nYsKFedvttvI/zoXfv3pw9e5Zdu3a99opEajLrtc+bNy8ffvghQKYMZitUqICzszPfffedzs1NGbF+/Xr8/f1ZsmQJTZs2pXLlylSoUCHVm9569erF8ePHiYqKYvfu3SiKQrNmzbTZRltbWyZPnsyVK1e4d+8eS5Ys4eTJkzRv3vyd2vlCzpw53/i3IjM/u15wcnLCxMTEIJ/ZwjjIYNbIjRkzBkVR6NevX6o3TCUmJrJr1y7g+d3FgPYGrhdCQkK4fPnyG2/myIgXd7W+OtB4EUtqTE1NqVy5svZmpDNnzqRZt27duhw6dEjv7ta1a9diY2OT4RuL0it37tx88sknNG/enB49eqRZT6PRYGZmpnMpNzY2lnXr1unVzaxsd3JyMp07d0aj0bB3716CgoJYuHAh27Zte6v9FSlShNy5c7Nx40adP/TR0dFs3bpV+4SDtzF06FBsbW0JCAhI9aYjRVHe+GzkUaNG4ezszOjRo9N1TBcXF0aNGsW3337Lb7/99lZxv0qj0eg9RWT37t06P/jwqhw5ctCuXTsGDhzIo0ePMuXmprp163Lp0iW9c2bt2rVoNBpq1679zseoWrUqvXv3pnXr1rRu3TpD22b0tX/69CnPnj1Ldd2LS/eZ8Zxfc3NzRo0axZUrV/Seo/vCgwcP+OWXX9LcR2rvgXPnzr32ZlhbW1saN27MuHHjSEhISHVg7ubmRs+ePencuTNXr17NlC89jRs35tq1a3rTTl6WFZ9dtra2VK5cmW3btunUT0lJYf369eTJkyfLn2st1E1+NMHIVa1alSVLlhAQEED58uUZMGAAJUqUIDExkd9//53ly5dTsmRJmjdvTpEiRfjwww9ZuHAhJiYmNG7cmFu3bjF+/Hi8vLzeeMd0RjRp0gRnZ2f69OnDlClTMDMzY/Xq1fz999869ZYuXcqhQ4do2rQpefPmJS4uTnvX9OvmdU2cOJHvv/+e2rVrM2HCBJydndmwYQO7d+8mODg41cc9ZZaZM2e+sU7Tpk2ZN28eXbp04cMPPyQiIoI5c+ak+vi0UqVKsXnzZrZs2UKBAgWwsrJ6q3muEydO5Oeff2b//v24u7szfPhwjh49Sp8+fShXrhz58+fP0P5MTEwIDg6ma9euNGvWjP79+xMfH8/s2bN5/Phxul6HtOTPn5/NmzfTsWNHypYtq/3RBHj+OLGVK1eiKMprB00ODg6MGzcuQ+/bwMBAFi1axN69e9869pc1a9aM1atXU7RoUUqXLs3p06eZPXu23qXc5s2bU7JkSSpUqEDOnDm5ffs28+fPx9vbGx8fn3eOY+jQoaxdu5amTZsyZcoUvL292b17N4sXL2bAgAGZNlBYsWLFW2+bkdf+6tWrNGzYkE6dOlGrVi08PDyIjIxk9+7dLF++HH9/f/z8/N46lpd98sknXL58mYkTJ/Lbb7/RpUsX7Y8m/PTTTyxfvpzJkydTrVq1VLdv1qwZU6dOZeLEidSqVYurV68yZcoU8ufPr33qBUC/fv2wtramWrVqeHh4cO/ePYKCgnB0dKRixYoAVK5cmWbNmlG6dGmcnJy4fPky69ate6cvji8LDAxky5YttGzZktGjR1OpUiViY2M5evQozZo1o3bt2ln22RUUFET9+vWpXbs2I0aMwMLCgsWLF3PhwgU2bdr01k+TEP8Rhrv3TLxPZ8+eVXr06KHkzZtXsbCw0D6QfsKECcqDBw+09ZKTk5VZs2YphQsXVszNzRVXV1flgw8+UP7++2+d/dWqVUspUaKE3nF69OiheHt765SRytMMFOX5HdJ+fn6Kra2tkjt3bmXixInKV199pfM0gxMnTiitW7dWvL29FUtLS8XFxUWpVauW3h3LpHIH8fnz55XmzZsrjo6OioWFhVKmTBmdu2cVJfW79BUl9bttU5Peu6NTu6t35cqVSpEiRRRLS0ulQIECSlBQkLJixQqd9iuKoty6dUtp0KCBYm9vrwDa1zet2F9e9+JpBvv371dMTEz0XqOIiAglb968SsWKFZX4+Pg043/dsXbs2KFUrlxZsbKyUmxtbZW6desqv/zyi06dt33g/40bN5SAgAClUKFCiqWlpWJtba0UL15cGTZsmM5rlNbd2vHx8Ur+/Pnf+DSDly1fvlx7h/y7/mhCZGSk0qdPHyVXrlyKjY2NUr16deXnn39WatWqpfN+mDt3ruLn56e4uroqFhYWSt68eZU+ffoot27d0tZJ6zV89QkgiqJ/l7qiKMrt27eVLl26KC4uLoq5ublSpEgRZfbs2TpPonjd6/LqOZbePn3T0wxeltZr/2r/RkZGKtOmTVPq1Kmj5M6dW/uZVrZsWWXatGl6P5rxwrv8yMV3332nNG3aVMmZM6diZmamODk5KbVr11aWLl2qc+68+jrFx8crI0aMUHLnzq1YWVkpvr6+yo4dO/Q+K9esWaPUrl1bcXNzUywsLBRPT0+lQ4cOOk8bGT16tFKhQgXFyclJ+7kxdOhQJTw8XFvnXZ5moCjPX9uPP/5YyZs3r2Jubq7kypVLadq0qXLlyhVtnXf97Err8/Xnn39W6tSpo9ja2irW1tZKlSpVlF27dunUSasP03qCi/hv0CjKW04EEkIIIYQQwsBkzqwQQgghhFAtGcwKIYQQQgjVksGsEEIIIYRQLRnMCiGEEEII1ZLBrBBCCCGEUC0ZzAohhBBCCNWSwawQQgghhFAt+QUwA3LotNbQIWSKB+u7GzqEd5aUbByPWzYxgh/JMTGGRhiRiGf6P4OtNi52FoYOIVMYw1PhjeWHvKwMOHqyLjcoy/Yd+/sXWbbvrCSZWSGEEEIIoVqSmRVCCCGEUAuN5CFfJa+IEEIIIYRQLcnMCiGEEEKohbFMPM5EMpgVQgghhFALmWagR14RIYQQQgihWjKYFUIIIYRQC40m65YMmDRpEhqNRmdxd3fXrlcUhUmTJuHp6Ym1tTX+/v5cvHhRZx/x8fEMHjwYV1dXbG1tadGiBXfu3MnwSyKDWSGEEEIIkWElSpQgLCxMu5w/f167Ljg4mHnz5vHFF18QEhKCu7s79evX5+nTp9o6gYGBbN++nc2bN3Ps2DGePXtGs2bNSE5OzlAcMmdWCCGEEEItstGcWTMzM51s7AuKojB//nzGjRtHmzZtAFizZg1ubm5s3LiR/v37ExUVxYoVK1i3bh316tUDYP369Xh5eXHgwAEaNmyY7jiyzysihBBCCCEMJj4+nidPnugs8fHxadb/888/8fT0JH/+/HTq1Im//voLgJs3b3Lv3j0aNGigrWtpaUmtWrU4fvw4AKdPnyYxMVGnjqenJyVLltTWSS8ZzAohhBBCqEUWzpkNCgrC0dFRZwkKCko1jMqVK7N27Vp++OEHvvzyS+7du4efnx8RERHcu3cPADc3N51t3NzctOvu3buHhYUFTk5OadZJL5lmIIQQQgghGDNmDMOGDdMps7S0TLVu48aNtf8uVaoUVatWpWDBgqxZs4YqVaoAoHnlpjJFUfTKXpWeOq+SzKwQQgghhFpoTLJssbS0xMHBQWdJazD7KltbW0qVKsWff/6pnUf7aob1wYMH2mytu7s7CQkJREZGplknvWQwK4QQQgihFtnk0Vyvio+P5/Lly3h4eJA/f37c3d358ccftesTEhI4evQofn5+AJQvXx5zc3OdOmFhYVy4cEFbJ71kmoEQQgghhMiQESNG0Lx5c/LmzcuDBw+YNm0aT548oUePHmg0GgIDA5kxYwY+Pj74+PgwY8YMbGxs6NKlCwCOjo706dOH4cOH4+LigrOzMyNGjKBUqVLapxuk1392MOvv70/ZsmWZP39+mnU0Gg3bt2+nVatW7y0uIYQQQog0ZZNHc925c4fOnTsTHh5Ozpw5qVKlCidPnsTb2xuAkSNHEhsbS0BAAJGRkVSuXJn9+/djb2+v3cdnn32GmZkZHTp0IDY2lrp167J69WpMTU0zFEv2eEVe49Vfl3h16dmzp942ycnJBAUFUbRoUaytrXF2dqZKlSqsWrUqQ8cOCwvTmeCcmtWrV5MjR44M7TezDGtZkiebuzOzewVt2Zh2ZTg1tyVhqztz+6uOfDeuPhUKuaa5j62j6/Jkc3eaVvB6HyFn2JZNG2jcoA4Vy5WiU/s2nDl9ytAhvdaZUyEEDvqIhnVrUL50UQ4fOqCzXlEUli1eSMO6NfCrWIYPe3fjxvU/DRRt+jRpWIdypYrqLUHTphg6tAw5fSqEwQEfUc+/OmVKFOHQwQNv3iibUtt58bKNq7+iTuVSfDFvlrYsNiaGz2dPp0OzujSqWYGeHVvw3dYtBowy/dTcFwBfb95I+9bNqVbZl2qVfenetSPHfj5q6LDemtr7Q002b97M3bt3SUhI4J9//mHr1q0UL15cu16j0TBp0iTCwsKIi4vj6NGjlCxZUmcfVlZWLFy4kIiICGJiYti1axdeXhkfj2T7wezLvywxf/58HBwcdMo+//xzvW0mTZrE/PnzmTp1KpcuXeLw4cP069dPb5Lxm7i7u7924nNiYmKG25NZfAu40LOuD+dvP9Ipvx72hBGrfqPqyF00nLSP0IfP2D62Hi72+u0Y2KQYiqK8r5AzbN/ePQTPDKLfhwPY8u0OfH3LE9C/H2F37xo6tDTFxsZSuEhRRo0Zn+r6Nau+YsO61YwaM561G7/BxTUnAf17Ex397D1Hmn7rN33Lj4d/1i5Llq8EoH4GHmidHcTGxlCkSBFGj5tg6FDeiRrPixeuXLrA9zu+pUChwjrli+YHE3LyF8ZOnsnqzd/RrlM3Fs4N4pejhwwUafqouS9ecHN3Z8jQEWzcspWNW7ZSsVIVAgcP5Ho2/5KdGmPoj3TJpnNmDSnbD2bd3d21i6Ojo/a3f18ue9WuXbsICAigffv25M+fnzJlytCnTx+9x02kpKQwcuRInJ2dcXd3Z9KkSTrrNRoNO3bsAODWrVtoNBq+/vpr/P39sbKyYv369fTq1YuoqChtpvjVfWQFW0szvhpcgyHLT/I4OkFn3Te/3OTIhTBuPXjGlTtRjF13CkcbC0p66z7HrWReJwY2LU7A0ow9mPh9WrdmFa3btqVNu/YUKFiQkWPG4e7hztdbNhk6tDRVq1GTgMGB1KnXQG+doihsXL+W3v0+ok69BhTyKczkaTOJi4tj357vDRBt+jg7O+PqmlO7/PzTEby88lK+QiVDh5Yh1WvUYtDHQ6lXX79v1ESN5wU8z77OmDCa4WMnYu/goLPu0vk/aNikBWXLV8TdMzfNWrenYKHCXL18MY29ZQ9q7YuX1fKvQ42atfDOlx/vfPkZ/PFQbGxsOP/HWUOHlmHG0B/i7WT7wezbcHd359ChQzx8+PC19dasWYOtrS2//vorwcHBTJkyReeuutSMGjWKIUOGcPnyZerWrauXLR4xYkRmNiVVc3tX5off73DkQthr65mbmtCzrg+PoxM4f/vfrLS1hSkrh9RgxMrfeBAVl9XhvpXEhAQuX7pIVb/qOuVV/arxx9nfDRTVu/nnnztEhD+kStVq2jILCwvKl6+omjYlJiaw5/udtGzdJsPPARTvTs3nxeezp1O5Wg3KV6qqt65UmXIc//kIDx/cR1EUfj/1G3f+vk3FKtX0d5RNqLkv0pKcnMy+PbuJjY2hdNlyhg4nQ4yxP9KUhY/mUiujvAFs3rx5tGvXDnd3d0qUKIGfnx8tW7bUm/9aunRpJk6cCICPjw9ffPEFBw8epH79+mnuOzAwUPs7w4BOtvh9aFs1H2XyO+M/bneadRr55mblkJrYWJhx73Esrab/yKOn//4cXVD3ivx67SF7Tv/9PkJ+K5GPI0lOTsbFxUWn3MXFlfDw139Jya4i/h/3q21ydnEhLEwdl8EOHzzI06dPad6ytaFD+U9S63lxaP9e/rx6iSWrNqe6ftDwMcydMYmOzethamqGiYmG4WMnU6qs73uONP3U2hep+fPaVbp37URCQjzWNjbM+3wRBQsWMnRYGWJM/SEyTr3D8P+zs7PTLh999BEAxYsX58KFC5w8eZJevXpx//59mjdvTt++fXW2LV26tM7/PTw8ePDgwWuPV6FChdeuT0tqv3esJGdszm1uFxtm9ahIvy+OEZ+Ykma9ny7ep/qo76k/YS8H/viH1YE1cXWwAqBx+TzUKuHO6DUhb9WO9+1tfj0k29NrE2hQR5t2bP+WatVrkCtXxh5oLTKXms6LB/fvsWjeTMZOmolFGvcgbNuygUsXzjFtzkKWrtnMRx+P4PPZ0zj924n3HG3Gqakv0pIvf362bN3B2g1b6NChMxPGjeLGjeuGDuutGEN/vJHMmdWj+szs2bNntf92eGkelomJCRUrVqRixYoMHTqU9evX061bN8aNG0f+/PkBMDc319mXRqMhJSXtQSI8/4WLtxEUFMTkyZN1yixKtMKyZPozXGXzu5ArhzU/BTXVlpmZmlCtqBsfNiyK6wcbSFEUYuKT+Ov+U/66/5SQ6+H8/lkrutcuxLzvLlCrhDv53ez5e2UnnX2vH1aL41ce0HTK/rdqX2ZzyuGEqakp4eHhOuWPHkXg4pL20xmyMxfXnABEhIeTM2cubXnkowicX8kmZEd37/7DrydPMOezhYYO5T9LjefFtSsXiYx8RP+eHbVlKcnJnPv9NDu+3cSug8dZseRzpsz6nCrVawJQ0KcIN65d5esNa1KdlpAdqLEv0mJubkHevM8fp1SiZCkuXjzPxvVrGT9RPU8sMab+eCMVTwfIKqofzBYqlL5LIS8eFxEdHZ2px7ewsCA5OfmN9VL7vePcfb7J0LGOXgij8oidOmVLBvhx7W4Un313kZQ0nkyg0YCl+fNnts377gJrDul+4/51TgvGrD3F3tN3MhRPVjK3sKBY8RKcPP4Ldev9O+3j5PHj+Nepa8DI3l7u3Hlwcc3JryeOU7TY8/djYmICp0+HMCRwuIGje7OdO7bh7OxCjZq1DB3Kf5YazwvfClVYsXGbTlnw1PF4eeenc/fepCSnkJSUhMZENytkYmLyxuSCIamxL9JLURQSEhLeXDEbMeb+EG+m+sFsatq1a0e1atXw8/PD3d2dmzdvMmbMGAoXLkzRokUz9Vj58uXj2bNnHDx4kDJlymBjY4ONjY1ePUtLS73HfGlMzfXqvc6zuCQu33msUxYdn8Sjp/FcvvMYG0szRrQuxd5Tf3PvcSzOdpb0bVAET2dbtp+8BcCDqLhUb/r6Ozya2w+z1+OhuvXoxbjRIylesiRlypRj6zdbCAsLo33HTm/e2EBiYqL5OzRU+/+7/9zh6pXLODg64uHhSZcPurNyxTK8vL3Jm9eblV8tw8rKikZNmhkw6jdLSUnhux3badaiFWZm6vzYiImOJvSlvvnnzh2uXL6Mo6MjHp6eBowsY9R2XtjY2pK/oI9OmZW1NQ6OObTlZXwrsGzhPCwtrXDz8OCPM6fYv3cXAz7+xBAhp5va+iI1C+bPo3qNmri5uxMTHc2+vXs4FfIbi5Z+ZejQMswY+iNdJDOrR51/ld6gYcOGbNq0iaCgIKKionB3d6dOnTpMmjQp0/8Q+/n58dFHH9GxY0ciIiKYOHHie3k8V2qSU1Io7OlAl2H+uNhb8uhpPGf+iqDRpH1cuRNlkJjeRaPGTYh6HMnyJYt5+PABhXwKs2jpcjw9cxs6tDRduniB/n16aP8/b/ZMAJq1aMXkaTPp0asv8XFxzJw+hadPoihZqjSLlq7A1tbOUCGny68nj3Mv7C6tWrd5c+Vs6uLFC/Tt1V37/znBQQC0aNmaqTNmGiqsDFPjefEm46fN5stF85k+cTRPn0Th5u5Bn48G06JNB0OH9lrG0BePIsIZN2Yk4Q8fYGdvT+HCRVi09Cuq+mXfJ0mkxRj6Q7wdjZKdn5pv5Bw6rTV0CJniwfrub66UzSUlG8dpYKLe+ftaJsbQCCMS8Uxdl5tT42JnYegQMoUx/LVW8T1GOqwMmAq0rj01y/Ydezj1H/zJ7iRXLYQQQgghVMsopxkIIYQQQhglmTOrR14RIYQQQgihWpKZFUIIIYRQC2OZeJyJJDMrhBBCCCFUSzKzQgghhBBqIXNm9chgVgghhBBCLWSagR4Z3gshhBBCCNWSzKwQQgghhFrINAM98ooIIYQQQgjVksysEEIIIYRayJxZPZKZFUIIIYQQqiWZWSGEEEIItZA5s3rkFRFCCCGEEKolmVkhhBBCCLWQObN6ZDArhBBCCKEWMs1Aj7wiQgghhBBCtSQzK4QQQgihFjLNQI8MZg3owfruhg4hUzj5jzd0CO8s8shUQ4eQKVIUxdAhCCPjYmdh6BDE/8kYRojUyWBWCCGEEEItZM6sHnlFhBBCCCGEaklmVgghhBBCLSQzq0deESGEEEIIoVqSmRVCCCGEUAu5E1CPDGaFEEIIIdRCphnokVdECCGEEEKolmRmhRBCCCHUQqYZ6JHMrBBCCCGEUC3JzAohhBBCqIXMmdUjr4gQQgghhFAtycwKIYQQQqiFzJnVI5lZIYQQQgihWpKZFUIIIYRQCY1kZvXIYFYIIYQQQiVkMKtPphkIIYQQQgjVksysEEIIIYRaSGJWT7bJzGo0mtcuPXv2NHSIqnH6VAiDAz6inn91ypQowqGDBwwd0muN+KAmscemMntIY22ZrbUFnw1tyvVtI3h0cAK/rx9Cv1YVdbbL7+nElhmdCd01mvs/jGP9lI7kcrJ93+G/0ZZNG2jcoA4Vy5WiU/s2nDl9ytAhZcjSRQspV7KozlKvVnVDh/XW1N4fYBxtAONohzG0QW1/M17HGPpDZFy2GcyGhYVpl/nz5+Pg4KBT9vnnn+vUT0xMNFCkr5eQkGDoEIiNjaFIkSKMHjfB0KG8UfmiuenTogLnrt/TKQ8e3Jj6lX3oNfVbynZdwMKvjzMvsCnNqhcFwMbKnO8/64miQOOPV1FnwFdYmJmyddYH2Wo+0b69ewieGUS/Dwew5dsd+PqWJ6B/P8Lu3jV0aBlSsJAPPx75Wbt8vX2noUN6K8bQH8bQBjCOdhhDG0BdfzNex1j6403elPx7l0Wtss1g1t3dXbs4Ojqi0Wi0/4+LiyNHjhx8/fXX+Pv7Y2Vlxfr160lJSWHKlCnkyZMHS0tLypYty759+7T7PHLkCBqNhsePH2vLzp49i0aj4datWwDcvn2b5s2b4+TkhK2tLSVKlGDPnj3a+pcuXaJJkybY2dnh5uZGt27dCA8P16739/dn0KBBDBs2DFdXV+rXr5/lr9WbVK9Ri0EfD6Ve/QaGDuW1bK0tWDWxHQHBO3j8NFZnXeWSXqzfe5aff79F6L3HrNx5inM37uFbNDcAVUvlxds9B/2mb+PiX/e5+Nd9PgzaRoXiefAvn98QzUnVujWraN22LW3atadAwYKMHDMOdw93vt6yydChZYipqSmurjm1i7Ozs6FDeivG0B/G0AYwjnYYQxtAPX8z3sRY+kNkXLYZzKbHqFGjGDJkCJcvX6Zhw4Z8/vnnzJ07lzlz5nDu3DkaNmxIixYt+PPPP9O9z4EDBxIfH89PP/3E+fPnmTVrFnZ2dsDzbHGtWrUoW7Ysp06dYt++fdy/f58OHTro7GPNmjWYmZnxyy+/sGzZskxtszGbP6wZ+45f4/Cpv/TWHT93m2bVi+Dpag9AzXL58fFy5cBvz/vW0sIMRVGIT0zSbhMXn0Rycgp+pb3fTwPeIDEhgcuXLlLVT/eSfFW/avxx9ncDRfV2QkNvU792DZo2rMuoEcO48/ffhg4pw4yhP4yhDWAc7TCGNhiT/1J/SGZWn6puAAsMDKRNmzba/8+ZM4dRo0bRqVMnAGbNmsXhw4eZP38+ixYtStc+Q0NDadu2LaVKlQKgQIEC2nVLlizB19eXGTNmaMtWrlyJl5cX165do3DhwgAUKlSI4ODg1x4nPj6e+Ph4nTLF1BJLS8t0xWls2tctRdnCnlTvtzTV9cPn72HxqJbc2DGSxKRkUlIUBszawfFzoQD8dvFvouMSmT6gAROWHUCjgekDGmBqaoK7i/37bEqaIh9HkpycjIuLi065i4sr4eEPDRRVxpUsXYapM2bi7Z2PiIgIvlq2hJ4fdObb73aRI4eTocNLN2PoD2NoAxhHO4yhDcZE+uO/TVWZ2QoVKmj//eTJE+7evUu1atV06lSrVo3Lly+ne59Dhgxh2rRpVKtWjYkTJ3Lu3DntutOnT3P48GHs7Oy0S9Giz+ds3rhxI9W40hIUFISjo6POMntWULrjNCZ5cjkw++Mm9J76LfEJSanWGdi+CpVKeNF21Hr8+ixh9Bf7+Hx4c2pXeP5lI/xxDF3Hb6ZJtaKE//gp9/eNw8HWijNX/yE5JeV9NueNXv22qyiKqr4BV69Rk3r1G+JTuAhVqvqxcPHzqw+7vtth2MDektr7A4yjDWAc7TCGNhiT/0J/SGZWn6oys7a2+neqv+6Na2Jioi174dUbx/r27UvDhg3ZvXs3+/fvJygoiLlz5zJ48GBSUlJo3rw5s2bN0juuh4fHa+N61ZgxYxg2bJhurKb/zaxsuSK5cXO24/hXH2nLzMxMqV7Gm4/aVMat0XQmf1iPjmM3se/ENQAu3LhPaR93AjtX105LOBhygxIdP8PF0Yak5BSinsVx87uR3L4baZB2vcophxOmpqY6c6wBHj2KwMXF1UBRvTtrGxsK+RQm9PZtQ4eSIcbQH8bQBjCOdhhDG4zJf6k/1DzozCqqysy+zMHBAU9PT44dO6ZTfvz4cYoVKwZAzpw5gedzX184e/as3r68vLz46KOP2LZtG8OHD+fLL78EwNfXl4sXL5IvXz4KFSqks6RnAPsyS0tLHBwcdJb/6hSDw6duUL7bQir3WqxdTl++w+b956jcazGmJiZYmJuR8tKXEIDkFAWTVE7iiKgYop7FUcs3P7mcbPn+2NX31ZTXMrewoFjxEpw8/otO+cnjxylTtpyBonp3CQkJ3Lx5A9f/n19qYQz9YQxtAONohzG0wZhIf/y3qSoz+6pPPvmEiRMnUrBgQcqWLcuqVas4e/YsGzZsAJ7PZfXy8mLSpElMmzaNP//8k7lz5+rsIzAwkMaNG1O4cGEiIyM5dOiQdjA8cOBAvvzySzp37swnn3yCq6sr169fZ/PmzXz55ZeYmpq+9zanR0x0NKGhodr//3PnDlcuX8bR0REPT08DRvbcs9gELt18oFMWHZfIoycx2vKffr/JjICGxMYnEnrvMTXK5qdro7KMWrhXu023JuW4evshDyOjqVwyL3M+bsLCr0/w59+638wNqVuPXowbPZLiJUtSpkw5tn6zhbCwMNp37GTo0NJt3uxZ1PSvjYeHJ48ePZ8zG/3sGc1btjJ0aBlmDP1hDG0A42iHMbQBsv/fjPQylv54I0nM6lH1YHbIkCE8efKE4cOH8+DBA4oXL87OnTvx8fEBwNzcnE2bNjFgwADKlClDxYoVmTZtGu3bt9fuIzk5mYEDB3Lnzh0cHBxo1KgRn332GQCenp788ssvjBo1ioYNGxIfH4+3tzeNGjXSTmHIji5evEDfXt21/58T/HxubouWrZk6Y6ahwsqQ7hO/Zkr/+qye0B4nB2tC7z1m0vIDfLkjRFuncF5XpvSvj7ODNbfvPSZ47VEWbDluwKj1NWrchKjHkSxfspiHDx9QyKcwi5Yux9Mzt6FDS7f79+8zZuRwHkc+xsnZiVKly7Bm4xZVteEFY+gPY2gDGEc7jKENYBx/M8B4+kNknEZRXrmWK96buNTvfVIdJ//xhg7hnUUemWroEDLFq1Mz1Ci1qSRCCJGdWBkwFZij6/os2/fjDR9k2b6zUvZNLwohhBBCCPEGqp5mIIQQQgjxXyJPM9AnmVkhhBBCCKFakpkVQgghhFAJyczqk8GsEEIIIYRKyGBWn0wzEEIIIYQQqiWZWSGEEEIItZDErB7JzAohhBBCCNWSzKwQQgghhErInFl9kpkVQgghhBCqJZlZIYQQQgiVkMysPsnMCiGEEEII1ZLMrBBCCCGESkhmVp8MZoUQQggh1ELGsnpkmoEQQgghhFAtycwKIYQQQqiETDPQJ5lZIYQQQgihWpKZNaDkFMXQIWSKyCNTDR3CO3NqvdjQIWSKR9sCDB3CO4tLTDZ0CJnCytzU0CFkivjEFEOH8M4szSVvI4yHZGb1yRkuhBBCCCFUSzKzQgghhBAqIZlZfZKZFUIIIYQQqiWZWSGEEEIIlZDMrD7JzAohhBBCqIUmC5e3FBQUhEajITAwUFumKAqTJk3C09MTa2tr/P39uXjxos528fHxDB48GFdXV2xtbWnRogV37tzJ8PFlMCuEEEIIId5KSEgIy5cvp3Tp0jrlwcHBzJs3jy+++IKQkBDc3d2pX78+T58+1dYJDAxk+/btbN68mWPHjvHs2TOaNWtGcnLGnmojg1khhBBCCJXQaDRZtmTUs2fP6Nq1K19++SVOTk7ackVRmD9/PuPGjaNNmzaULFmSNWvWEBMTw8aNGwGIiopixYoVzJ07l3r16lGuXDnWr1/P+fPnOXDgQIbikMGsEEIIIYQgPj6eJ0+e6Czx8fFp1h84cCBNmzalXr16OuU3b97k3r17NGjQQFtmaWlJrVq1OH78OACnT58mMTFRp46npyclS5bU1kkvGcwKIYQQQqhEVmZmg4KCcHR01FmCgoJSjWPz5s2cOXMm1fX37t0DwM3NTafczc1Nu+7evXtYWFjoZHRfrZNe8jQDIYQQQgjBmDFjGDZsmE6ZpaWlXr2///6bjz/+mP3792NlZZXm/l6duqAoyhunM6SnzqskMyuEEEIIoRJZmZm1tLTEwcFBZ0ltMHv69GkePHhA+fLlMTMzw8zMjKNHj7JgwQLMzMy0GdlXM6wPHjzQrnN3dychIYHIyMg066SXDGaFEEIIIUS61a1bl/Pnz3P27FntUqFCBbp27crZs2cpUKAA7u7u/Pjjj9ptEhISOHr0KH5+fgCUL18ec3NznTphYWFcuHBBWye9ZJqBEEIIIYRaZIPfTLC3t6dkyZI6Zba2tri4uGjLAwMDmTFjBj4+Pvj4+DBjxgxsbGzo0qULAI6OjvTp04fhw4fj4uKCs7MzI0aMoFSpUno3lL2JDGaFEEIIIUSmGjlyJLGxsQQEBBAZGUnlypXZv38/9vb22jqfffYZZmZmdOjQgdjYWOrWrcvq1asxNTXN0LE0iqIomd0AkT7RCcbx0puaZIOvie/IqfViQ4eQKR5tCzB0CO8sPiljD8vOrqzMM/ZhnF3FJ6YYOoR3ZmkuM+pE5rIyYCow7+CdWbbv0IUtsmzfWUkys0IIIYQQKvE2P25g7FT9ddXf31/nd4Dz5cvH/PnzDRaPEEIIIYR4vww6mO3Zs6f2cRDm5uYUKFCAESNGEB0dbciwVC86+hmzZ82gSYM6VK1Qhp4fdOLihfOGDitDTp8KYXDAR9Tzr06ZEkU4dDBjP233vo1o50vsrgBm962mLVseWIfYXQE6y9HZbbTr8+ay11v/YmlTraAhmpGqFV8uo0vHtvhVKkftmlUJHBLArZt/GTqs11q9Yjk9u3Sgtl8FGtWuzieBg7h966ZOHUVR+HLJFzStX4ualcsxoE8P/rr+p4EizpgtmzbQuEEdKpYrRaf2bThz+pShQ3qtM6dDGDZkAE3q16RS2WIcOaR7PkdEhDN5/Bia1K9JjSrlGBLQj9DbtwwTbAaprS/SIu1Qj+z0c7bZhcEzs40aNSIsLIy//vqLadOmsXjxYkaMGGHosN5aQkKCoUNgysTx/HriOFNnzGLLtp1U8avGgH69eHD/vqFDS7fY2BiKFCnC6HETDB3KG5X3yUWfRsU5dzNcb90Pp2+Tr9sq7dJq8m7tujvhz3TW5eu2iikbfuNZbCI/nL79PpvwWqdP/UbHzl1Zu/Frli5fRXJSMgM+7ENsTIyhQ0vT76dP0a5jZ1as3cSCpV+RnJzMkAF9iY39N+Z1q1ewcf0aRoz+lFUbvsbZ1ZXBA/pm+y/T+/buIXhmEP0+HMCWb3fg61uegP79CLt719ChpSkuNhafwkX4ZPSneusUReGToYP455+/mfPZItZv3oaHhyeDPuqt01/ZkRr7IjXSDqF2Bh/MWlpa4u7ujpeXF126dKFr167s2LGDnj170qpVK526gYGB+Pv7p3vfoaGhtGzZEjs7OxwcHOjQoQP3/z+gu3r1KhqNhitXruhsM2/ePPLly8eL++IuXbpEkyZNsLOzw83NjW7duhEe/u+gxd/fn0GDBjFs2DBcXV2pX7/+270QmSQuLo5DB/bz8bARlK9Qkbx5vfkoYDCeufPwzZZNBo0tI6rXqMWgj4dSr36DN1c2IFsrM1YNr0fAwiM8fqb/+9UJicncfxyrXSJfqpOSouisu/84lhZV8vPtz9eJjkt6j614vcXLVtCyVRsKFfKhSNGiTJ4WRFjYXS5dumjo0NL0+eLlNGvZmgKFfChcpCjjJ0/nXlgYVy5dAp4PoDZvWEuvvv2pXbc+BQv5MHFqEHGxcfyw93sDR/9669asonXbtrRp154CBQsycsw43D3c+Tobn99+1WsyYFAgtevqn8+hobe4cO4PRo2dSPGSpfDOl5+RYycQExPDD3t3p7K37EONfZEaaYe6SGZWn8EHs6+ytrYmMTHxnfejKAqtWrXi0aNHHD16lB9//JEbN27QsWNHAIoUKUL58uXZsGGDznYbN26kS5cuaDQawsLCqFWrFmXLluXUqVPs27eP+/fv06FDB51t1qxZg5mZGb/88gvLli1759jfRXJyEsnJyVhY6P5ih6WlJWd/P22gqIzX/I9qsu/UbQ7/cSfV9TVK5ub2up6cW9qFRYP8yelonea+yhXMSdmCOVnz4+WsCjdTPHv2FHj+jEC1eBGzw/9jvvvPHSLCw6lc9d8Hc1tYWFCuQgXOnz1riBDTJTEhgcuXLlLVr7pOeVW/avxx9ncDRfVuEhOef96//CtDpqammJub88fvZwwV1hsZS19IO4QxyFZPM/jtt9/YuHEjdevWfed9HThwgHPnznHz5k28vLwAWLduHSVKlCAkJISKFSvStWtXvvjiC6ZOnQrAtWvXOH36NGvXrgVgyZIl+Pr6MmPGDO1+V65ciZeXF9euXaNw4cIAFCpUiODg4HeOOTPY2tpRukxZvlq2mAIFCuDs4sq+Pbu5cP4ceb29DR2eUWlfoxBlC+ak+rBvU12//1Qo247dIPTBU/K52TPhg8rsnd4Cv8BvSEjSf9xRjwbFuBz6iJNX7qWyt+xBURTmBgdRzrc8hXwKGzqcdFEUhc/nBlOmnC8FC/kAEPH/qyvOzq46dZ2dXbkXln0vSUY+jiQ5ORkXFxedchcXV8LDHxooqneTL19+PDw8WbTgM8aMn4S1tTUb160hIjw8W7fJWPpC2qFC6k2gZhmDZ2a///577OzssLKyomrVqtSsWZOFCxe+834vX76Ml5eXdiALULx4cXLkyMHly88zX506deL27ducPHkSgA0bNlC2bFmKFy8OPP/t4cOHD2NnZ6ddihYtCsCNGze0+61QocIb44mPj+fJkyc6S3y8/mXpzDA1KBhFUWhYtxZVypdm88Z1NGrSDBMT43juZXaQx9WO2f2q03vuAeITU38u6rfHrrPv1G0uhT5iT8htWk36Hh/PHDSumE+vrpWFKR1r+mT7rGzQ9Clcu3aNmcHzDB1Kus0Omsb1a1eZOnOO3jq9y2qKoopLba/GqKgk7tSYmZszc+4CQm/fol7NKtSs4svpU7/hV60GpiYG/xP1RsbSF9IOoWYGz8zWrl2bJUuWYG5ujqenJ+bm5gCYmJjw6u85ZGT6QVpv4JfLPTw8qF27Nhs3bqRKlSps2rSJ/v37a+umpKTQvHlzZs2apbcfDw8P7b9tbW3fGE9QUBCTJ0/WKRvz6QTGjZ+U3ialm5dXXr5avZ7YmBieRT8jZ85cjBoxlNy582T6sf6ryhXKiZuTDcfnt9eWmZmaUL2EJx81K4Vjm2WkpOi+f+9FxhD68CmFPPUvz7euVhAbSzM2HLqa5bG/rZkzpnL08CFWrlmPm7u7ocNJlzkzp/Hz0cMsW7kWN7d/Y3ZxfZ6RjYh4iGvOnNryR5ERODu76O0nu3DK4YSpqanOvH2AR48icHFxTWOr7K9Y8RJs+Ho7z54+JTExESdnZ3p90JFixUsYOrQ0GUtfSDvURwbn+gz+tdfW1pZChQrh7e2tHcgC5MyZk7CwMJ26ZzMwl6148eKEhoby999/a8suXbpEVFQUxYoV05Z17dqVLVu2cOLECW7cuEGnTp2063x9fbl48SL58uWjUKFCOkt6BrAvGzNmDFFRUTrLiJFjMrSPjLK2sSFnzlw8iYrixPFj1KpdJ0uP919y+I87lB+4mcpDvtYup/98wOaj16g85Gu9gSyAs70leVztCHukf4d2z/rF2P3bLcKfxL2P8DNEURSCpk/h4IH9LF+5htx5vN68kYEpisLsoGkcOXiARctX4vnKFznP3HlwcXXltxMntGWJiQn8fuoUpcqWfc/Rpp+5hQXFipfg5PFfdMpPHj9OmbLlDBRV5rGzt8fJ2ZnQ27e4fOkCNf3ffcpZVjGWvpB2qI/cAKbP4JnZtNSpU4fZs2ezdu1aqlatyvr167lw4QLlyqXvTVmvXj1Kly5N165dmT9/PklJSQQEBFCrVi2daQFt2rRhwIABDBgwgNq1a5M7d27tuoEDB/Lll1/SuXNnPvnkE1xdXbl+/TqbN2/myy+/zNBvB1taWurc4ABZ93O2x3/5GUV5Phft79DbzJ83m3z58tOiVZs3b5xNxERHExoaqv3/P3fucOXyZRwdHfHw9DRgZM89i03kUugjnbLouEQePYnjUugjbK3M+LRLJXb8coOwyBi8c9kzpXsVIp7EsfOk7jNaC3g4UL2EJ60mZ8+76GdMm8zePd8zf8FibG1ttfPP7OzssbKyMnB0qZs9Yyo/7N3N7PlfYGtrS8T/Y7b9f8wajYZOXbuzesVyvLy98crrzeqvlmNlbUXDxs0MHP3rdevRi3GjR1K8ZEnKlCnH1m+2EBYWRvuOnd68sYHExERz56Xz+e4/d7h25TIOjo64e3hyYP8+nJyccffw4Pqf15gXPINatetSxa/aa/ZqeGrsi9RIO4TaZdvBbMOGDRk/fjwjR44kLi6O3r170717d86fT9/D/zUaDTt27GDw4MHUrFkTExMTGjVqpDcf18HBgebNm/PNN9+wcuVKnXWenp788ssvjBo1ioYNGxIfH4+3tzeNGjXCJBvP5Xr29BlffD6P+/fv4eiYgzr16jNwyFCdzHd2d/HiBfr26q79/5zgIABatGzN1BkzDRVWuiWnKJTwdqZL7cLksLXkXmQMR8//Q7fg/TyL1Z0u06NeMe5GRHPg97/T2JthvXikW99e3XTKJ08LomU2/YK09ZvNAAzo20OnfPzk6TRr2RqAbj37EB8XR/CMKTx98oQSpUqzYMlXGb7q8r41atyEqMeRLF+ymIcPH1DIpzCLli7H0zP3mzc2kMsXLzKg3799MX/u86lbTZu3YuLUICLCHzJ/7iweRUTgmtOVJs1a0ufDAYYKN93U2BepkXaoi4oTqFlGo7w6MVW8N1mVmX3fTE3Uf2Y5tV5s6BAyxaNtAYYO4Z3FJ6V+Q53aWJkbxw2X8Yn6T95QG0vz7Jt8EOpkZcBUYKERe7Ns39fnNM6yfWelbJuZFUIIIYQQutQ8tzWryNdVIYQQQgihWpKZFUIIIYRQCUnM6pPMrBBCCCGEUC3JzAohhBBCqITMmdUng1khhBBCCJWQsaw+mWYghBBCCCFUSzKzQgghhBAqYWIEz3bPbJKZFUIIIYQQqiWZWSGEEEIIlZA5s/okMyuEEEIIIVRLMrNCCCGEECohj+bSJ5lZIYQQQgihWpKZFUIIIYRQCUnM6pPBrBBCCCGESsg0A30yzUAIIYQQQqiWZGaFEEIIIVRCMrP6JDMrhBBCCCFUSzKzBmQsX64UxdARvLtH2wIMHUKmcG443dAhvLOHe8caOgQhhMi2jGXskJkkMyuEEEIIIVRLMrNCCCGEECohc2b1SWZWCCGEEEKolmRmhRBCCCFUQhKz+mQwK4QQQgihEjLNQJ9MMxBCCCGEEKolmVkhhBBCCJWQxKw+ycwKIYQQQgjVksysEEIIIYRKyJxZfZKZFUIIIYQQqiWZWSGEEEIIlZDErD7JzAohhBBCCNWSzKwQQgghhErInFl9MpgVQgghhFAJGcvqk2kGQgghhBBCtSQzK4QQQgihEjLNQJ/qM7M9e/ZEo9Gg0WgwNzfHzc2N+vXrs3LlSlJSUgwdnsGt+HIZ5UoWZfbMGYYOJUNWfLmMLh3b4lepHLVrViVwSAC3bv5l6LAyTG3tGNHZj9hD45g9sL62LPbQuFSXoR2raOvk98zBlintCN0WyP1dI1g/oTW5nGwN0QStM6dCCBz0EQ3r1qB86aIcPnRAZ72iKCxbvJCGdWvgV7EMH/buxo3rfxoo2ozZsmkDjRvUoWK5UnRq34Yzp08ZOqTXOnM6hGFDBtCkfk0qlS3GkVf6IiYmmtlBU2nWwJ8alcvSoXVTvv16k4GizRi19UVapB1CzVQ/mAVo1KgRYWFh3Lp1i71791K7dm0+/vhjmjVrRlJSUqrbJCYmvuco37+L58+z7duv8SlcxNChZNjpU7/RsXNX1m78mqXLV5GclMyAD/sQGxNj6NAyRE3tKF/Egz7NynHuxn2d8nxt5+ssHwbvIiVFYftPVwCwsTLn++AuKIpC4+EbqDNkDRZmpmyd3sGgc7tiY2MpXKQoo8aMT3X9mlVfsWHdakaNGc/ajd/g4pqTgP69iY5+9p4jzZh9e/cQPDOIfh8OYMu3O/D1LU9A/36E3b1r6NDSFBcbi0/hInwy+tNU1382eyYnjh9j8vRgtmzbTeeuPZg7azpHDx98z5FmjBr7IjXSDnXRaLJuUSujGMxaWlri7u5O7ty58fX1ZezYsXz33Xfs3buX1atXA8/T8kuXLqVly5bY2toybdo0AHbt2kX58uWxsrKiQIECTJ48WWcAPGnSJPLmzYulpSWenp4MGTJEu27x4sX4+PhgZWWFm5sb7dq1e6/tfp2YmGjGjh7B+ElTcXBwMHQ4GbZ42QpatmpDoUI+FClalMnTgggLu8ulSxcNHVqGqKUdtlbmrBrbkoC5u3n8NE5n3f3IaJ2luV9hjp69xa2wxwBULZkHbzdH+s3axcWbD7l48yEfBn9PhaKe+JfL9/4b83/VatQkYHAgdeo10FunKAob16+ld7+PqFOvAYV8CjN52kzi4uLYt+d7A0SbfuvWrKJ127a0adeeAgULMnLMONw93Pl6S/bNZPpVr8mAQYHUrqvfFwDnz52lafOWlK9YCc/cuWndrgM+hYtw+dKF9xxpxqixL1Ij7RBqZxSD2dTUqVOHMmXKsG3bNm3ZxIkTadmyJefPn6d379788MMPfPDBBwwZMoRLly6xbNkyVq9ezfTp0wH49ttv+eyzz1i2bBl//vknO3bsoFSpUgCcOnWKIUOGMGXKFK5evcq+ffuoWbOmQdqamqBpU6hR058qVf0MHUqmePbsKQCOjo4GjuTdZNd2zP+4Eft+vc7hM7deWy+Xky2NqhRizZ4/tGWW5mYoQHxisrYsLiGJ5OQU/Ep5ZVHE7+aff+4QEf6QKlWracssLCwoX74if5z93YCRvV5iQgKXL12kql91nfKqftWyddxvUqZceX46cpgH9++jKAqnQn4l9PYtqrzSzuzEWPpC2qE+L6ZWZsWiVkZ9A1jRokU5d+6c9v9dunShd+/e2v9369aN0aNH06NHDwAKFCjA1KlTGTlyJBMnTiQ0NBR3d3fq1auHubk5efPmpVKlSgCEhoZia2tLs2bNsLe3x9vbm3Llyr3fBqZh357dXLl8ifWbvzV0KJlCURTmBgdRzrc8hXwKGzqct5Zd29G+dnHK+rhTfcDKN9b9oEEpnsYksOPnK9qy3y79Q3RsAtM/rMOErw6j0WiY/mEdTE1NcHe2y8rQ31pE+EMAXFxcdMqdXVwIC8u+lyQjH0eSnJysF7eLiyvh/2+TGo0YNZbpkyfQrKE/pmZmmGg0jJs4lbLlyhs6tDQZS19IO4QxMOrBrKIoOt80KlSooLP+9OnThISEaDOxAMnJycTFxRETE0P79u2ZP38+BQoUoFGjRjRp0oTmzZtjZmZG/fr18fb21q5r1KgRrVu3xsbGJtVY4uPjiY+P1ylLNrHA0tIyE1sM98LCmD1zBouXr8j0fRtK0PQpXLt2jdVrNxo6lHeSHduRJ6c9swfWp/nITTqZ1bR0b1yGLQcv6NQNj4qh65RtLAhsTEDriqQoCl8fusiZa2EkpyhZGf67eyUToSigIftnJ17NoLz6Wac2Wzau58L5P5j7+WLcPTz5/cwpgmdMwdU1J5WqZO+rS8bSF9IO9TCy5mQKox7MXr58mfz582v/b2ure3d1SkoKkydPpk2bNnrbWllZ4eXlxdWrV/nxxx85cOAAAQEBzJ49m6NHj2Jvb8+ZM2c4cuQI+/fvZ8KECUyaNImQkBBy5Miht7+goCAmT56sUzb20wmMmzApU9r6wuVLF3n0KIKuHdtqy5KTkzlz+hRbNm3g1zPnMDU1zdRjZqWZM6Zy9PAhVq5Zj5u7u6HDeWvZtR3lCnvg5mzH8WV9tGVmpiZUL52Xj1pVwLHhTFL+PyCtVsqLInld6TZlu95+Dp66SYkPFuPiYE1ScgpR0fHc/PZjbt97/L6akiEurjkBiAgPJ2fOXNryyEcROL+S2clOnHI4YWpqSnh4uE75o0cRuLi4GiiqdxMXF8fihfMJnreA6jX9AfApXIRrVy+zfu2qbDuYNZa+kHaoj7ENzjOD0Q5mDx06xPnz5xk6dGiadXx9fbl69SqFChVKs461tTUtWrSgRYsWDBw4kKJFi3L+/Hl8fX0xMzOjXr161KtXj4kTJ5IjRw4OHTqU6uB4zJgxDBs2TKcs2cTi7RuYhkpVqvDN9p06ZRM/HUv+/AXo2aevagayiqIwc8ZUDh38ka9WrSN3nuw59/JNsns7Dp+5Rfney3XKlo9sxtW/I5i76YR2IAvQo3EZTl8N4/xfD9LcX8STWABqlfMmVw5bvj9+LWsCf0e5c+fBxTUnv544TtFixQFITEzg9OkQhgQON3B0aTO3sKBY8RKcPP4Ldev9+/i0k8eP41+nrgEje3tJSUkkJSViYqJ7C4epiSlKNn68orH0hbRDGAOjGMzGx8dz7949kpOTuX//Pvv27SMoKIhmzZrRvXv3NLebMGECzZo1w8vLi/bt22NiYsK5c+c4f/4806ZNY/Xq1SQnJ1O5cmVsbGxYt24d1tbWeHt78/333/PXX39Rs2ZNnJyc2LNnDykpKRQpkvpjsCwtLfUu+8ckZv4lWFtbO735mNbW1jjmyJGt5mm+yYxpk9m753vmL1iMra2tds6TnZ09VlZWBo4u/bJ7O57FJnDplu58sui4RB49idUpt7exoE2tYoxemvqjkro1Ks3V2+E8jIqhcvE8zBlYn4Xf/sqffz/K0vhfJyYmmr9DQ7X/v/vPHa5euYyDoyMeHp50+aA7K1csw8vbm7x5vVn51TKsrKxo1KSZwWJOj249ejFu9EiKlyxJmTLl2PrNFsLCwmjfsZOhQ0tTTEw0d17pi2v/7wt3D098y1dkwWezsbS0wt3Tk99PhbDn++/4ePgoA0b9Zmrsi9RIO9RFMrP6jGIwu2/fPjw8PDAzM8PJyYkyZcqwYMECevToofdt/2UNGzbk+++/Z8qUKQQHB2Nubk7RokXp27cvADly5GDmzJkMGzaM5ORkSpUqxa5du3BxcSFHjhxs27aNSZMmERcXh4+PD5s2baJEiRLvq9lG7Zv/P0qlb69uOuWTpwXRspV+5ju7MpZ2tK9dAo1Gw9eHUn+kWGEvF6b0rY2zvTW37z0meMMvLPj2t/ccpa5LFy/Qv08P7f/nzZ4JQLMWrZg8bSY9evUlPi6OmdOn8PRJFCVLlWbR0hXY2mbPm9ZeaNS4CVGPI1m+ZDEPHz6gkE9hFi1djqdnbkOHlqbLFy8yoN+/fTF/7iwAmjZvxcSpQUybNZfFCz5jwthPePIkCncPTz4aFEjb9tl7EKLGvkiNtEOonUZRlGx+h4bxyorMrCGo4YaZ/wrnhtPfXCmbe7h3rKFDyBRmpsZxXsQnZt9L/ellaW60T6EUBmJlwFRgrc9+ybJ9Hx1a7c2VsiE5w4UQQgghhGoZxTQDIYQQQoj/Apkzq08ys0IIIYQQQrUkMyuEEEIIoRKSmNUng1khhBBCCJWQaQb6ZJqBEEIIIYRQLcnMCiGEEEKohCRm9UlmVgghhBBCqJZkZoUQQgghVMJEUrN6JDMrhBBCCCFUSzKzQgghhBAqIYlZfZKZFUIIIYQQqiWZWSGEEEIIlZDnzOqTwawQQgghhEqYyFhWj0wzEEIIIYQQqiWZWSGEEEIIlZBpBvokMyuEEEIIIVRLMrNCCCGEECohiVl9Mpg1IPkVj+wjMTnF0CFkiod7xxo6hHeWs9MKQ4eQKSK/6WvoEDKFfEwJIbI7GcwKIYQQQqiEBvmG+SqZMyuEEEIIITJkyZIllC5dGgcHBxwcHKhatSp79+7VrlcUhUmTJuHp6Ym1tTX+/v5cvHhRZx/x8fEMHjwYV1dXbG1tadGiBXfu3MlwLDKYFUIIIYRQCRNN1i0ZkSdPHmbOnMmpU6c4deoUderUoWXLltoBa3BwMPPmzeOLL74gJCQEd3d36tevz9OnT7X7CAwMZPv27WzevJljx47x7NkzmjVrRnJycoZi0SiKomQsfJFZ4pIMHYF4wVjmzBrD5SeZM5u9JCSp/9ywMJO8jchcVgacpNlieUiW7XvnhxXfaXtnZ2dmz55N79698fT0JDAwkFGjRgHPs7Bubm7MmjWL/v37ExUVRc6cOVm3bh0dO3YE4O7du3h5ebFnzx4aNmyY7uPKGS6EEEIIoRIajSbLlvj4eJ48eaKzxMfHvzGm5ORkNm/eTHR0NFWrVuXmzZvcu3ePBg0aaOtYWlpSq1Ytjh8/DsDp06dJTEzUqePp6UnJkiW1ddJLBrNCCCGEECqh0WTdEhQUhKOjo84SFBSUZiznz5/Hzs4OS0tLPvroI7Zv307x4sW5d+8eAG5ubjr13dzctOvu3buHhYUFTk5OadZJL3magRBCCCGEYMyYMQwbNkynzNLSMs36RYoU4ezZszx+/JitW7fSo0cPjh49ql3/6q+VKYryxl8wS0+dV8lgVgghhBBCJbLyGfWWlpavHby+ysLCgkKFCgFQoUIFQkJC+Pzzz7XzZO/du4eHh4e2/oMHD7TZWnd3dxISEoiMjNTJzj548AA/P78MxS3TDIQQQgghxDtTFIX4+Hjy58+Pu7s7P/74o3ZdQkICR48e1Q5Uy5cvj7m5uU6dsLAwLly4kOHBrGRmhRBCCCFUIrv8Kt/YsWNp3LgxXl5ePH36lM2bN3PkyBH27duHRqMhMDCQGTNm4OPjg4+PDzNmzMDGxoYuXboA4OjoSJ8+fRg+fDguLi44OzszYsQISpUqRb169TIUiwxmhRBCCCFEhty/f59u3boRFhaGo6MjpUuXZt++fdSvXx+AkSNHEhsbS0BAAJGRkVSuXJn9+/djb2+v3cdnn32GmZkZHTp0IDY2lrp167J69WpMTU0zFIs8Z9aA5Dmz2Yc8Zzb7kOfMZi/ynFkh9BnyObPtVp3Jsn1/28s3y/adleQMF0IIIYQQqiXTDIQQQgghVCK7zJnNTmQwK4QQQgihEln5aC61kmkGQgghhBBCtYx+MHvv3j0GDx5MgQIFsLS0xMvLi+bNm3Pw4MFMO0a+fPmYP39+pu0vM2zZtIHGDepQsVwpOrVvw5nTpwwdUoacPhXC4ICPqOdfnTIlinDo4AFDh5QuZ06FMHTQABrVrUmF0sU4cujfuJMSE1nw2Rw6tmlB9Uq+NKpbkwljR/HwwQMDRqzvzKkQAgd9RMO6NShfuiiHD+m+9ocO7GfgR32oU7MK5UsX5eqVywaKNG0j2pQhdntfZveukur6hR9VI3Z7XwY1K6FTnt/dni2j6hG6uiv3N3Rn/Yg65HK0fh8hZ4jazu8zp0MYOngAjevVpGIZ3fMCoGKZYqku61Zn/5sB1dYXaZF2qIcmCxe1MurB7K1btyhfvjyHDh0iODiY8+fPs2/fPmrXrs3AgQMNHV6W2bd3D8Ezg+j34QC2fLsDX9/yBPTvR9jdu4YOLd1iY2MoUqQIo8dNMHQoGRIbG4tPkSKMHPOp3rq4uDiuXL5E3/4DWL9lK7PnLSD09i2GDQkwQKRpi42NpXCRoowaMz7N9WXK+jL44+HvObL0KV/IlT4NinLuZkSq65tX8qZi4VzcjYjWKbexNOP7iY1RgMYT9lBnzC4szEzYOq5+tpqjpsbz+/l7qgifjNY/LwD2HvxJZxk/eToajYba9Rq850gzRo19kRpph1A7ox7MBgQEoNFo+O2332jXrh2FCxemRIkSDBs2jJMnTwIQGhpKy5YtsbOzw8HBgQ4dOnD//n3tPm7cuEHLli1xc3PDzs6OihUrcuDAv1kFf39/bt++zdChQ9FoNBn+PeGssG7NKlq3bUubdu0pULAgI8eMw93Dna+3bDJ0aOlWvUYtBn08lHr1s/cfs1dVq1GTgMGB1Enlj7CdvT2Ll6+kfsPG5Mufn1JlyvLJmE+5fOki98Kyz4ft69oA0LR5Sz78aCCVq1R9z5G9ma2VGauG1iZg8c88jk7QW+/pbMNn/fzo9dlhvcexVS3qhndOO/otOMrF0Eguhkby4cKfqOCTC/9Snu+rCW+kxvO7WvWaDBiU9nvK1TWnzvLTkUOUr1iZPHm83nOkGaPGvkiNtENdXow1smJRK6MdzD569Ih9+/YxcOBAbG1t9dbnyJEDRVFo1aoVjx494ujRo/z444/cuHGDjh07aus9e/aMJk2acODAAX7//XcaNmxI8+bNCQ0NBWDbtm3kyZOHKVOmEBYWRlhY2HtrY2oSExK4fOkiVf2q65RX9avGH2d/N1BUIi3Pnj1Fo9FgZ+9g6FCMwvwP/dh3KpTD5/S/HGg0sCLQn8++O8flvx/rrbc0N0UB4hOTtWVxickkJ6fgV8w9C6NOv//C+R0REc6xn4/SsnVbQ4fyWsbSF9IOYQyM9mkG169fR1EUihYtmmadAwcOcO7cOW7evImX1/MMwLp16yhRogQhISFUrFiRMmXKUKZMGe0206ZNY/v27ezcuZNBgwbh7OyMqakp9vb2uLsb/g9e5ONIkpOTcXFx0Sl3cXElPPyhgaISqYmPj+eL+fNo1KQZdnZ2hg5H9dpXL0DZAq5U/+S7VNcPb12GpOQUFn1/MdX1v117QHRcEtO7V2LC+hA0Gg3Tu1fE1NQEd6fsMW/2v3B+7965A1sbW2rXrW/oUF7LWPpC2qE+JupNoGYZo83Mvvhhs9elzS9fvoyXl5d2IAtQvHhxcuTIweXLz29qiY6OZuTIkdpyOzs7rly5os3Mpld8fDxPnjzRWeLj49+iZenzarsVRVH1JQRjk5SYyNiRw0lJSWGUyuYFZ0d5XGyZ3acqvecf0cmsvlCugAsDm5XgwwU/pbmP8CdxdJ19kCYV8xK+qSf3N3THwcaCMzfCSU7JXj+UaMzn984d22jUpBmWlpaGDiVdjKUvpB1CzYw2M+vj44NGo+Hy5cu0atUq1TppvclfLv/kk0/44YcfmDNnDoUKFcLa2pp27dqRkKA/H+91goKCmDx5sk7ZuPET+XTCpAzt502ccjhhampKeHi4TvmjRxG4uLhm6rHE20lKTGT0J0O5+88dlny1SrKymaBcQVfcclhzfE4rbZmZqQnVi7vzUZPifLo2hFyO1lz7spPO+pk9KzOoeUmK9t8CwME//qHEgK9xsbckKVkhKiaBmyu7cPvB0/fdpFQZ+/n9+5lT3L51kxnB8wwdyhsZS19IO9RHBuf6jHYw6+zsTMOGDVm0aBFDhgzRmzf7+PFjihcvTmhoKH///bc2O3vp0iWioqIoVqwYAD///DM9e/akdevWwPM5tLdu3dLZl4WFBcnJ+tmgl40ZM4Zhw4bplCmmmZ95MLewoFjxEpw8/gt16/17me7k8eP416mb6ccTGfNiIBt6+zbLVqwhRw4nQ4dkFA6fu0v5j7fqlC0fVJOr/zxm7vZz3IuM4cezd3TW75rQiI1Hr7P24DW9/UU8fX7VpFYpD3I5WvP9bxm7EpNVjP38/m77VooVL0HhImlPD8sujKUvpB3qI2NZfUY7mAVYvHgxfn5+VKpUiSlTplC6dGmSkpL48ccfWbJkCZcuXaJ06dJ07dqV+fPnk5SUREBAALVq1aJChQoAFCpUiG3bttG8eXM0Gg3jx48nJUX3Luh8+fLx008/0alTJywtLXF11f8WaGlpqXfZLC4pa9rdrUcvxo0eSfGSJSlTphxbv9lCWFgY7Tt2evPG2URMdLTOVI5/7tzhyuXLODo64uGZfe4sf1VMTDR/vxz3P3e4euV53K45czFyeCBXL1/isy+WkJySrJ3L5ejoiLm5haHC1vFqG+7+vw0Ojo54eHgSFfWYe2FhPHz4/Pm4t2/dBMDF1RVX15wGiflZXCKXQiN1yqLjk3j0NF5b/uip7rSexOQU7kfG8OfdKG1Ztzo+XL3zmIdP4qhcxI05faqwcNcFnTqGpsbzO633lKOjI+4ez8/nZ8+ecXD/DwQOH2moMDNMjX2RGmmHUDujHszmz5+fM2fOMH36dIYPH05YWBg5c+akfPnyLFmyBI1Gw44dOxg8eDA1a9bExMSERo0asXDhQu0+PvvsM3r37o2fnx+urq6MGjWKJ0+e6BxnypQp9O/fn4IFCxIfH6+dr2sojRo3IepxJMuXLObhwwcU8inMoqXL8fTMbdC4MuLixQv07dVd+/85wUEAtGjZmqkzZhoqrDe6dPEiH/Xpof3/Z7NnAdCsRSs+HDCIn44cAqBL+9Y62y1dsYYKFSu9v0Bf49LFC/R/qQ3zZj9/vZu1aMXkaTM5euQQk8eP1a4fM/L5FYcPPxpI/4DB7zfYTFY4dw6mfFARZztLbj98RvC3Z1mw84Khw9KhxvP78sWLfNT3pfNizvPzommLVkya+vzc3r9vDwoKDRs3NUiMb0ONfZEaaYe6yDQDfRrF0COv/7CsysyKjHv1maNqpVH1b7g8l7NT9v/Vp/SI/KavoUPIFAlJ6j83LMyM9l5nYSBWBkwFdt94Lsv2vbZL6Szbd1Yy6sysEEIIIYQxkUdz6ZOvq0IIIYQQQrUkMyuEEEIIoRIyZ1afZGaFEEIIIYRqSWZWCCGEEEIlJC+rTwazQgghhBAqYSLTDPTINAMhhBBCCKFakpkVQgghhFAJSczqk8ysEEIIIYRQLcnMCiGEEEKohDyaS59kZoUQQgghhGpJZlYIIYQQQiUkMatPMrNCCCGEEEK1JDMrhBBCCKES8pxZfTKYFUIIIYRQCRnL6pNpBkIIIYQQQrUkMyuEEEIIoRLyaC59kpkVQgghhBCqJZlZIQBzU/lel11EftPX0CFkCqda4wwdQqaIPDrd0CG8s6RkxdAhZAozU8nICclCpkZeEyGEEEIIoVqSmRVCCCGEUAmZM6tPMrNCCCGEEEK1JDMrhBBCCKESJpKY1SODWSGEEEIIlZDBrD6ZZiCEEEIIIVRLMrNCCCGEECohN4Dpk8ysEEIIIYRQLcnMCiGEEEKohMyZ1SeZWSGEEEIIoVqSmRVCCCGEUAmZMqtPMrNCCCGEEEK1JDMrhBBCCKESJpKa1SODWSGEEEIIlZBL6vrkNRFCCCGEEKolg9k09OzZk1atWqW7/q1bt9BoNJw9ezbLYhJCCCHEf5tGk3WLWmX7weyDBw/o378/efPmxdLSEnd3dxo2bMiJEycMHVq2tmXTBho3qEPFcqXo1L4NZ06fMnRIb8UY2mEMbQBpx/s2oltNYn+ZzuyPm2jLcjnZsnxcW/76bhQRByfy3dweFMzjorOdhbkp84Y24+/dYwk/MJFvZn1A7pwO7zv8dFFLX7xw5lQIgYM+omHdGpQvXZTDhw7orFcUhWWLF9Kwbg38Kpbhw97duHH9TwNFmzFq64tXnT4VwuCAj6jnX50yJYpw6OCBN28kjEa2H8y2bduWP/74gzVr1nDt2jV27tyJv78/jx49MnRo2da+vXsInhlEvw8HsOXbHfj6liegfz/C7t41dGgZYgztMIY2gLTjfStfNDd9WlTk3J9hOuVfz/yA/J5OtB+1niq9FhF67zF7Pu+FjZW5ts7sj5vSomZxuk/cQt0By7GztmDr7O6YZLMnraulL14WGxtL4SJFGTVmfKrr16z6ig3rVjNqzHjWbvwGF9ecBPTvTXT0s/ccacaosS9eFRsbQ5EiRRg9boKhQ8lyJhpNli1qla0Hs48fP+bYsWPMmjWL2rVr4+3tTaVKlRgzZgxNmzYFYN68eZQqVQpbW1u8vLwICAjg2bN/PzhWr15Njhw5+OGHHyhWrBh2dnY0atSIsLB//0gkJyczbNgwcuTIgYuLCyNHjkRRFJ1Y9u3bR/Xq1bV1mjVrxo0bN97PC5FB69asonXbtrRp154CBQsycsw43D3c+XrLJkOHliHG0A5jaANIO94nW2sLVk3sQMCsHTx+GqstL+TlQuWSeRkyZyenr/zDn6HhfDx3J7bWlnSoXwYAB1tLejYrz+gv9nL41A3++DOM3lO+oWQBN+pUKGioJqVKDX3xqmo1ahIwOJA69RrorVMUhY3r19K730fUqdeAQj6FmTxtJnFxcezb870Bok0/NfbFq6rXqMWgj4dSr75+3wjjl60Hs3Z2dtjZ2bFjxw7i4+NTrWNiYsKCBQu4cOECa9as4dChQ4wcOVKnTkxMDHPmzGHdunX89NNPhIaGMmLECO36uXPnsnLlSlasWMGxY8d49OgR27dv19lHdHQ0w4YNIyQkhIMHD2JiYkLr1q1JSUnJ/Ia/g8SEBC5fukhVv+o65VX9qvHH2d8NFFXGGUM7jKENIO143+YPb86+E1c5fEr3y7Kl+fOHz8QlJGnLUlIUEhKT8SvtDUC5IrmxMDfjwG//XtoOC3/Kxb/uU6WU93uIPn3U0hcZ8c8/d4gIf0iVqtW0ZRYWFpQvXzFbt8kY+8LYyZxZfdl6MGtmZsbq1atZs2YNOXLkoFq1aowdO5Zz585p6wQGBlK7dm3y589PnTp1mDp1Kl9//bXOfhITE1m6dCkVKlTA19eXQYMGcfDgQe36+fPnM2bMGNq2bUuxYsVYunQpjo6OOvto27Ytbdq0wcfHh7Jly7JixQrOnz/PpUuXsvZFyKDIx5EkJyfj4qI7j87FxZXw8IcGiirjjKEdxtAGkHa8T+3rlqJsYU/GL92vt+7q7YfcDotkav8G5LC3wtzMlBEf1MTD1R53F3sA3F3siE9I4vHTOJ1tH0Q+w83Z7r20IT3U0BcZFfH/uF9tk7OLCxER4YYIKV2MsS/Ef0+2HszC80Hk3bt32blzJw0bNuTIkSP4+vqyevVqAA4fPkz9+vXJnTs39vb2dO/enYiICKKjo7X7sLGxoWDBfy+xeXh48ODBAwCioqIICwujatWq2vVmZmZUqFBBJ44bN27QpUsXChQogIODA/nz5wcgNDQ0Xe2Ij4/nyZMnOkta2ebMoHnlK5aiKHplamAM7TCGNoC0I6vlyeXI7MBm9J7yDfEvZV9fSEpOofO4jRTK60rYvvE8OjiRGuXys+/EVZLfcIVIo9GgvLaGYWTXvngnem0CDdm/TUbZF0bKRJN1i1pl+8EsgJWVFfXr12fChAkcP36cnj17MnHiRG7fvk2TJk0oWbIkW7du5fTp0yxatAh4no19wdzcXGd/Go1Gb07smzRv3pyIiAi+/PJLfv31V3799VcAEhIS0rV9UFAQjo6OOsvsWUEZiiE9nHI4YWpqSni4bibg0aMIXFxcM/14WcUY2mEMbQBpx/tSrognbs52HF8RwNOjU3h6dAo1fQsQ0K4qT49OwcREw+9X71Kl5xe4NZhC/pYzaTl8DS4ONty6GwnAvYhnWFqYkcPeSmffOXPY8uBR9rkJKbv3xdtwcc0JQMQrbYp8FIHzK1nP7MQY+8LYyQ1g+lQxmH1V8eLFiY6O5tSpUyQlJTF37lyqVKlC4cKFuZvBuy8dHR3x8PDg5MmT2rKkpCROnz6t/X9ERASXL1/m008/pW7duhQrVozIyMgMHWfMmDFERUXpLJ+MGpOhfaSHuYUFxYqX4OTxX3TKTx4/Tpmy5TL9eFnFGNphDG0Aacf7cvj0Dcp/8DmVe36hXU5fvsPm/X9QuecXpKT8+wX8SXQ84Y9jKJjHBd+iufn+2GUAfr/6DwmJSdStWEhb193FnhIF3Dh5/vZ7b1NasntfvI3cufPg4pqTX08c15YlJiZw+nRItm6TMfaF+O/J1j9nGxERQfv27enduzelS5fG3t6eU6dOERwcTMuWLSlYsCBJSUksXLiQ5s2b88svv7B06dIMH+fjjz9m5syZ+Pj4UKxYMebNm8fjx4+1652cnHBxcWH58uV4eHgQGhrK6NGjM3QMS0tLLC0tdcri9K8kZopuPXoxbvRIipcsSZky5dj6zRbCwsJo37FT1hwwixhDO4yhDSDteB+exSRw6eYDnbLo2AQePYnRlrepXZKHj6P5+/5jShZwZ05gU3b9fImDv10Hng9yV39/mpmDGhMRFUPkk1iCBjXmwl/3OXQqez19JTv3RVpiYqL5+6WpZXf/ucPVK5dxcHTEw8OTLh90Z+WKZXh5e5M3rzcrv1qGlZUVjZo0M2DUb6bGvnhVTHS0zrS/f+7c4crly88TVp6eBows86k4gZplsvVg1s7OjsqVK/PZZ59x48YNEhMT8fLyol+/fowdOxZra2vmzZvHrFmzGDNmDDVr1iQoKIju3btn6DjDhw8nLCyMnj17YmJiQu/evWndujVRUVHA8ycmbN68mSFDhlCyZEmKFCnCggUL8Pf3z4JWv7tGjZsQ9TiS5UsW8/DhAwr5FGbR0uV4euY2dGgZYgztMIY2gLQju3B3sWfW4MbkcrbjXsRTNuw7S9Cqwzp1Ri7YQ3JyCuundsba0ozDp/7iw+nrdDK72YEa++LSxQv079ND+/95s2cC0KxFKyZPm0mPXn2Jj4tj5vQpPH0SRclSpVm0dAW2ttnn5rvUqLEvXnXx4gX69vr3b/+c4OfT+Fq0bM3UGTMNFZZ4TzRKRiePikyTVZlZIYThOdUaZ+gQMkXk0emGDuGdJSUbx585M1NJyWUXVgZMBU4/eD3L9j2ubqE3V8qGVDlnVgghhBBCCMjm0wyEEEIIIcS/1PCot/dNMrNCCCGEEEK1JDMrhBBCCKESav5xg6wimVkhhBBCCKFakpkVQgghhFAJyczqk8GsEEIIIYRKaORXE/TINAMhhBBCCKFakpkVQgghhFAJmWagTzKzQgghhBBCtSQzK4QQQgihEjJlVp9kZoUQQgghhGpJZlYIIYQQQiVMJDWrRzKzQgghhBBCtSQzK4QQQgihEvI0A30ymBVCCCGEUAmZZaBPphkIIYQQQgjVksysEEIIIYRKmCCp2VfJYNaAnsQmGjqETGFvZW7oEN5ZfFKyoUPIFGYm6r/YEh2fZOgQMkXk0emGDiFTlBy919AhvLMLMxsbOgQhjE5QUBDbtm3jypUrWFtb4+fnx6xZsyhSpIi2jqIoTJ48meXLlxMZGUnlypVZtGgRJUqU0NaJj49nxIgRbNq0idjYWOrWrcvixYvJkydPumNR/18+IYQQQoj/CI0m65aMOHr0KAMHDuTkyZP8+OOPJCUl0aBBA6Kjo7V1goODmTdvHl988QUhISG4u7tTv359nj59qq0TGBjI9u3b2bx5M8eOHePZs2c0a9aM5OT0J5k0iqIoGQtfZJYHTyUzm11IZjb7MJbMrKON+s8LkMysEKmxMuB17cXHb2XZvgP88r31tg8fPiRXrlwcPXqUmjVroigKnp6eBAYGMmrUKOB5FtbNzY1Zs2bRv39/oqKiyJkzJ+vWraNjx44A3L17Fy8vL/bs2UPDhg3TdWz1/+UTQgghhPiPMNFk3fIuoqKiAHB2dgbg5s2b3Lt3jwYNGmjrWFpaUqtWLY4fPw7A6dOnSUxM1Knj6elJyZIltXXSQ+bMCiGEEEII4uPjiY+P1ymztLTE0tLytdspisKwYcOoXr06JUuWBODevXsAuLm56dR1c3Pj9u3b2joWFhY4OTnp1XmxfXpIZlYIIYQQQiVMNJosW4KCgnB0dNRZgoKC3hjToEGDOHfuHJs2bdJbp3llMq6iKHplr0pPnZdJZlYIIYQQQiWy8kcTxowZw7Bhw3TK3pSVHTx4MDt37uSnn37SeQKBu7s78Dz76uHhoS1/8OCBNlvr7u5OQkICkZGROtnZBw8e4Ofnl+64JTMrhBBCCCGwtLTEwcFBZ0lrMKsoCoMGDWLbtm0cOnSI/Pnz66zPnz8/7u7u/Pjjj9qyhIQEjh49qh2oli9fHnNzc506YWFhXLhwIUODWcnMCiGEEEKohEk2+T3bgQMHsnHjRr777jvs7e21c1wdHR2xtrZGo9EQGBjIjBkz8PHxwcfHhxkzZmBjY0OXLl20dfv06cPw4cNxcXHB2dmZESNGUKpUKerVq5fuWGQwK4QQQgghMmTJkiUA+Pv765SvWrWKnj17AjBy5EhiY2MJCAjQ/mjC/v37sbe319b/7LPPMDMzo0OHDtofTVi9ejWmpqbpjkWeM2tA8pzZ7EOeM5t9yHNmsxd5zqwQ+gz5nNmVIaFZtu/eFfNm2b6zkvr/8gkhhBBCiP8smWYghBBCCKESkoXUJ6+JEEIIIYRQLcnMCiGEEEKoREZ+TOC/QgazQgghhBAqIUNZfTLNQAghhBBCqNZ/ejCr0WjYsWNHmuuPHDmCRqPh8ePH7y0mIYQQQoi0mGg0WbaolVFPM3jw4AHjx49n79693L9/HycnJ8qUKcOkSZOoWrXqG7f38/MjLCwMR0fH19br2bMnjx8/fu3AOKts/3YzO77dwr2wuwDkL1CInn0/okq1GgDUqFAy1e0GDBlGl+6931ucGbXiy2UcPLCfWzf/wtLKijJlyxE4dAT58hcwdGhpWr1iOUcOHuD2rb+wtLSiVJmyDAocjne+f3/i7/DBH9n+7ddcuXyRqMePWbd5K4WLFjNg1PrOnAph7eoVXL58kfCHD5kz/wtq1/n3l1gURWH5ki/YtvVrnj55QslSpRk1dgIFC/kYMGpdO77dzI6tuudFjz7/nhcxMTEs++Izjh09RFTUY9w9PGnXsSut2nUyZNjptmXTBlavWkH4w4cULOTDyNFj8S1fwdBhAdClal66VPUij7MNAH/ee8rCA9f56Uo4ANfnpP7M15nfX+GrIzcByOtiw+hmRaiQ3xkLMxN+uvqQydsvEfEs4f00IgOyc1+k1+lTIaxeuYLLly7w8OFDPluwiDp10//rS9mJMfSHyDijzsy2bduWP/74gzVr1nDt2jV27tyJv78/jx49Stf2FhYWuLu7pznZOjk5mZSUlMwMOcNy5XLno0FD+XLtFr5cuwXfCpUYM3wwN29cB2DHviM6y+gJU9FoNPjXqW/QuN/k9Knf6Ni5K2s3fs3S5atITkpmwId9iI2JMXRoafr99CnadezMirWbWLD0K5KTkxkyoC+xsf/GHBsbS+my5Rg4ZJgBI3292NhYChcpyqgx41Ndv2bVV2xYt5pRY8azduM3uLjmJKB/b6Kjn73nSNOWM5c7/QcN5cs1W/hyzfPzYuyIf8+LL+bN4rcTx/h0ShDrvt5Jh87d+XxOED8fPWTgyN9s3949BM8Mot+HA9jy7Q58fcsT0L8fYXfvGjo0AO5FxTF7zzVazf+FVvN/4cT1CJb2LI+Pmx0AVSYf1FlGbTlHSorCD+ee/xSmtYUpq/tVRAE+WPorHb44gbmpCct7lye7JY6ye1+kV2xsDEWKFGH0uAmGDuWdGEt/vIkmCxe1MtrB7OPHjzl27BizZs2idu3aeHt7U6lSJcaMGUPTpk219cLDw2ndujU2Njb4+Piwc+dO7bpXpxmsXr2aHDly8P3331O8eHEsLS3p1asXa9as4bvvvkOj0aDRaDhy5Mh7a2e1mv5UrV6TvN75yOudjw8Hfoy1jQ0Xz/8BgIurq85y7OhhylWohGcer/cW49tYvGwFLVu1oVAhH4oULcrkaUGEhd3l0qWLhg4tTZ8vXk6zlq0pUMiHwkWKMn7ydO6FhXHl0iVtnSbNWtC3fwAVK7/5yoChVKtRk4DBgdSp10BvnaIobFy/lt79PqJOvQYU8inM5GkziYuLY9+e7w0Qbeqq1fSnarWaeHnnw8s7H/0C/n9eXHh+Xlw8/weNmrakXPlKeHjmpkWb9hT0KcLVbPz+emHdmlW0btuWNu3aU6BgQUaOGYe7hztfb9lk6NAAOHTpAUevPORWeAy3wmOYt+9PYhKSKOudA4Dwpwk6S70Sbpy8EcHfj2IBKJ/PidzO1ozafJ5r955x7d4zRm05R5m8OahayMWALdOX3fsivarXqMWgj4dSr77+Oa8mxtIfIuOMdjBrZ2eHnZ0dO3bsID4+Ps16kydPpkOHDpw7d44mTZrQtWvX12ZuY2JiCAoK4quvvuLixYssWLCADh060KhRI8LCwggLC8PPzy8rmvRGycnJHPhhD3GxsZQoXVZv/aOIcE4c+4lmLdu8/+De0bNnTwHeOOUjO3kRs4OKYn6Tf/65Q0T4Q6pUraYts7CwoHz5ivxx9ncDRpa25ORkDu5/fl6ULFUWgFJly/HLT4d5+OA+iqJw5tRv/B16i0ovtSs7SkxI4PKli1T1q65TXtWvWrZ8/U000LSsBzYWZvx++7Heehc7C/yL5eSb3+5oyyzMTFAUhYSkf696xSemkJyiUCG/0/sIO13U1hfG7r/UHxpN1i1qZbRzZs3MzFi9ejX9+vVj6dKl+Pr6UqtWLTp16kTp0qW19Xr27Ennzp0BmDFjBgsXLuS3336jUaNGqe43MTGRxYsXU6ZMGW2ZtbU18fHxuLu7pxlPfHy83qA6PsEES0vLd2kmADeuX2NAr64kJCRgbW3D9Nmfk79AQb16e7/fiY2tDTVrq2sulKIozA0OopxveQr5FDZ0OOmiKAqfzw2mTDnfbDWX9F1FhD8EwMVFN0Pm7OJCWFj2upR34/o1Anr/e15Mm/05+f5/Xnw8YizB0yfStmldTE3NMDHRMPLTyZQu62vgqF8v8nEkycnJeq+/i4sr4f/vm+ygsLsd3wyuiqWZCTEJyQxYfYbr9/WnobSpkJvo+CR+OH9fW3b29mNiE5L5pGkR5u69ikajYWTTIpiaaMhp/+6fl5lFLX3xXyH98d9mtJlZeD5n9u7du+zcuZOGDRty5MgRfH19Wb16tbbOywNbW1tb7O3tefDgQZr7tLCw0NkmvYKCgnB0dNRZFsydleH9pCavd35WbtzK0lUbaNmuA9MnjePmXzf06u3ZuZ36jZplygD6fQqaPoVr164xM3ieoUNJt9lB07h+7SpTZ84xdChZ45Wv8IoCmmw24yqvd35WbNjKkpUbaNm2AzMmjePW/8+Lbzev59L5cwTN/YKv1m0hIPAT5s2axqlfTxg46vR5dR6/oijZ6kHqNx9G02LeL7RbeIKNx0OZ3ak0hf4/Z/Zl7SrlYeeZuzpZ2EfRCQxed5a6xXNxbnoDfp9aD3srMy7ciSJFUd5nM9Ilu/fFf81/oT9eTGnMikWtjDYz+4KVlRX169enfv36TJgwgb59+zJx4kR69uwJgLm5uU59jUbz2pu6rK2t36rDx4wZw7Bhujf9RCVkzncJc3Nz8njlBaBo8ZJcuXSRbzet55NxE7V1/vj9NKG3bzI5aHamHPN9mTljKkcPH2LlmvW4vSbznZ3MmTmNn48eZtnKtbi5qSPm9HJxzQlARHg4OXPm0pZHPorA2SV7zWdM7bz4ZvN6hgwbxZeLP2f67M+pWr0WAAV9inD92hU2r19NhWw8n9kphxOmpqaEh4frlD96FIGLi6uBotKXmKxwO+L5jY8X7jyhlJcjPap7M37rv3OSK+R3omAuOz5ed1Zv+2PXwqkz8yhONuYkpSg8jUvixIQ62nm12YFa+uK/4r/UH0adhXxL/7nXpHjx4kRHR2fqPi0sLEhOTn5tHUtLSxwcHHSWrMqQKopCQqLuI2y+/24bRYoVp1DhollyzMymKApB06dw8MB+lq9cQ+5sfsMaPI95dtA0jhw8wKLlK/HMncfQIWW63Lnz4OKak19PHNeWJSYmcPp0CGXKljNgZG+mKAqJCQkkJSWRlJSERqP78WdiYkqKYtink7yJuYUFxYqX4OTxX3TKTx4/nq1ff43m+VzYl7WvlIfzf0dxJexpmttFxiTyNC6JKoWccbGz4ODFtK+avW9q7QtjJf3x32a0mdmIiAjat29P7969KV26NPb29pw6dYrg4GBatmyZqcfKly8fP/zwA1evXsXFxQVHR0e9jG9WWbZoPlX8apDLzZ2YmGgO/rCXs6dDmLNgqbZO9LNnHDmwn4GBI95LTJlhxrTJ7N3zPfMXLMbW1lY758nOzh4rKysDR5e62TOm8sPe3cye/wW2trba+aW2L8UcFfWY+2FhPHz4/I/y7du3gBdPnchpkLhfFRMTzd+hodr/3/3nDlevXMbB0REPD0+6fNCdlSuW4eXtTd683qz8ahlWVlY0atLMgFHrWr5oPpVfOi8O7d/L2TMhzF6wFFs7O8r6VmDJgrlYWlni5u7JH2dO8cOenQwK/MTQob9Rtx69GDd6JMVLlqRMmXJs/WYLYWFhtO+YPZ6RO7xxYY5eeUjY4zhsLU1pVtaDygVd6P1liLaOnaUZjcu4E7TrSqr7aFsxNzfuR/MoOoFy3jn4tGUxVv18i5sPMzcR8a6ye1+kV0x0NKEvnfP/3LnDlcuXcXR0xMPT04CRZYyx9MebqHk6QFYx2sGsnZ0dlStX5rPPPuPGjRskJibi5eVFv379GDt2bKYeq1+/fhw5coQKFSrw7NkzDh8+jL+/f6YeIy2RERFMmzCGiPCH2NrZU9CnMHMWLKVilX+fqHBw/14URaFeoybvJabM8M3/H6XSt1c3nfLJ04Jo2Sp7Po1h6zebARjQt4dO+fjJ02nWsjUAPx85zNSJ47TrPh01HIC+/QPoN2DQe4r09S5dvED/Pv+2Yd7smQA0a9GKydNm0qNXX+Lj4pg5fQpPn0RRslRpFi1dga2t/pxIQ3n0KILpE186LwoVZvaCpVSs/Py8mDh9DssXzWfq+NE8eRKFu7sn/QYMoWXbjgaO/M0aNW5C1ONIli9ZzMOHDyjkU5hFS5fj6Znb0KEB4GpnwZzOpcnlYMXTuESu3H1K7y9D+OXPCG2dpmU90KBh1+9hqe6jQE5bRjQugqONOf9ExrLk4A1W/nTrPbUg/bJ7X6TXxYsX6Nuru/b/c4KDAGjRsjVTZ8w0VFgZZiz9ITJOoyjZcEb9f8SDp4mGDiFT2Fu9nyx0VopPev00EbUwM1H/zKHo+CRDh5ApHG3Uf14AlBy919AhvLMLM1P/1TEh3paVAVOB35zNuifHtC+rnkz8y9T/l08IIYQQQvxnGe00AyGEEEIIYyNzZvVJZlYIIYQQQqiWZGaFEEIIIVRCspD6ZDArhBBCCKESMs1AnwzwhRBCCCGEaklmVgghhBBCJSQvq08ys0IIIYQQQrUkMyuEEEIIoRIyZVafZGaFEEIIIYRqSWZWCCGEEEIlTGTWrB7JzAohhBBCCNWSzKwQQgghhErInFl9MpgVQgghhFAJjUwz0CPTDIQQQgghhGpJZlYIIYQQQiVkmoE+ycwKIYQQQgjV0iiKohg6iP+quCRDRyCEyCrG8slqDFkgp4qDDB1CpogM+cLQIbwzYzkvrM0Nd+x9Fx9m2b4blciZZfvOSpKZFUIIIYQQqiVzZoUQQgghVMIYrpZkNsnMCiGEEEII1ZLMrBBCCCGESkhmVp8MZoUQQgghVEJ+NEGfTDMQQgghhBCqJZlZIYQQQgiVMJHErB7JzAohhBBCCNWSzKwQQgghhErInFl9kpkVQgghhBCqJZlZIYQQQgiVkEdz6ZPMrBBCCCGEUC3JzAohhBBCqITMmdUnmVkhhBBCCKFakpkVQgghhFAJec6sPhnMCiGEEEKohEwz0Pefn2Zw69YtNBoNZ8+eNXQoQgghhBAigww6mH3w4AH9+/cnb968WFpa4u7uTsOGDTlx4oQhwzIKWzZtoHGDOlQsV4pO7dtw5vQpQ4f0VoyhHcbQBpB2ZAcrvlxGl45t8atUjto1qxI4JIBbN/8ydFhvLTv3xbj+TYj9/Qud5eaPM7TrW9Ypw85FA/n70Exif/+C0oVz62zv5GDDvFHt+WP7eCKOz+PaninMHdkOBzur992UdMnOfZEeX2/eSPvWzalW2ZdqlX3p3rUjx34+auiwsoRGk3WLWhl0MNu2bVv++OMP1qxZw7Vr19i5cyf+/v48evTIkGG9s8TERIMef9/ePQTPDKLfhwPY8u0OfH3LE9C/H2F37xo0rowyhnYYQxtA2pFdnD71Gx07d2Xtxq9ZunwVyUnJDPiwD7ExMYYOLcPU0BcXr98lX70x2qVih38HszbWFpz44wbjF36X6rYeOR3xyOnImM+2U6HDDPpNXE99v+Isndj1fYWfbmroizdxc3dnyNARbNyylY1btlKxUhUCBw/k+vU/DR2aeA8MNph9/Pgxx44dY9asWdSuXRtvb28qVarEmDFjaNq0KQAajYavvvqK1q1bY2Njg4+PDzt37tTZz6VLl2jSpAl2dna4ubnRrVs3wsPDtev37dtH9erVyZEjBy4uLjRr1owbN26kGVdKSgr9+vWjcOHC3L59G4Bdu3ZRvnx5rKysKFCgAJMnTyYpKUm7jUajYenSpbRs2RJbW1umTZuWmS9Vhq1bs4rWbdvSpl17ChQsyMgx43D3cOfrLZsMGldGGUM7jKENIO3ILhYvW0HLVm0oVMiHIkWLMnlaEGFhd7l06aKhQ8swNfRFUnIK9yOeapfwyGfadZt2hxC0fB+HTl5NddtLN8LoPOIr9vx0gZt3wjkaco1JX+yiSc2SmJpmrxl+auiLN6nlX4caNWvhnS8/3vnyM/jjodjY2HD+j7OGDi3TabJwUSuDnVF2dnbY2dmxY8cO4uPj06w3efJkOnTowLlz52jSpAldu3bVZm7DwsKoVasWZcuW5dSpU+zbt4/79+/ToUMH7fbR0dEMGzaMkJAQDh48iImJCa1btyYlJUXvWAkJCXTo0IFTp05x7NgxvL29+eGHH/jggw8YMmQIly5dYtmyZaxevZrp06frbDtx4kRatmzJ+fPn6d27dya9ShmXmJDA5UsXqepXXae8ql81/jj7u4GiyjhjaIcxtAGkHdnZs2dPAXB0dDRwJBmjlr4olDcnf+2fzuXvJ7F2Zi/y5XZ5p/052FvxJDqO5GT9vz+Gopa+yIjk5GT27dlNbGwMpcuWM3Q44j0w2NMMzMzMWL16Nf369WPp0qX4+vpSq1YtOnXqROnSpbX1evbsSefOnQGYMWMGCxcu5LfffqNRo0YsWbIEX19fZsz499LPypUr8fLy4tq1axQuXJi2bdvqHHfFihXkypWLS5cuUbJkSW35s2fPaNq0KbGxsRw5ckT7x2H69OmMHj2aHj16AFCgQAGmTp3KyJEjmThxonb7Ll26GHQQ+0Lk40iSk5NxcdH90HVxcSU8/KGBoso4Y2iHMbQBpB3ZlaIozA0OopxveQr5FDZ0OBmihr4IuXCLvuPX8eftB+RysWd030YcXj2c8u2m8ygqOsP7c3a0ZUy/xqz49pcsiPbtqaEv0uvPa1fp3rUTCQnxWNvYMO/zRRQsWMjQYWU6EzVPbs0iBp8ze/fuXXbu3EnDhg05cuQIvr6+rF69Wlvn5YGtra0t9vb2PHjwAIDTp09z+PBhbZbXzs6OokWLAminEty4cYMuXbpQoEABHBwcyJ8/PwChoaE6sXTu3Jlnz56xf/9+nSzH6dOnmTJlis4x+vXrR1hYGDEvzVOrUKHCa9saHx/PkydPdJbXZaTfleaVN7uiKHplamAM7TCGNoC0I7sJmj6Fa9euMTN4nqFDeWvZuS/2/3KJHQfPcvH6XQ7/epXWg5cA8EHzyhne1//au++4KI6HDeDP0auAJRSDKIIodiAae42AYo8lsWtMYsOWWF4LgiiWqBgTe0VNwCRqInbFFtEYUQRFxRqMggUQ6Qjs+wc/L7kcKiCwt8vzzWc/8Wb37p7h7rhhdnbG1NgAe779EtfvxmPB+gOlHbVUaPJrUVQ1a9VCyC97EbQzBP37f4K5s6bjzp3bYseiciD6wB0DAwN89NFHmDt3LsLDwzF8+HCVHk9dXV2V4xUKhXKIQH5+Prp3747IyEiV7datW2jbti0AoHv37khMTMSGDRvwxx9/4I8//gBQMKTg37p27YqoqCicP39epTw/Px++vr4qjx8dHY1bt27BwOCfq1KNjY3fWM+AgACYmZmpbEsXBxTzp/V2FuYW0NbWVhk3DABJSYmoUqVqqT9fWZFDPeRQB4D10ESLFs7HqRNh2Lh5GyytrMSOU2xSfC0ysnJw7fYj1K5RrVj3MzHSx2/fj0VaZjYGTNmA3FzNGWIASPO1eB1dXT3UqGGH+g0awnvyVNRxqosfdgSJHavUccysOtEbs//l7OyM9PSincJxcXHBtWvXULNmTTg4OKhsxsbGSExMxPXr1zF79mx06tQJ9erVQ3JycqGPNWbMGCxatAg9evTAqVOnVJ7j5s2bao/v4OAALa2i//hmzpyJlJQUle3r6TOLfP+i0tXTQz3n+jgfrnoq63x4OBpLaOyQHOohhzoArIcmEQQBAQv8cPzYEazfvA3V37cVO1KJSPG10NPVQd1alkh4llLk+5gaGyB0zXjkvMzDx5PWITsn9+13KmdSfC2KShAEtY4rWWBrVo1oY2YTExPRr18/jBw5Eo0aNYKpqSkuXryIJUuWoGfPnkV6jHHjxmHDhg345JNP8PXXX6Nq1aq4ffs2goODsWHDBlhYWKBKlSpYv349rK2tERcXhxkzZrz28SZMmIC8vDx4eXnh4MGDaN26NebOnQsvLy/Y2tqiX79+0NLSQlRUFKKjo4s1a4G+vj709fVVyrLK6PfakGEjMGvGNDg3aIDGjZvil59CEB8fj34DBpbNE5YROdRDDnUAWA9NsdDfFwcPhCLw29UwNjZWjmk0MTFVOVMkBZr+WgRM7o39p6PxID4Z71U2wfTPPGBqbICd+wrO7llUMoKtlQWs3ysYllanpiUA4HHiCzxOTIWJkT5CV4+DoYEeRszahkrGBqhkXPAaPU1OQ36+IE7FCqHpr0VRfBu4HK3btIWllRUy0tNx6OABXPzzAr5fu1HsaFQORGvMmpiYoHnz5lixYgXu3LmDly9fwtbWFqNHj8b//d//FekxbGxscPbsWUyfPh3u7u7Izs6GnZ0dPDw8oKWlBYVCgeDgYHh7e6NBgwZwcnLCt99+i/bt27/2MSdNmoT8/Hx07doVhw4dgru7O0JDQ+Hn54clS5ZAV1cXdevWxWeffVZKP4nS5+HZFSnPk7F+zWo8ffoEDo518P3a9bCxqf72O2sQOdRDDnUAWA9N8dP/pkr6bMQQlXJf/wD07NVHjEglpumvRXVLcwQFjEAVc2M8S07Dhej7aDdsGeLiC87udWvXEBv8/nkdti8uuADYf+0BLFh3AE3r1UCzRgXXaMTsm6fy2E5d5yIuXnPmU9f016IokhKfYdbMaXj29AlMTE1Rp44Tvl+7ES1athI7WqnjcrbqFIIgaM6fhxVMWfXMEpH45PKbVWLXABXK4oPxYkcoFcl/fid2hHcml8+Foe7bjykrf9wp+lCX4mpeW1rT/L0iWs8sERERERWPHP7ALG0adwEYEREREVFRsWeWiIiISCLYMauOPbNEREREJFnsmSUiIiKSCnbNqmFjloiIiEgiODWXOg4zICIiIiLJYs8sERERkURwai517JklIiIiIslizywRERGRRLBjVh17ZomIiIhIstgzS0RERCQV7JpVw55ZIiIiIpIs9swSERERSQTnmVXHxiwRERGRRHBqLnUcZkBEREREksXGLBEREZFEKMpwK47Tp0+je/fusLGxgUKhwN69e1X2C4KAefPmwcbGBoaGhmjfvj2uXbumckx2djYmTJiAqlWrwtjYGD169MDff/9dzCRszBIRERFRMaWnp6Nx48b47rvvCt2/ZMkSLF++HN999x3+/PNPWFlZ4aOPPkJqaqrymEmTJmHPnj0IDg7G77//jrS0NHh5eSEvL69YWRSCIAjvVBsqsaxcsRMQEb2ZHL4h5DLG0KLjPJETvLvksHliRygVBiJecXTlQerbDyqhxramJbqfQqHAnj170KtXLwAFvbI2NjaYNGkSpk+fDqCgF9bS0hKLFy/GF198gZSUFFSrVg3bt2/HgAEDAACPHj2Cra0tDhw4AHd39yI/P3tmiYiIiAjZ2dl48eKFypadnV3sx7l37x4SEhLQpUsXZZm+vj7atWuH8PBwAEBERARevnypcoyNjQ0aNGigPKao2JglIiIikghFGf4XEBAAMzMzlS0gIKDYGRMSEgAAlpaWKuWWlpbKfQkJCdDT04OFhcVrjykqTs1FRERERJg5cyamTJmiUqavr1/ix1P8Z4yPIAhqZf9VlGP+iz2zRERERBKhUJTdpq+vj0qVKqlsJWnMWllZAYBaD+uTJ0+UvbVWVlbIyclBcnLya48pKjZmiYiIiCRCU6bmepNatWrBysoKR48eVZbl5OTg1KlTaNmyJQDA1dUVurq6KsfEx8fj6tWrymOKisMMiIiIiKhY0tLScPv2beXte/fuITIyEpUrV0aNGjUwadIkLFy4EI6OjnB0dMTChQthZGSETz/9FABgZmaGUaNGYerUqahSpQoqV66Mr776Cg0bNkTnzp2LlYWNWSIiIiKp0JCp5i5evIgOHToob78aazts2DBs3boV06ZNQ2ZmJsaOHYvk5GQ0b94cR44cganpP9N/rVixAjo6Oujfvz8yMzPRqVMnbN26Fdra2sXKwnlmRcR5ZolI08nhG4LzzGoOzjP77q4+TCuzx25Q3aTMHrsssWeWiIiISCIUmtI1q0F4ARgRERERSRZ7ZomIiIgkQi7DZkoTe2aJiIiISLLYM0tEREQkEeyYVcfGLBEREZFUsDWrhsMMiIiIiEiy2DNLREREJBGcmktdheuZHT58OBQKhXKrUqUKPDw8EBUVJXa0UhXy4054dumID5o2xMB+fXAp4qLYkUpEDvWQQx0iLv6JCWO/ROf2rdG4vhPCjh8TO1KxyaEOr0j9PbVpwzp8OqAvWjZrig5tW2CS91jcv3dX7FjFJrX31FeDWiPz9DwsneChLDM21MOKSV1x++cpSDo6C5e3j8Ponm4q9zu8cjgyT89T2YJ8Pi7H5EUn9c8GlUyFa8wCgIeHB+Lj4xEfH4/jx49DR0cHXl5eYscqNYcOHsCSRQEY/fkYhPy8Fy4urhj7xWjEP3okdrRikUM95FAHAMjMzICTkxNmzJordpQSk0MdAHm8pyIuXsCATwYh6IddWLt+C/Jy8zDm81HIzMgQO1qxSOk95VrXBqN6uCLqdoJK+ZLx7viomQNG+O9GkyHfY9Wu81g+sSu8WjupHLfptwjU7PWNchv/zb7yjF8kcvhsFIVCUXabVFXIxqy+vj6srKxgZWWFJk2aYPr06Xjw4AGePn0KAJg+fTrq1KkDIyMj2NvbY86cOXj58qXKY/j7++O9996DqakpPvvsM8yYMQNNmjQRoTbqtm/bgt59+6LPx/1gX7s2ps2cBStrK+wK+VHsaMUih3rIoQ4A0LpNO4yfOBmdP+oidpQSk0MdAHm8p1av24SevfrAwcERTnXrwtc/APHxjxATc03saMUilfeUsaEetszpi7FL9uF5apbKvub1bbHjUCTORN5HXMJzbN4Xgag7CXBxslE5LjP7JR4npSm3F+nZ5VmFIpHDZ4NKpkI2Zv8tLS0NO3fuhIODA6pUqQIAMDU1xdatWxETE4OVK1diw4YNWLFihfI+O3fuxIIFC7B48WJERESgRo0aWLNmjVhVUPEyJwfXY66hRcvWKuUtWrbClcjLIqUqPjnUQw51IM0i1/dUWloqAMDMzEzkJPIUOLkrDp2LxYkI9aEc4dFx8GrlBJuqpgCAtk1rwtG2Co5duKNy3ICPGuLBb9MQsW0sAsZ2gYmhXrlkLyq5fjYKoyjDTaoq5AVgoaGhMDExAQCkp6fD2toaoaGh0NIqaNvPnj1beWzNmjUxdepUhISEYNq0aQCAVatWYdSoURgxYgQAYO7cuThy5AjS0tLKuSbqkp8nIy8vT9kwf6VKlap49uypSKmKTw71kEMdSLPI8T0lCAKWLQlAUxdXODjWETuO7PTr2ABN6lij9ecbCt0/deVBrJ7WHXd2T8XL3Dzk5wsYs+Q3hEfHKY8JPhqF+/HP8TgpDfVrvQe/LzqhYW1LeE3dXl7VeCs5fjao6CpkY7ZDhw7KntSkpCSsXr0anp6euHDhAuzs7PDzzz8jMDAQt2/fRlpaGnJzc1GpUiXl/W/evImxY8eqPGazZs0QFhb22ufMzs5GdrbqaRlBWx/6+vqlWLN/KP4z+EUQBLUyKZBDPeRQB9IscnpPBSzwQ2xsLLYG/SB2FNl5/71KWOrtge5TtyM7J7fQY8Z93BzNnN9H3xk/IC4hBa2b2GHllG5ISExT9uRuCb2kPD7m3hPc/jsR4Ru/QJM61oiMjS+XuhSVnD4bryWz6pSGCtmYNTY2hoODg/K2q6srzMzMsGHDBnh5eWHgwIHw9fWFu7s7zMzMEBwcjGXLlqk8RmEfmDcJCAiAr6+vStmsOT6YPXfeu1XmPyzMLaCtrY1nz56plCclJaJKlaql+lxlSQ71kEMdSLPI7T21aOF8nDoRhs3bdsDSykrsOLLTtI4NLCubIHzDF8oyHR0ttG5shy97N4Nl1wD4ju6EAbOCcej8LQDA1buP0cjBCpMGtix0WAIAXI6NR87LPDi8X1ljGrNy+2y8CafmUlfhx8wCBQ1TLS0tZGZm4uzZs7Czs8OsWbPg5uYGR0dH/PXXXyrHOzk54cKFCyplFy++efqPmTNnIiUlRWX7evrMUq+Lrp4e6jnXx/nwsyrl58PD0bhJ01J/vrIih3rIoQ6kWeTynhIEAQEL/HD82BGs37wN1d+3FTuSLJ2IuAvXYavRfNRa5RZx/SGCj0ah+ai10NbSgp6uNvL/0xmTl58PLa3XN5ica70HPV1txCeKP7TuFbl8NqhkKmTPbHZ2NhISCqYnSU5OxnfffYe0tDR0794dKSkpiIuLQ3BwMD744APs378fe/bsUbn/hAkTMHr0aLi5uaFly5YICQlBVFQU7O3tX/uc+vrqQwqyCj/r886GDBuBWTOmwblBAzRu3BS//BSC+Ph49BswsGyesIzIoR5yqAMAZKSnIy7unzF0D//+GzeuX4eZmRmsbWzecE/NIYc6APJ4Ty3098XBA6EI/HY1jI2NlWMaTUxMYWBgIHK6otP091RaZg5i7j1RKUvPeomkF5nK8tOX72PhmC7IzM5F3OPnaNO4Jga5N8b07w4DAGrZWGDgR41w+PwtPEvJQL2a1bBoXBdcjo3HuX+Nq9UEcvhsFIXcRk2UhgrZmD106BCsra0BFMxcULduXfz0009o3749AGDy5MkYP348srOz0a1bN8yZMwfz5s1T3n/QoEG4e/cuvvrqK2RlZaF///4YPny4Wm+tWDw8uyLleTLWr1mNp0+fwMGxDr5fux42NtXFjlYscqiHHOoAANeuXcVnI4Yqb3+zJAAA0KNnb8xfuEisWMUihzoA8nhP/fS/qZI+GzFEpdzXPwA9e/URI1KJyOE9NdT3Z/h93glb5/SBRSVDxCWkYN6GMGz4teBs48vcPHRwrYVxHzeHiaEe/n7yAofOx2LBllPIz3/z8LryJofPBpWMQnjbYE8qko8++ghWVlbYvr3oV3eWVc8sEVFpkcM3hFx6siw6zhM5wbtLDpsndoRSYSBiV+CdJ5ll9ti13zMss8cuSxWyZ/ZdZWRkYO3atXB3d4e2tjZ+/PFHHDt2DEePHhU7GhEREVGFwsZsCSgUChw4cAD+/v7Izs6Gk5MTfvnlF3Tu3FnsaERERCRnMjnTUJrYmC0BQ0NDHDt2TOwYRERERBUeG7NEREREEsF5ZtWxMUtEREQkEXK5oLE0cdEEIiIiIpIs9swSERERSQQ7ZtWxZ5aIiIiIJIs9s0RERERSwa5ZNeyZJSIiIiLJYs8sERERkURwai517JklIiIiIslizywRERGRRHCeWXVszBIRERFJBNuy6jjMgIiIiIgkiz2zRERERBLBYQbq2DNLRERERJLFnlkiIiIiyWDX7H8pBEEQxA5RUWXlip2AiMqKXH6zyuGUZtbLPLEjlAoDXW2xI7wzi65LxY5QKjKPfC3ac/+dnFNmj/2+hV6ZPXZZYs8sERERkUTI4Q/M0sYxs0REREQkWeyZJSIiIpIIdsyqY88sEREREUkWe2aJiIiIJIJjZtWxMUtEREQkEQoONFDDYQZEREREJFnsmSUiIiKSCnbMqmHPLBERERFJFntmiYiIiCSCHbPq2DNLRERERJLFnlkiIiIiieDUXOrYM0tEREREksWeWSIiIiKJ4Dyz6tiYJSIiIpIKtmXViDLMYN68eWjSpMlr92/duhXm5ublloeIiIiIpKlEjdnw8HBoa2vDw8OjtPOUmnnz5kGhUCg3MzMztGnTBqdOnSr151IoFNi7d2+pP+67CPlxJzy7dMQHTRtiYL8+uBRxUexIJSKHesihDgDroQk2bViHTwf0RctmTdGhbQtM8h6L+/fuih2rxKT0WmzdtB7DP+2PDi3d4NGhNb6eNB5/3b+ncowgCNiw5jt0+6gd2jZvijGjhuHu7VsiJS4eKb0WXw1sjswjX2Pplx2UZZlHvi50m9zvA+UxerraWD62Ex78NA7PfpuIn3x7o3pVEzGq8E4UZbhJVYkas5s3b8aECRPw+++/Iy4urrQzlZr69esjPj4e8fHxOHfuHBwdHeHl5YWUlBSxo5WpQwcPYMmiAIz+fAxCft4LFxdXjP1iNOIfPRI7WrHIoR5yqAPAemiKiIsXMOCTQQj6YRfWrt+CvNw8jPl8FDIzMsSOVmxSey0uR1zExwM+waagH/Ht2o3Iy8uD95jPkJn5z89++9ZN+GHHNnw1Yza27NyFylWrYsKYz5Ceni5i8reT0mvhWscKo7o2QtSdJyrlNQesVtk+/+Yg8vMF7DkTqzxm6Zcd0aOVI4YuDEWnyT/CxFAXv8zvCy0tKTfjCChBYzY9PR27du3CmDFj4OXlha1bt6rsP3nyJBQKBY4fPw43NzcYGRmhZcuWuHnz5msf8969e3BwcMCYMWOQn59f6DH79u2Dq6srDAwMYG9vD19fX+Tm5r4xq46ODqysrGBlZQVnZ2f4+voiLS0NsbH/vLnj4uLQs2dPmJiYoFKlSujfvz8eP36s8jhr1qxB7dq1oaenBycnJ2zfvl25r2bNmgCA3r17Q6FQKG+Lafu2Lejdty/6fNwP9rVrY9rMWbCytsKukB/FjlYscqiHHOoAsB6aYvW6TejZqw8cHBzhVLcufP0DEB//CDEx18SOVmxSey1Wrl4Pr569Ye/giDpOdTHHdwES4uNxIyYGQEGvbPDOIIz47At06PQRajs4wmd+ALIys3D4YKjI6d9MKq+FsYEutszohrErjuB5WpbKvsfJ6Spb95YOOHUlDvcTCjqvKhnpYbhHQ8xYfwInLv+FK3eeYOSi/WhQsyo6NrUTozolplCU3SZVxW7MhoSEwMnJCU5OThg8eDC2bNkCQRDUjps1axaWLVuGixcvQkdHByNHjiz08a5evYpWrVqhX79+WLNmDbS01CMdPnwYgwcPhre3N2JiYrBu3Tps3boVCxYsKHLu7Oxs5VhcJycnAAW/fHr16oWkpCScOnUKR48exZ07dzBgwADl/fbs2YOJEydi6tSpuHr1Kr744guMGDECJ06cAAD8+eefAIAtW7YgPj5eeVssL3NycD3mGlq0bK1S3qJlK1yJvCxSquKTQz3kUAeA9dBkaWmpAAAzMzORkxSPHF6LVz/7Sv/72T96+DcSnz1D8xYtlcfo6emhqZsboiMjxYhYJFJ6LQIndMahC3dx4vJfbzzuPXMjeDSzx7ZD0cqypnWsoKerjWMR95Vl8UnpuHb/GT50rl5WkamcFHs2g02bNmHw4MEAAA8PD6SlpeH48ePo3LmzynELFixAu3btAAAzZsxAt27dkJWVBQMDA+Ux586dg5eXF2bOnImvvvrqtc+5YMECzJgxA8OGDQMA2NvbY/78+Zg2bRp8fHxee7/o6GiYmBSMh8nIyICpqSlCQkJQqVIlAMCxY8cQFRWFe/fuwdbWFgCwfft21K9fH3/++Sc++OADfPPNNxg+fDjGjh0LAJgyZQrOnz+Pb775Bh06dEC1atUAAObm5rCysir6D7KMJD9PRl5eHqpUqaJSXqVKVTx79lSkVMUnh3rIoQ4A66GpBEHAsiUBaOriCgfHOmLHKRapvxaCIGDlsiVo3NQFtR0cAQCJz54BACpXrqpybOXKVZEQr3mn61+RymvRr31dNHGwROvx29967OCPGiA1Iwd7f//nLKyVhTGyc3LxPC1b5dgnzzNgWdm41POWJU7Npa5YPbM3b97EhQsXMHDgQAAFp/EHDBiAzZs3qx3bqFEj5b+tra0BAE+e/DPGJS4uDp07d8bs2bPf2JAFgIiICPj5+cHExES5jR49GvHx8ch4w1gxJycnREZGIjIyEhERERgzZgz69euHixcLBrZfv34dtra2yoYsADg7O8Pc3BzXr19XHtOqVSuVx23VqpVyf1FlZ2fjxYsXKlt2dvbb71hCiv+cLxAEQa1MCuRQDznUAWA9NE3AAj/ExsZi0ZLlYkcpMam+FksD/HE79ibmL/pGbZ9afonUSZNfi/ermWLpmI4YuXg/sl/mvfX4oR4NEBJ2vUjHKhQo9OwySUuxemY3bdqE3NxcVK/+T5e8IAjQ1dVFcnIyLCwslOW6urrKf7/6QPx7PGy1atVgY2OD4OBgjBo1StlbWpj8/Hz4+vqiT58+avv+3dP7X3p6enBwcFDebtq0Kfbu3YvAwEDs2LHjtR/W/5aXxoc8ICAAvr6+KmWz5vhg9tx5xXqct7Ewt4C2tjae/a+X4JWkpERUqVL1NffSPHKohxzqALAemmjRwvk4dSIMm7ftgKUGnBEqLim/Ft8s8seZUyewbnMQLC3/+dlXqVqQOzHxKar+74wdACQlJ6Jy5Spqj6MppPBaNHW0hKWFMcK/H6os09HWQuuGtviypwvMui1Hfn5Bg7RVg+pwsq2CIQv2qTxGQnI69PV0YG6ir9I7W83MCOevaW7PeWE05G8MjVLkntnc3FwEBQVh2bJlyt7OyMhIXLlyBXZ2dti5c2exntjQ0BChoaEwMDCAu7s7UlNTX3usi4sLbt68CQcHB7WtsDG2b6KtrY3MzEwABb2wcXFxePDggXJ/TEwMUlJSUK9ePQBAvXr18Pvvv6s8Rnh4uHI/UNBwz8t781+AM2fOREpKisr29fSZxcpeFLp6eqjnXB/nw8+qlJ8PD0fjJk1L/fnKihzqIYc6AKyHJhEEAQEL/HD82BGs37wN1d+3ffudNJAUXwtBELA0wB8njx/D9+s3w6b6+yr7baq/jypVq+LCuXPKspcvc3D54kU0fMO86mKTwmtx4vJfcP18C5qP2abcIm7GIzgsBs3HbFM2ZAFgmEcjRMQmIPqu6hCJy7EJyHmZh04uNZVlVpWNUb9mVZyPeVheVaEyUuSe2dDQUCQnJ2PUqFFqFxt8/PHH2LRpE8aPH1+sJzc2Nsb+/fvh6ekJT09PHDp0SDnG9d/mzp0LLy8v2Nraol+/ftDS0kJUVBSio6Ph7+//2sfPzc1FQkICACA1NRUhISGIiYnB9OnTAQCdO3dGo0aNMGjQIAQGBiI3Nxdjx45Fu3bt4ObmBgD4+uuv0b9/f7i4uKBTp07Yt28fdu/ejWPHjimfp2bNmjh+/DhatWoFfX19lR7qV/T19aGvr69SlvXmyRhKbMiwEZg1YxqcGzRA48ZN8ctPIYiPj0e/AQPL5gnLiBzqIYc6AKyHpljo74uDB0IR+O1qGBsbK8c0mpiYvvEslSaS2muxdOF8HD64H0sDv4OxsTES//ezN/7fz16hUGDgoKHYumk9bO3sYFvDDls3roeBoQHcPb1ETv9mmv5apGW+RMx91Z7j9KyXSHqRqVJuaqSHPm3rYMa6k2qP8SIjB1sPRWPRF+2R+CITyalZCPi8Pa7ef4awt1xQRpqvyI3ZTZs2oXPnzoVeNdu3b18sXLgQly5dKnYAExMTHDx4EO7u7ujatSsOHjyodoy7uztCQ0Ph5+eHJUuWQFdXF3Xr1sVnn332xse+du2acryukZERateujTVr1mDo0IJTFa8WO5gwYQLatm0LLS0teHh4YNWqVcrH6NWrF1auXImlS5fC29sbtWrVwpYtW9C+fXvlMcuWLcOUKVOwYcMGVK9eHffv3y/2z6E0eXh2RcrzZKxfsxpPnz6Bg2MdfL92PWxspHXFphzqIYc6AKyHpvjpf1MlfTZiiEq5r38AevZSH4alyaT2WvzyUzAAYMxnw1TK5/gugFfP3gCAIcNHITsrC0sW+iH1xQvUb9gI367ZCGNjzb7ASGqvxev0a18XCiiw60Th17RMWxuGvLx87JjdA4Z6OjgR+Rc+n3tQpWdXCjjMQJ1C4Mhn0ZRVzywRiU8uv1nl8MWZVYQLgaTAQFdb7AjvzKLrUrEjlIrMI1+L9tzPM8vu/WxuKM33WLGn5iIiIiIicXBqLnUlWs6WiIiIiEgTsGeWiIiISCLkMPSntLFnloiIiIgkiz2zRERERBLBjll1bMwSERERSQVbs2o4zICIiIiIJIs9s0REREQSwam51LFnloiIiIgkiz2zRERERBLBqbnUsWeWiIiIiCSLPbNEREREEsGOWXXsmSUiIiIiyWLPLBEREZFUsGtWDXtmiYiIiCRCUYb/lcTq1atRq1YtGBgYwNXVFWfOnCnlGr8dG7NEREREVGwhISGYNGkSZs2ahcuXL6NNmzbw9PREXFxcueZQCIIglOszklJWrtgJiKisyOU3qxymAcp6mSd2hFJhoKstdoR3ZtF1qdgRSkXmka9Fe+6ybDsYFHPwafPmzeHi4oI1a9Yoy+rVq4devXohICCglNO9HntmiYiIiAjZ2dl48eKFypadnV3osTk5OYiIiECXLl1Uyrt06YLw8PDyiPsPgWQrKytL8PHxEbKyssSOUmJyqIMgsB6aRA51EAR51EMOdRAE1kOTyKEOYvLx8REAqGw+Pj6FHvvw4UMBgHD27FmV8gULFgh16tQph7T/4DADGXvx4gXMzMyQkpKCSpUqiR2nRORQB4D10CRyqAMgj3rIoQ4A66FJ5FAHMWVnZ6v1xOrr60NfX1/t2EePHqF69eoIDw9HixYtlOULFizA9u3bcePGjTLP+wqn5iIiIiKi1zZcC1O1alVoa2sjISFBpfzJkyewtLQsi3ivxTGzRERERFQsenp6cHV1xdGjR1XKjx49ipYtW5ZrFvbMEhEREVGxTZkyBUOGDIGbmxtatGiB9evXIy4uDl9++WW55mBjVsb09fXh4+NT5FMGmkgOdQBYD00ihzoA8qiHHOoAsB6aRA51kJIBAwYgMTERfn5+iI+PR4MGDXDgwAHY2dmVaw5eAEZEREREksUxs0REREQkWWzMEhEREZFksTFLRERERJLFxiwRERERSRYbszLi5+eHjIwMtfLMzEz4+fmJkKh4Xr58iQ4dOiA2NlbsKEREJAH29vZITExUK3/+/Dns7e1FSERiYGNWRnx9fZGWlqZWnpGRAV9fXxESFY+uri6uXr0KhUIhdhSSGW1tbTx58kStPDExEdra2iIkqrhOnz6N3NxctfLc3FycPn1ahETFl5ubC19fXzx48EDsKBXe/fv3kZeXp1aenZ2Nhw8fipCIxMB5ZmVEEIRCG4JXrlxB5cqVRUhUfEOHDsWmTZuwaNEisaO8k5cvX6JLly5Yt24d6tSpI3acUvXixQuEhYXByckJ9erVEztOkbxuBsLs7Gzo6emVc5riiYqKKvKxjRo1KsMkpaNDhw6Ij4/He++9p1KekpKCDh06FNow0TQ6OjpYunQphg0bJnaUCuu3335T/vvw4cMwMzNT3s7Ly8Px48dRs2ZNEZKRGNiYlQELCwsoFAooFArUqVNHpUGbl5eHtLS0cl+No6RycnKwceNGHD16FG5ubjA2NlbZv3z5cpGSFY+cepn79++Ptm3bYvz48cjMzISbmxvu378PQRAQHByMvn37ih3xtb799lsAgEKhwMaNG2FiYqLcl5eXh9OnT6Nu3bpixSuSJk2aQKFQvLZB/mqfQqGQREPwdX90JyYmqn3eNVnnzp1x8uRJDB8+XOwo76Rp06aFvh4KhQIGBgZwcHDA8OHD0aFDBxHSvV6vXr0AFOT87x8Vurq6qFmzJpYtWyZCMhIDG7MyEBgYCEEQMHLkSPj6+qr8haqnp4eaNWuiRYsWIiYsuqtXr8LFxQUA1MbOSq1hKJde5tOnT2PWrFkAgD179kAQBDx//hzbtm2Dv7+/RjdmV6xYAaCgAbV27VqVIQWvPhtr164VK16R3Lt3T+wIpaJPnz4ACj7Hw4cPV1mhKS8vD1FRUeW+nvu78PT0xMyZM3H16lW4urqqNcR79OghUrLi8fDwwJo1a9CwYUM0a9YMgiDg4sWLiIqKwvDhwxETE4POnTtj9+7d6Nmzp9hxlfLz8wEAtWrVwp9//omqVauKnIjExBXAZCI3Nxc7duxA586d8f7774sdhwBMmDABQUFBcHBwkHQvs6GhIWJjY2Fra4uhQ4fCxsYGixYtQlxcHJydnQsdp61pOnTogN27d8PCwkLsKBXWiBEjAADbtm1D//79YWhoqNz36g+L0aNHS6ZRoqX1+ktOpNJLDgCjR49GjRo1MGfOHJVyf39//PXXX9iwYQN8fHywf/9+XLx4UaSURG/GxqyMGBkZ4fr16+W+JnJZuH37Nu7cuYO2bdvC0NDwtacmNdmbTsspFAqEhYWVY5qSq1OnDvz9/dGtWzfUqlULwcHB6NixI65cuYJOnTrh2bNnYkescGJiYhAXF4ecnByVck3vDRQEASNGjMCqVatgamoqdhwCYGZmhoiICDg4OKiU3759G66urkhJScGNGzfwwQcfIDU1VaSUb3bq1Cl88803uH79OhQKBerVq4evv/4abdq0ETsalRMOM5CR5s2b4/Lly5JuzCYmJqJ///44ceIEFAoFbt26BXt7e3z22WcwNzeX1BioEydOiB2hVEyaNAmDBg2CiYkJ7Ozs0L59ewAFww8aNmwobrhi+Pvvv/Hbb78V2giUSi/53bt30bt3b0RHR6uMo331h56m9wYKgoAffvgBs2bNklVjNisrCwYGBmLHKBEDAwOEh4erNWbDw8OVdcrPz1cZFqJJduzYgREjRqBPnz7w9vaGIAgIDw9Hp06dsHXrVnz66adiR6TyIJBs7Nq1S7C3txdWrVolhIeHC1euXFHZpGDIkCGCu7u78ODBA8HExES4c+eOIAiCcPjwYcHZ2VnkdCVz69Yt4dChQ0JGRoYgCIKQn58vcqLi+/PPP4Xdu3cLqampyrLQ0FDh999/FzFV0R07dkwwMjIS6tevL+jo6AhNmjQRzM3NBTMzM6FDhw5ixysyLy8voWfPnsKTJ08EExMTISYmRjhz5ozQrFkz4fTp02LHKxJnZ2fh3LlzYsd4Z7m5uYKfn59gY2MjaGtrK39XzZ49W9i4caPI6Ypu/vz5gqGhoeDt7S1s375d2LFjh+Dt7S0YGRkJ/v7+giAIwvLly4XOnTuLnLRwdevWFZYvX65WvmzZMqFu3boiJCIxsDErIwqFQm3T0tJS/l8KLC0thcjISEEQBJXG7N27dwVjY2MxoxXbs2fPhI4dOyp//q/qMnLkSGHKlCkipyu53Nxc4fLly0JSUpLYUYrsgw8+EObMmSMIwj/vq9TUVKFHjx7C6tWrRU5XdFWqVFH+YVqpUiXhxo0bgiAIwvHjx4UmTZqIGa3IQkNDhdatWwvR0dFiR3knvr6+gr29vbBjxw7B0NBQ+fkOCQkRPvzwQ5HTFc+OHTuEDz/8ULCwsBAsLCyEDz/8UNi5c6dyf0ZGhpCZmSliwtfT09MTbt26pVZ+69YtQV9fX4REJAYumiAj9+7dU9vu3r2r/L8UpKenw8jISK382bNnGnua63UmT54MXV1dxMXFqdRpwIABOHTokIjJimfSpEnYtGkTgILT2O3atYOLiwtsbW1x8uRJccMV0fXr15XT9+jo6CAzMxMmJibw8/PD4sWLRU5XdHl5ecrpxapWrYpHjx4BAOzs7HDz5k0xoxXZ4MGDceHCBTRu3BiGhoaoXLmyyiYVQUFBWL9+PQYNGqQyS0ajRo1w48YNEZMV36BBg3Du3DkkJSUhKSkJ586dUzk9b2hoqLHDKGxtbXH8+HG18uPHj8PW1laERCQGjpmVESmPlX2lbdu2CAoKwvz58wEUjAXMz8/H0qVLNW6ew7c5cuQIDh8+rDa7hKOjI/766y+RUhXfzz//jMGDBwMA9u3bh3v37uHGjRsICgrCrFmzcPbsWZETvp2xsTGys7MBADY2Nrhz5w7q168PAJK6gK1BgwaIioqCvb09mjdvjiVLlkBPTw/r16+XzNKdgYGBYkcoFQ8fPlQbZwoUjC99+fKlCIkqpqlTp8Lb2xuRkZFo2bIlFAoFfv/9d2zduhUrV64UOx6VEzZmJe63336Dp6cndHV1VVZEKYymX+kMAEuXLkX79u1x8eJF5OTkYNq0abh27RqSkpIk0Wj6N7n0Mj979gxWVlYAgAMHDqBfv36oU6cORo0apVyUQNN9+OGHOHv2LJydndGtWzdMnToV0dHR2L17Nz788EOx4xXZ7NmzkZ6eDqBg6iQvLy+0adMGVapUQUhIiMjpikYuq2bVr18fZ86cUetE+Omnn9C0aVORUhVfXl4eVqxYgV27dhV6cWRSUpJIyYpmzJgxsLKywrJly7Br1y4AQL169RASEqJR8+JS2WJjVuJ69eqFhIQEvPfee8oVUQojlXkPnZ2dERUVhTVr1kBbWxvp6eno06cPxo0bB2tra7HjFYtcepktLS0RExMDa2trHDp0CKtXrwYAZGRkqJxe1WTLly9Xzoc7b948pKWlISQkBA4ODsqFFaTA3d1d+W97e3vExMQgKSlJuQqgpnrx4gUqVaqk/PebvDpO0/n4+GDIkCF4+PAh8vPzsXv3bty8eRNBQUEIDQ0VO16R+fr6YuPGjZgyZQrmzJmDWbNm4f79+9i7dy/mzp0rdrwi6d27N3r37i12DBIR55klKiMxMTFo3749XF1dERYWhh49eqj0MteuXVvsiEUyb948BAYGwtraGhkZGYiNjYW+vj42b96MDRs24Ny5c2JHJA2nra2N+Ph4vPfee9DS0iq04S1IaEneVw4fPoyFCxciIiIC+fn5cHFxwdy5c9GlSxexoxVZ7dq18e2336Jbt24wNTVFZGSksuz8+fP44YcfxI5YJBEREcp5Zp2dnSXVO07vjo1Z0iiHDh2CiYkJWrduDQD4/vvvsWHDBjg7O+P777+X3ApOCQkJWLNmjcqXnRR7mX/++Wc8ePAA/fr1U44B3rZtG8zNzSV1Kk+KX3ivloEtit27d5dhkpI7deoUWrVqBR0dHZw6deqNx7Zr166cUr2bBw8evPYCo/Pnz0tm+IqxsTGuX7+OGjVqwNraGvv374eLiwvu3r2Lpk2bIiUlReyIb/TkyRMMHDgQJ0+ehLm5OQRBQEpKCjp06IDg4GBUq1ZN7IhUDtiYlZnjx4/j+PHjePLkiXLt6lc2b94sUqqia9iwIRYvXoyuXbsiOjoabm5umDp1KsLCwlCvXj1s2bJF7IgVmlQnh5fyF96rZWCBgt7LPXv2wMzMDG5ubgAKGujPnz9Hnz59+PkoR3Xr1sXZs2dRpUoVlfKzZ8+iW7dueP78uTjBisnJyQlBQUFo3rw52rRpg27dumHGjBkICQnBhAkT8OTJE7EjvtGAAQNw584dbN++HfXq1QNQcFZs2LBhcHBwwI8//ihyQioPHDMrI76+vvDz84Obmxusra01egzd69y7dw/Ozs4AgF9++QXdu3fHwoULcenSJXTt2lXkdMVTq1YtDB48GIMHD4aTk5PYcUosLy8PCxcuxNq1a/H48WPExsbC3t4ec+bMQc2aNTFq1CixI77VhAkT8OLFC1y7dk3tC8/b21ujv/D+3UCdPn06+vfvj7Vr1yrHK+fl5WHs2LGSGWsKAM+fP8eFCxcK/aN76NChIqUqnjZt2qBLly44efKkcjWz06dPo3v37pg3b5644Yqhd+/eOH78OJo3b46JEyfik08+waZNmxAXF4fJkyeLHe+tDh06hGPHjik/1wCUZ/KkNNyD3pFYE9xS6bOyshKCgoLEjvFOLCwshGvXrgmCIAitWrUS1q1bJwiCINy7d08wNDQUM1qxLVu2THBzcxMUCoXg4uIirFixQnj06JHYsYpNDpPDV6pUSbhw4YJa+R9//CGYmZmVf6ASqlq1qnKhhH+7ceOGULlyZRESFd9vv/0mmJqaClpaWoKZmZlgbm6u3CwsLMSOV2T5+flC3759hTZt2giZmZlCWFiYYGJiIgQGBood7Z2cP39eWLZsmfDrr7+KHaVITExMhMuXL6uVX7p0STA1NS3/QCQKLpogIzk5OWjZsqXYMd5J69atMWXKFMyfPx8XLlxAt27dAACxsbFq87VquilTpuDPP//EjRs34OXlhTVr1qBGjRro0qULgoKCxI5XZHKYHD4/Px+6urpq5bq6umo9g5osNzcX169fVyu/fv26ZOoxdepUjBw5EqmpqXj+/DmSk5OVm6ZPA/VvCoUCP/74IwwMDNCpUyf06NEDAQEBmDhxotjRiiUxMVH57wcPHmD//v2Ij4+Hubm5eKGKoWPHjpg4caJyARGgYA7gyZMno1OnTiImo/LEMbMyMn36dJiYmGDOnDliRymxuLg4jB07Fg8ePIC3t7fyFPbkyZORl5cnmXlNX+f8+fMYM2YMoqKiJHPVtqGhIW7cuAE7OzuYmpriypUrymmhmjVrppzySpP17NkTz58/x48//ggbGxsABV94gwYNgoWFBfbs2SNywqKZMmUKtm7div/7v/9TXmB0/vx5LFq0CEOHDsXy5ctFTvh2xsbGiI6OlswiD/8WFRWlVpaamopPPvkE3bp1w5gxY5TljRo1Ks9oxRYdHY3u3bvjwYMHcHR0RHBwMDw8PJCeng4tLS2kp6fj559/fuOUj5rgwYMH6NmzJ65evQpbW1soFArExcWhYcOG+PXXXyXXCUIlw8asxE2ZMkX57/z8fGzbtg2NGjVCo0aN1HqipPBFJ1cXLlzADz/8gJCQEKSkpKB79+6SmeTezc0NkyZNwuDBg1Uas76+vjh27BjOnDkjdsS3kssXXn5+Pr755husXLkS8fHxAABra2tMnDgRU6dOlcS8v3369MHAgQPRv39/saMU26tpxf79tfnv26/+LYUpxjw9PaGjo4Pp06djx44dCA0NRZcuXbBx40YABePMIyIicP78eZGTFs3Ro0dx48YNCIIAZ2dndO7cWexIVI7YmJW4ok6+r1AoEBYWVsZp3l1cXNwb99eoUaOckry72NhY7Ny5Ez/88APu37+PDh06YNCgQejTp4/yghEp2LdvH4YMGYKZM2fCz88Pvr6+KpPDf/TRR2JHLDI5feG9WnxAChd+/Xt1wqdPn8LPzw8jRoxAw4YN1f7o1uSVCouzDLWmLy9etWpVhIWFoVGjRkhLS0OlSpVw4cIF5SwZN27cwIcffiiZWRmoYmNjljTK6yZUf0XTezv+TUtLC25ubvj0008xcOBA5ZKwUiTVyeHDwsIwfvx4nD9/Xq3Rl5KSgpYtW2Lt2rVo06aNSAkrBi2tol2eIYUeTbnQ0tJSrh4JQOWsCwA8fvwYNjY2Gv16pKamIjY2Fk5OTjAxMcGlS5cQGBiIzMxM9OrVC4MGDRI7IpUTTs0lY3/99RfS09NRt27dIn+ZiO3y5csqt1++fInLly9j+fLlWLBggUipSubGjRuoU6eO2DHeSW5uLhYsWICRI0e+dbJ7TRQYGIjRo0cX2ntpZmaGL774AsuXL5dMY/bx48f46quvlHNJ/7cvQlMbHlK5OK0kYmJiEBcXh5ycHJVyTe5hfuW/HQdSms7x9OnT8PLyQlpaGiwsLPDjjz/i448/RvXq1aGtrY3du3cjIyMDo0ePFjsqlQP2zMrAtm3bkJycjEmTJinLPv/8c2zatAlAwaTYhw8ffu1qNVKwf/9+LF26FCdPnhQ7SrH9e9WpevXqwcXFRexIxWJiYoKrV6+iZs2aYkcpNjs7Oxw6dEhlDsp/u3HjBrp06fLW4S2awtPTE3FxcRg/fnyhc0lr8mpsf/zxB5KSkuDp6aksCwoKgo+PD9LT09GrVy+sWrUK+vr6IqYsurt376J3796Ijo5WGzcLaO4fFq9oaWnB09NT+fPet28fOnbsCGNjYwBAdnY2Dh06pLH1aNu2LRwdHeHr64stW7Zg+fLlGDNmDBYuXAgA8Pf3x88//4zIyEhxg1K5YGNWBlq0aIHPP/9cuVLQoUOH0L17d2zduhX16tXD+PHj4ezsrBzYL0W3bt1CkyZNkJ6eLnaUIpPyqlP/1qtXL/Tq1QvDhw8XO0qxGRgY4OrVq3BwcCh0/+3bt9GwYUNkZmaWc7KSMTU1xZkzZ9CkSROxoxSbh4cHOnTogOnTpwMouJrexcUFw4cPR7169bB06VJ88cUXkllwoHv37tDW1saGDRtgb2+PCxcuIDExEVOnTsU333yj8b39/15Z7k00dVU5c3NznD9/HnXr1kVOTg4MDQ1x6dIlNG7cGEDBZ7tp06ZITU0VOSmVBw4zkIHY2FjloH0A+PXXX9GjRw/leKGFCxcW+ReX2F5d1PKKIAiIj4/HvHnz4OjoKFKqkpHyqlP/5unpiZkzZ+Lq1atwdXVV9ty8osmnU6tXr47o6OjXNmajoqJgbW1dzqlKztbWVm1ogVRcuXIF/v7+ytvBwcFo3rw5NmzYAKCgbj4+PpJpzJ47dw5hYWGoVq0atLS0oKWlhdatWyMgIADe3t5qQ6Y0jaY2UovqxYsXqFy5MgBAT08PRkZGKhfWmpqaIiMjQ6x4VM7YmJWBzMxMlTGB4eHhGDlypPK2vb09EhISxIhWbObm5mqnTgVBgK2tLYKDg0VKVTJyWWbx1dyZhU3tpukX7HTt2hVz586Fp6cnDAwMVPZlZmbCx8cHXl5eIqUrvsDAQMyYMQPr1q2T3LCP5ORkWFpaKm+fOnUKHh4eytsffPABHjx4IEa0EsnLy4OJiQmAgpkBHj16BCcnJ9jZ2eHmzZsip5M/hUKh8l3x39tUsbAxKwN2dnaIiIiAnZ0dnj17hmvXrqF169bK/QkJCTAzMxMxYdGdOHFC5baWlhaqVasGBwcH6OhI6+0ql1WnpJT1v2bPno3du3ejTp06GD9+PJycnKBQKHD9+nV8//33yMvLw6xZs8SOWWQDBgxARkYGateuDSMjI7X3lyavoGVpaYl79+7B1tYWOTk5uHTpEnx9fZX7U1NTC/28aKoGDRogKioK9vb2aN68OZYsWQI9PT2sX79ekgtCSI0gCOjUqZPyeyEjIwPdu3eHnp4egIKLV6nikFbrgAo1dOhQjBs3DteuXUNYWBjq1q0LV1dX5f7w8HA0aNBAxIRF165dO7EjlJpXyyz+d9UpKS2zmJ+fj61bt2L37t24f/8+FAoF7O3t0bdvXwwZMkTje0IsLS0RHh6OMWPGYObMmSoX6bi7u2P16tUqvYWaLjAwUOwIJebh4YEZM2Zg8eLF2Lt3L4yMjFTGlUZFRaF27doiJiye2bNnK8fw+/v7w8vLC23atEGVKlUksyCKlPn4+KjcLuzix759+5ZXHBIZLwCTgfz8fPj4+CA0NBRWVlZYvny5yqntfv36wcPDQ7k0rKa7c+cOAgMDVWYAmDhxoqS+6ADprzolCAK6d++OAwcOoHHjxqhbty4EQcD169cRHR2NHj16YO/evWLHLLLk5GTcvn0bgiDA0dERFhYWYkeqUJ4+fYo+ffrg7NmzMDExwbZt29C7d2/l/k6dOuHDDz+U3BR8/5aUlAQLCwuN/yOPSG7YmCWNcvjwYfTo0QNNmjRBq1atIAgCwsPDceXKFezbt09Sq029ItVVp7Zs2YKJEyfi119/VVtpLiwsDL169cJ3332HoUOHipSwYsvMzMTLly9VyqSwGlhKSgpMTEzUlt5NSkqCiYmJ8jSxVNy+fRt37txB27ZtYWhoqFzOlojKDxuzMjV27Fj4+fmhatWqYkcplqZNm8Ld3R2LFi1SKZ8xYwaOHDmCS5cuiZSs4unSpQs6duyIGTNmFLp/4cKFOHXqFA4fPlzOySqu9PR0TJ8+Hbt27UJiYqLafk2+GE9uEhMT0b9/f5w4cQIKhQK3bt2Cvb09Ro0aBXNzcyxbtkzsiEQVBhuzMlWpUiVERkZK7kIEAwMDREdHq03DFRsbi0aNGiErK0ukZEXz7bffFvlYb2/vMkzy7qysrHDo0KHXzml6+fJleHp6SmamDDkYN24cTpw4AT8/PwwdOhTff/89Hj58iHXr1mHRokVcvrMcDR06FE+ePMHGjRtRr1495VKwR44cweTJk3Ht2jWxIxJVGLwATKak+jdKtWrVEBkZqdaYjYyMVK4hrslWrFhRpOMUCoXGN2aTkpLeeHGUpaUlkpOTyzER7du3D0FBQWjfvj1GjhyJNm3awMHBAXZ2dti5cycbs+XoyJEjOHz4sNrYd0dHR/z1118ipSKqmNiYlYGRI0di5cqVKhNGS9Xo0aPx+eef4+7du2jZsiUUCgV+//13LF68GFOnThU73lvdu3dP7AilJi8v743ToWlra3P6m3KWlJSEWrVqASg4+/JqKq7WrVsr5wOm8pGeng4jIyO18mfPnklmSV65ycrKUptPmioGNmZlYNu2bVi0aJFKY1aqS/jNmTMHpqamWLZsGWbOnAkAsLGxwbx58zS+J1NuBEHA8OHDX/vFnJ2dXc6JyN7eHvfv34ednR2cnZ2xa9cuNGvWDPv27YO5ubnY8SqUtm3bIigoCPPnzwdQcLYlPz8fS5cuVbtgkspOfn4+FixYgLVr1+Lx48eIjY2Fvb095syZg5o1a0pmFh96NxwzKwNaWlpISEiQxGn4N8nNzcXOnTvh7u4OKysrZYNcSj3OU6ZMKfKxha2opUmkvna7HK1YsQLa2trw9vbGiRMn0K1bN+Tl5SE3NxfLly/HxIkTxY5YYcTExKB9+/ZwdXVFWFgYevTogWvXriEpKQlnz56V3FSCUuXn54dt27bBz88Po0ePxtWrV2Fvb49du3ZhxYoVOHfunNgRqRywMSsDWlpaePz4MapVqyZ2lHdmZGSE69evw87OTuwoJVLUHhmFQoGwsLAyTkNyFxcXh4sXL6J27dpo3Lix2HEqnISEBKxZswYRERHIz8+Hi4sLxo0bB2tra7GjVRgODg5Yt24dOnXqBFNTU+WFeDdu3ECLFi04rr+C4DADmahTp85b5zbU5KUuX2nevDkuX74s2cbsf5fjJSoNf/zxB5KSkuDp6aksCwoKgo+PD9LT09GrVy+sWrWKYzXLmZWVlcqSvFT+Hj58CAcHB7Xy/Px8tXmYSb7YmJUJX19fmJmZiR3jnY0dOxZTp07F33//DVdXVxgbG6vsb9SokUjJiMQzb948tG/fXtmYjY6OxqhRozB8+HA4OztjyZIlyrHlVH6eP3+OCxcu4MmTJ8jPz1fZx8VEykf9+vVx5swZtQ6Qn376CU2bNhUpFZU3DjOQATmMmR05ciQCAwMLvYhFoVAoV9WR0qTwHTp0eGNvOYcZUFFZW1tj3759cHNzAwDMmjULp06dwu+//w6g4Ivbx8cHMTExYsasUPbt24dBgwYhPT0dpqamKp91hUIhiTNhcrBv3z4MGTIEM2fOhJ+fH3x9fXHz5k0EBQUhNDRUkqtGUvGxMSsD2traiI+Pl3Rj9lUdMjMz33iclIYfTJ48WeX2y5cvERkZiatXr2LYsGFYuXKlSMlIagwMDHDr1i3Y2toCKJiKy8PDA7NnzwYA3L9/Hw0bNpTsLCZSVKdOHXTt2hULFy4sdIouKj+HDx/GwoULVcYuz507F126dBE7GpUTDjOQATn8PfKqDlJqrL7N6xZQmDdvHtLS0so5DUmZpaUl7t27B1tbW+Tk5ODSpUsqYzVTU1Ohq6srYsKK5+HDh/D29mZDVgO4u7vD3d1d7BgkIi2xA9C7y8/Pl3Sv7Ctvu4BNLgYPHozNmzeLHYMkxMPDAzNmzMCZM2cwc+ZMGBkZoU2bNsr9UVFRnAqqnLm7u+PixYtixyAisGeWNIhcZmR4m3PnznGVGioWf39/9OnTB+3atYOJiQm2bdsGPT095f7NmzfzlGo569atG77++mvExMSgYcOGaj3jPXr0ECmZ/FlYWBS580MO3xn0dhwzSxpBS0sLgYGBb52RYdiwYeWU6N317t1b5ReuIAiIj4/HxYsXMWfOHPj4+IiYjqQoJSUFJiYm0NbWVilPSkqCiYmJSgOXypaW1utPbErtYlWp2bZtW5GPldJ3BpUcG7OkEeQwI8N/jRgxQjkTA1BQx2rVqqFjx47sRSMiKmOZmZkwNDQUOwaVA46ZJY0gp/GyGRkZGDduHA4fPozQ0FBkZWVh6dKl2LRpExYtWsSGLJGE/fHHHzh48KBKWVBQEGrVqoX33nsPn3/+ObKzs0VKV/GMGzeu0PL09HSVRUZI3tiYJY0gpxMEPj4+2Lp1K7y8vPDJJ5/g2LFjGDNmjNixiKgUzJs3D1FRUcrbrxaw6Ny5M2bMmIF9+/YhICBAxIQVy5EjR5RT1L2Snp4ODw8PDvWoQDjMgKiU1a5dGwsWLMDAgQMBABcuXECrVq2QlZWlNtaRiKSFC1holnv37qF169b46quvMHnyZKSmpsLd3R06Ojo4ePCg2iqSJE+czYColD148EBl2qRmzZpBR0cHjx49Uk56T0TSlJycDEtLS+XtU6dOwcPDQ3n7gw8+wIMHD8SIViHVqlULhw8fRvv27aGlpYXg4GDo6+tj//79bMhWIBxmQFTK8vLy1K4q19HRQW5urkiJiKi0vFrAAoByAYsWLVoo93MBi/LXoEEDhIaGYtasWTAyMmKPbAXEnlmiUiYIAoYPHw59fX1lWVZWFr788kuVX7C7d+8WIx4RvYNXC1gsXrwYe/fu5QIWImjatGmhFw3r6+vj0aNHaNWqlbLs0qVL5RmNRMLGLFEpK2xew8GDB4uQhIhKGxewEF+vXr3EjkAahheAERERFRMXsCDSHGzMEhERkaTl5OTgyZMnyM/PVymvUaOGSImoPHGYAREREUlSbGwsRo0ahfDwcJVyQRC4rHAFwsYsERERSdKIESOgo6OD0NBQWFtby2o1SSo6DjMgIiIiSTI2NkZERATq1q0rdhQSEeeZJSIiIklydnbGs2fPxI5BImNjloiIiCRp8eLFmDZtGk6ePInExES8ePFCZaOKgcMMiIiISJK0tAr65P47VpYXgFUsvACMiIiIJOnEiRNiRyANwJ5ZIiIikp3IyEg0adJE7BhUDjhmloiIiGQhJSUFq1evhouLC1xdXcWOQ+WEjVkiIiKStLCwMAwePBjW1tZYtWoVunbtiosXL4odi8oJx8wSERGR5Pz999/YunUrNm/ejPT0dPTv3x8vX77EL7/8AmdnZ7HjUTlizywRERFJSteuXeHs7IyYmBisWrUKjx49wqpVq8SORSJhzywRERFJypEjR+Dt7Y0xY8bA0dFR7DgkMvbMEhERkaScOXMGqampcHNzQ/PmzfHdd9/h6dOnYscikXBqLiIiIpKkjIwMBAcHY/Pmzbhw4QLy8vKwfPlyjBw5EqampmLHo3LCxiwRERFJ3s2bN7Fp0yZs374dz58/x0cffYTffvtN7FhUDtiYJSIiItnIy8vDvn37sHnzZjZmKwg2ZomIiIhIsngBGBERERFJFhuzRERERCRZbMwSERERkWSxMUtEREREksXGLBERERFJFhuzRERERCRZbMwSERERkWSxMUtEREREkvX/ulrsbtnKuLMAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Confusion Matrix of FashionMNIST model\n", + "\n", + "# load best model\n", + "\n", + "model.load_state_dict(torch.load(save_path))\n", + "model.eval()\n", + "\n", + "label_dict = {\n", + " 0: \"T-Shirt\",\n", + " 1: \"Trouser\",\n", + " 2: \"Pullover\",\n", + " 3: \"Dress\",\n", + " 4: \"Coat\",\n", + " 5: \"Sandal\",\n", + " 6: \"Shirt\",\n", + " 7: \"Sneaker\",\n", + " 8: \"Bag\",\n", + " 9: \"Ankle Boot\"\n", + "}\n", + "\n", + "all_labels = []\n", + "predictions = []\n", + "\n", + "for data, labels in test_loader:\n", + " data = data.to(device)\n", + " with torch.no_grad():\n", + " outputs = model(data)\n", + " predictions.append(outputs.argmax(dim=1).cpu().numpy())\n", + " all_labels.append(labels.numpy())\n", + "\n", + "cm = confusion_matrix(np.concatenate(all_labels), np.concatenate(predictions))\n", + "\n", + "plt.figure(figsize=(8, 6))\n", + "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=label_dict.values(), yticklabels=label_dict.values())\n", + "plt.title(\"Confusion Matrix for CNN FashionMNIST Classification\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "701c4bfe", + "metadata": {}, + "source": [ + "Wie können wir das eventuell noch weiter verbessern?" + ] + }, + { + "cell_type": "markdown", + "id": "cdd6e01b", + "metadata": {}, + "source": [ + "Wir verwenden ein Pretrained Model und finetunen es auf unseren Daten." + ] + }, + { + "cell_type": "markdown", + "id": "9aee01d0", + "metadata": {}, + "source": [ + "> **Übung:** Lade dir ein Pretrained Model herunter (zBsp *ResNet18*) und trainiere es nochmal mit unserer Trainingsmethode für weitere 20 Epochen." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dsai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.23" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/06_NN/code/nn_1_numpy_pytorch.ipynb b/06_NN/code/nn_1_numpy_pytorch.ipynb new file mode 100644 index 0000000..b83178a --- /dev/null +++ b/06_NN/code/nn_1_numpy_pytorch.ipynb @@ -0,0 +1,3506 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3892507e", + "metadata": {}, + "source": [ + "
\n", + "\n", + "

Neural Networks: Numpy, PyTorch, Google Colab

\n", + "

DSAI

\n", + "

Jakob Eggl

\n", + "\n", + "
\n", + " \"Logo\"\n", + "
\n", + " © 2025/26 Jakob Eggl. Nutzung oder Verbreitung nur mit ausdrücklicher Genehmigung des Autors.\n", + "
\n", + "
\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "138e9e63", + "metadata": {}, + "source": [ + "# Numpy" + ] + }, + { + "cell_type": "markdown", + "id": "4e58901d", + "metadata": {}, + "source": [ + "Wir wollen uns nun ``numpy`` widmen. Es ist eine grundlegende - wenn nicht sogar *DIE* - Bibliothek, auf der die meisten weiteren (und von uns auch bisher schon verwendeten) Bibliotheken aufbauen.\n", + "\n", + "Für numpy gibt es viele Tutorials online, unter anderem wurden [Tutorial1](https://numpy.org/doc/stable/user/quickstart.html) und [Tutorial2](https://cs231n.github.io/python-numpy-tutorial/) für diesen Abschnitt des Notebooks verwendet." + ] + }, + { + "cell_type": "markdown", + "id": "cbc12329", + "metadata": {}, + "source": [ + "## Was ist Numpy?" + ] + }, + { + "cell_type": "markdown", + "id": "85b0ddf5", + "metadata": {}, + "source": [ + "* **Num**erical **Py**thon\n", + "* Open Source\n", + "* ermöglicht numerisches Rechnen mit Python\n", + "* Hauptobjekte in Numpy: (mehrdimensionale) *Arrays*" + ] + }, + { + "cell_type": "markdown", + "id": "db4f4f5e", + "metadata": {}, + "source": [ + "Bisher haben wir uns manchmal schon implizit mit ``numpy`` Objekten beschäftigt:\n", + "* Bei scikit-learn (``sklearn``) werden haben wir oft mit numpy Objekten gearbeitet (zBsp.: in vielen Fällen waren *X* und *y* numpy arrays).\n", + "* Pandas Dataframes können auch einfach in numpy arrays konvertiert werden.\n", + "\n", + "Bisher war das aber bei uns immer ein eher unwichtiges Detail. Wir betrachten nun weitere Gründe, warum wir uns gut in numpy auskennen sollen." + ] + }, + { + "cell_type": "markdown", + "id": "88dc22e9", + "metadata": {}, + "source": [ + "Wir importieren ``numpy`` ganz einfach mit:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "717661df", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np" + ] + }, + { + "cell_type": "markdown", + "id": "d40fe311", + "metadata": {}, + "source": [ + "## Warum Numpy?" + ] + }, + { + "cell_type": "markdown", + "id": "34300d89", + "metadata": {}, + "source": [ + "#### Vorteile:\n", + "* **sehr schnell** und **effizient** (ist in *C* geschrieben)\n", + "* Viele vektorisierte (vectorized) Operationen\n", + "* Große Anzahl an mathematischen Funktionen und Operationen implementiert\n", + "* Super integriert in alle gängigen Bibliotheken\n", + "* Effizienter Speicherverbrauch (überall gleicher Datentyp in einem array)\n", + "\n", + "#### Nachteile:\n", + "* (Fast) nur numerische Datentypen (Integer, Float, Double, etc.) möglich\n", + "* Pro array nur ein Datentyp erlaubt (viele Datasets bei uns haben bisher oft auch andere Features gehabt (Name, Adresse, etc.))\n", + "* Namen der Features und Label(s) gehen verloren." + ] + }, + { + "cell_type": "markdown", + "id": "50ba1dd8", + "metadata": {}, + "source": [ + "Betrachten wir den Geschwindigkeitsunterschied anhand von einem kurzem Beispiel:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e9e6441e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Listenzeit: 0.11680150032043457\n", + "NumPy-Zeit: 0.005227804183959961\n", + "Numpy ist in diesem Fall 22.342363296392577 mal schneller\n" + ] + } + ], + "source": [ + "import time\n", + "\n", + "n_elements = 3000000\n", + "\n", + "a = list(range(n_elements))\n", + "b = list(range(n_elements))\n", + "\n", + "start = time.time()\n", + "z = zip(a, b)\n", + "c = [x + y for x, y in z]\n", + "time_1 = time.time() - start\n", + "print(f\"Listenzeit: {time_1}\")\n", + "\n", + "a_np = np.array(a)\n", + "b_np = np.array(b)\n", + "\n", + "start = time.time()\n", + "c_np = a_np + b_np\n", + "time_2 = time.time() - start\n", + "print(f\"NumPy-Zeit: {time_2}\")\n", + "\n", + "print(f\"Numpy ist in diesem Fall {time_1/time_2} mal schneller\")" + ] + }, + { + "cell_type": "markdown", + "id": "e4f48f44", + "metadata": {}, + "source": [ + "![Vectored_Meme](../resources/Vectored.png)\n", + "\n", + "(von https://imageresizer.com/pt/gerador-de-memes/editar/you-just-got-vectored)" + ] + }, + { + "cell_type": "markdown", + "id": "0750b461", + "metadata": {}, + "source": [ + "## Numpy Arrays" + ] + }, + { + "cell_type": "markdown", + "id": "e07283cf", + "metadata": {}, + "source": [ + "Nun beschäftigen wir uns mit dem zentralen Element in numpy: Das **numpy array**." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "78b046b0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n" + ] + } + ], + "source": [ + "# Short Recap on Python Lists\n", + "\n", + "a_list = list(range(10))\n", + "\n", + "print(a_list)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "8d30d466", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 2, 2, 3, 4, 5, 6, 7, 8, 9]\n" + ] + } + ], + "source": [ + "a_list[1]=2\n", + "print(a_list)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "fb7fd093", + "metadata": {}, + "outputs": [], + "source": [ + "# Now to numpy arrays. We now convert our list to a numpy array.\n", + "\n", + "a_array = np.array(a_list) # we only need to pass a list to the np.array function" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "db374151", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0 2 2 3 4 5 6 7 8 9]\n", + "\n", + "int64\n" + ] + } + ], + "source": [ + "print(a_array)\n", + "print(type(a_array))\n", + "print(a_array.dtype)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f1637084", + "metadata": {}, + "outputs": [], + "source": [ + "# Now what about this list?\n", + "a_second_list = ['hello', 'world!']" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "b7fb78f5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['hello', 'world!']" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a_second_list" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "5728c71b", + "metadata": {}, + "outputs": [], + "source": [ + "# Converting to numpy?\n", + "a_second_array = np.array(a_second_list)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "07bc23c8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['hello' 'world!']\n", + "\n", + " **Übung:** Was wäre mit den bekannten Methoden also der schnellste Weg, um eine $3\\times 3$ Matrix zu erstellen, welche aufsteigend die Zahlen 1-9 beinhaltet?" + ] + }, + { + "cell_type": "markdown", + "id": "1a4115e3", + "metadata": {}, + "source": [ + "## Indexing von Numpy Arrays" + ] + }, + { + "cell_type": "markdown", + "id": "e00496f0", + "metadata": {}, + "source": [ + "Es gibt viele Möglichkeiten, wie in numpy ein Array indiziert werden kann. Im einfachsten Fall ist das gleich wie in den meisten gängigen Programmiersprachen. Es gibt aber auch Eigenheiten (zBsp slicing) für numpy." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "7dac2775", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[2 3]\n", + " [6 7]]\n", + "2\n", + "77\n" + ] + } + ], + "source": [ + "# Create the following rank 2 array with shape (3, 4)\n", + "# [[ 1 2 3 4]\n", + "# [ 5 6 7 8]\n", + "# [ 9 10 11 12]]\n", + "a = np.array([[1,2,3,4], [5,6,7,8], [9,10,11,12]]) # this can be done more elegantly (see previous exercise)\n", + "\n", + "# Use slicing to pull out the subarray consisting of the first 2 rows\n", + "# and columns 1 and 2; b is the following array of shape (2, 2):\n", + "# [[2 3]\n", + "# [6 7]]\n", + "b = a[:2, 1:3]\n", + "print(b) # prints [[2 3], [6 7]]\n", + "\n", + "# A slice of an array is a view into the same data, so modifying it\n", + "# will modify the original array.\n", + "print(a[0, 1]) # Prints \"2\"\n", + "b[0, 0] = 77 # b[0, 0] is the same piece of data as a[0, 1]\n", + "print(a[0, 1]) # Prints \"77\"" + ] + }, + { + "cell_type": "markdown", + "id": "7b4bf033", + "metadata": {}, + "source": [ + "Wir sehen also auch, dass wir durch slicing immer noch auf das gleiche Array referenzieren! Es können auch beide Indexmethoden gemischt werden:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "0ab228f2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[5 6 7 8] (4,)\n", + "[[5 6 7 8]] (1, 4)\n", + "[ 2 6 10] (3,)\n", + "[[ 2]\n", + " [ 6]\n", + " [10]] (3, 1)\n" + ] + } + ], + "source": [ + "# Create the following rank 2 array with shape (3, 4)\n", + "# [[ 1 2 3 4]\n", + "# [ 5 6 7 8]\n", + "# [ 9 10 11 12]]\n", + "a = np.array([[1,2,3,4], [5,6,7,8], [9,10,11,12]])\n", + "\n", + "# Two ways of accessing the data in the middle row of the array.\n", + "# Mixing integer indexing with slices yields an array of lower rank,\n", + "# while using only slices yields an array of the same rank as the\n", + "# original array:\n", + "row_r1 = a[1, :] # Rank 1 view of the second row of a\n", + "row_r2 = a[1:2, :] # Rank 2 view of the second row of a\n", + "print(row_r1, row_r1.shape) # Prints \"[5 6 7 8] (4,)\"\n", + "print(row_r2, row_r2.shape) # Prints \"[[5 6 7 8]] (1, 4)\"\n", + "\n", + "# We can make the same distinction when accessing columns of an array:\n", + "col_r1 = a[:, 1]\n", + "col_r2 = a[:, 1:2]\n", + "print(col_r1, col_r1.shape) # Prints \"[ 2 6 10] (3,)\"\n", + "print(col_r2, col_r2.shape) # Prints \"[[ 2]\n", + " # [ 6]\n", + " # [10]] (3, 1)\"" + ] + }, + { + "cell_type": "markdown", + "id": "a42724a6", + "metadata": {}, + "source": [ + "Wir können auch numpy arrays mit anderen numpy arrays (zbsp integer-array) indizieren:" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "80643fac", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1 2]\n", + " [3 4]\n", + " [5 6]]\n", + "[1 4 5]\n", + "[1 4 5]\n", + "[2 2]\n", + "[2 2]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "\n", + "a = np.array([[1,2], [3, 4], [5, 6]])\n", + "print(a)\n", + "\n", + "# An example of integer array indexing.\n", + "# The returned array will have shape (3,) and\n", + "print(a[[0, 1, 2], [0, 1, 0]]) # Prints \"[1 4 5]\"\n", + "\n", + "# The above example of integer array indexing is equivalent to this:\n", + "print(np.array([a[0, 0], a[1, 1], a[2, 0]])) # Prints \"[1 4 5]\"\n", + "\n", + "# When using integer array indexing, you can reuse the same\n", + "# element from the source array:\n", + "print(a[[0, 0], [1, 1]]) # Prints \"[2 2]\"\n", + "\n", + "# Equivalent to the previous integer array indexing example\n", + "print(np.array([a[0, 1], a[0, 1]])) # Prints \"[2 2]\"" + ] + }, + { + "cell_type": "markdown", + "id": "9ddd9330", + "metadata": {}, + "source": [ + "Dies kann auch verwendet werden um zBsp ein Element von jeder Zeile zu erhalten oder verändern" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "d549ccbd", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 1 2 3]\n", + " [ 4 5 6]\n", + " [ 7 8 9]\n", + " [10 11 12]]\n", + "[ 1 6 7 11]\n", + "[[11 2 3]\n", + " [ 4 5 16]\n", + " [17 8 9]\n", + " [10 21 12]]\n" + ] + } + ], + "source": [ + "# Create a new array from which we will select elements\n", + "a = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])\n", + "\n", + "print(a) # prints \"array([[ 1, 2, 3],\n", + " # [ 4, 5, 6],\n", + " # [ 7, 8, 9],\n", + " # [10, 11, 12]])\"\n", + "\n", + "# Create an array of indices\n", + "b = np.array([0, 2, 0, 1])\n", + "\n", + "# Select one element from each row of a using the indices in b\n", + "print(a[np.arange(4), b]) # Prints \"[ 1 6 7 11]\"\n", + "\n", + "# Mutate one element from each row of a using the indices in b\n", + "a[np.arange(4), b] += 10\n", + "\n", + "print(a) # prints \"array([[11, 2, 3],\n", + " # [ 4, 5, 16],\n", + " # [17, 8, 9],\n", + " # [10, 21, 12]])" + ] + }, + { + "cell_type": "markdown", + "id": "bbf1586f", + "metadata": {}, + "source": [ + "Auch mit einem Boolean-Array kann indiziert werden." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "b668187f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1 2]\n", + " [3 4]\n", + " [5 6]]\n", + "[[False False]\n", + " [ True True]\n", + " [ True True]]\n", + "[3 4 5 6]\n", + "[3 4 5 6]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "\n", + "a = np.array([[1,2], [3, 4], [5, 6]])\n", + "print(a)\n", + "\n", + "bool_idx = (a > 2) # Find the elements of a that are bigger than 2;\n", + " # this returns a numpy array of Booleans of the same\n", + " # shape as a, where each slot of bool_idx tells\n", + " # whether that element of a is > 2.\n", + "\n", + "print(bool_idx) # Prints \"[[False False]\n", + " # [ True True]\n", + " # [ True True]]\"\n", + "\n", + "# We use boolean array indexing to construct a rank 1 array\n", + "# consisting of the elements of a corresponding to the True values\n", + "# of bool_idx\n", + "print(a[bool_idx]) # Prints \"[3 4 5 6]\"\n", + "\n", + "# We can do all of the above in a single concise statement:\n", + "print(a[a > 2]) # Prints \"[3 4 5 6]\"" + ] + }, + { + "cell_type": "markdown", + "id": "af7674b1", + "metadata": {}, + "source": [ + "## Mathematische Funktionen und Matrizen/Vektorrechnung (Lineare Algebra)" + ] + }, + { + "cell_type": "markdown", + "id": "e65b3f7b", + "metadata": {}, + "source": [ + "Nachdem ja numpy vorrangig für die Durchführung von numerischen Simulationen/Programmen entwickelt wurde, möchten wir uns jetzt noch spannende und vor allem für den Machine Learning Bereich wichtige Funktionen/Operationen ansehen." + ] + }, + { + "cell_type": "markdown", + "id": "357800bf", + "metadata": {}, + "source": [ + "#### Addition und Subtraktion von Arrays" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "7c759fbc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[5 7 9]\n", + "[-3 -3 -3]\n", + "[13 17 21]\n" + ] + } + ], + "source": [ + "a = np.array([1,2,3])\n", + "b = np.array([4,5,6])\n", + "\n", + "print(a+b)\n", + "print(a-b)\n", + "print(a+3*b)" + ] + }, + { + "cell_type": "markdown", + "id": "ab14d4e6", + "metadata": {}, + "source": [ + "#### Transponieren" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "504d93d5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1 2]\n", + " [3 4]\n", + " [5 6]]\n" + ] + } + ], + "source": [ + "A = np.array([[1, 2], [3, 4], [5, 6]])\n", + "print(A)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "09bce2af", + "metadata": {}, + "outputs": [], + "source": [ + "# We can transpose the array using the .T attribute\n", + "transposed_A = A.T" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "c6dd9b06", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1 3 5]\n", + " [2 4 6]]\n" + ] + } + ], + "source": [ + "print(transposed_A)" + ] + }, + { + "cell_type": "markdown", + "id": "360f2a4a", + "metadata": {}, + "source": [ + "#### Matrix-Matrix und Matrix-Vektor und Skalarprodukt und Betrag vom Vektor" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "295ff193", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 23 29 35]\n", + " [ 53 67 81]\n", + " [ 83 105 127]]\n", + "[[ 23 29 35]\n", + " [ 53 67 81]\n", + " [ 83 105 127]]\n" + ] + } + ], + "source": [ + "A = np.array([[1, 2], [3, 4], [5, 6]])\n", + "B = np.array([[7, 8], [9, 10], [11, 12]])\n", + "\n", + "# Matrix multiplication (dot product) using numpy\n", + "matrix_product = np.dot(A, B.T)\n", + "matrix_product_2 = A @ B.T\n", + "\n", + "print(matrix_product)\n", + "print(matrix_product_2)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "9180ea27", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[32 77]\n" + ] + } + ], + "source": [ + "A = np.array([[1, 2, 3], [4, 5, 6]])\n", + "x = np.array([4,5,6])\n", + "\n", + "print(A@x) # calculates A*x" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "10d617ff", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "13\n" + ] + } + ], + "source": [ + "# Scalarproduct (Innerproduct)\n", + "a = np.array([5,6,7])\n", + "b = np.array([0,1,1])\n", + "print(np.inner(a,b))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "cb35b966", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10.488088481701515\n" + ] + } + ], + "source": [ + "# Length of a vector\n", + "a = np.array([5,6,7])\n", + "print(np.linalg.norm(a)) # Calculates with Pythagorean theorem" + ] + }, + { + "cell_type": "markdown", + "id": "b194550a", + "metadata": {}, + "source": [ + "#### Lösen von Gleichungssystemen" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "bd28f3ae", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Solution x: [0.8 1.3]\n" + ] + } + ], + "source": [ + "# We can also solve a system of linear equations using numpy\n", + "A = np.array([[3, 2], [1, 4]])\n", + "b = np.array([5, 6])\n", + "x = np.linalg.solve(A, b) # Solves the equation Ax = b\n", + "print(\"Solution x:\", x) # Prints the solution to the system of equations" + ] + }, + { + "cell_type": "markdown", + "id": "5aed967d", + "metadata": {}, + "source": [ + "> **Übung:** Wie sieht das Gleichungssystem hier aus?" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "712b2694", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The matrix A is singular, cannot solve the system of equations.\n" + ] + } + ], + "source": [ + "# Note that this is not possible if the determinant of A is zero, e.g.\n", + "A = np.array([[1, 2], [2, 4]])\n", + "b = np.array([3, 6])\n", + "try:\n", + " x = np.linalg.solve(A, b) # This will raise an error\n", + "except:\n", + " print(\"The matrix A is singular, cannot solve the system of equations.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "4efaa740", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Determinant of A: 0.0\n" + ] + } + ], + "source": [ + "# We can check that directly:\n", + "det_A = np.linalg.det(A)\n", + "print(\"Determinant of A:\", det_A) # Prints the determinant of A" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "53ab1c64", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inverse of A:\n", + " [[-2. 1. ]\n", + " [ 1.5 -0.5]]\n", + "[[1.00000000e+00 1.11022302e-16]\n", + " [0.00000000e+00 1.00000000e+00]]\n" + ] + } + ], + "source": [ + "# And we can also calculate the inverse of a matrix, if it is not singular\n", + "A = np.array([[1, 2], [3, 4]])\n", + "A_inv = np.linalg.inv(A) # Calculates the inverse of A\n", + "print(\"Inverse of A:\\n\", A_inv) # Prints the inverse of A\n", + "\n", + "# Sanity check:\n", + "print(A@A_inv)" + ] + }, + { + "cell_type": "markdown", + "id": "258398a5", + "metadata": {}, + "source": [ + "#### (Arg)Min und (Arg)Max" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "aa18cb5c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[1 2 3 4 5]\n", + "Max value: 5 at index 4\n", + "Min value: 1 at index 0\n" + ] + } + ], + "source": [ + "# We can also look for the maximum and minimum values in an array and its indices\n", + "a = np.array([1, 2, 3, 4, 5])\n", + "print(a)\n", + "max_value = np.max(a) # Maximum value\n", + "max_index = np.argmax(a) # Index of maximum value\n", + "min_value = np.min(a) # Minimum value\n", + "min_index = np.argmin(a) # Index of minimum value\n", + "print(\"Max value:\", max_value, \"at index\", max_index)\n", + "print(\"Min value:\", min_value, \"at index\", min_index)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "c8ad38f7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1 2 3]\n", + " [4 5 6]\n", + " [7 8 9]]\n", + "Max value in matrix: 9 at index 8\n", + "Min value in matrix: 1 at index 0\n" + ] + } + ], + "source": [ + "# This can also be done in matrices\n", + "A = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])\n", + "print(A)\n", + "max_value = np.max(A) # Maximum value in the matrix\n", + "max_index = np.argmax(A) # Index of maximum value in the matrix\n", + "min_value = np.min(A) # Minimum value in the matrix\n", + "min_index = np.argmin(A) # Index of minimum value in the matrix\n", + "print(\"Max value in matrix:\", max_value, \"at index\", max_index)\n", + "print(\"Min value in matrix:\", min_value, \"at index\", min_index)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "000e5eec", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1 2 3]\n", + " [4 5 6]\n", + " [7 8 9]]\n", + "Max values in each row: [3 6 9]\n", + "Min values in each row: [1 4 7]\n", + "Max values in each column: [7 8 9]\n" + ] + } + ], + "source": [ + "# And we can also get the maximum and minimum values and indices in each row or column\n", + "\n", + "print(A)\n", + "\n", + "max_values_row = np.max(A, axis=1) # Maximum values in each row\n", + "print(\"Max values in each row:\", max_values_row)\n", + "min_values_row = np.min(A, axis=1) # Minimum values in each row\n", + "print(\"Min values in each row:\", min_values_row)\n", + "# We can also get the maximum and minimum values in each column\n", + "max_values_col = np.max(A, axis=0) # Maximum values in each column\n", + "print(\"Max values in each column:\", max_values_col)\n" + ] + }, + { + "cell_type": "markdown", + "id": "8870e317", + "metadata": {}, + "source": [ + "#### Mathematische Funktionen und komplexe Zahlen" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "741d77e9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Original array: [ 1 2 3 -4 -5]\n", + "Sine of array: [0.84147098 0.90929743 0.14112001 0.7568025 0.95892427]\n", + "Absolute values of array: [1 2 3 4 5]\n", + "Square root of array: [1. 1.41421356 1.73205081 2. 2.23606798]\n", + "[[1 2 3]\n", + " [4 5 6]\n", + " [7 8 9]]\n", + "Sine of matrix:\n", + " [[ 0.84147098 0.90929743 0.14112001]\n", + " [-0.7568025 -0.95892427 -0.2794155 ]\n", + " [ 0.6569866 0.98935825 0.41211849]]\n", + "Absolute values of matrix:\n", + " [[1 2 3]\n", + " [4 5 6]\n", + " [7 8 9]]\n", + "Complex array: [1.+2.j 3.+4.j 5.+6.j]\n", + "Real part of complex array: [1. 3. 5.]\n", + "Imaginary part of complex array: [2. 4. 6.]\n" + ] + } + ], + "source": [ + "# We can also just apply functions like sin, abs, etc. to numpy arrays\n", + "a = np.array([1, 2, 3, -4, -5])\n", + "print(\"Original array:\", a)\n", + "print(\"Sine of array:\", np.sin(a)) # Applies sine function to each element\n", + "print(\"Absolute values of array:\", np.abs(a)) # Applies absolute value function to each\n", + "print(\"Square root of array:\", np.sqrt(np.abs(a))) # Applies square root function to each element (after taking absolute value)\n", + "\n", + "# We can also apply functions to matrices\n", + "A = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])\n", + "print(A)\n", + "print(\"Sine of matrix:\\n\", np.sin(A)) # Applies sine function to each element\n", + "print(\"Absolute values of matrix:\\n\", np.abs(A)) # Applies absolute value function to each element\n", + "\n", + "\n", + "# And we can also work with complex numbers in numpy\n", + "complex_array = np.array([1 + 2j, 3 + 4j, 5 + 6j])\n", + "print(\"Complex array:\", complex_array)\n", + "print(\"Real part of complex array:\", np.real(complex_array)) # Extracts real part\n", + "print(\"Imaginary part of complex array:\", np.imag(complex_array)) # Extracts" + ] + }, + { + "cell_type": "markdown", + "id": "6ac7d19a", + "metadata": {}, + "source": [ + "# PyTorch" + ] + }, + { + "cell_type": "markdown", + "id": "c3125881", + "metadata": {}, + "source": [ + "Nun beschäftigen wir uns mit ``PyTorch``. Es baut auf ``numpy`` und ist eine sehr weit verbreitete Bibliothek, die das Entwickeln von neuronalen Netzwerken sehr einfach ermöglicht." + ] + }, + { + "cell_type": "markdown", + "id": "5e866ad8", + "metadata": {}, + "source": [ + "Das folgende basiert hauptsächlich auf einem sehr gutem Tutorium von *Philipp Lippe* ([link](https://github.com/phlippe/uvadlc_notebooks/blob/master/docs/tutorial_notebooks/tutorial2/Introduction_to_PyTorch.ipynb))." + ] + }, + { + "cell_type": "markdown", + "id": "3298ca1d", + "metadata": {}, + "source": [ + "Wir starten mit dem Import von dem Paket, genannt ``torch``." + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "2c0b4dd6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using torch 2.8.0+cpu\n" + ] + } + ], + "source": [ + "import torch\n", + "print(\"Using torch\", torch.__version__)" + ] + }, + { + "cell_type": "markdown", + "id": "27b54dcb", + "metadata": {}, + "source": [ + "Sollte die obige Zelle nicht ausführbar sein, so müssen wir PyTorch zuerst installieren. Einen Command dazu kann man sich [hier](https://pytorch.org/get-started/locally/) erstellen lassen. Wichtig ist die GPU (Cuda) Funktionalität, welche man unbedingt auswählen sollte, sofern man eine (unterstützte) NVIDIA Grafikkarte besitzt." + ] + }, + { + "cell_type": "markdown", + "id": "4b2a6273", + "metadata": {}, + "source": [ + "**Hinweis:** Fast alle gängigen NVIDIA Grafikkarten werden unterstützt. Es wird auch Apple-Silicon unterstützt." + ] + }, + { + "cell_type": "markdown", + "id": "afc7f17e", + "metadata": {}, + "source": [ + "Nun kommen wir auch schon zu den wichtigsten Elementen in PyTorch (und somit quasi auch für unsere Neuronalen Netzwerke später), den ***Tensoren***." + ] + }, + { + "cell_type": "markdown", + "id": "098d808d", + "metadata": {}, + "source": [ + "#### Tensoren" + ] + }, + { + "cell_type": "markdown", + "id": "9b7f6432", + "metadata": {}, + "source": [ + "* sind eine Generalisierung von Matrizen bzw. Vektoren (also mathematische Objekte)\n", + "* prinzipiell komplizierte Objekte, jedoch für unsere Zwecke mit Kenntnis von Matrizen und Vektoren recht leicht zu bedienen.\n", + "* namensgebend auch zum Beispiel für die Bibliothek ``Tensorflow`` (hat den gleichen Zweck wie zBsp PyTorch)" + ] + }, + { + "cell_type": "markdown", + "id": "3d7a78dc", + "metadata": {}, + "source": [ + "![Tensor](../resources/Tensor_1.png)\n", + "\n", + "(von https://databasecamp.de/python/tensor)" + ] + }, + { + "cell_type": "markdown", + "id": "cb3d930a", + "metadata": {}, + "source": [ + "Wie könnte der ganz rechte Tensor vom obigen Bild aussehen in der Praxis?" + ] + }, + { + "cell_type": "markdown", + "id": "c9df1ee8", + "metadata": {}, + "source": [ + "![Columbus_Tensored](../resources/Columbus_Tensor.png)" + ] + }, + { + "cell_type": "markdown", + "id": "2ca1adcc", + "metadata": {}, + "source": [ + "Eine andere Ansicht zeigt das nochmal allgemeiner (und von einer anderen Sichtweise im Vergleich zu vorher)" + ] + }, + { + "cell_type": "markdown", + "id": "3d2518ca", + "metadata": {}, + "source": [ + "![Tensor_2](../resources/Tensor_2.png)\n", + "\n", + "(von https://brainpy.readthedocs.io/en/brainpy-1.1.x/tutorial_math/tensors.html)" + ] + }, + { + "cell_type": "markdown", + "id": "35b7d883", + "metadata": {}, + "source": [ + "Wir siehen hier auch die verschiedenen Dimensionen, genannt ``axis``. In welcher Reihenfolge die axis numeriert werden, hängt von der *shape* ab. \n", + "\n", + "So ist im obigen Bild beim \"3D tensor\" *axis 0* jene axis, die zum ersten Eintrag der Shape *(4, 3, 2)* gehört.\n", + "\n", + "**Wichtig:** Wir können auch in NumPy mit *Tensoren* arbeiten, jedoch werden sie dort nicht Tensoren genannt, sondern nach wie vor einfach *arrays*. Sprich auch in Numpy können wir 3d arrays (können sogar $n$-dimensional sein) arbeiten!\n", + "\n", + "\n", + "\n", + "Man muss sich in beiden Fällen (NumPy und PyTorch) aber immer bewusst sein, welche Dimension zu welcher Richtung gehört. Das folgende Code-Beispiel sollte das verdeutlichen. Wir verwenden nochmal kurz NumPy am Anfang, danach sehen wir uns das ganze in PyTorch an." + ] + }, + { + "cell_type": "markdown", + "id": "0c51bae7", + "metadata": {}, + "source": [ + "> **Übung:** Finde Beispiele für 3D und 4D Tensoren im Bereich Machine Learning. **Tipp**: Welche Art von Tensor waren unsere bisherigen \"tabellarischen Daten\" immer?" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "e03bb183", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.13.1\n", + "1.6.1\n" + ] + } + ], + "source": [ + "import scipy\n", + "import sklearn\n", + "print(scipy.__version__)\n", + "print(sklearn.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "c3ed7024", + "metadata": {}, + "outputs": [], + "source": [ + "import sklearn.datasets as sklearn_datasets\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "ddcc5ad3", + "metadata": {}, + "outputs": [], + "source": [ + "iris_data = sklearn_datasets.load_iris()" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "4b87663b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': array([[5.1, 3.5, 1.4, 0.2],\n", + " [4.9, 3. , 1.4, 0.2],\n", + " [4.7, 3.2, 1.3, 0.2],\n", + " [4.6, 3.1, 1.5, 0.2],\n", + " [5. , 3.6, 1.4, 0.2],\n", + " [5.4, 3.9, 1.7, 0.4],\n", + " [4.6, 3.4, 1.4, 0.3],\n", + " [5. , 3.4, 1.5, 0.2],\n", + " [4.4, 2.9, 1.4, 0.2],\n", + " [4.9, 3.1, 1.5, 0.1],\n", + " [5.4, 3.7, 1.5, 0.2],\n", + " [4.8, 3.4, 1.6, 0.2],\n", + " [4.8, 3. , 1.4, 0.1],\n", + " [4.3, 3. , 1.1, 0.1],\n", + " [5.8, 4. , 1.2, 0.2],\n", + " [5.7, 4.4, 1.5, 0.4],\n", + " [5.4, 3.9, 1.3, 0.4],\n", + " [5.1, 3.5, 1.4, 0.3],\n", + " [5.7, 3.8, 1.7, 0.3],\n", + " [5.1, 3.8, 1.5, 0.3],\n", + " [5.4, 3.4, 1.7, 0.2],\n", + " [5.1, 3.7, 1.5, 0.4],\n", + " [4.6, 3.6, 1. , 0.2],\n", + " [5.1, 3.3, 1.7, 0.5],\n", + " [4.8, 3.4, 1.9, 0.2],\n", + " [5. , 3. , 1.6, 0.2],\n", + " [5. , 3.4, 1.6, 0.4],\n", + " [5.2, 3.5, 1.5, 0.2],\n", + " [5.2, 3.4, 1.4, 0.2],\n", + " [4.7, 3.2, 1.6, 0.2],\n", + " [4.8, 3.1, 1.6, 0.2],\n", + " [5.4, 3.4, 1.5, 0.4],\n", + " [5.2, 4.1, 1.5, 0.1],\n", + " [5.5, 4.2, 1.4, 0.2],\n", + " [4.9, 3.1, 1.5, 0.2],\n", + " [5. , 3.2, 1.2, 0.2],\n", + " [5.5, 3.5, 1.3, 0.2],\n", + " [4.9, 3.6, 1.4, 0.1],\n", + " [4.4, 3. , 1.3, 0.2],\n", + " [5.1, 3.4, 1.5, 0.2],\n", + " [5. , 3.5, 1.3, 0.3],\n", + " [4.5, 2.3, 1.3, 0.3],\n", + " [4.4, 3.2, 1.3, 0.2],\n", + " [5. , 3.5, 1.6, 0.6],\n", + " [5.1, 3.8, 1.9, 0.4],\n", + " [4.8, 3. , 1.4, 0.3],\n", + " [5.1, 3.8, 1.6, 0.2],\n", + " [4.6, 3.2, 1.4, 0.2],\n", + " [5.3, 3.7, 1.5, 0.2],\n", + " [5. , 3.3, 1.4, 0.2],\n", + " [7. , 3.2, 4.7, 1.4],\n", + " [6.4, 3.2, 4.5, 1.5],\n", + " [6.9, 3.1, 4.9, 1.5],\n", + " [5.5, 2.3, 4. , 1.3],\n", + " [6.5, 2.8, 4.6, 1.5],\n", + " [5.7, 2.8, 4.5, 1.3],\n", + " [6.3, 3.3, 4.7, 1.6],\n", + " [4.9, 2.4, 3.3, 1. ],\n", + " [6.6, 2.9, 4.6, 1.3],\n", + " [5.2, 2.7, 3.9, 1.4],\n", + " [5. , 2. , 3.5, 1. ],\n", + " [5.9, 3. , 4.2, 1.5],\n", + " [6. , 2.2, 4. , 1. ],\n", + " [6.1, 2.9, 4.7, 1.4],\n", + " [5.6, 2.9, 3.6, 1.3],\n", + " [6.7, 3.1, 4.4, 1.4],\n", + " [5.6, 3. , 4.5, 1.5],\n", + " [5.8, 2.7, 4.1, 1. ],\n", + " [6.2, 2.2, 4.5, 1.5],\n", + " [5.6, 2.5, 3.9, 1.1],\n", + " [5.9, 3.2, 4.8, 1.8],\n", + " [6.1, 2.8, 4. , 1.3],\n", + " [6.3, 2.5, 4.9, 1.5],\n", + " [6.1, 2.8, 4.7, 1.2],\n", + " [6.4, 2.9, 4.3, 1.3],\n", + " [6.6, 3. , 4.4, 1.4],\n", + " [6.8, 2.8, 4.8, 1.4],\n", + " [6.7, 3. , 5. , 1.7],\n", + " [6. , 2.9, 4.5, 1.5],\n", + " [5.7, 2.6, 3.5, 1. ],\n", + " [5.5, 2.4, 3.8, 1.1],\n", + " [5.5, 2.4, 3.7, 1. ],\n", + " [5.8, 2.7, 3.9, 1.2],\n", + " [6. , 2.7, 5.1, 1.6],\n", + " [5.4, 3. , 4.5, 1.5],\n", + " [6. , 3.4, 4.5, 1.6],\n", + " [6.7, 3.1, 4.7, 1.5],\n", + " [6.3, 2.3, 4.4, 1.3],\n", + " [5.6, 3. , 4.1, 1.3],\n", + " [5.5, 2.5, 4. , 1.3],\n", + " [5.5, 2.6, 4.4, 1.2],\n", + " [6.1, 3. , 4.6, 1.4],\n", + " [5.8, 2.6, 4. , 1.2],\n", + " [5. , 2.3, 3.3, 1. ],\n", + " [5.6, 2.7, 4.2, 1.3],\n", + " [5.7, 3. , 4.2, 1.2],\n", + " [5.7, 2.9, 4.2, 1.3],\n", + " [6.2, 2.9, 4.3, 1.3],\n", + " [5.1, 2.5, 3. , 1.1],\n", + " [5.7, 2.8, 4.1, 1.3],\n", + " [6.3, 3.3, 6. , 2.5],\n", + " [5.8, 2.7, 5.1, 1.9],\n", + " [7.1, 3. , 5.9, 2.1],\n", + " [6.3, 2.9, 5.6, 1.8],\n", + " [6.5, 3. , 5.8, 2.2],\n", + " [7.6, 3. , 6.6, 2.1],\n", + " [4.9, 2.5, 4.5, 1.7],\n", + " [7.3, 2.9, 6.3, 1.8],\n", + " [6.7, 2.5, 5.8, 1.8],\n", + " [7.2, 3.6, 6.1, 2.5],\n", + " [6.5, 3.2, 5.1, 2. ],\n", + " [6.4, 2.7, 5.3, 1.9],\n", + " [6.8, 3. , 5.5, 2.1],\n", + " [5.7, 2.5, 5. , 2. ],\n", + " [5.8, 2.8, 5.1, 2.4],\n", + " [6.4, 3.2, 5.3, 2.3],\n", + " [6.5, 3. , 5.5, 1.8],\n", + " [7.7, 3.8, 6.7, 2.2],\n", + " [7.7, 2.6, 6.9, 2.3],\n", + " [6. , 2.2, 5. , 1.5],\n", + " [6.9, 3.2, 5.7, 2.3],\n", + " [5.6, 2.8, 4.9, 2. ],\n", + " [7.7, 2.8, 6.7, 2. ],\n", + " [6.3, 2.7, 4.9, 1.8],\n", + " [6.7, 3.3, 5.7, 2.1],\n", + " [7.2, 3.2, 6. , 1.8],\n", + " [6.2, 2.8, 4.8, 1.8],\n", + " [6.1, 3. , 4.9, 1.8],\n", + " [6.4, 2.8, 5.6, 2.1],\n", + " [7.2, 3. , 5.8, 1.6],\n", + " [7.4, 2.8, 6.1, 1.9],\n", + " [7.9, 3.8, 6.4, 2. ],\n", + " [6.4, 2.8, 5.6, 2.2],\n", + " [6.3, 2.8, 5.1, 1.5],\n", + " [6.1, 2.6, 5.6, 1.4],\n", + " [7.7, 3. , 6.1, 2.3],\n", + " [6.3, 3.4, 5.6, 2.4],\n", + " [6.4, 3.1, 5.5, 1.8],\n", + " [6. , 3. , 4.8, 1.8],\n", + " [6.9, 3.1, 5.4, 2.1],\n", + " [6.7, 3.1, 5.6, 2.4],\n", + " [6.9, 3.1, 5.1, 2.3],\n", + " [5.8, 2.7, 5.1, 1.9],\n", + " [6.8, 3.2, 5.9, 2.3],\n", + " [6.7, 3.3, 5.7, 2.5],\n", + " [6.7, 3. , 5.2, 2.3],\n", + " [6.3, 2.5, 5. , 1.9],\n", + " [6.5, 3. , 5.2, 2. ],\n", + " [6.2, 3.4, 5.4, 2.3],\n", + " [5.9, 3. , 5.1, 1.8]]),\n", + " 'target': array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n", + " 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]),\n", + " 'frame': None,\n", + " 'target_names': array(['setosa', 'versicolor', 'virginica'], dtype='\n", + "2\n" + ] + } + ], + "source": [ + "print(X.shape)\n", + "print(type(X))\n", + "print(X.ndim)" + ] + }, + { + "cell_type": "markdown", + "id": "ed4eebcc", + "metadata": {}, + "source": [ + "Welche Ordnung hat also dieses Array (bzw. dieser Tensor)?" + ] + }, + { + "cell_type": "markdown", + "id": "d5329e3f", + "metadata": {}, + "source": [ + "Nun hat die erste Axis (axis 0), also 150 Einträge und besteht also aus unseren verschiedenen Datenpunkten. Wobei jeder Datenpunkt ein Vektor von 4 Einträgen ist. Also haben wir 4 Features. Es ist also in unserem Fall die erste Axis (axis 0) die Datenpunkt-Axis und die zweite Axis (axis 1) die Feature axis." + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "d3e67c00", + "metadata": {}, + "outputs": [], + "source": [ + "# What happens when we run this code?\n", + "X_aggregated = np.mean(X, axis=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "ada8690f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[5.84333333 3.05733333 3.758 1.19933333]\n", + "(4,)\n" + ] + } + ], + "source": [ + "# How large do you think is now the resulting array \"X_aggregated\"?\n", + "print(X_aggregated)\n", + "print(X_aggregated.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "05a6dc60", + "metadata": {}, + "source": [ + "Was wäre, wenn wir ``axis=1`` verwendet hätten?" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "dd5e0707", + "metadata": {}, + "outputs": [], + "source": [ + "X_aggregated_2 = np.mean(X, axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "id": "790181da", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2.55 2.375 2.35 2.35 2.55 2.85 2.425 2.525 2.225 2.4 2.7 2.5\n", + " 2.325 2.125 2.8 3. 2.75 2.575 2.875 2.675 2.675 2.675 2.35 2.65\n", + " 2.575 2.45 2.6 2.6 2.55 2.425 2.425 2.675 2.725 2.825 2.425 2.4\n", + " 2.625 2.5 2.225 2.55 2.525 2.1 2.275 2.675 2.8 2.375 2.675 2.35\n", + " 2.675 2.475 4.075 3.9 4.1 3.275 3.85 3.575 3.975 2.9 3.85 3.3\n", + " 2.875 3.65 3.3 3.775 3.35 3.9 3.65 3.4 3.6 3.275 3.925 3.55\n", + " 3.8 3.7 3.725 3.85 3.95 4.1 3.725 3.2 3.2 3.15 3.4 3.85\n", + " 3.6 3.875 4. 3.575 3.5 3.325 3.425 3.775 3.4 2.9 3.45 3.525\n", + " 3.525 3.675 2.925 3.475 4.525 3.875 4.525 4.15 4.375 4.825 3.4 4.575\n", + " 4.2 4.85 4.2 4.075 4.35 3.8 4.025 4.3 4.2 5.1 4.875 3.675\n", + " 4.525 3.825 4.8 3.925 4.45 4.55 3.9 3.95 4.225 4.4 4.55 5.025\n", + " 4.25 3.925 3.925 4.775 4.425 4.2 3.9 4.375 4.45 4.35 3.875 4.55\n", + " 4.55 4.3 3.925 4.175 4.325 3.95 ]\n", + "(150,)\n" + ] + } + ], + "source": [ + "# How large do you think is now the resulting array \"X_aggregated\"?\n", + "print(X_aggregated_2)\n", + "print(X_aggregated_2.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "4c3fe9ba", + "metadata": {}, + "source": [ + "Und natürlich können wir auch die Richtung umdrehen. Sprich in unserem Fall die Matrix $X$ transponieren.\n", + "\n", + "**Hinweis:** Das Transponieren von Tensoren funktioniert auch in höheren Dimensionen. Hier muss man jedoch eine Reihenfolge angeben, in der die verschiedenen *axis* getauscht werden sollen." + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "bb9b954a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[5.1 4.9 4.7 4.6 5. 5.4 4.6 5. 4.4 4.9 5.4 4.8 4.8 4.3 5.8 5.7 5.4 5.1\n", + " 5.7 5.1 5.4 5.1 4.6 5.1 4.8 5. 5. 5.2 5.2 4.7 4.8 5.4 5.2 5.5 4.9 5.\n", + " 5.5 4.9 4.4 5.1 5. 4.5 4.4 5. 5.1 4.8 5.1 4.6 5.3 5. 7. 6.4 6.9 5.5\n", + " 6.5 5.7 6.3 4.9 6.6 5.2 5. 5.9 6. 6.1 5.6 6.7 5.6 5.8 6.2 5.6 5.9 6.1\n", + " 6.3 6.1 6.4 6.6 6.8 6.7 6. 5.7 5.5 5.5 5.8 6. 5.4 6. 6.7 6.3 5.6 5.5\n", + " 5.5 6.1 5.8 5. 5.6 5.7 5.7 6.2 5.1 5.7 6.3 5.8 7.1 6.3 6.5 7.6 4.9 7.3\n", + " 6.7 7.2 6.5 6.4 6.8 5.7 5.8 6.4 6.5 7.7 7.7 6. 6.9 5.6 7.7 6.3 6.7 7.2\n", + " 6.2 6.1 6.4 7.2 7.4 7.9 6.4 6.3 6.1 7.7 6.3 6.4 6. 6.9 6.7 6.9 5.8 6.8\n", + " 6.7 6.7 6.3 6.5 6.2 5.9]\n", + " [3.5 3. 3.2 3.1 3.6 3.9 3.4 3.4 2.9 3.1 3.7 3.4 3. 3. 4. 4.4 3.9 3.5\n", + " 3.8 3.8 3.4 3.7 3.6 3.3 3.4 3. 3.4 3.5 3.4 3.2 3.1 3.4 4.1 4.2 3.1 3.2\n", + " 3.5 3.6 3. 3.4 3.5 2.3 3.2 3.5 3.8 3. 3.8 3.2 3.7 3.3 3.2 3.2 3.1 2.3\n", + " 2.8 2.8 3.3 2.4 2.9 2.7 2. 3. 2.2 2.9 2.9 3.1 3. 2.7 2.2 2.5 3.2 2.8\n", + " 2.5 2.8 2.9 3. 2.8 3. 2.9 2.6 2.4 2.4 2.7 2.7 3. 3.4 3.1 2.3 3. 2.5\n", + " 2.6 3. 2.6 2.3 2.7 3. 2.9 2.9 2.5 2.8 3.3 2.7 3. 2.9 3. 3. 2.5 2.9\n", + " 2.5 3.6 3.2 2.7 3. 2.5 2.8 3.2 3. 3.8 2.6 2.2 3.2 2.8 2.8 2.7 3.3 3.2\n", + " 2.8 3. 2.8 3. 2.8 3.8 2.8 2.8 2.6 3. 3.4 3.1 3. 3.1 3.1 3.1 2.7 3.2\n", + " 3.3 3. 2.5 3. 3.4 3. ]\n", + " [1.4 1.4 1.3 1.5 1.4 1.7 1.4 1.5 1.4 1.5 1.5 1.6 1.4 1.1 1.2 1.5 1.3 1.4\n", + " 1.7 1.5 1.7 1.5 1. 1.7 1.9 1.6 1.6 1.5 1.4 1.6 1.6 1.5 1.5 1.4 1.5 1.2\n", + " 1.3 1.4 1.3 1.5 1.3 1.3 1.3 1.6 1.9 1.4 1.6 1.4 1.5 1.4 4.7 4.5 4.9 4.\n", + " 4.6 4.5 4.7 3.3 4.6 3.9 3.5 4.2 4. 4.7 3.6 4.4 4.5 4.1 4.5 3.9 4.8 4.\n", + " 4.9 4.7 4.3 4.4 4.8 5. 4.5 3.5 3.8 3.7 3.9 5.1 4.5 4.5 4.7 4.4 4.1 4.\n", + " 4.4 4.6 4. 3.3 4.2 4.2 4.2 4.3 3. 4.1 6. 5.1 5.9 5.6 5.8 6.6 4.5 6.3\n", + " 5.8 6.1 5.1 5.3 5.5 5. 5.1 5.3 5.5 6.7 6.9 5. 5.7 4.9 6.7 4.9 5.7 6.\n", + " 4.8 4.9 5.6 5.8 6.1 6.4 5.6 5.1 5.6 6.1 5.6 5.5 4.8 5.4 5.6 5.1 5.1 5.9\n", + " 5.7 5.2 5. 5.2 5.4 5.1]\n", + " [0.2 0.2 0.2 0.2 0.2 0.4 0.3 0.2 0.2 0.1 0.2 0.2 0.1 0.1 0.2 0.4 0.4 0.3\n", + " 0.3 0.3 0.2 0.4 0.2 0.5 0.2 0.2 0.4 0.2 0.2 0.2 0.2 0.4 0.1 0.2 0.2 0.2\n", + " 0.2 0.1 0.2 0.2 0.3 0.3 0.2 0.6 0.4 0.3 0.2 0.2 0.2 0.2 1.4 1.5 1.5 1.3\n", + " 1.5 1.3 1.6 1. 1.3 1.4 1. 1.5 1. 1.4 1.3 1.4 1.5 1. 1.5 1.1 1.8 1.3\n", + " 1.5 1.2 1.3 1.4 1.4 1.7 1.5 1. 1.1 1. 1.2 1.6 1.5 1.6 1.5 1.3 1.3 1.3\n", + " 1.2 1.4 1.2 1. 1.3 1.2 1.3 1.3 1.1 1.3 2.5 1.9 2.1 1.8 2.2 2.1 1.7 1.8\n", + " 1.8 2.5 2. 1.9 2.1 2. 2.4 2.3 1.8 2.2 2.3 1.5 2.3 2. 2. 1.8 2.1 1.8\n", + " 1.8 1.8 2.1 1.6 1.9 2. 2.2 1.5 1.4 2.3 2.4 1.8 1.8 2.1 2.4 2.3 1.9 2.3\n", + " 2.5 2.3 1.9 2. 2.3 1.8]]\n", + "(4, 150)\n" + ] + } + ], + "source": [ + "flipped_X = X.T \n", + "print(flipped_X)\n", + "print(flipped_X.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "59ac3246", + "metadata": {}, + "source": [ + "Was ist nun die axis0 und was ist die axis1?" + ] + }, + { + "cell_type": "markdown", + "id": "e18d373f", + "metadata": {}, + "source": [ + "Wir können aber auch ohne angegebener Axis die Daten aggregieren. Was ist das Ergebnis von der folgenden Code-Zeile?" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "372db3a4", + "metadata": {}, + "outputs": [], + "source": [ + "X_aggregated_3 = np.mean(X)" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "7852f8ee", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3.4644999999999997\n", + "()\n" + ] + } + ], + "source": [ + "print(X_aggregated_3)\n", + "print(X_aggregated_3.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "48e09a80", + "metadata": {}, + "source": [ + "Wie sieht das ganze in PyTorch aus?" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "id": "b47afa75", + "metadata": {}, + "outputs": [], + "source": [ + "import torch" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "id": "ad8d55f8", + "metadata": {}, + "outputs": [], + "source": [ + "X = iris_data.data" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "id": "c0dd17ff", + "metadata": {}, + "outputs": [], + "source": [ + "# Now we convert the numpy array to a PyTorch tensor\n", + "X_tensor = torch.tensor(X, dtype=torch.float32) # We can also skip the dtype, then it is inferred from the numpy array but it is a good practice to specify the dtype\n", + "\n", + "# Another possibility is\n", + "X_tensor_2 = torch.from_numpy(X)" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "a17f4f3e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[5.1000, 3.5000, 1.4000, 0.2000],\n", + " [4.9000, 3.0000, 1.4000, 0.2000],\n", + " [4.7000, 3.2000, 1.3000, 0.2000],\n", + " [4.6000, 3.1000, 1.5000, 0.2000],\n", + " [5.0000, 3.6000, 1.4000, 0.2000],\n", + " [5.4000, 3.9000, 1.7000, 0.4000],\n", + " [4.6000, 3.4000, 1.4000, 0.3000],\n", + " [5.0000, 3.4000, 1.5000, 0.2000],\n", + " [4.4000, 2.9000, 1.4000, 0.2000],\n", + " [4.9000, 3.1000, 1.5000, 0.1000],\n", + " [5.4000, 3.7000, 1.5000, 0.2000],\n", + " [4.8000, 3.4000, 1.6000, 0.2000],\n", + " [4.8000, 3.0000, 1.4000, 0.1000],\n", + " [4.3000, 3.0000, 1.1000, 0.1000],\n", + " [5.8000, 4.0000, 1.2000, 0.2000],\n", + " [5.7000, 4.4000, 1.5000, 0.4000],\n", + " [5.4000, 3.9000, 1.3000, 0.4000],\n", + " [5.1000, 3.5000, 1.4000, 0.3000],\n", + " [5.7000, 3.8000, 1.7000, 0.3000],\n", + " [5.1000, 3.8000, 1.5000, 0.3000],\n", + " [5.4000, 3.4000, 1.7000, 0.2000],\n", + " [5.1000, 3.7000, 1.5000, 0.4000],\n", + " [4.6000, 3.6000, 1.0000, 0.2000],\n", + " [5.1000, 3.3000, 1.7000, 0.5000],\n", + " [4.8000, 3.4000, 1.9000, 0.2000],\n", + " [5.0000, 3.0000, 1.6000, 0.2000],\n", + " [5.0000, 3.4000, 1.6000, 0.4000],\n", + " [5.2000, 3.5000, 1.5000, 0.2000],\n", + " [5.2000, 3.4000, 1.4000, 0.2000],\n", + " [4.7000, 3.2000, 1.6000, 0.2000],\n", + " [4.8000, 3.1000, 1.6000, 0.2000],\n", + " [5.4000, 3.4000, 1.5000, 0.4000],\n", + " [5.2000, 4.1000, 1.5000, 0.1000],\n", + " [5.5000, 4.2000, 1.4000, 0.2000],\n", + " [4.9000, 3.1000, 1.5000, 0.2000],\n", + " [5.0000, 3.2000, 1.2000, 0.2000],\n", + " [5.5000, 3.5000, 1.3000, 0.2000],\n", + " [4.9000, 3.6000, 1.4000, 0.1000],\n", + " [4.4000, 3.0000, 1.3000, 0.2000],\n", + " [5.1000, 3.4000, 1.5000, 0.2000],\n", + " [5.0000, 3.5000, 1.3000, 0.3000],\n", + " [4.5000, 2.3000, 1.3000, 0.3000],\n", + " [4.4000, 3.2000, 1.3000, 0.2000],\n", + " [5.0000, 3.5000, 1.6000, 0.6000],\n", + " [5.1000, 3.8000, 1.9000, 0.4000],\n", + " [4.8000, 3.0000, 1.4000, 0.3000],\n", + " [5.1000, 3.8000, 1.6000, 0.2000],\n", + " [4.6000, 3.2000, 1.4000, 0.2000],\n", + " [5.3000, 3.7000, 1.5000, 0.2000],\n", + " [5.0000, 3.3000, 1.4000, 0.2000],\n", + " [7.0000, 3.2000, 4.7000, 1.4000],\n", + " [6.4000, 3.2000, 4.5000, 1.5000],\n", + " [6.9000, 3.1000, 4.9000, 1.5000],\n", + " [5.5000, 2.3000, 4.0000, 1.3000],\n", + " [6.5000, 2.8000, 4.6000, 1.5000],\n", + " [5.7000, 2.8000, 4.5000, 1.3000],\n", + " [6.3000, 3.3000, 4.7000, 1.6000],\n", + " [4.9000, 2.4000, 3.3000, 1.0000],\n", + " [6.6000, 2.9000, 4.6000, 1.3000],\n", + " [5.2000, 2.7000, 3.9000, 1.4000],\n", + " [5.0000, 2.0000, 3.5000, 1.0000],\n", + " [5.9000, 3.0000, 4.2000, 1.5000],\n", + " [6.0000, 2.2000, 4.0000, 1.0000],\n", + " [6.1000, 2.9000, 4.7000, 1.4000],\n", + " [5.6000, 2.9000, 3.6000, 1.3000],\n", + " [6.7000, 3.1000, 4.4000, 1.4000],\n", + " [5.6000, 3.0000, 4.5000, 1.5000],\n", + " [5.8000, 2.7000, 4.1000, 1.0000],\n", + " [6.2000, 2.2000, 4.5000, 1.5000],\n", + " [5.6000, 2.5000, 3.9000, 1.1000],\n", + " [5.9000, 3.2000, 4.8000, 1.8000],\n", + " [6.1000, 2.8000, 4.0000, 1.3000],\n", + " [6.3000, 2.5000, 4.9000, 1.5000],\n", + " [6.1000, 2.8000, 4.7000, 1.2000],\n", + " [6.4000, 2.9000, 4.3000, 1.3000],\n", + " [6.6000, 3.0000, 4.4000, 1.4000],\n", + " [6.8000, 2.8000, 4.8000, 1.4000],\n", + " [6.7000, 3.0000, 5.0000, 1.7000],\n", + " [6.0000, 2.9000, 4.5000, 1.5000],\n", + " [5.7000, 2.6000, 3.5000, 1.0000],\n", + " [5.5000, 2.4000, 3.8000, 1.1000],\n", + " [5.5000, 2.4000, 3.7000, 1.0000],\n", + " [5.8000, 2.7000, 3.9000, 1.2000],\n", + " [6.0000, 2.7000, 5.1000, 1.6000],\n", + " [5.4000, 3.0000, 4.5000, 1.5000],\n", + " [6.0000, 3.4000, 4.5000, 1.6000],\n", + " [6.7000, 3.1000, 4.7000, 1.5000],\n", + " [6.3000, 2.3000, 4.4000, 1.3000],\n", + " [5.6000, 3.0000, 4.1000, 1.3000],\n", + " [5.5000, 2.5000, 4.0000, 1.3000],\n", + " [5.5000, 2.6000, 4.4000, 1.2000],\n", + " [6.1000, 3.0000, 4.6000, 1.4000],\n", + " [5.8000, 2.6000, 4.0000, 1.2000],\n", + " [5.0000, 2.3000, 3.3000, 1.0000],\n", + " [5.6000, 2.7000, 4.2000, 1.3000],\n", + " [5.7000, 3.0000, 4.2000, 1.2000],\n", + " [5.7000, 2.9000, 4.2000, 1.3000],\n", + " [6.2000, 2.9000, 4.3000, 1.3000],\n", + " [5.1000, 2.5000, 3.0000, 1.1000],\n", + " [5.7000, 2.8000, 4.1000, 1.3000],\n", + " [6.3000, 3.3000, 6.0000, 2.5000],\n", + " [5.8000, 2.7000, 5.1000, 1.9000],\n", + " [7.1000, 3.0000, 5.9000, 2.1000],\n", + " [6.3000, 2.9000, 5.6000, 1.8000],\n", + " [6.5000, 3.0000, 5.8000, 2.2000],\n", + " [7.6000, 3.0000, 6.6000, 2.1000],\n", + " [4.9000, 2.5000, 4.5000, 1.7000],\n", + " [7.3000, 2.9000, 6.3000, 1.8000],\n", + " [6.7000, 2.5000, 5.8000, 1.8000],\n", + " [7.2000, 3.6000, 6.1000, 2.5000],\n", + " [6.5000, 3.2000, 5.1000, 2.0000],\n", + " [6.4000, 2.7000, 5.3000, 1.9000],\n", + " [6.8000, 3.0000, 5.5000, 2.1000],\n", + " [5.7000, 2.5000, 5.0000, 2.0000],\n", + " [5.8000, 2.8000, 5.1000, 2.4000],\n", + " [6.4000, 3.2000, 5.3000, 2.3000],\n", + " [6.5000, 3.0000, 5.5000, 1.8000],\n", + " [7.7000, 3.8000, 6.7000, 2.2000],\n", + " [7.7000, 2.6000, 6.9000, 2.3000],\n", + " [6.0000, 2.2000, 5.0000, 1.5000],\n", + " [6.9000, 3.2000, 5.7000, 2.3000],\n", + " [5.6000, 2.8000, 4.9000, 2.0000],\n", + " [7.7000, 2.8000, 6.7000, 2.0000],\n", + " [6.3000, 2.7000, 4.9000, 1.8000],\n", + " [6.7000, 3.3000, 5.7000, 2.1000],\n", + " [7.2000, 3.2000, 6.0000, 1.8000],\n", + " [6.2000, 2.8000, 4.8000, 1.8000],\n", + " [6.1000, 3.0000, 4.9000, 1.8000],\n", + " [6.4000, 2.8000, 5.6000, 2.1000],\n", + " [7.2000, 3.0000, 5.8000, 1.6000],\n", + " [7.4000, 2.8000, 6.1000, 1.9000],\n", + " [7.9000, 3.8000, 6.4000, 2.0000],\n", + " [6.4000, 2.8000, 5.6000, 2.2000],\n", + " [6.3000, 2.8000, 5.1000, 1.5000],\n", + " [6.1000, 2.6000, 5.6000, 1.4000],\n", + " [7.7000, 3.0000, 6.1000, 2.3000],\n", + " [6.3000, 3.4000, 5.6000, 2.4000],\n", + " [6.4000, 3.1000, 5.5000, 1.8000],\n", + " [6.0000, 3.0000, 4.8000, 1.8000],\n", + " [6.9000, 3.1000, 5.4000, 2.1000],\n", + " [6.7000, 3.1000, 5.6000, 2.4000],\n", + " [6.9000, 3.1000, 5.1000, 2.3000],\n", + " [5.8000, 2.7000, 5.1000, 1.9000],\n", + " [6.8000, 3.2000, 5.9000, 2.3000],\n", + " [6.7000, 3.3000, 5.7000, 2.5000],\n", + " [6.7000, 3.0000, 5.2000, 2.3000],\n", + " [6.3000, 2.5000, 5.0000, 1.9000],\n", + " [6.5000, 3.0000, 5.2000, 2.0000],\n", + " [6.2000, 3.4000, 5.4000, 2.3000],\n", + " [5.9000, 3.0000, 5.1000, 1.8000]])\n" + ] + } + ], + "source": [ + "print(X_tensor)" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "1aa6ff20", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[5.1000, 3.5000, 1.4000, 0.2000],\n", + " [4.9000, 3.0000, 1.4000, 0.2000],\n", + " [4.7000, 3.2000, 1.3000, 0.2000],\n", + " [4.6000, 3.1000, 1.5000, 0.2000],\n", + " [5.0000, 3.6000, 1.4000, 0.2000],\n", + " [5.4000, 3.9000, 1.7000, 0.4000],\n", + " [4.6000, 3.4000, 1.4000, 0.3000],\n", + " [5.0000, 3.4000, 1.5000, 0.2000],\n", + " [4.4000, 2.9000, 1.4000, 0.2000],\n", + " [4.9000, 3.1000, 1.5000, 0.1000],\n", + " [5.4000, 3.7000, 1.5000, 0.2000],\n", + " [4.8000, 3.4000, 1.6000, 0.2000],\n", + " [4.8000, 3.0000, 1.4000, 0.1000],\n", + " [4.3000, 3.0000, 1.1000, 0.1000],\n", + " [5.8000, 4.0000, 1.2000, 0.2000],\n", + " [5.7000, 4.4000, 1.5000, 0.4000],\n", + " [5.4000, 3.9000, 1.3000, 0.4000],\n", + " [5.1000, 3.5000, 1.4000, 0.3000],\n", + " [5.7000, 3.8000, 1.7000, 0.3000],\n", + " [5.1000, 3.8000, 1.5000, 0.3000],\n", + " [5.4000, 3.4000, 1.7000, 0.2000],\n", + " [5.1000, 3.7000, 1.5000, 0.4000],\n", + " [4.6000, 3.6000, 1.0000, 0.2000],\n", + " [5.1000, 3.3000, 1.7000, 0.5000],\n", + " [4.8000, 3.4000, 1.9000, 0.2000],\n", + " [5.0000, 3.0000, 1.6000, 0.2000],\n", + " [5.0000, 3.4000, 1.6000, 0.4000],\n", + " [5.2000, 3.5000, 1.5000, 0.2000],\n", + " [5.2000, 3.4000, 1.4000, 0.2000],\n", + " [4.7000, 3.2000, 1.6000, 0.2000],\n", + " [4.8000, 3.1000, 1.6000, 0.2000],\n", + " [5.4000, 3.4000, 1.5000, 0.4000],\n", + " [5.2000, 4.1000, 1.5000, 0.1000],\n", + " [5.5000, 4.2000, 1.4000, 0.2000],\n", + " [4.9000, 3.1000, 1.5000, 0.2000],\n", + " [5.0000, 3.2000, 1.2000, 0.2000],\n", + " [5.5000, 3.5000, 1.3000, 0.2000],\n", + " [4.9000, 3.6000, 1.4000, 0.1000],\n", + " [4.4000, 3.0000, 1.3000, 0.2000],\n", + " [5.1000, 3.4000, 1.5000, 0.2000],\n", + " [5.0000, 3.5000, 1.3000, 0.3000],\n", + " [4.5000, 2.3000, 1.3000, 0.3000],\n", + " [4.4000, 3.2000, 1.3000, 0.2000],\n", + " [5.0000, 3.5000, 1.6000, 0.6000],\n", + " [5.1000, 3.8000, 1.9000, 0.4000],\n", + " [4.8000, 3.0000, 1.4000, 0.3000],\n", + " [5.1000, 3.8000, 1.6000, 0.2000],\n", + " [4.6000, 3.2000, 1.4000, 0.2000],\n", + " [5.3000, 3.7000, 1.5000, 0.2000],\n", + " [5.0000, 3.3000, 1.4000, 0.2000],\n", + " [7.0000, 3.2000, 4.7000, 1.4000],\n", + " [6.4000, 3.2000, 4.5000, 1.5000],\n", + " [6.9000, 3.1000, 4.9000, 1.5000],\n", + " [5.5000, 2.3000, 4.0000, 1.3000],\n", + " [6.5000, 2.8000, 4.6000, 1.5000],\n", + " [5.7000, 2.8000, 4.5000, 1.3000],\n", + " [6.3000, 3.3000, 4.7000, 1.6000],\n", + " [4.9000, 2.4000, 3.3000, 1.0000],\n", + " [6.6000, 2.9000, 4.6000, 1.3000],\n", + " [5.2000, 2.7000, 3.9000, 1.4000],\n", + " [5.0000, 2.0000, 3.5000, 1.0000],\n", + " [5.9000, 3.0000, 4.2000, 1.5000],\n", + " [6.0000, 2.2000, 4.0000, 1.0000],\n", + " [6.1000, 2.9000, 4.7000, 1.4000],\n", + " [5.6000, 2.9000, 3.6000, 1.3000],\n", + " [6.7000, 3.1000, 4.4000, 1.4000],\n", + " [5.6000, 3.0000, 4.5000, 1.5000],\n", + " [5.8000, 2.7000, 4.1000, 1.0000],\n", + " [6.2000, 2.2000, 4.5000, 1.5000],\n", + " [5.6000, 2.5000, 3.9000, 1.1000],\n", + " [5.9000, 3.2000, 4.8000, 1.8000],\n", + " [6.1000, 2.8000, 4.0000, 1.3000],\n", + " [6.3000, 2.5000, 4.9000, 1.5000],\n", + " [6.1000, 2.8000, 4.7000, 1.2000],\n", + " [6.4000, 2.9000, 4.3000, 1.3000],\n", + " [6.6000, 3.0000, 4.4000, 1.4000],\n", + " [6.8000, 2.8000, 4.8000, 1.4000],\n", + " [6.7000, 3.0000, 5.0000, 1.7000],\n", + " [6.0000, 2.9000, 4.5000, 1.5000],\n", + " [5.7000, 2.6000, 3.5000, 1.0000],\n", + " [5.5000, 2.4000, 3.8000, 1.1000],\n", + " [5.5000, 2.4000, 3.7000, 1.0000],\n", + " [5.8000, 2.7000, 3.9000, 1.2000],\n", + " [6.0000, 2.7000, 5.1000, 1.6000],\n", + " [5.4000, 3.0000, 4.5000, 1.5000],\n", + " [6.0000, 3.4000, 4.5000, 1.6000],\n", + " [6.7000, 3.1000, 4.7000, 1.5000],\n", + " [6.3000, 2.3000, 4.4000, 1.3000],\n", + " [5.6000, 3.0000, 4.1000, 1.3000],\n", + " [5.5000, 2.5000, 4.0000, 1.3000],\n", + " [5.5000, 2.6000, 4.4000, 1.2000],\n", + " [6.1000, 3.0000, 4.6000, 1.4000],\n", + " [5.8000, 2.6000, 4.0000, 1.2000],\n", + " [5.0000, 2.3000, 3.3000, 1.0000],\n", + " [5.6000, 2.7000, 4.2000, 1.3000],\n", + " [5.7000, 3.0000, 4.2000, 1.2000],\n", + " [5.7000, 2.9000, 4.2000, 1.3000],\n", + " [6.2000, 2.9000, 4.3000, 1.3000],\n", + " [5.1000, 2.5000, 3.0000, 1.1000],\n", + " [5.7000, 2.8000, 4.1000, 1.3000],\n", + " [6.3000, 3.3000, 6.0000, 2.5000],\n", + " [5.8000, 2.7000, 5.1000, 1.9000],\n", + " [7.1000, 3.0000, 5.9000, 2.1000],\n", + " [6.3000, 2.9000, 5.6000, 1.8000],\n", + " [6.5000, 3.0000, 5.8000, 2.2000],\n", + " [7.6000, 3.0000, 6.6000, 2.1000],\n", + " [4.9000, 2.5000, 4.5000, 1.7000],\n", + " [7.3000, 2.9000, 6.3000, 1.8000],\n", + " [6.7000, 2.5000, 5.8000, 1.8000],\n", + " [7.2000, 3.6000, 6.1000, 2.5000],\n", + " [6.5000, 3.2000, 5.1000, 2.0000],\n", + " [6.4000, 2.7000, 5.3000, 1.9000],\n", + " [6.8000, 3.0000, 5.5000, 2.1000],\n", + " [5.7000, 2.5000, 5.0000, 2.0000],\n", + " [5.8000, 2.8000, 5.1000, 2.4000],\n", + " [6.4000, 3.2000, 5.3000, 2.3000],\n", + " [6.5000, 3.0000, 5.5000, 1.8000],\n", + " [7.7000, 3.8000, 6.7000, 2.2000],\n", + " [7.7000, 2.6000, 6.9000, 2.3000],\n", + " [6.0000, 2.2000, 5.0000, 1.5000],\n", + " [6.9000, 3.2000, 5.7000, 2.3000],\n", + " [5.6000, 2.8000, 4.9000, 2.0000],\n", + " [7.7000, 2.8000, 6.7000, 2.0000],\n", + " [6.3000, 2.7000, 4.9000, 1.8000],\n", + " [6.7000, 3.3000, 5.7000, 2.1000],\n", + " [7.2000, 3.2000, 6.0000, 1.8000],\n", + " [6.2000, 2.8000, 4.8000, 1.8000],\n", + " [6.1000, 3.0000, 4.9000, 1.8000],\n", + " [6.4000, 2.8000, 5.6000, 2.1000],\n", + " [7.2000, 3.0000, 5.8000, 1.6000],\n", + " [7.4000, 2.8000, 6.1000, 1.9000],\n", + " [7.9000, 3.8000, 6.4000, 2.0000],\n", + " [6.4000, 2.8000, 5.6000, 2.2000],\n", + " [6.3000, 2.8000, 5.1000, 1.5000],\n", + " [6.1000, 2.6000, 5.6000, 1.4000],\n", + " [7.7000, 3.0000, 6.1000, 2.3000],\n", + " [6.3000, 3.4000, 5.6000, 2.4000],\n", + " [6.4000, 3.1000, 5.5000, 1.8000],\n", + " [6.0000, 3.0000, 4.8000, 1.8000],\n", + " [6.9000, 3.1000, 5.4000, 2.1000],\n", + " [6.7000, 3.1000, 5.6000, 2.4000],\n", + " [6.9000, 3.1000, 5.1000, 2.3000],\n", + " [5.8000, 2.7000, 5.1000, 1.9000],\n", + " [6.8000, 3.2000, 5.9000, 2.3000],\n", + " [6.7000, 3.3000, 5.7000, 2.5000],\n", + " [6.7000, 3.0000, 5.2000, 2.3000],\n", + " [6.3000, 2.5000, 5.0000, 1.9000],\n", + " [6.5000, 3.0000, 5.2000, 2.0000],\n", + " [6.2000, 3.4000, 5.4000, 2.3000],\n", + " [5.9000, 3.0000, 5.1000, 1.8000]], dtype=torch.float64)\n" + ] + } + ], + "source": [ + "print(X_tensor_2)" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "5568c323", + "metadata": {}, + "outputs": [], + "source": [ + "# Now we can do the same things as before\n", + "X_tensor_agg = torch.mean(X_tensor, dim=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "8f76834f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([5.8433, 3.0573, 3.7580, 1.1993])\n" + ] + } + ], + "source": [ + "print(X_tensor_agg)" + ] + }, + { + "cell_type": "markdown", + "id": "79e1f809", + "metadata": {}, + "source": [ + "Ein kleiner Unterschied ist, dass nun die Dimension wirklich mit ``dim`` abgekürzt wird und nicht mit ``axis``." + ] + }, + { + "cell_type": "markdown", + "id": "df2f9432", + "metadata": {}, + "source": [ + "Wie kann man nun einen Tensor erstellen (ohne Daten)?\n", + "\n", + "-> (Fast) genauso wie in numpy!" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "3eea711e", + "metadata": {}, + "outputs": [], + "source": [ + "zeros_tensor = torch.zeros((3,5))\n", + "ones_tensor = torch.ones((2,5))\n", + "my_list = [1,2,3.0, 1.123345, -3]\n", + "my_tensor = torch.tensor(my_list)" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "4f67937b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0.]])\n", + "tensor([[1., 1., 1., 1., 1.],\n", + " [1., 1., 1., 1., 1.]])\n", + "tensor([ 1.0000, 2.0000, 3.0000, 1.1233, -3.0000])\n" + ] + } + ], + "source": [ + "print(zeros_tensor)\n", + "print(ones_tensor)\n", + "print(my_tensor)" + ] + }, + { + "cell_type": "markdown", + "id": "6aa6dfb3", + "metadata": {}, + "source": [ + "Und natürlich geht das auch mit mehreren Dimensionen (nicht nur 1 oder 2)." + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "id": "73817492", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[[[[[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]],\n", + "\n", + "\n", + " [[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]]],\n", + "\n", + "\n", + "\n", + " [[[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]],\n", + "\n", + "\n", + " [[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]]]],\n", + "\n", + "\n", + "\n", + "\n", + " [[[[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]],\n", + "\n", + "\n", + " [[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]]],\n", + "\n", + "\n", + "\n", + " [[[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]],\n", + "\n", + "\n", + " [[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]]]],\n", + "\n", + "\n", + "\n", + "\n", + " [[[[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]],\n", + "\n", + "\n", + " [[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]]],\n", + "\n", + "\n", + "\n", + " [[[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]],\n", + "\n", + "\n", + " [[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]]]]],\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " [[[[[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]],\n", + "\n", + "\n", + " [[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]]],\n", + "\n", + "\n", + "\n", + " [[[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]],\n", + "\n", + "\n", + " [[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]]]],\n", + "\n", + "\n", + "\n", + "\n", + " [[[[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]],\n", + "\n", + "\n", + " [[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]]],\n", + "\n", + "\n", + "\n", + " [[[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]],\n", + "\n", + "\n", + " [[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]]]],\n", + "\n", + "\n", + "\n", + "\n", + " [[[[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]],\n", + "\n", + "\n", + " [[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]]],\n", + "\n", + "\n", + "\n", + " [[[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]],\n", + "\n", + "\n", + " [[[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]],\n", + "\n", + " [[0., 0.],\n", + " [0., 0.],\n", + " [0., 0.]]]]]]])\n" + ] + } + ], + "source": [ + "# Create 7d Tensor:\n", + "seven_d_tensor = torch.zeros((2, 3, 2, 2, 2, 3, 2)) # Example of a 7-dimensional tensor\n", + "print(seven_d_tensor)" + ] + }, + { + "cell_type": "markdown", + "id": "5a2f705e", + "metadata": {}, + "source": [ + "Bei solchen Dimensionen macht es dann natürlich wenig Sinn, sich die Daten direkt anzusehen, aber man kann sich natürlich die shape ansehen." + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "id": "960129ee", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([2, 3, 2, 2, 2, 3, 2])\n" + ] + } + ], + "source": [ + "# Shape of 7d tensor\n", + "print(seven_d_tensor.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "14a0250d", + "metadata": {}, + "source": [ + "> **Übung:** Welche Shape hat ein RGB FullHD Film, welcher in 24 FPS abgespielt wird und eine Länge von 90min hat? Wie viele Einträge hat der Tensor? Welche Reihenfolge ist sinnvoll?" + ] + }, + { + "cell_type": "markdown", + "id": "ef7077ea", + "metadata": {}, + "source": [ + "> **Übung:** Was wäre, wenn wir nun 12 so Videos gleichzeitig in einem Tensor speichern möchten? Können die Videos eine unterschiedliche Länge haben?" + ] + }, + { + "cell_type": "markdown", + "id": "8faa5e00", + "metadata": {}, + "source": [ + "Ok, aber warum nutzen wir jetzt genau PyTorch und nicht einfach numpy, wenn in beiden Bibliotheken Matrizen erstellt werden können?\n", + "\n", + "Grund dafür sind (mitunter) folgende 3 Dinge:\n", + "* Die wichtigsten Machine Learning Funktionen sind bereits implementiert (werden wir noch lernen)\n", + "* Automatische Differenzierung, welche das Trainieren von neuronalen Netzwerken ermöglicht\n", + "* Die sehr einfache Möglichkeit, die Berechnungen auf einem Hardware-beschleunigten Gerät (zBsp. die GPU) auszuführen" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Dynamic Computation Graph und Automatic Differentiation" + ] + }, + { + "cell_type": "markdown", + "id": "0c23fb95", + "metadata": {}, + "source": [ + "Es ist also möglich in PyTorch, sich komplizierte Funktionen zu basteln (zum Beispiel ein Neuronales Netz) und PyTorch kann automatisch die Ableitungen bzgl. den verschiedenen Parametern berechnen.\n", + "\n", + "Warum das Berechnen der Ableitungen wichtig ist, werden wir in den nächsten Notebooks noch genauer erfahren (Gradient Descent). Kurz gesagt, gibt uns die Ableitung der Fehlerfunktion eine Richtung vor, in die wir uns bewegen müssen." + ] + }, + { + "cell_type": "markdown", + "id": "907a55f2", + "metadata": {}, + "source": [ + "Sehen wir uns einmal an, wie wir auf die Gradienten zugreifen können." + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "id": "5523cd69", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([1., 1., 1.])\n", + "False\n" + ] + } + ], + "source": [ + "x = torch.ones((3))\n", + "print(x)\n", + "print(x.requires_grad)" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "id": "6ef67d7b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], + "source": [ + "# We can also equip the tensor with the ability to calculate gradients\n", + "x.requires_grad_(True)\n", + "print(x.requires_grad)" + ] + }, + { + "cell_type": "markdown", + "id": "2b131de4", + "metadata": {}, + "source": [ + "Um nun eine Ableitung einer Funktion zu berechnen, definieren wir uns einmal eine beliebige Funktion, in diesem Fall:\n", + "$$y(x) = \\frac{1}{n}\\sum_{i=1}^{n} \\left[(x_i + 2)^2 + 3\\right],$$\n", + "\n", + "wobei hier $n$ die Anzahl der Elemente von $x$ darstellt." + ] + }, + { + "cell_type": "markdown", + "id": "0528fd8d", + "metadata": {}, + "source": [ + "In unserem Fall ist nun $x$ der Parameter, den wir optimieren (also verändern) wollen, sodass $y(x)$ maximal (oder minimal) wird, sprich ein Extremum erreicht.\n", + "\n", + "**Wichtig:** Später wird $y(x)$ die *Loss*-Funktion sein, und diese wollen wir natürlich minimieren. Somit kann man sich auch jetzt schon vorstellen, dass $y(x)$ eine Loss-Funktion ist." + ] + }, + { + "cell_type": "markdown", + "id": "d842a4c0", + "metadata": {}, + "source": [ + "Wir wissen, dass bei einem Extremum die Ableitung $0$ ist, somit müssen wir die Ableitung von $y(x)$ bezüglich $x$ finden, sprich $\\frac{\\mathrm d\\, y(x)}{\\mathrm d\\, x}$.\n", + "\n", + "Dabei kann uns jetzt PyTorch helfen." + ] + }, + { + "cell_type": "markdown", + "id": "100b98d0", + "metadata": {}, + "source": [ + "Die Art, wie PyTorch die Ableitung berechnet ist mit sogenannten *Computational Graphs*. Um zu verstehen, was das ist, zerlegen wir nun unsere Funktion in kleinere Bestandteile." + ] + }, + { + "cell_type": "markdown", + "id": "c2e487a2", + "metadata": {}, + "source": [ + "Als Input verwenden wir den Vektor $x=[1,2,3]$ (zum Beispiel eine Person mit $1$ Geschwister, $2$ Urlauben pro Jahr und $3$ Mahlzeiten pro Tag)." + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "id": "6f8e2b65", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([0., 1., 2.], requires_grad=True)\n" + ] + } + ], + "source": [ + "x = torch.arange(3, dtype=torch.float32, requires_grad=True) # Create a tensor with requires_grad set to True (we want to calculate the gradient later)\n", + "print(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "id": "161d1ab8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "y = 12.666666984558105\n", + "Y tensor(12.6667, grad_fn=)\n" + ] + } + ], + "source": [ + "a = x + 2\n", + "b = a ** 2\n", + "c = b + 3\n", + "y = c.mean()\n", + "print(f'y = {y}')\n", + "print(\"Y\", y)" + ] + }, + { + "cell_type": "markdown", + "id": "e4351e02", + "metadata": {}, + "source": [ + "Hier ist der *Computational Graph* nun auch grafisch dargestellt." + ] + }, + { + "cell_type": "markdown", + "id": "4e6d53d9", + "metadata": {}, + "source": [ + "![Computational_Graph](../resources/pytorch_computation_graph.svg)\n", + "\n", + "(von https://github.com/phlippe/uvadlc_notebooks/blob/master/docs/tutorial_notebooks/tutorial2/Introduction_to_PyTorch.ipynb)" + ] + }, + { + "cell_type": "markdown", + "id": "5fcb5ece", + "metadata": {}, + "source": [ + "Nun berechnen wir die Ableitung $\\frac{\\mathrm d\\, y(x)}{\\mathrm d\\, x}$. Diese setzt sich (aufgrund der Kettenregel) aus den folgenden Ableitungen zusammen:\n", + "\n", + "$$\\frac{\\mathrm d\\, y(x)}{\\mathrm d\\, x_i} = \\frac{\\mathrm d\\, y}{\\mathrm d\\, c_i}\\cdot\\frac{\\mathrm d\\, c_i}{\\mathrm d\\, b_i}\\cdot \\frac{\\mathrm d\\, b_i}{\\mathrm d\\, a_i}\\cdot \\frac{\\mathrm d\\, a_i}{\\mathrm d\\, x_i}.$$\n", + "\n", + "**Hinweis:** Nachdem hier $x$ ein Vektor ist, müssen wir die Ableitung für jede Komponente einzeln berechnen. Dementsprechend wurde oben der Index $i$ verwendet." + ] + }, + { + "cell_type": "markdown", + "id": "2aa67ccc", + "metadata": {}, + "source": [ + "Wie können wir die Ableitung händisch berechnen?\n", + "\n", + "$$\n", + "\\frac{\\mathrm d\\, a_i}{\\mathrm d\\, x_i} = 1,\\hspace{1cm}\n", + "\\frac{\\mathrm d\\, b_i}{\\mathrm d\\, a_i} = 2\\cdot a_i,\\hspace{1cm}\n", + "\\frac{\\mathrm d\\, c_i}{\\mathrm d\\, b_i} = 1,\\hspace{1cm}\n", + "\\frac{\\mathrm d\\, y}{\\mathrm d\\, c_i} = \\frac{1}{3}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "id": "52884584", + "metadata": {}, + "source": [ + "Somit ergibt sich für $x=[0,1,2]$ und $a_i=x_i+2$ die Ableitung\n", + "\n", + "$$\\frac{\\mathrm d\\, y(x)}{\\mathrm d\\, x} =\\left[\\frac43,2,\\frac83\\right].$$\n", + "\n", + "Wie kann das nun PyTorch machen?" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "id": "7c98a812", + "metadata": {}, + "outputs": [], + "source": [ + "# For calculating the gradient, we need to call the backward() method on the output tensor\n", + "y.backward()" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "id": "29a22f7d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Gradient of y with respect to x: tensor([1.3333, 2.0000, 2.6667])\n" + ] + } + ], + "source": [ + "# Then we get the gradient of y with respect to x by invoking x.grad\n", + "print(\"Gradient of y with respect to x:\", x.grad)" + ] + }, + { + "cell_type": "markdown", + "id": "bc2bffca", + "metadata": {}, + "source": [ + "**Wichtig:** Später werden wir die Ableitungen nicht bezüglich $x$ berechnen (wir können ja unsere Daten nicht ändern), sondern werden die Parameter bezüglich der Parameter von unserem Netzwerk berechnen. Diese können wir dann jeden Schritt dementsprechend anpassen." + ] + }, + { + "cell_type": "markdown", + "id": "fe0653d6", + "metadata": {}, + "source": [ + "**Hinweis:** Das heißt, bei einer Loss-Funktion sind die Daten fix und wir haben die Parameter der Funktion/des neuronalen Netzwerks als Parameter." + ] + }, + { + "cell_type": "markdown", + "id": "2a9ea7ec", + "metadata": {}, + "source": [ + "**Wichtig:** Nachdem hier unser Input ein Vektor ist, sprechen wir eigentlich nicht von einer Ableitung, sondern von einem **Gradient**. Ein Gradient ist ein Vektor, bei dem jede Komponente, die Ableitung bzgl. der Komponente ist." + ] + }, + { + "cell_type": "markdown", + "id": "71758179", + "metadata": {}, + "source": [ + "#### GPU Support" + ] + }, + { + "cell_type": "markdown", + "id": "60077273", + "metadata": {}, + "source": [ + "Wie bereits oben angekündigt, wollen wir hier nun die GPU (leider nur NVIDIA (und Apple Silicon) möglich) benutzen, sofern eine zur Verfügung steht. Mit ihr können wir die Berechnungen dann mit *Cuda* durchführen. \n", + "\n", + "Ob eine GPU zur Verfügung steht, können wir mit folgenden Command testen.\n", + "\n", + "**Hinweis:** Es kann nur dann die NVIDIA GPU verwendet werden, wenn auch die entsprechende PyTorch Version installiert ist." + ] + }, + { + "cell_type": "markdown", + "id": "a4fa29bd", + "metadata": {}, + "source": [ + "**Hinweis:** Für alle ohne NVIDIA GPU gibt es auch einige Online Tools bei denen man gratis hardwarebeschleunigt Notebooks ausführen kann (siehe Ende vom Notebook)." + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "id": "e774adec", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "GPU is available: False\n" + ] + } + ], + "source": [ + "gpu_avail = torch.cuda.is_available()\n", + "print(f'GPU is available: {gpu_avail}')" + ] + }, + { + "cell_type": "markdown", + "id": "2a15900a", + "metadata": {}, + "source": [ + "Im Allgemeinen ist es gängig, folgenden command am Beginn jedes Notebooks zu verwenden." + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "id": "d7ddbb44", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cpu\n" + ] + } + ], + "source": [ + "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('mps') if torch.backends.mps.is_available() else torch.device('cpu')\n", + "print(device)" + ] + }, + { + "cell_type": "markdown", + "id": "2b1c4349", + "metadata": {}, + "source": [ + "Um nun auch wirklich die Berechnungen auf der GPU durchzuführen, müssen wir unsere Berechnungen auf dieses Gerät schieben. Dies geht mit dem `to(device)` command." + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "id": "3b6a40ac", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[1., 1.],\n", + " [1., 1.],\n", + " [1., 1.]])\n" + ] + } + ], + "source": [ + "x = torch.ones((3,2))\n", + "x = x.to(device) # Move the tensor to the GPU if available\n", + "print(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "id": "f86a010e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[1., 1.],\n", + " [1., 1.],\n", + " [1., 1.]])\n" + ] + } + ], + "source": [ + "# We can also directly create a tensor on the GPU\n", + "x_gpu = torch.ones((3, 2), device=device) # Create a tensor directly on the GPU\n", + "print(x_gpu)" + ] + }, + { + "cell_type": "markdown", + "id": "3485380b", + "metadata": {}, + "source": [ + "Warum sollten wir Code überhaupt auf der GPU ausführen? Weil es viel **schneller** ist. (Faktor > 100 ohne Probleme möglich)\n", + "\n", + "Vergleichen wir nun die Geschwindigkeit." + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "id": "586f80ab", + "metadata": {}, + "outputs": [], + "source": [ + "if torch.cuda.is_available():\n", + " size = 5000\n", + "\n", + " x = torch.randn(size, size)\n", + "\n", + " ## CPU version\n", + " start_time = time.time()\n", + " _ = torch.matmul(x, x)\n", + " end_time = time.time()\n", + " print(f\"CPU time: {(end_time - start_time):6.5f}s\")\n", + "\n", + " ## GPU version\n", + " x = x.to(device)\n", + " _ = torch.matmul(x, x) # First operation to 'burn in' GPU\n", + " # CUDA is asynchronous, so we need to use different timing functions\n", + " start = torch.cuda.Event(enable_timing=True)\n", + " end = torch.cuda.Event(enable_timing=True)\n", + " start.record()\n", + " _ = torch.matmul(x, x)\n", + " end.record()\n", + " torch.cuda.synchronize() # Waits for everything to finish running on the GPU\n", + " print(f\"GPU time: {0.001 * start.elapsed_time(end):6.5f}s\") # Milliseconds to seconds" + ] + }, + { + "cell_type": "markdown", + "id": "c6e26cd5", + "metadata": {}, + "source": [ + "Wir sehen also, mit *cuda* ist wirklich alles **viel** schneller." + ] + }, + { + "cell_type": "markdown", + "id": "c886862b", + "metadata": {}, + "source": [ + "**Hinweis:** Mit *cuda*, also auf der GPU müssen wir nun aber auch beachten, dass nun die Daten den GPU Speicher befüllen (GPU Memory). Dieser ist meistens geringer als der RAM, somit müssen die Modelle ggf. kleiner gemacht werden, bzw. das Dataset muss aufgeteilt werden (Details dazu gibt in einem anderen Notebook (Datasets und Dataloader))." + ] + }, + { + "cell_type": "markdown", + "id": "2bd0aed8", + "metadata": {}, + "source": [ + "![Tensors_Everywhere](../resources/Tensors_Everywhere.jpg)" + ] + }, + { + "cell_type": "markdown", + "id": "77e318c9", + "metadata": {}, + "source": [ + "# Bonus: Google Colab" + ] + }, + { + "cell_type": "markdown", + "id": "995c383d", + "metadata": {}, + "source": [ + "Was, wenn wir keine Grafikkarte haben und trotzdem ein Model hardwarebeschleunigt trainieren wollen?\n", + "\n", + "Es gibt zahlreiche Online Anbieter, wo man gratis Notebooks laufen lassen kann inkl. GPU. Zum Beispiel [Google Colab](https://colab.research.google.com/).\n", + "\n", + "Wir laden nun unser Notebook hier hoch und testen Colab ein wenig." + ] + }, + { + "cell_type": "markdown", + "id": "3f753b93", + "metadata": {}, + "source": [ + "**Vorteile**:\n", + "* Gratis\n", + "* Gute GPU's\n", + "* Ermöglicht Verwendung auch mit wenig-leistungsfähigem Laptop" + ] + }, + { + "cell_type": "markdown", + "id": "44553735", + "metadata": {}, + "source": [ + "**Nachteile:**\n", + "* Runtime nur für begrenzte (inaktive) Zeit\n", + "* Hochladen von Daten nervig" + ] + }, + { + "cell_type": "markdown", + "id": "d93c9eee", + "metadata": {}, + "source": [ + "## Aufgabe:" + ] + }, + { + "cell_type": "markdown", + "id": "8e72bbbc", + "metadata": {}, + "source": [ + "Programme ein mehrdimensionales Linear Regression Modell für 4 Input Features $x_1, x_2, x_3, x_4$ und berechne den Gradient bzgl. der Parameter $w_1, w_2, w_3, w_4, x_1, x_2, x_3, x_4, b$.\n", + "\n", + "Bearbeite dafür die folgenden Schritte:\n", + "* Eingabe: Vektor $x$ der mit 4 Features (Tensor mit Shape (4))\n", + "* Gewichtsmatrix $W$ (Größe $1\\times 4$)\n", + "* Offset $b$ (Größe $1$)\n", + "\n", + "Berechne:\n", + "* $y = Wx+b = w_1\\cdot x_1 + w_2 \\cdot x_2 + w_3 \\cdot x_3 + w_4 \\cdot x_4 + b$\n", + "* a = $\\max(y, 0)$\n", + "* $L=f(a)$ mit $f(a) = a^2$ (*Dummy*-Loss Funktion)\n", + "* Ableitungen bzgl. $W$, $x$ und $b$.\n", + "\n", + "\n", + "Die Gewichtsvektoren und Input kann zufällig gewählt werden." + ] + }, + { + "cell_type": "markdown", + "id": "84103a14", + "metadata": {}, + "source": [ + "**Bonus:** Verwende dazu das Attribut **device**, sodass immer die GPU verwendet wird, falls eine verfügbar ist." + ] + }, + { + "cell_type": "markdown", + "id": "2d8d80f6", + "metadata": {}, + "source": [ + "**Bonus:** Lade nochmal das Iris Dataset und berechne den Output hier? Was muss alles geändert werden?" + ] + }, + { + "cell_type": "markdown", + "id": "5233833a", + "metadata": {}, + "source": [ + "### Lösung:" + ] + }, + { + "cell_type": "markdown", + "id": "b471f56d", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "055c55dc", + "metadata": {}, + "source": [ + "Die vorige Aufgabenstellung war der Türöffner für \"echte\" neuronale Netze, mit welchen wir uns von nun an befassen wollen." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dsai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.23" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/06_NN/code/nn_2_mlp_activation_fct.ipynb b/06_NN/code/nn_2_mlp_activation_fct.ipynb new file mode 100644 index 0000000..e13145f --- /dev/null +++ b/06_NN/code/nn_2_mlp_activation_fct.ipynb @@ -0,0 +1,1581 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3892507e", + "metadata": {}, + "source": [ + "
\n", + "\n", + "

Neural Networks: Feed Forward Neural Networks (MLP)

\n", + "

DSAI

\n", + "

Jakob Eggl

\n", + "\n", + "
\n", + " \"Logo\"\n", + "
\n", + " © 2025/26 Jakob Eggl. Nutzung oder Verbreitung nur mit ausdrücklicher Genehmigung des Autors.\n", + "
\n", + "
\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "e88622c2", + "metadata": {}, + "source": [ + "## Wiederholung Lineare Regression" + ] + }, + { + "cell_type": "markdown", + "id": "58948c41", + "metadata": {}, + "source": [ + "Wir haben am Ende vom letzten Notebook selber eine lineare Regression programmiert. Wir wiederholen kurz die wichtigsten Punkte:\n", + "\n", + "Im eindimensionalen Fall haben wir als Input unsere $Z$ Datenpaare $\\{(x_i,y_i)\\}_{i=1}^{Z}$, wobei jedes $x_i\\in\\mathbb R$ das (einzige) Input Feature ist und $y_i\\in\\mathbb R$ der dazugehörige Output (Label). Ziel ist es $k,d\\in\\mathbb R$ zu finden, sodass ein vorgegebener Fehler (zum Beispiel Mean Squared Error) für unser gefundenes Modell\n", + "$$\\hat f(x) = k\\cdot x + d$$\n", + "klein ist.\n", + "\n", + "Im mehrdimensionalen ist die Idee gleich, wir sprechen auch hier noch von einer **Linearen Regression**. Das Modell ist in diesem Fall dann\n", + "$$\\hat f(X) = WX+b$$\n", + "und wir suchen diesmal eine Matrix $W$ (in diesem Fall ein Vektor, weil der Output eine Zahl ist) und einen Vektor $b$ (auch hier kein Vektor sondern eine Zahl, weil Output ein Vektor ist). Hier entspricht $W$ quasi unserem $k$ von vorher und $b$ unserem $d$.\n", + "\n", + "**Hinweis:** In diesem Fall sind unsere Daten eine Matrix (quasi wie das Dataframe, bestehend aus $Z$ Zeilen und $d$ Spalten). Jede Zeile repräsentiert ein $x_i$ und ist dann ein Vektor bestehend aus den $d$ (Dimension) reellen Zahlen (**=Features**) $x_{i1}, x_{i2}, \\ldots, x_{id}$." + ] + }, + { + "cell_type": "markdown", + "id": "4753462a", + "metadata": {}, + "source": [ + "**WICHTIG:** Lineare Regression ist offensichtlich ein *Supervised Machine Learning* Problem. Wir werden uns im folgenden (für Neuronale Netzwerke) hauptsächlich mit Supervised Machine Learning beschäftigen. Sprich wir haben immer das Label $y$ gegeben. Dies kann eine (oder mehrere) Klasse(n) sein, oder auch in der Form von Zahlenwerten gegeben sein." + ] + }, + { + "cell_type": "markdown", + "id": "923fc2f0", + "metadata": {}, + "source": [ + "## Das Perzeptron" + ] + }, + { + "cell_type": "markdown", + "id": "4bd3ee69", + "metadata": {}, + "source": [ + "Die mehrdimensionale, lineare Regression ist auch die Grundidee des sogenannten **Perzeptrons**, erfunden von *Frank Rosenblatt* im Jahr 1957.\n", + "\n", + "Das folgende Bild zeigt den (leicht modifizierten) Aufbau eines einzigen Perzeptrons. Wir sehen, der Output $z$ ist nichts anderes als wie bei uns $\\hat f (x)$ bzw. $\\hat f (X)$." + ] + }, + { + "cell_type": "markdown", + "id": "7b90c5d2", + "metadata": {}, + "source": [ + "![Perzeptron_Zugeschnitten](../resources/Perzeptron_zugeschnitten.png)\n", + "\n", + "(von https://www.mql5.com/de/articles/8908)" + ] + }, + { + "cell_type": "markdown", + "id": "268cdb1e", + "metadata": {}, + "source": [ + "Es gilt also:\n", + "\n", + "\\begin{align*}\n", + " z &= w_1 \\cdot x_1 + w_2 \\cdot x_2 + \\ldots + w_N \\cdot x_N + b\\\\\n", + " &= \\underbrace{\\begin{pmatrix}\n", + " w_1 & w_2 & \\ldots & w_N\n", + " \\end{pmatrix}}_{=W}\\cdot \n", + " \\underbrace{\\begin{pmatrix}\n", + " x_1 \\\\ x_2 \\\\ \\vdots \\\\ x_N\n", + " \\end{pmatrix}}_{=X} + b\n", + "\\end{align*}" + ] + }, + { + "cell_type": "markdown", + "id": "e2f9463e", + "metadata": {}, + "source": [ + "**Wichtig:** Wie bereits vorher erwähnt, wird im obigen Bild und jetzt generell immer nur ein Datenpunkt betrachtet, also zum Beispiel beim Iris Dataset eine bestimmte Blume mit den Eigenschaften $x_1, x_2, x_3, x_4$ (die 4 Längen). Ansonsten bräuchten wir eine Doppel-Indizierung ($x_{ij}$). " + ] + }, + { + "cell_type": "markdown", + "id": "efba0b54", + "metadata": {}, + "source": [ + "Den Bias können wir auch in der Weight-Matrix $W$ verstecken:\n", + "\n", + "$$z = w_1 \\cdot x_1 + w_2 \\cdot x_2 + \\ldots + w_N \\cdot x_N + b = \\begin{pmatrix}\n", + " w_1 & w_2 & \\ldots & w_N\n", + " \\end{pmatrix}\\cdot \n", + " \\begin{pmatrix}\n", + " x_1 \\\\ x_2 \\\\ \\vdots \\\\ x_N\n", + " \\end{pmatrix} + b = \\begin{pmatrix}\n", + " w_1 & w_2 & \\ldots & w_N & b\n", + " \\end{pmatrix}\\cdot \n", + " \\begin{pmatrix}\n", + " x_1 \\\\ x_2 \\\\ \\vdots \\\\ x_N \\\\ 1\n", + " \\end{pmatrix}.$$" + ] + }, + { + "cell_type": "markdown", + "id": "839c3d56", + "metadata": {}, + "source": [ + "### Notation:" + ] + }, + { + "cell_type": "markdown", + "id": "cb6aea6b", + "metadata": {}, + "source": [ + "* Wir nennen ab jetzt die Parametermatrix $W$ **Weights**\n", + "* Die Matrix $X$ ist unser **Input**\n", + "* $b$ ist der **Bias**" + ] + }, + { + "cell_type": "markdown", + "id": "d95b54c3", + "metadata": {}, + "source": [ + "## Vom Perzeptron zum Linearen Layer" + ] + }, + { + "cell_type": "markdown", + "id": "028728cb", + "metadata": {}, + "source": [ + "*Aber wie hängt das jetzt mit Künstlicher Intelligenz bzw. mit Neuronalen Netzwerken zusammen?*" + ] + }, + { + "cell_type": "markdown", + "id": "10c2e160", + "metadata": {}, + "source": [ + "Wir werden jetzt unser Perzeptron immer weiter verallgemeinern und schlussendlich beim **Multi-Layer Perzeptron** (*MLP*) landen. Dieses ist bei einem normalen Feed-Forward Neural Network der Standard und bildet quasi schon ein vollständiges *neuronales Netzwerk*!" + ] + }, + { + "cell_type": "markdown", + "id": "5abc25b1", + "metadata": {}, + "source": [ + "Der erste Schritt dazu ist, dass wir nun mehrerere Perzeptrons nebeneinander verwenden können, falls wir mehrere Werte am Ausgang haben wollen.\n", + "\n", + "Was wäre ein möglicher Anwendungsfall? (Wir werden später sehen, warum das Sinn macht)" + ] + }, + { + "cell_type": "markdown", + "id": "16be11d4", + "metadata": {}, + "source": [ + "In so einem Fall sieht dann unser Aufbau folgendermaßen aus." + ] + }, + { + "cell_type": "markdown", + "id": "122c0b2c", + "metadata": {}, + "source": [ + "![Single_Layer_Perceptron](../resources/single_layer_perceptron.jpg)\n", + "\n", + "(von https://www.upgrad.com/blog/perceptron-learning-algorithm-how-it-works/)" + ] + }, + { + "cell_type": "markdown", + "id": "ae6ba41c", + "metadata": {}, + "source": [ + "Sprich wir haben hier einfach 3 (rote) Perzeptrons nebeneinander, welche jeweils 5 Input-Features (blau) verarbeiten. Dabei hat jedes Perzeptron seine eigenen Gewichte und Bias. Die Outputs in rot lassen sich dann so berechnen:" + ] + }, + { + "cell_type": "markdown", + "id": "e75fe14b", + "metadata": {}, + "source": [ + "\\begin{align*}\n", + " z_1 &= W_1X+b_1 \\\\\n", + " z_2 &= W_2X+b_2 \\\\\n", + " z_3 &= W_3X+b_3\n", + "\\end{align*}" + ] + }, + { + "cell_type": "markdown", + "id": "a89a45aa", + "metadata": {}, + "source": [ + "Dies können wir aber auch in Vektorschreibweise schreiben mit $\\mathbf z = (z_1, z_2, z_3)$ und $\\mathbf W = (W_1, W_2, W_3)$ und $\\mathbf b = (b_1, b_2, b_3)$. Wir erhalten dann\n", + "\n", + "$$\\mathbf z = \\mathbf W \\mathbf x + \\mathbf b.$$" + ] + }, + { + "cell_type": "markdown", + "id": "8b30c234", + "metadata": {}, + "source": [ + "Hier ist nochmal dargestellt, wie das ganze in Matrix schreibweise aussieht für ein ähnliches Problem.\n", + "\n", + "![MLP_Matrix_Multiplication](../resources/mlp_matrix_mul.png)\n", + "\n", + "(von https://community.deeplearning.ai/t/matrix-multiplication-in-neural-network/685083)" + ] + }, + { + "cell_type": "markdown", + "id": "ba403835", + "metadata": {}, + "source": [ + "**Hinweis:** Auch hier könnten wir den Bias wieder verstecken in der Weight-Matrix, ist aber erneut wieder nur eine rein optische Änderung." + ] + }, + { + "cell_type": "markdown", + "id": "64580d7c", + "metadata": {}, + "source": [ + "Wir sehen im obigen Bild auch, dass die Perzeptrons hier **Neuronen** genannt werden. Um zu verstehen, warum das Sinn macht, sehen wir uns folgende Grafik an." + ] + }, + { + "cell_type": "markdown", + "id": "7f2ba829", + "metadata": {}, + "source": [ + "![Biological_vs_Artificial_Neuron](../resources/Biological_Neuron_vs_Artificial_Neuron.png)\n", + "\n", + "(von https://towardsdatascience.com/the-concept-of-artificial-neurons-perceptrons-in-neural-networks-fab22249cbfc/)" + ] + }, + { + "cell_type": "markdown", + "id": "57662646", + "metadata": {}, + "source": [ + "Unser *Perzeptron* ist also ein Konstrukt, dass dem biologischen Neuron ähneln soll.\n", + "\n", + "Wir sehen, dass rechts auch noch eine sogenannte **Activation Function** $f$ ist. Diese ignorieren wir noch zwischenzeitlich und kümmern uns vorher noch um den nächsten Punkt." + ] + }, + { + "cell_type": "markdown", + "id": "e84d64c8", + "metadata": {}, + "source": [ + "## Vom Linearen Layer zum MLP" + ] + }, + { + "cell_type": "markdown", + "id": "0dba6c51", + "metadata": {}, + "source": [ + "Um nun ein neuronales Netz zu erhalten, schalten wir mehrere Schichten von den vorigen Perzeptronen hintereinander. Eine Schicht wird dabei **Single-Layer Perzeptron** genannt. Mehrere Schichten nennen wir dann **Multi-Layer Perzeptron**." + ] + }, + { + "cell_type": "markdown", + "id": "8b22d4fd", + "metadata": {}, + "source": [ + "![MLP](../resources/mlp.png)\n", + "\n", + "(von https://machinelearninggeek.com/multi-layer-perceptron-neural-network-using-python/)" + ] + }, + { + "cell_type": "markdown", + "id": "090fba0c", + "metadata": {}, + "source": [ + "Obiges Beispiel zeigt also ein Multi-Layer Perceptron (also eine mögliche Form eines neuronalen Netzes) mit $3$ **Inputs** (das Dataset hat also 3 Features) und einem **Output** Neuron (Rot). Dazwischen (grün) sind die sogenannten **Hidden Layers**." + ] + }, + { + "cell_type": "markdown", + "id": "e31ae53e", + "metadata": {}, + "source": [ + "Die Anzahl der Hidden Layers beschreibt die **Tiefe des Neuronalen Netzwerkes**. Von hier kommt auch der Begriff **Deep Learning**, welcher im Vergleich zu **Machine Learning** für (sehr) tiefe neuronale Netze verwendet wird." + ] + }, + { + "cell_type": "markdown", + "id": "4efc5fcb", + "metadata": {}, + "source": [ + "**Hinweis:** Alle modernen großen KI-Modelle sind \"Deep-Learning\"-Modelle." + ] + }, + { + "cell_type": "markdown", + "id": "1fef54a0", + "metadata": {}, + "source": [ + "Beim Aufbau so eines Netzwerkes haben wir relativ viele Freiheitsgrade (Hyperparameter), da wir die Anzahl der Hidden Layers und die dazugehörigen Anzahl an Neuronen pro Hidden Layer frei wählen können. Fix vorgegeben sind nur die Anzahl der Input Neuronen (=Anzahl der Features im Dataset) und je nach Task (Regression/Klassifikation) ist die Anzahl der Output-Neuronen festgelegt." + ] + }, + { + "cell_type": "markdown", + "id": "935d5b51", + "metadata": {}, + "source": [ + "Wir sehen in der Grafik auch, dass der Output von den Neuronen in Schicht $n$ der Input für die Neuronen aus Schicht $n+1$ darstellt. Somit ist also jedes Neuron aus Schicht $n$ verbunden mit alle Neuronen aus Schicht $n+1$." + ] + }, + { + "cell_type": "markdown", + "id": "2a2fd40a", + "metadata": {}, + "source": [ + "Wie können wir den Output nun berechnen anhand von diesem schematischen Beispiel?\n", + "\n", + "Wir bezeichnen die einzelnen Elemente nun folgendermaßen:\n", + "* Input-Layer: $\\mathbf X = (x_1, x_2, x_3)$\n", + "* Hidden-Layer 1: $\\mathbf h_1 = (h_{11}, h_{12}, h_{13}, h_{14})$\n", + "* Hidden-Layer 2: $\\mathbf h_2 = (h_{21}, h_{22}, h_{23}, h_{24})$\n", + "* Hidden-Layer 3: $\\mathbf h_3 = (h_{31}, h_{32}, h_{33}, h_{34})$\n", + "* Output: $\\mathbf z$\n", + "\n", + "\n", + "Es gilt\n", + "\\begin{align*}\n", + " \\mathbf h_{1} &= \\mathbf W_{1}\\mathbf X \\\\\n", + " \\mathbf h_{2} &= \\mathbf W_{2}\\mathbf h_1 \\\\\n", + " \\mathbf h_{3} &= \\mathbf W_{3}\\mathbf h_2 \\\\\n", + " \\mathbf z &= \\mathbf W_4 \\mathbf h_3\n", + "\\end{align*}" + ] + }, + { + "cell_type": "markdown", + "id": "f75aff2e", + "metadata": {}, + "source": [ + "> **Übung:** Welche Dimensionen haben hier die Elemente?" + ] + }, + { + "cell_type": "markdown", + "id": "a062a2e6", + "metadata": {}, + "source": [ + "> **Übung:** Wie viele Parameter hat das Netzwerk, welche wir später lernen wollen?" + ] + }, + { + "cell_type": "markdown", + "id": "882bdd8c", + "metadata": {}, + "source": [ + "## Lineare Layer in PyTorch" + ] + }, + { + "cell_type": "markdown", + "id": "43b57480", + "metadata": {}, + "source": [ + "Um nun diese Funktionalität in PyTorch zu erreichen, könnte man meinen, man initialisiert einfach die verschiedenen Weight-Matrizen und die Bias-Vektoren als Tensoren mit Gradient und berechnet die Output \"händisch\". Das ist prinzipiell nicht falsch, aber tatsächlich ist es viel einfacher, wie wir im folgenden sehen." + ] + }, + { + "cell_type": "markdown", + "id": "bcaabe7d", + "metadata": {}, + "source": [ + "**Hinweis:** Der Grund warum wir Gradienten brauchen, wird spätestens klar, wenn wir über die Optimierung (=das Lernen von Neuronale Netzwerke) lernen." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "9016d86c", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn" + ] + }, + { + "cell_type": "markdown", + "id": "e4b6fa16", + "metadata": {}, + "source": [ + "Um nun ein Neuronales Netzwerk zu designed, müssen wir zuerst eine **Klasse erstellen**, welche von `nn.Module` erbt. Danach müssen wir nur noch die `init` und `forward` **Methoden implementieren** und unser Netzwerk ist schon fertig.\n", + "\n", + "Wir bauen jetzt das Netzwerk von gerade eben nach (3 Input Features, 3 Hidden Layers mit jeweils 4 Features, Output Layer mit einem Neuron)." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "310aecd6", + "metadata": {}, + "outputs": [], + "source": [ + "class MyFirstNeuralNetwork(nn.Module):\n", + " \"\"\"A simple feedforward neural network, that expects 3 input features and produces 1 Output by using 3 Hidden Layers with 4 Features each.\"\"\"\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.hidden_layer_1 = nn.Linear(3, 4) # transforms the 3 features into 4 features\n", + " self.hidden_layer_2 = nn.Linear(4, 4) # transforms the 4 features into 4 features\n", + " self.hidden_layer_3 = nn.Linear(4, 4) # transforms the 4 features into 4 features\n", + " self.output_layer = nn.Linear(4, 1) # transforms the 4 features into 1 feature\n", + "\n", + " def forward(self, x):\n", + " \"\"\"\n", + " :param x: Input tensor with shape (batch_size, 3)\n", + " :return: Output tensor with shape (batch_size, 1)\n", + " \"\"\"\n", + " x = self.hidden_layer_1(x)\n", + " x = self.hidden_layer_2(x)\n", + " x = self.hidden_layer_3(x)\n", + " x = self.output_layer(x)\n", + " return x" + ] + }, + { + "cell_type": "markdown", + "id": "1c066dc9", + "metadata": {}, + "source": [ + "Damit ist unser Neuronales Netzwerk schon fertig. Natürlich ist das soeben erstellte Neuronale Netzwerk sehr einfach. Man könnte es noch weiter vergrößern. Es ist auch möglich, in der Forward Methode zu Normalisieren, bzw. die Daten zu bearbeiten. Wir können unser Netzwerk auch variabel machen, indem wir im Konstruktor (`init`) noch Parameter übergeben." + ] + }, + { + "cell_type": "markdown", + "id": "6df741f0", + "metadata": {}, + "source": [ + "> **Übung:** Warum speichern wir hier 2x das gleiche Lineare Layer (`nn.Linear(4,4)`) als 2 verschiedene Variablen (`self.hidden_layer_2` und `self.hidden_layer_3`)?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ändern wir unser Netzwerk nun so, dass wir keinen Bias haben und die Anzahl der Input Features angepasst werden kann." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0eaef635", + "metadata": {}, + "outputs": [], + "source": [ + "class MyVariableFirstNeuralNetwork(nn.Module):\n", + " \"\"\"A simple feedforward neural network, that expects n_input_features input features and produces 1 Output by using 3 Hidden Layers with 4 Features each.\"\"\"\n", + " def __init__(self, n_input_features: int):\n", + " super().__init__()\n", + " self.hidden_layer_1 = nn.Linear(n_input_features, 4, bias=False) # transforms the n_input_features features into 4 features\n", + " self.hidden_layer_2 = nn.Linear(4, 4, bias=False) # transforms the 4 features into 4 features\n", + " self.hidden_layer_3 = nn.Linear(4, 4, bias=False) # transforms the 4 features into 4 features\n", + " self.output_layer = nn.Linear(4, 1, bias=False) # transforms the 4 features into 1 feature\n", + "\n", + " def forward(self, x):\n", + " \"\"\"\n", + " :param x: Input tensor with shape (batch_size, n_input_features)\n", + " :return: Output tensor with shape (batch_size, 1)\n", + " \"\"\"\n", + " x = self.hidden_layer_1(x)\n", + " x = self.hidden_layer_2(x)\n", + " x = self.hidden_layer_3(x)\n", + " x = self.output_layer(x)\n", + " return x" + ] + }, + { + "cell_type": "markdown", + "id": "b8d51eac", + "metadata": {}, + "source": [ + "Unser Netzwerk können wir nun folgendermaßen erstellen (die Gewichte werden am Anfang zufällig gewählt)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d3926ee8", + "metadata": {}, + "outputs": [], + "source": [ + "model = MyVariableFirstNeuralNetwork(n_input_features=3)" + ] + }, + { + "cell_type": "markdown", + "id": "7d5ce043", + "metadata": {}, + "source": [ + "Um nun den Output zu berechnen, können wir das Model wie eine Funktion aufrufen. Dabei wird dann die `forward` Methode ausgeführt." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "4c076bb9", + "metadata": {}, + "outputs": [], + "source": [ + "aux_input = torch.tensor([[1.0, 2.0, 3.0]])\n", + "output = model(aux_input)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "727b57b6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.1016]], grad_fn=)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output" + ] + }, + { + "cell_type": "markdown", + "id": "b9a5a49b", + "metadata": {}, + "source": [ + "Wie oben in der `forward` Methode als Kommentar geschrieben, ist die *Shape*, die sich das Modell bei der `forward` Methode erwartet, `(batch_size, n_input_features)`. Dabei steht die `batch_size` für die Anzahl der Datenpunkte gleichzeitig. Sprich, wenn wir nun als Input 2 Vektoren der Länge 3 haben, dann werden beide Elemente durch das Netzwerk geschickt." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5a4003e3", + "metadata": {}, + "outputs": [], + "source": [ + "aux_input = torch.tensor([[1.0, 2.0, 3.0], [-1.0, 4, 0.1111]])\n", + "output = model(aux_input)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "4189433a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 0.1016],\n", + " [-0.2725]], grad_fn=)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output" + ] + }, + { + "cell_type": "markdown", + "id": "6587ec67", + "metadata": {}, + "source": [ + "Wir sehen, der Output besteht nun auch aus 2 Einträgen (jeweils das eine Output Neuron für beide Datenpunkte).\n", + "\n", + "Wir können also stand jetzt--unser Modell ist sehr klein--in vielen Fällen auch das ganze Dataset durch das Modell schicken. Später, zum Beispiel bei Bildern, wird uns das nicht mehr möglich sein, da werden wir dann sogenannte Teile des Datasets verwenden (Batches). Dies werden wir uns genauer anschauen, wenn wir über Datasets und Dataloader lernen. " + ] + }, + { + "cell_type": "markdown", + "id": "e1640e5e", + "metadata": {}, + "source": [ + "**Hinweis:** Für die Inferenz (= spätere Verwendung vom Modell) ist uns die Möglichkeit, mehrere Datenpunkte gleichzeitig verwenden zu können relativ \"egal\", da wir meistens immer nur an einer Vorhersage interessiert sind. Für das Training jedoch ist es sehr praktisch, da wir dann die Parameter für mehrere Samples gleichzeitig bearbeiten können." + ] + }, + { + "cell_type": "markdown", + "id": "cad6300f", + "metadata": {}, + "source": [ + "Betrachten wir nun nochmal die Formel von vorher und schreiben alles in eine Zeile. Was fällt uns auf?" + ] + }, + { + "cell_type": "markdown", + "id": "861e365e", + "metadata": {}, + "source": [ + "$$\\mathbf z = \\mathbf W_4 \\mathbf W_3 \\mathbf W_2 \\mathbf W_1 \\mathbf X$$" + ] + }, + { + "cell_type": "markdown", + "id": "21ea6670", + "metadata": {}, + "source": [ + "Hat es wirklich einen Vorteil, wenn wir mehrere lineare Layer einfach hintereinander schalten?" + ] + }, + { + "cell_type": "markdown", + "id": "7dbe1680", + "metadata": {}, + "source": [ + "Natürlich **NICHT**, genauso wie im 1d Fall $\\hat f(x) = kx+d$ wär die Hintereinanderausführung umsonst, da wieder eine lineare Funktion entsteht. Also unsere Netzwerk (also unsere Funktion) ist nach wie vor einfach eine lineare (im mehrdimenionalen eine Ebene). Dies reicht nicht aus, um komplizierte Zusammenhänge zu lernen. Deswegen werden wir nun die sogenannten **Activation Functions** (*Aktivierungsfunktionen*) dazwischen schalten. Sie sind **nicht-linear** und erlauben so dem Modell, kompliziertere Zusammenhänge zu lernen." + ] + }, + { + "cell_type": "markdown", + "id": "018a5d71", + "metadata": {}, + "source": [ + "> **Übung:** Zeige, dass für zwei lineare Funktionen, sagen wir $f(x)=a\\cdot x + b$ und $g(x) = c\\cdot x + d$ die Hintereinanderausführung $h(x):=f(g(x))$ auch wieder von der Form $h(x)=u\\cdot x + v$ ist. Welchen Wert haben $u$ und $v$?" + ] + }, + { + "cell_type": "markdown", + "id": "a48da39d", + "metadata": {}, + "source": [ + "## Die Aktivierungsfunktion" + ] + }, + { + "cell_type": "markdown", + "id": "e817fd78", + "metadata": {}, + "source": [ + "Nachdem wir nun gesehen haben, dass wir zwischen den einzelnen Layern eine Aktivierungsfunktion brauchen, sehen wir uns nochmal den vollständigen Aufbau eines Perzeptrons an." + ] + }, + { + "cell_type": "markdown", + "id": "740dfbb1", + "metadata": {}, + "source": [ + "![Perzeptron](../resources/Perzeptron.png)\n", + "\n", + "(von https://www.mql5.com/de/articles/8908)" + ] + }, + { + "cell_type": "markdown", + "id": "ed2c116a", + "metadata": {}, + "source": [ + "Es ist also $z$ nur ein sogenanntes **Logit** und $y=\\sigma(z)$ der \"echte\" Output.\n", + "\n", + "**Wichtig:** In diesem Bild ist für die Aktivierungsfunktion eine *Sigmoidfunktion* $\\sigma(x)$ eingezeichnet, prinzipiell kann es aber jede beliebige Aktivierungsfunktion sein $f$. (Dies ist bei dem Bild mit dem biologischen Neuron besser eingezeichnet). " + ] + }, + { + "cell_type": "markdown", + "id": "0be5c763", + "metadata": {}, + "source": [ + "**Wichtig:** Für ein MLP ist *pro Schicht* immer die selbe Aktivierungsfunktion vorgesehen. Also nach jedem Layer können wir auswählen, welche Aktivierungsfunktion verwendet werden soll für diesen Layer." + ] + }, + { + "cell_type": "markdown", + "id": "add96255", + "metadata": {}, + "source": [ + "### Anforderungen an eine Aktivierungsfunktion" + ] + }, + { + "cell_type": "markdown", + "id": "f82193c9", + "metadata": {}, + "source": [ + "Prinzipiell können wir jede Funktion als Aktivierungsfunktion verwenden, jedoch wollen wir ein paar (nicht zwingend erschöpfend) Eigenschaften sammeln:\n", + "\n", + "* Nicht-Linearität: Ansonsten würde sich unsere Ausgangssituation nicht ändern\n", + "* Differenzierbarkeit: Für jeden Wert $x\\in\\mathbb R$ muss die Ableitung bekannt (berechenbar) sein\n", + "* Beschränkt oder \"nicht-explodierend\": Der Output muss in einem gewissen Bereich liegen oder nicht explodieren\n", + "\n", + "\n", + "Ein paar weitere nette Eigenschaften:\n", + "* Der Gradient von der Ableitung sollte idealerweise überall im Bereich $1$ sein\n", + "* Die Ableitung sollte ohne viel Rechenaufwand berechenbar sein" + ] + }, + { + "cell_type": "markdown", + "id": "a8e4078c", + "metadata": {}, + "source": [ + "## Übersicht der gängigsten Aktivierungsfunktionen" + ] + }, + { + "cell_type": "markdown", + "id": "5a19bbd4", + "metadata": {}, + "source": [ + "Wir werden uns nun die gängigsten Aktivierungsfunktionen, deren Ableitungen, die Eigenschaften und die Vor- und Nachteile ansehen. Im Anschluss werden wir diese dann auch plotten. " + ] + }, + { + "cell_type": "markdown", + "id": "0dc8a290", + "metadata": {}, + "source": [ + "Dazu bereiten wir uns kurz eine Plotting Methode vor." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "c6464ff0", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "a705b3d3", + "metadata": {}, + "outputs": [], + "source": [ + "def plot_function_and_derivative(func, deriv, f_name, x_range):\n", + " x = np.linspace(x_range[0], x_range[1], 1000)\n", + " y = func(x)\n", + " dy = deriv(x)\n", + "\n", + " fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5), sharey=True)\n", + "\n", + " ax1.plot(x, y, label=f\"{f_name}(x)\")\n", + " ax1.set_title(f\"{f_name}(x)\")\n", + " ax1.set_xlabel(\"x\")\n", + " ax1.set_ylabel(\"y\")\n", + " ax1.grid()\n", + " ax1.legend()\n", + "\n", + " ax2.plot(x, dy, label=f\"{f_name}'(x)\", color=\"orange\")\n", + " ax2.set_title(f\"{f_name}'(x)\")\n", + " ax2.set_xlabel(\"x\")\n", + " ax2.grid()\n", + " ax2.legend()\n", + "\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "92f653c9", + "metadata": {}, + "source": [ + "### Sigmoid Funktion" + ] + }, + { + "cell_type": "markdown", + "id": "37f2fcb2", + "metadata": {}, + "source": [ + "\\begin{align*}\n", + " f(x) &= (1+e^{-x})^{-1}\\\\\n", + " f'(x) &= \\sigma(x) \\cdot (1-\\sigma(x))\n", + "\\end{align*}" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "5237d9d1", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+kAAAHUCAYAAABGRmklAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABy1ElEQVR4nO3dd3gU5f7+8Xuz6Y2QBEKAEEIH6aEIiIgICNg9iooCCipiA46N4xEVC+dY8zsqqF9RREWxi4pKbBQDSFV6DwGSAKEkIXWTnd8fC4GQUBaSzJb367r2CjuZ2f3Mw26evXdmnsdiGIYhAAAAAABgOh+zCwAAAAAAAA6EdAAAAAAAXAQhHQAAAAAAF0FIBwAAAADARRDSAQAAAABwEYR0AAAAAABcBCEdAAAAAAAXQUgHAAAAAMBFENIBAAAAAHARhHTATY0cOVKNGzc2u4wzslgseuqpp8643owZM2SxWJSamlpuuc1mU6tWrfSf//zH6ed+4okn1LlzZ9ntdqe3BQDADN7Qv19yySUaOXJkhXUXLlyogIAA7dy506labDabmjZtqqSkJKe2A1wVIR1wU0888YS++uors8s4o8WLF2v06NHnvP3UqVN16NAh3X///U5v+9BDD2nHjh16//33z/n5AQCoSd7Sv5/MMAyNGzdOd955p+Lj453a1s/PT5MmTdLkyZN14MCBKqsJMAshHXBTTZs2VadOncwu44wuvPBCNWzY8Jy2LSkp0Ysvvqg77rhDISEhTm9fq1Yt3XrrrfrPf/4jwzDOqQYAAGqSN/Tvlfnxxx+1cuXKc/pSXpJuvvlmWSwWvfXWW1VWE2AWQjrgovbv36+77rpLcXFxCggIUJ06ddSrVy/9/PPPkio/He7w4cMaNWqUIiMjFRoaqiFDhmj79u0VTkl76qmnZLFY9Pfff+uGG25QrVq1FBkZqQkTJqikpESbNm3S5ZdfrrCwMDVu3FgvvPBChfrS0tJ06623qm7dugoICFDr1q318ssvVzi1vLLT4ZYsWaJevXopMDBQ9evX18SJE2Wz2So8x5w5c7Rnzx7ddtttZcsKCwvVqVMnNWvWTNnZ2WXLMzMzVa9ePV1yySUqLS0tW37bbbdp8+bN+u23387Y5gAAVDf698pNmzZNXbt2VcuWLcuWLVq0SH5+fnrooYfKrXvsFPrp06eXLfP399fQoUP19ttv88U83J6v2QUAqNxtt92mlStX6rnnnlOLFi10+PBhrVy58pSncdntdl155ZVavny5nnrqKXXu3FmLFy/W5ZdffsrnuPHGG3Xrrbfq7rvvVnJysl544QXZbDb9/PPPGjt2rB566CHNmjVLjz76qJo1a6brrrtOkuMDRs+ePVVcXKxnnnlGjRs31nfffaeHHnpI27Zt09SpU0/5nOvXr1e/fv3UuHFjzZgxQ8HBwZo6dapmzZpVYd3vv/9edevWVZs2bcqWBQYG6tNPP1ViYqLuuOMOffHFF7Lb7Ro2bJgMw9DHH38sq9Vatn5iYqJCQ0P1/fff69JLLz1juwMAUJ3o36Xff/+93P3i4mL9/PPPFY6iX3TRRXr22Wf12GOP6eKLL9ZVV12ldevW6d5779Wtt96qUaNGlVv/kksu0bRp07R27Vq1a9fulLUCLs8A4JJCQ0ONcePGnfL3I0aMMOLj48vuf//994YkY9q0aeXWmzJliiHJePLJJ8uWPfnkk4Yk4+WXXy63bseOHQ1Jxpdfflm2zGazGXXq1DGuu+66smWPPfaYIclYunRpue3vuecew2KxGJs2bSpbdvJzDx061AgKCjIyMzPLlpWUlBitWrUyJBk7duwoW966dWvj8ssvr3T/Z8+ebUgykpKSjEmTJhk+Pj7GvHnzKl23V69eRvfu3Sv9HQAANYn+vaKlS5cakoxPPvmkwu/sdrsxePBgIyIiwli7dq3Rpk0bo1WrVsaRI0cqrLtly5ZK2wpwN5zuDriobt26acaMGXr22We1ZMmSM54uNn/+fEmOb89PdPPNN59ymyuuuKLc/datW8tisWjQoEFly3x9fdWsWbNyI63++uuvatOmjbp161Zu+5EjR8owDP3666+nfM7ffvtN/fr1U0xMTNkyq9WqoUOHVlg3PT1ddevWrfRxbrzxRt1zzz16+OGH9eyzz+pf//qX+vfvX+m6devW1Z49e05ZEwAANYX+vaL09HRJqrTPt1gsmjlzpsLCwtSlSxft2LFDn376aaVj1Rzbnj4f7o6QDrio2bNna8SIEXrnnXfUo0cPRUZGavjw4crMzKx0/QMHDsjX11eRkZHllp/YWZ7s5HX9/f0VHByswMDACssLCwvLPVdsbGyFx6tfv37Z70/lwIEDqlevXoXllS0rKCioUMuJ7rjjDtlsNvn6+uqBBx445XqBgYEqKCg45e8BAKgp9O8VHeujT9XnR0VF6aqrrlJhYaEuv/zyU57Kfmx7+ny4O0I64KKio6OVlJSk1NRU7dy5U1OmTNGXX35Z6byikqMDKykp0cGDB8stP1Wnfz6ioqKUkZFRYfmxb8Kjo6NPu21lNVW2LDo6usL+HJOXl6fbbrtNLVq0UFBQ0GmngTl48OBpawIAoKbQv1d07HFP1ecnJydr2rRp6tatm7766it98cUXla53bHv6fLg7QjrgBho1aqT77rtP/fv318qVKytdp0+fPpIc39Cf6JNPPqnyevr166f169dXqGXmzJmyWCzq27fvKbft27evfvnlF+3du7dsWWlpaYW6JalVq1batm1bpY8zZswYpaWl6csvv9T06dM1Z84cvfrqq5Wuu3379nKDzwEA4Aq8tX8/WevWrSWp0j4/IyNDt956q/r06aOUlBRdddVVGjVqlHbs2FFh3e3bt0sSfT7cHqO7Ay4oOztbffv21S233KJWrVopLCxMy5Yt048//lg2AuvJLr/8cvXq1Uv//Oc/lZOTo8TERC1evFgzZ86UJPn4VN13cuPHj9fMmTM1ZMgQTZ48WfHx8fr+++81depU3XPPPWrRosUpt/33v/+tOXPm6NJLL9WkSZMUHBysN954Q3l5eRXWveSSSzR58mTl5+crODi4bPk777yjDz/8UO+9954uuOACXXDBBbrvvvv06KOPqlevXuWupTtw4IC2bNlyzvOuAgBQVejfK9ewYUM1adJES5YsKXf5Wmlpadn857NmzZLVatWMGTPUsWNHDR06VIsWLZK/v3/Z+kuWLJHVatXFF198fg0BmIwj6YALCgwMVPfu3fXBBx9o2LBhGjRokN555x09+uij+r//+79Kt/Hx8dG3336rm266Sf/5z3909dVXa+HChfrwww8lSREREVVWX506dZSSkqJLL71UEydO1BVXXKGffvpJL7zwgl577bXTbtu2bVv9/PPPCg8P14gRI3TXXXepffv2euKJJyqse8stt6i0tFTff/992bI1a9bogQce0IgRI8qdGvjSSy+pffv2Gjp0qA4fPly2/JtvvpGfn1+FAXcAAKhp9O+nNmzYMP34448qKioqW/bkk09q4cKFmjVrVtm17bVr19Ynn3yiVatW6ZFHHin3GF9//bUGDx5cpW0CmMFiGIZhdhEAqs+sWbM0bNgw/fHHH+rZs6fZ5TjtyiuvVElJiX744Ydz2r53795q1KiRPvrooyquDAAA87h7/36y9PR0JSQkaObMmWc1IvzJtm3bpubNm+unn3465WwvgLsgpAMe5OOPP9aePXvUrl07+fj4aMmSJXrxxRfVqVOnsilc3M3atWvVqVMnpaSkqGvXrk5tu2DBAg0YMEDr169XkyZNqqlCAACqlyf275V59NFH9cMPP2j16tVOn8Z/++23a/fu3UpOTq6m6oCawzXpgAcJCwvTJ598omeffVZ5eXmKjY3VyJEj9eyzz5pd2jlr27at3nvvvXMaxfbAgQOaOXMmAR0A4NY8sX+vzL///W8FBwdrz549iouLO+vtSkpK1LRpU02cOLEaqwNqDkfSAQAAAABwEQwcBwAAAACAiyCkAwAAAADgIgjpAAAAAAC4CK8bOM5utys9PV1hYWGyWCxmlwMAgAzDUG5ururXr+/0iMaoHP09AMCVONPXe11IT09Pd2q0SAAAasquXbvUsGFDs8vwCPT3AABXdDZ9vdeF9LCwMEmOxgkPDze5mupjs9k0b948DRgwQH5+fmaX4xZoM+fRZs6jzZzjLe2Vk5OjuLi4sj4K588b+ntveX9UJdrMebSZ82gz53lDmznT13tdSD92ylt4eLjHdtqS44UeHBys8PBwj32hVzXazHm0mfNoM+d4W3txWnbV8Yb+3tveH1WBNnMebeY82sx53tRmZ9PXc+EbAAAAAAAugpAOAAAAAICLIKQDAAAAAOAivO6a9LNhGIZKSkpUWlpqdinnzGazydfXV4WFhW69H9XFarXK19eX6z8BAABQLQzDkM1m47P4WfCU7OLn5yer1Xrej0NIP0lxcbEyMjKUn59vdinnxTAM1atXT7t27SKInkJwcLBiY2Pl7+9vdikAAADwID4+PtqzZ48KCwvNLsUteEp2sVgsatiwoUJDQ8/rcQjpJ7Db7dqxY4esVqvq168vf39/t32R2O12HTlyRKGhofLx4aqGExmGoeLiYu3fv187duxQ8+bNaSMAAABUCbvdrjp16qikpMTtM0VN8YTsYhiG9u/fr927d6t58+bndUSdkH6C4uJi2e12xcXFKTg42OxyzovdbldxcbECAwPd9oVenYKCguTn56edO3eWtRMAAABwvmw2m/z8/BQbG3veR1S9hadklzp16ig1NVU2m+28Qrr7tkA1cucXBs4e/88AAACoaoZhSOKzpjeqqjMmTH3lLFiwQFdeeaXq168vi8Wir7/++ozbzJ8/X4mJiQoMDFSTJk305ptvVn+hAAAAAADUAFNDel5enjp06KDXX3/9rNbfsWOHBg8erN69e2vVqlX617/+pQceeEBffPFFNVcKAAAAAED1MzWkDxo0SM8++6yuu+66s1r/zTffVKNGjZSUlKTWrVtr9OjRuuOOO/TSSy9Vc6Xu6/bbb9c111xjdhmSpMaNGyspKem061R2RsX06dM1YMCAs36e7777Tp06dZLdbj+HKgEAAACcaOTIkW6fKU72xBNP6K677jrr53399dd11VVXnfX658OtBo5bvHhxhbA2cOBATZ8+vWyAhpMVFRWpqKio7H5OTo4kx4AONput3Lo2m02GYchut7t9wDt2Lcyrr74qSS6xP0uXLlVISMgZazmx/YuKijRp0iTNmjXrrPdh8ODBmjRpkj788EPdeuutp32eY/NXWq3WstfDya8LnBpt5jzazDne0l6evn81wZn+3lN4y/ujKtFmzqPNnFdSUiJJZbnCE7z66qvVuj/HssvZPMe5ZAqr1apt27apcePGkqS9e/fq//2//6fVq1ef9T6NGjVKzz33nBYsWKCLLrrolM95Yr44kTPvIbcK6ZmZmYqJiSm3LCYmRiUlJcrKylJsbGyFbaZMmaKnn366wvJ58+ZVGMHd19dX9erV05EjR1RcXFy1xZvk2IAVxz6smCkgIEAlJSVnrKWgoKBsnc8++0zBwcHq0KGDU/tw00036X//+99pv+0qLi5WQUGBFixYUPbHVJKSk5PP+nngQJs5jzZzjqe3V35+vtkluD1n+ntP4+nvj+pAmzmPNjt7xzJFXl6ex3y5YbFYZLFYqj1T5ObmnnGdc8kUknTkyJGy+1OnTlXXrl0VGRnp1D5df/31SkpKUvv27Sv9/anyheRcX+9WIV2qOGLesW9dTjWS3sSJEzVhwoSy+zk5OYqLi9OAAQMUHh5ebt3CwkLt2rVLoaGhZVNyGYahAltpVe7CWQvys571CIGff/65nnnmGW3dulXBwcHq2LGjZs6cqX/96186fPiwvvrqK0mOF/4999yjb775RuHh4Xr44Yc1Z84cdejQoeyoe5MmTTRq1Cht3rxZX331laKiopSUlKSePXvqzjvv1K+//qqEhARNnz5dXbp0Kavhiy++0FNPPaWtW7cqNjZW9913X7m2b9KkiR588EE9+OCDkqQtW7bozjvv1J9//qkmTZqUPX9QUFDZ/82cOXN09dVXl90vLCxU165d1bNnT7311luSHGMVdO7cWS+88ILuvPNOSdINN9ygRx99VFlZWWrSpEmlbVZYWKigoCBdfPHFCgwMlM1mU3Jysvr371/pWRmoiDZzHm3mnJpuL7vd0KECmw4eKdahgmLlFJTocIFN2QU2ZefbdLjAVm6Zj0X6YsyF5/28rvBFqrtzpr/3FPw9cR5t5jzazHlHjhzR9u3bFRISoqCgIMkwpFKTvoy1BktOjDh+cqbo1KmTvvrqK913333VmikSExOVm5ursLAwffnll1WeKSQpNDS07P4333yju+66q+z+/v371aFDB91///2aOHGiJMcR+z59+mjOnDllZ3Nff/31uvzyy+Xn5+f4vz3JyfniRM709W4V0uvVq6fMzMxyy/bt2ydfX19FRUVVuk1AQIACAgIqLPfz86vwh6a0tFQWi0U+Pj5lR6Dzi0vU9ilzvjlcP3mggv3PPL9eRkaGhg0bphdeeEHXXnutcnNztWDBgnJfYBzbn4ceekgpKSmaM2eOYmJiNGnSJK1cuVIdO3YsN01EUlKSnn/+eU2aNEmvvvqqRowYoV69epWNAfDoo49q5MiRWrdunSwWi1asWKGbbrpJTz31lIYOHaqUlBSNHTtW0dHRGjlyZNnjHqvFbrfrH//4h6Kjo7VkyRLl5ORo3LhxklSu/RctWqRbb7217H5wcLA++ugjde/eXUOGDNGVV16pESNGqG/fvrr77rvLnichIUF169bVH3/8oWbNmlXabj4+PrJYLBVeC5W9NnB6tJnzaDPnnG975RTalJldqPTDBdqXW6QDR4qVdaRIB44UKevov7OOFOtgXpHshhN1WS3y9fU97ylXeC2cP2f6e0/jDftY1Wgz59FmZ8/X1xGxyj6Dl+RJn5v0ZeGNRyRryFmtWlmmWLhwYdlR9OrMFGvWrJEkrVy5sloyxYn3Dx06pLVr16pr165lv4+JidG7776ra665RgMHDlSrVq00fPhwjR07VpdffnnZY3Tr1k02m03Lly9Xnz59KrThqfKF5Fxf71YhvUePHvr222/LLZs3b566dOni1X80MjIyVFJSouuuu07x8fGSpAsuuKDCtzW5ubl6//33NWvWLPXr10+S9N5776l+/foVHnPw4MFloXfSpEmaNm2aunbtqhtuuEGS9Oijj6pHjx7au3ev6tWrp1deeUX9+vXTE088IUlq0aKF1q9frxdffLHcG+qYn3/+WRs2bFBqaqoaNmwoSXr++ec1aNCgsnUOHz6sw4cPV6ivY8eOevbZZ3XnnXfq5ptv1rZt2yodGKJBgwZKTU09ixYE4M5K7YbSDxco7WC+dh/KV/rhQmVkFygju1AZ2YXKzC7UkaKSMz/QCWoH+6l2sL9qBfspIshPEcH+qhXkp1pBfooIdtwc9/2raa8AAKhZlWWKdu3aVVivujJFcHCwXn311SrPFNLxs68laefOnTIMo0K9gwcP1p133qlhw4apa9euCgwM1H/+859y64SEhCgiIkKpqamVhvSqYmpIP3LkiLZu3Vp2f8eOHVq9erUiIyPVqFEjTZw4UXv27NHMmTMlSWPGjNHrr7+uCRMm6M4779TixYs1ffp0ffzxx9VWY5CfVesnD6y2xz/Tc5+NDh06qF+/fmrXrp0GDhyoAQMG6LrrrqswWMH27dtls9nUrVu3smW1atVSy5YtKzzmiddZHBsH4MQ36bFl+/btU7169bRhwwZdffXV5R6jV69eSkpKUmlpaYVaNmzYoEaNGpW9mSTHlzAnKigokKQKp4pI0j//+U998803eu211/TDDz8oOjq6wjpBQUFc5wl4iFK7oT1ZedqedUQ7D+QfveVp54F87TqUL1vpmQ9/1wryU2ytQNUND1R0qL/qhAYoOjRAUaH+ZT/rhAYoMsRfvlZTJz8BAHgSa7DjiLZZz32WKssU//jHP1S7du1y61VXpmjcuLE2btxY5ZniZKfLGC+99JLatm2rTz/9VMuXL690nZrIGKaG9OXLl6tv375l949dazBixAjNmDFDGRkZSktLK/t9QkKC5s6dq/Hjx+uNN95Q/fr19b///U/XX399tdVosVgU7O/aJxxYrVYlJycrJSVF8+bN02uvvabHH3+8wgAfp7p+/8Rvlo458cyEY+tXtuzYaIiGYZzV457udydvHxUVJYvFokOHDlVYd9++fdq0aZOsVqu2bNlS7jSUYw4ePKg6deqcsgYArqfUbmjXwXxt2XdEm/fmalNGjlZsteqRZb+oqOTUo6/6WS2Kqx2suMhg1Y8IVGytINWrFaj6R3/G1gpUSIBr/y0HAHgoi0XyPbtTzs10qkyxdOnScuu5W6Y42bGDe4cOHaqQFbZv36709HTZ7Xbt3Lmz0gHiaiJjmPqJ5ZJLLjlto8+YMaPCsj59+mjlypXVWJV7slgs6tWrl3r16qVJkyYpPj5e3333Xbl1mjZtKj8/P/3555+Ki4uT5BjAYMuWLed9ukabNm20aNGicstSUlLUokWLCt94HVs/LS1N6enpZaeaLF68uNw6/v7+atOmjdavX19h6r077rhDbdu21Z133qlRo0apX79+atOmTdnvCwsLtW3bNnXq1Om89gtA9SkptWvb/jyt2ZOttXuytWZPttan51QyWKdFkl0Bvj5qUidUjaOC1SgqWPGRIYqPClZ8VLBiawXJ6nN+14UDAODtKssUxwaLO6Y6M0Xr1q2rPFOcrGnTpgoPD9f69evVokWLsuXFxcUaNmyYhg4dqlatWmnUqFFas2ZNudnFtm3bpsLCwmrPGBxW8ABLly7VL7/8ogEDBqhu3bpaunSp9u/frxYtWmjz5s1l64WFhWnEiBF6+OGHFRkZqbp16+rJJ58sG+DgfPzzn/9U165d9cwzz2jo0KFavHixXn/9dU2dOrXS9S+77DK1bNlSw4cP18svv6ycnBw9/vjjFdYbOHCgFi1aVDYAhCS98cYbWrx4sf7++2/FxcXphx9+0LBhw7R06VL5+zuuD12yZIkCAgLOeLoLgJqzP7dIK3Ye1LLUQ1qVdkjrM3JUaKt4dNzf10fN6oSqeUyomkYHK2fXJt00qI8S6oYTxAEAqCanyhStW7fW33//XbZedWaKCRMmqHv37lWeKU7k4+Ojyy67TIsWLdI111xTtvzxxx9Xdna2/ve//yk0NFQ//PCDRo0aVe7A58KFC9WkSRM1bdr0vPbzTAjpHiA8PFwLFixQUlKScnJyFB8fr5deekn9+/evcDT9lVde0ZgxY3TFFVcoPDxcjzzyiHbt2lXp9RbO6Ny5sz799FNNmjRJzzzzjGJjYzV58uRKB3iQHG+Or776SqNGjVK3bt3UuHFj/e9//6tw2vqdd96pzp07Kzs7W7Vq1dLGjRv18MMPa/r06WXf3L3xxhvq0KGDnnjiCf33v/+VJH388ccaNmyYx8+NC7gqwzC080C+lu44oGWph7Q89aBSD1S8fivE36oL6tdS2wa11K5huNo1qKWE6NCyMG6z2TR37kbFRwUT0AEAqEaVZYqXX35ZgwYN0uzZs8ut626Z4mR33XWXRo0apRdeeEE+Pj76/ffflZSUpN9++61sWrYPPvhA7du317Rp03TPPfdIcmSMY1M+VydCugdo3bq1fvzxx3LL7Ha7cnJy9N5775WbeiAsLEwfffRR2f28vDw9/fTTuuuuu8qWVTYi+smXJTRu3LjCsuuvv/604wOc/LgtWrTQwoULT/s8rVq10hVXXKGpU6dq4sSJatWqVYWBGsLDw7Vjx46y+/v379fnn3+u5cuXn7IWAFXvUF6x/tiWpUVbsrRwS5b2HC4o93uLRWoZE6YujWsrMb622jWIUJPoEPkQvgEAMF1lmeKYky9DrupMcSy7SNWTKU42YMAANWjQQLNnz9bNN9+sSy65RDabrdw6jRo10uHDh8vur127VqtXr9ann3562seuCoR0L7Nq1Spt3LhR3bp1U3Z2tiZPnixJFUZRdCUvvvii5syZc9br79ixQ1OnTlVCQkI1VgXAMAytS8/RvPV79dvGfVqbnq0T+0Q/q0WdGtVWt8aRSmxcW50b1VatIO+dLhMAAE/hjpniRBaLRW+//Xa50/jPJD09XTNnzlStWrWqsTIHQroXeumll7Rp0yb5+/srMTFRCxcurHQKM1cRHx+v+++//6zX79atW7kpIQBUneISu5buOKDk9Xv18/q9Ss8uLPf7ljFhuqh5tC5qHq3uCZEuPzsGAAA4N+6WKU7WoUMHdejQ4azXP3kg6+rEpycv06lTJ61YscLsMgC4Ebvd0NIdBzXnrz2auyZT2QXHTwcL8rPq4hbRuqx1jC5uUUcx4ed3LRoAAHB9ZIrqRUgHAFRw7FT2b1bv0bd/ZSgz5/gR8+jQAF3Wuq76t4lRr2bRCvSrOCUKAAAAzg0hvRJnGmgAnoH/Z6CinEKbvlm1Rx//uUvrM3LKlocH+mpwu1hd1bG+uidEMdI6AACncGwaMj5rep+q+j8npJ/Az88xoFF+fr6CgoJMrgbV7dgo8cf+3wFvZRiGVu06rI+Xpunbv9PL5i739/VR/zYxurpDffVpWUcBvhwxBwDgTHx9fWW325Wfn6+QkBCzy0ENKi4uliRZref3mYmQfgKr1aqIiAjt27dPkhQcHFz2TZi7sdvtKi4uVmFhYbkp2OAIJPn5+dq3b58iIiLO+00EuKuSUrvmrs3U9IXb9dfu7LLlLWJCdXO3Rrq2UwNFBPubWCEAAO7HarUqNzdX+/fvl4+Pj1tnipriCdnFbrdr//79Cg4Olq/v+cVsQvpJ6tWrJ0llQd1dGYahgoICBQUF8UfhFCIiIsr+vwFvklNo0+w/d2lGSmrZXOb+vj66sn193dI9Tp0b1ebvBgAA5yE3N1ctWrRw+0xRUzwlu/j4+KhRo0bnvQ+E9JNYLBbFxsaqbt26FSa0dyc2m00LFizQxRdfzOnclfDz8+MIOrzOobxiTV+0QzNSUnWkqESSFB3qr9subKxbL2ykqNAAkysEAMBzxMTEKDY21q0zRU3xlOzi7+9fJWcCENJPwWq1unWIs1qtKikpUWBgoFu/0AGcv8rCefO6oRrdO0FXd2zA6OwAAFQTd88UNYXsUh4hHQA8VG6hTW8v2K73/jgeztvEhuvBy5qrf+sY+TBCOwAAgMshpAOAhykpteuTZbuU9PNmZR1xjDJ6LJwPaBPj1td6AQAAeDpCOgB4CMMw9OvGfXp+7gZt258nSUqIDtGjl7fUwAvqEc4BAADcACEdADxAalaenvhmrRZuyZIk1Q7207jLWuiW7o3kZ3XPqUwAAAC8ESEdANxYoa1Ub87fpqm/b1NxiV3+Vh/dcVGCxvZtqvBABl4BAABwN4R0AHBTf2zN0uNfrVHqgXxJUu/m0Zp8dVslRIeYXBkAAADOFSEdANzMkaISTZm7QR8tTZMk1Q0L0KQr22hIu1iuOwcAAHBzhHQAcCMp27L0yOd/a/ehAknSbRfG65HLWyqMU9sBAAA8AiEdANxAoa1U//lho2akpEqSGkQE6cV/tFfPZtHmFgYAAIAqRUgHABe3bf8R3TdrlTZk5EiSbu7WSI8Paa3QAP6EAwAAeBo+4QGAC/t8xW5N+mat8otLFRXir5du7KC+LeuaXRYAAACqCSEdAFxQfnGJ/v31Wn25co8kqUeTKCXd1FEx4YEmVwYAAIDqREgHABez62C+7vpghTZk5MjHIo27rIXu7dtMVh9GbgcAAPB0hHQAcCEp27J070crdSjfpuhQf71+S2dd2CTK7LIAAABQQwjpAOACDMPQjJRUPfv9BpXaDbVrUEtv3Zao+hFBZpcGAACAGkRIBwCTlZTa9cQ36/Txn2mSpOs6NdDz17VToJ/V5MoAAABQ0wjpAGCivKIS3TdrpX7btF8+Fulfg1tr1EUJsli4/hwAAMAbEdIBwCT7c4s06v1l+nt3tgL9fPS/mzppwAX1zC4LAAAAJiKkA4AJdmTladQHK7XrYIEiQ/z1zogu6tyottllAQAAwGSEdACoYXvypKff+VMH82yKjwrWjNu7KSE6xOyyAAAA4AII6QBQg/7ana3X1llVUGpT2wbhmnF7N0WHBphdFgAAAFwEIR0AasifOw7q9hnLVVBqUae4WppxR3fVCvIzuywAAAC4EEI6ANSARVuydOfM5Sqwlap5uF3vjUgkoAMAAKACQjoAVLOUrVka9f4yFZXYdXHzKF1Ze69CAvjzCwAAgIp8zC4AADzZ8tSDGvX+chWV2HVZ6xhNvaWT/K1mVwUAAABXxaEcAKgmf+06rJHvLVOBrVQXt6ijN4Z1ko9hN7ssAAAAuDCOpANANVifnqPh7/6pI0UlurBJpN66NVEBvhxCBwAAwOkR0gGgiu08kKfh7y5VdoFNnRtF6J0RXRXEOe4AAAA4C4R0AKhCWUeKNOLdP5V1pFitY8P13u3dFMogcQAAADhLhHQAqCJ5RSUaNWOZUg/kq2HtIL1/e1emWQMAAIBTCOkAUAVspXbdO2ul/tqdrdrBfnr/jm6qGx5odlkAAABwM4R0ADhPhmHo31+t1e+b9ivQz0fTR3ZV0zqhZpcFAAAAN0RIB4DzNH3RDs1evks+Fun1mzurc6PaZpcEAAAAN0VIB4Dz8NumfXp+7gZJ0uND2uiyNjEmVwQAAAB3RkgHgHO0ZW+uHpi1SnZDuqlrnO7o1djskgAAAODmCOkAcA4O5RVr9Mzlyi0qUbeESE2+uq0sFovZZQEAAMDNEdIBwEklpXaN/Wildh7IV1xkkN68NVH+vvw5BQAAwPnjUyUAOOnFeZu0ePsBhfhbNX1EV0WG+JtdEgAAADwEIR0AnDBvXabemr9dkvTiDR3UIibM5IoAAADgSQjpAHCWUrPy9M/P/pIk3dErQYPbxZpcEQAAADwNIR0AzkKhrVT3fLRSuYUlSoyvrYmDW5ldEgAAADwQIR0AzsITX6/VhowcRYX4641bOsvPyp9PAAAAVD0+ZQLAGXy1arc+W7FbPhbptZs7qV6tQLNLAgAAgIcipAPAaew8kKcnvl4nSXqwXwv1bBZtckUAAADwZIR0ADgFW6ldD36yWkeKStStcaTuu7SZ2SUBAADAwxHSAeAU/t/PW7R612GFBfrq1Zs6yupjMbskAAAAeDhCOgBUYsn2A3rj962SpCnXtVODiCCTKwIAAIA3MD2kT506VQkJCQoMDFRiYqIWLlx42vU/+ugjdejQQcHBwYqNjdXtt9+uAwcO1FC1ALzB4fxijZ+9WoYh3diloa5oX9/skgAAAOAlTA3ps2fP1rhx4/T4449r1apV6t27twYNGqS0tLRK11+0aJGGDx+uUaNGad26dfrss8+0bNkyjR49uoYrB+DJnpyzThnZhUqIDtGTV15gdjkAAADwIqaG9FdeeUWjRo3S6NGj1bp1ayUlJSkuLk7Tpk2rdP0lS5aocePGeuCBB5SQkKCLLrpId999t5YvX17DlQPwVD+uzdA3q9PlY5FeHdpRIQG+ZpcEAAAAL2Lap8/i4mKtWLFCjz32WLnlAwYMUEpKSqXb9OzZU48//rjmzp2rQYMGad++ffr88881ZMiQUz5PUVGRioqKyu7n5ORIkmw2m2w2WxXsiWs6tm+evI9VjTZznqe12cG8Yj3+1VpJ0l29E3RBvZAq3zdPa7Pq5i3t5en7VxO8sb/3lvdHVaLNnEebOY82c543tJkz+2YxDMOoxlpOKT09XQ0aNNAff/yhnj17li1//vnn9f7772vTpk2Vbvf555/r9ttvV2FhoUpKSnTVVVfp888/l5+fX6XrP/XUU3r66acrLJ81a5aCg4OrZmcAeIQZm3206oCP6gUZerh9qXxNH7UD3iI/P1+33HKLsrOzFR4ebnY5bon+HgDgypzp600P6SkpKerRo0fZ8ueee04ffPCBNm7cWGGb9evX67LLLtP48eM1cOBAZWRk6OGHH1bXrl01ffr0Sp+nsm/W4+LilJWV5dEfhGw2m5KTk9W/f/9TfoGB8mgz53lSm/2wNlMPzP5bVh+LPr+ru9o2qJ6/D57UZjXBW9orJydH0dHRhPTz4I39vbe8P6oSbeY82sx5tJnzvKHNnOnrTTvdPTo6WlarVZmZmeWW79u3TzExMZVuM2XKFPXq1UsPP/ywJKl9+/YKCQlR79699eyzzyo2NrbCNgEBAQoICKiw3M/Pz2NfACfylv2sSrSZ89y9zbKOFOmp7xxfDI69pKk6NY6q9ud09zaraZ7eXp68bzXFm/t7b9jHqkabOY82cx5t5jxPbjNn9su0kzn9/f2VmJio5OTkcsuTk5PLnf5+ovz8fPn4lC/ZarVKkkw6IQCAB3hqzjodzCtWq3phuv/S5maXAwAAAC9m6hWXEyZM0DvvvKN3331XGzZs0Pjx45WWlqYxY8ZIkiZOnKjhw4eXrX/llVfqyy+/1LRp07R9+3b98ccfeuCBB9StWzfVr888xgCc9+vGvfru7wxZfSx66YYO8udCdAAAAJjI1LmFhg4dqgMHDmjy5MnKyMhQ27ZtNXfuXMXHx0uSMjIyys2ZPnLkSOXm5ur111/XP//5T0VEROjSSy/Vf//7X7N2AYAbyysq0RNfr5MkjbooQW0b1DK5IgAAAHg70ycAHjt2rMaOHVvp72bMmFFh2f3336/777+/mqsC4A1eTd6sPYcL1CAiSOMu4zR3AAAAmI/zOgF4pbV7svXuHzskSc9e21bB/qZ/ZwkAAAAQ0gF4n5JSux778m/ZDenKDvXVt2Vds0sCAAAAJBHSAXihGSmpWrsnR+GBvpp0RRuzywEAAADKENIBeJXM7EK9mrxZkjRxcGvVCas4rzIAAABgFkI6AK8y5YcNyisuVedGERraJc7scgAAAIByCOkAvMbS7Qf0zep0WSzS5KvbysfHYnZJAAAAQDmEdABeoaTUrifnOOZEv7lbI+ZEBwAAgEsipAPwCrP+TNPGzFzVCvLTwwNaml0OAAAAUClCOgCPd+BIkV76aZMk6aGBLVU7xN/kigAAAIDKEdIBeLwXf9qknMIStYkN1y3dGpldDgAAAHBKhHQAHu3v3Yc1e/kuSdLkqy+QlcHiAAAA4MII6QA8lmEYmvztehmGdG2nBurSONLskgAAAIDTIqQD8Fg/rs3U8p2HFOjno0cvb2V2OQAAAMAZEdIBeKTiErv+8+NGSdJdvZuoXq1AkysCAAAAzoyQDsAjzVycqp0H8lUnLEB392lqdjkAAADAWSGkA/A4h/OL9dqvWyVJ/+zfQiEBviZXBAAAAJwdQjoAj/Par1uVXWBTy5gw3dAlzuxyAAAAgLNGSAfgUVKz8jRzcaok6V9DWjPlGgAAANwKIR2AR/nvjxtlKzV0cYs66tOijtnlAAAAAE4hpAPwGCvTDumHtZnysUiPD25tdjkAAACA0wjpADyCYRh64eiUa9d3bqiW9cJMrggAAABwHiEdgEdYuCVLS7YflL/VR+P6tzC7HAAAAOCcENIBuD3DMPTiT5skSbdeGK8GEUEmVwQAAACcG0I6ALf3w9pMrdmTrRB/q+7t29TscgAAAIBzRkgH4NZKSu16aZ7jKPqo3k0UFRpgckUAAADAuSOkA3BrX67co+3781Q72E939k4wuxwAAADgvBDSAbitQlupkn7eLEkae0kzhQX6mVwRAAAAcH4I6QDc1kdL05SeXah64YG6rUe82eUAAAAA542QDsAt5RWV6I3ftkqSHrysuQL9rCZXBAAAAJw/QjoAt/T+4lQdzCtWQnSIbkhsaHY5AAAAQJUgpANwO0eKSvR/C7ZLkh7o10y+Vv6UAQAAwDPwyRaA25m5OFWH8m1qEh2iK9vXN7scAAAAoMoQ0gG4lROPot/PUXQAAAB4GD7dAnArHEUHAACAJyOkA3AbHEUHAACAp+MTLgC3wVF0AAAAeDpCOgC3wFF0AAAAeAM+5QJwCxxFBwAAgDcgpANweRxFBwAAgLfgky4Al/fB4p0cRQcAAIBXIKQDcGkFxaV6Z6HjKPq9fTmKDgAAAM/Gp10ALu3T5bt0IK9YDWsH6aqOHEUHAACAZyOkA3BZxSV2vTV/myTp7j5N5cdRdAAAAHg4PvECcFnfrN6j9OxCRYcG6IbEhmaXAwAAAFQ7QjoAl1RqNzTt6FH0O3snKNDPanJFAAAAQPUjpANwSfPWZWr7/jyFB/pq2IXxZpcDAAAA1AhCOgCXYxiG3vh9qyRpZK8EhQb4mlwRAAAAUDMI6QBczoItWVq7J0fB/lbd3rOx2eUAAAAANYaQDsDlvPGb4yj6Ld0aqXaIv8nVAAAAADWHkA7ApSxLPag/dxyUv9VHo3s3MbscAAAAoEYR0gG4lKlHj6Jfn9hQ9WoFmlwNAAAAULMI6QBcxrr0bP22ab98LNKYPhxFBwAAgPchpANwGW8v2C5JGtK+vuKjQkyuBgAAAKh5hHQALmH3oXx993eGJOnuizmKDgAAAO9ESAfgEt5dlKpSu6FezaLUtkEts8sBAAAATEFIB2C67HybPlmWJkm66+KmJlcDAAAAmIeQDsB0Hy7dqfziUrWqF6aLm0ebXQ4AAABgGkI6AFMVlZRqRkqqJOmui5vIYrGYWxAAAABgIkI6AFN9vWqP9ucWKbZWoK7sUN/scgAAAABTEdIBmMZuN8qmXbujV4L8rPxJAgAAgHcz/RPx1KlTlZCQoMDAQCUmJmrhwoWnXb+oqEiPP/644uPjFRAQoKZNm+rdd9+toWoBVKVfN+7Ttv15Cgvw1U3d4swuBwAAADCdr5lPPnv2bI0bN05Tp05Vr1699NZbb2nQoEFav369GjVqVOk2N954o/bu3avp06erWbNm2rdvn0pKSmq4cgBV4dhR9FsubKSwQD+TqwEAAADMZ2pIf+WVVzRq1CiNHj1akpSUlKSffvpJ06ZN05QpUyqs/+OPP2r+/Pnavn27IiMjJUmNGzeuyZIBVJFVaYf0Z+pB+VktuqNXgtnlAAAAAC7BtJBeXFysFStW6LHHHiu3fMCAAUpJSal0mzlz5qhLly564YUX9MEHHygkJERXXXWVnnnmGQUFBVW6TVFRkYqKisru5+TkSJJsNptsNlsV7Y3rObZvnryPVY02c975tNmbv2+VJF3ZPlaRQVavaXdeZ87xlvby9P2rCd7Y33vL+6Mq0WbOo82cR5s5zxvazJl9My2kZ2VlqbS0VDExMeWWx8TEKDMzs9Jttm/frkWLFikwMFBfffWVsrKyNHbsWB08ePCU16VPmTJFTz/9dIXl8+bNU3Bw8PnviItLTk42uwS3Q5s5z9k2218gzVtvlWRRC3ua5s5Nq57CXBivM+d4envl5+ebXYLb8+b+3tPfH9WBNnMebeY82sx5ntxmzvT1FsMwjGqs5ZTS09PVoEEDpaSkqEePHmXLn3vuOX3wwQfauHFjhW0GDBighQsXKjMzU7Vq1ZIkffnll/rHP/6hvLy8So+mV/bNelxcnLKyshQeHl4Ne+YabDabkpOT1b9/f/n5ca3v2aDNnHeubfbkt+s168/d6tMiWu/c1rkaK3Q9vM6c4y3tlZOTo+joaGVnZ3t031SdvLG/95b3R1WizZxHmzmPNnOeN7SZM329aUfSo6OjZbVaKxw137dvX4Wj68fExsaqQYMGZQFdklq3bi3DMLR79241b968wjYBAQEKCAiosNzPz89jXwAn8pb9rEq0mfOcabNDecX6clW6JGlMn2Ze29a8zpzj6e3lyftWU7y5v/eGfaxqtJnzaDPn0WbO8+Q2c2a/TJuCzd/fX4mJiRVOaUhOTlbPnj0r3aZXr15KT0/XkSNHypZt3rxZPj4+atiwYbXWC6BqzPozTYU2uy6oH64Lm0SaXQ4AAADgUkydJ33ChAl655139O6772rDhg0aP3680tLSNGbMGEnSxIkTNXz48LL1b7nlFkVFRen222/X+vXrtWDBAj388MO64447TjlwHADXUVxi18zFqZKkURclyGKxmFsQAAAA4GJMnYJt6NChOnDggCZPnqyMjAy1bdtWc+fOVXx8vCQpIyNDaWnHB5QKDQ1VcnKy7r//fnXp0kVRUVG68cYb9eyzz5q1CwCcMHdNhvbmFKluWICuaF/f7HIAAAAAl2NqSJeksWPHauzYsZX+bsaMGRWWtWrVyqNH/QM8lWEYmr5ohyRpeI94+fuaeiIPAAAA4JL4lAygRixLPaQ1e7IV4OujW7rHm10OAAAA4JII6QBqxLtHj6Jf17mBIkP8Ta4GAAAAcE2EdADVbtfBfM1b75hu8Y5eCSZXAwAAALguQjqAavfeH6myG9LFLeqoeUyY2eUAAAAALouQDqBa5Rba9OnyXZIc064BAAAAODVCOoBqNXvZLh0pKlGzuqG6uHm02eUAAAAALo2QDqDalNoNzUhJleS4Ft1isZhbEAAAAODiCOkAqk3y+kztPlSg2sF+uq5zA7PLAQAAAFweIR1AtZl+dNq1Yd3jFehnNbkaAAAAwPUR0gFUi793H9ay1EPys1o0vEe82eUAAAAAboGQDqBaHDuKfmX7+qobHmhyNQAAAIB7IKQDqHKZ2YX6/u8MSdIdTLsGAAAAnDVCOoAqN3NxqkrshrolRKptg1pmlwMAAAC4DUI6gCpVUFyqWX+mSZJGcRQdAAAAcAohHUCV+mLlbh3Ot6lRZLAuax1jdjkAAACAWyGkA6gydruhd/9wDBh3e6/GsvpYTK4IAAAAcC+EdABVZv6W/dq+P09hAb66oUuc2eUAAAAAboeQDqDKvHt02rWhXeMUGuBrcjUAAACA+yGkA6gSW/bmauGWLPlYpBE9G5tdDgAAAOCWCOkAqsR7KamSpP5tYhQXGWxuMQAAAICbIqQDOG+H8236cuVuSdLtvZh2DQAAADhXhHQA5+3TFbtVaLOrTWy4uidEml0OAAAA4LYI6QDOS6khfbh0lyTHtGsWC9OuAQAAAOeKkA7gvPx90KKM7EJFhfjryg71zS4HAAAAcGuEdADnZX6G48/IsO6NFOhnNbkaAAAAwL0R0gGcszV7srUj1yI/q0W3XhhvdjkAAACA23M6pI8cOVILFiyojloAuJn3F6dJkga3rae64YEmVwMAAAC4P6dDem5urgYMGKDmzZvr+eef1549e6qjLgAubl9OoeauzZQkjejRyORqAAAAAM/gdEj/4osvtGfPHt1333367LPP1LhxYw0aNEiff/65bDZbddQIwAV9uGSnbKWGEsIMtWtQy+xyAAAAAI9wTtekR0VF6cEHH9SqVav0559/qlmzZrrttttUv359jR8/Xlu2bKnqOgG4kEJbqT5a6jjVvU+s3eRqAAAAAM9xXgPHZWRkaN68eZo3b56sVqsGDx6sdevWqU2bNnr11VerqkYALubbv9J1IK9YsbUC1T7SMLscAAAAwGM4HdJtNpu++OILXXHFFYqPj9dnn32m8ePHKyMjQ++//77mzZunDz74QJMnT66OegGYzDAMvftHqiRpWLc4WS3m1gMAAAB4El9nN4iNjZXdbtfNN9+sP//8Ux07dqywzsCBAxUREVEF5QFwNUt3HNSGjBwF+vloaJeGSvl9g9klAQAAAB7D6ZD+6quv6oYbblBg4KmnW6pdu7Z27NhxXoUBcE3v/eF4b1/XuaEigv1MrgYAAADwLE6H9Ntuu6066gDgBnYdzFfy+r2SpNt7Nja3GAAAAMADndfAcQC8y/spqbIbUu/m0WoeE2Z2OQAAAIDHIaQDOCt5RSWavXyXJOn2Xo3NLQYAAADwUIR0AGfli5W7lVtYooToEF3Soq7Z5QAAAAAeiZAO4IzsdkPvHZ12bWTPxvLxYd41AAAAoDoQ0gGc0fzN+7UjK09hAb66PrGh2eUAAAAAHouQDuCM3j067dqNXeMUGuD0pBAAAAAAzhIhHcBpbdmbq4VbsuRjcZzqDgAAAKD6ENIBnNZ7KamSpMtaxyguMtjcYgAAAAAPR0gHcEqH84v15crdkqTbeyWYXA0AAADg+QjpAE7pk2W7VGizq1W9MF3YJNLscgAAAACPR0gHUKmSUrtmHj3V/Y6LEmSxMO0aAAAAUN0I6QAq9dO6vUrPLlRUiL+u6lDf7HIAAAAAr0BIB1Cp945Ou3ZL90YK9LOaXA0AAADgHQjpACpYveuwlu88JD+rRbdeGG92OQAAAIDXIKQDqGD6IsdR9Cs71FdMeKDJ1QAAAADeg5AOoJw9hws0d02GJGnURUy7BgAAANQkQjqAct5PSVWp3VDPplG6oH4ts8sBAAAAvAohHUCZI0Ul+nhpmiSOogMAAABmIKQDKPPpsl3KLSpRk+gQ9W1Z1+xyAAAAAK9DSAcgSSq1G3r36LRrd1yUIB8fi8kVAQAAAN6HkA5AkjRvXaZ2HypQRLCfru/c0OxyAAAAAK9ESAcgSXrn6LRrt3aPV5C/1eRqAAAAAO9ESAegVWmHtGLnIflZLRreI97scgAAAACvRUgHoOlHj6Jf1aGB6oYHmlwNAAAA4L0I6YCX230oXz+szZTEtGsAAACA2UwP6VOnTlVCQoICAwOVmJiohQsXntV2f/zxh3x9fdWxY8fqLRDwcO+npKrUbqhXsyi1qR9udjkAAACAVzM1pM+ePVvjxo3T448/rlWrVql3794aNGiQ0tLSTrtddna2hg8frn79+tVQpYBnyi206ZM/d0mSRl/UxORqAAAAAJga0l955RWNGjVKo0ePVuvWrZWUlKS4uDhNmzbttNvdfffduuWWW9SjR48aqhTwTJ8u363cohI1rROiPi3qmF0OAAAA4PV8zXri4uJirVixQo899li55QMGDFBKSsopt3vvvfe0bds2ffjhh3r22WfP+DxFRUUqKioqu5+TkyNJstlsstls51i96zu2b568j1XN29qs1G7ovUXbJUkje8SrtLREpaXOPYa3tVlVoM2c4y3t5en7VxO8sb/3lvdHVaLNnEebOY82c543tJkz+2ZaSM/KylJpaaliYmLKLY+JiVFmZmal22zZskWPPfaYFi5cKF/fsyt9ypQpevrppyssnzdvnoKDg50v3M0kJyebXYLb8ZY2W3XAot2HrQrxNRSY+bfmzv37nB/LW9qsKtFmzvH09srPzze7BLfnzf29p78/qgNt5jzazHm0mfM8uc2c6etNC+nHWCyWcvcNw6iwTJJKS0t1yy236Omnn1aLFi3O+vEnTpyoCRMmlN3PyclRXFycBgwYoPBwzx0ky2azKTk5Wf3795efn5/Z5bgFb2ozwzD0f28ulZSjO3o31TWXNjunx/GmNqsqtJlzvKW9jh31xbnzxv7eW94fVYk2cx5t5jzazHne0GbO9PWmhfTo6GhZrdYKR8337dtX4ei6JOXm5mr58uVatWqV7rvvPkmS3W6XYRjy9fXVvHnzdOmll1bYLiAgQAEBARWW+/n5eewL4ETesp9VyRvaLGVbltam5yjQz0cjezU57/31hjararSZczy9vTx532qKN/f33rCPVY02cx5t5jzazHme3GbO7JdpA8f5+/srMTGxwikNycnJ6tmzZ4X1w8PDtWbNGq1evbrsNmbMGLVs2VKrV69W9+7da6p0wO29Nd9xLfoNiXGKCq34oRYAAACAOUw93X3ChAm67bbb1KVLF/Xo0UNvv/220tLSNGbMGEmOU9f27NmjmTNnysfHR23bti23fd26dRUYGFhhOYBT25CRo/mb98vHIo3unWB2OQAAAABOYGpIHzp0qA4cOKDJkycrIyNDbdu21dy5cxUfHy9JysjIOOOc6QCc8/YCx1H0Qe1iFR8VYnI1AAAAAE5k+sBxY8eO1dixYyv93YwZM0677VNPPaWnnnqq6osCPNTuQ/ma81e6JOnui5uYXA0AAACAk5l2TTqAmvfuolSV2g31bBql9g0jzC4HAAAAwEkI6YCXOJxfrE+WOS4fubtPU5OrAQAAAFAZQjrgJT5cslP5xaVqVS9MFzePNrscAAAAAJUgpANeoNBWqhkpqZKkMX2aymKxmFsQAAAAgEoR0gEv8MXK3co6UqwGEUEa0j7W7HIAAAAAnAIhHfBwpXZD/3d02rVRFyXIz8rbHgAAAHBVfFoHPNxP6zKVeiBftYL8NLRrnNnlAAAAADgNQjrgwQzD0Bu/bZUkDe8Rr5AAX5MrAgAAAHA6hHTAg/2+eb/WpecoyM+q23slmF0OAAAAgDMgpAMeyjAMvfGr4yj6sO6NFBnib3JFAAAAAM6EkA54qKU7Dmr5zkPyt/rozoubmF0OAAAAgLNASAc81LFr0W/o0lAx4YEmVwMAAADgbBDSAQ/0167DWrglS1Yfi8b0aWp2OQAAAADOEiEd8EDHjqJf3bG+4iKDTa4GAAAAwNkipAMeZlNmruat3yuLRRp7STOzywEAAADgBEI64GGm/u44ij6obT01qxtqcjUAAAAAnEFIBzxIalaevv0rXRJH0QEAAAB3REgHPMib87fJbkh9W9ZR2wa1zC4HAAAAgJMI6YCH2HUwX5+v2C1Juu9SjqIDAAAA7oiQDniIN37bqhK7od7No5UYH2l2OQAAAADOASEd8AAnHkUfd1kLk6sBAAAAcK4I6YAHeO3XLSqxG7q4RR0lxtc2uxwAAAAA54iQDri5nQfy9MXKPZKkcZc1N7kaAAAAAOeDkA64udd/3apSu6E+LeqocyOOogMAAADujJAOuLHUrDx9uYqj6AAAAICnIKQDbuy1o0fRL2lZR504ig4AAAC4PUI64KZ2ZOXpq1WM6A4AAAB4EkI64KZe+2WL7IZ0aau66hgXYXY5AAAAAKoAIR1wQ5v35uqr1VyLDgAAAHgaQjrghl76aZMMQ7r8gnpq3zDC7HIAAAAAVBFCOuBmVqYd0rz1e+VjkR4ayLXoAAAAgCchpANuxDAMvfjjJknS9Z0bqlndMJMrAgAAAFCVCOmAG1m0NUuLtx+Qv9VH4/pzFB0AAADwNIR0wE0YhqEXjh5Fv/XCeDWICDK5IgAAAABVjZAOuIkf1mZqzZ5shfhbdW/fpmaXAwAAAKAaENIBN1BSatdL8xxH0Uf3bqKo0ACTKwIAAABQHQjpgBv4fMVubd+fp9rBfhrdO8HscgAAAABUE0I64OLyikr0cvJmSdJ9lzZXWKCfyRUBAAAAqC6EdMDFvTV/m/bnFqlxVLBuuzDe7HIAAAAAVCNCOuDCMrIL9PbC7ZKkxwa1kr8vb1kAAADAk/GJH3BhL8/brEKbXV0b19bAC+qZXQ4AAACAakZIB1zUuvRsfbFytyTp8SFtZLFYTK4IAAAAQHUjpAMuyDAMPff9BhmGdFWH+uoYF2F2SQAAAABqACEdcEG/btynlG0H5O/ro4cHtjS7HAAAAAA1xNfsAgCUZyu16/m5GyRJt/dqrLjIYJMrAgAAXqe0SCo+JBXlKMi+XypIlyx1JF8+lwDVjZAOuJj3U1K1bX+eIkP8NfaSZmaXAwAAPFlhlnRgiZS1RMrZIOVslvJ2SCV5kiQ/SQMk6buj6/uGSkGxUnhrKaKtVLujVOdiKSjGnPoBD0RIB1zIvtxCJf28RZL0yMCWqhXkZ3JFAADAoxh26eAKafc3jlv22lOva/GR4RMoe6lNPha7LEapVHJEyt3iuO2Zc3zd8NZS7ECp0T+k6B6ShatqgXNFSAdcyH9/2KQjRSVq37CWbuwSZ3Y5AADAU+SlSdvelba/J+Wnlf9deCspuqdUu4MU1kIKbSoF1pH8wlVSUqq5c+dq8KBB8rMUSgWZUv5uKXu9lL1GOvCndOivo0fhN0ibkqSgBlLjm6Vmd0thnBUIOIuQDriIFTsPlU259vRVF8jHhynXAADAeTAMad98acOLUvoPkgzHct9QKfZyqeHVjp+B0ad5kFLHD4tF8gt33MJbSPUuPb5K0UFp72/S7q+k3XOkgj3Shpcct9iBUov7pfqDHY8B4IwI6YALKLUbemrOOknSDYkN1alRbZMrAgAAbsswpD3fSeued1xvfkxMX6npnVLctZI1sOqeLyBSanS941ZaJGX8KG192/HFQMZPjlvtjlLbJ6SG13AqPHAGhHTABXy6fJfW7MlWWICvHrm8ldnlAAAAd7U/RVr1sJSV4rjvEyA1vUNqOV4Kb179z28NcByhb3i1dGS7tGWatOVN6dBqaeH1UkQ7qdNLUuyA6q8FcFN8jQWY7HB+sV74caMkaVz/FqoTFmByRQAAwO3kbnOE4ORejoBuDZJaPyJdnSp1nVozAf1koU2kTi86arjg345T5Q+vkX4bKP0+RMreUPM1AW6AkA6Y7D8/bNShfJua1w3V8B7xZpcDAADcSWmxtPY56fsLpF1fOk4lbzpKunKr1Om/UlA9syuUAqKkDs9IV+2QWj4oWXyl9LnS3HaOo/4l+WZXCLgUQjpgoj93HNQny3ZJkp6/rp38rLwlAQDAWdq3UPqho/T3vyV7kRTTTxr0l9T9HSm4vtnVVRQQKSUmSUPWSg2ulIxSx+By37eVMpLNrg5wGSQCwCRFJaX611drJEk3d4tT18aRJlcEAADcQkmBtGK89PPFjmnPAutKPT6ULk2WItqaXd2ZhbeU+syRLp4jBTeU8nZIvw2QltwuFWebXR1gOkI6YJK352/X1n1HFB3qr0cZLA4AAJyNgyulHxMd85FLjlPbh2yQEoa53xRnDa+Uhqx3TNEmi7R9hvRDB2nfIrMrA0xFSAdMsCMrT6/9tlWS9MQVbRQR7G9yRQAAwKXZSx1Tqv3U/ejR83pSn+8dp7YHuPHZeH5hUpf/Sf0XSiEJUt5O6Zc+0l+PS3ab2dUBpiCkAzXMMAz9++s1Ki6xq3fzaF3VwQWvGQMAAK6jcL/0+2BHcDVKpLjrpcFrpAaDza6s6tTpJQ1eLSWMkAy74wuJ5IscoR3wMoR0oIZ9sXKP/th6QAG+Pnr2mrayuNupaQAAoObsWyT90EnKnOeYVu3C96SLPpMCo82urOr5hUs9ZkgXfSr5RUgH/pR+6Cyl/2h2ZUCNIqQDNWhvTqEmf7tOkvTgZc0VHxVickUAAMAlGYa0/kXpl0ukgj1SeCtp4J9Sk5Hud+25sxrdIA1aJUUmSsUHHWcR/P2k45R/wAuYHtKnTp2qhIQEBQYGKjExUQsXLjzlul9++aX69++vOnXqKDw8XD169NBPP/1Ug9UC584wDE38co1yCkvUvmEt3dW7idklAQAAV2Q7Ii36h7T6Ecc0ZfG3SAOXucfI7VUltLHUf5HUbIwkQ1o72RHWC7PMrgyodqaG9NmzZ2vcuHF6/PHHtWrVKvXu3VuDBg1SWlpapesvWLBA/fv319y5c7VixQr17dtXV155pVatWlXDlQPO+3LlHv26cZ/8rT566YYO8mVOdAAAcLIjqVJyT2nXl5KPv9R1mtTzQ8kv1OzKap41UOo2TerxgWQNdpzy/1M36fA6sysDqpWpKeGVV17RqFGjNHr0aLVu3VpJSUmKi4vTtGnTKl0/KSlJjzzyiLp27armzZvr+eefV/PmzfXtt9/WcOWAc/bmFOrpE05zbxETZnJFAADA5eydL/3UVTq8RgqMkfr9LjUf4/mnt59Jwq3SwKVSaBPHnOrzLpR28/kfnsvXrCcuLi7WihUr9Nhjj5VbPmDAAKWkpJzVY9jtduXm5ioy8tTTThQVFamoqKjsfk5OjiTJZrPJZvPcaR2O7Zsn72NVq642MwxDj37+l+M09wbhuqNHnMf8v/A6cx5t5hxvaS9P37+a4I39vbe8P6qSK7eZz7a35bNqnCxGiey1O6u05+dScEPJ5Fpdps1CWkqX/iHr4pvks3++jAVXy97uWdlbPuRyX2K4TJu5EW9oM2f2zWIYhlGNtZxSenq6GjRooD/++EM9e/YsW/7888/r/fff16ZNm874GC+++KL+85//aMOGDapbt26l6zz11FN6+umnKyyfNWuWgoODz30HgLP0536LPtpqldVi6JH2parHyw7ASfLz83XLLbcoOztb4eHhZpfjlujv4a4sRonaFb+jhBLHCOa7rb21OuA+lVoCTK7MNZ3cXrusfbQ64F7ZLf4mVwacnjN9vekhPSUlRT169Chb/txzz+mDDz7Qxo0bT7v9xx9/rNGjR+ubb77RZZdddsr1KvtmPS4uTllZWR79Qchmsyk5OVn9+/eXn5+f2eW4hepos12H8nXlG4uVV1Sqh/o3190XJ1TJ47oKXmfOo82c4y3tlZOTo+joaEL6efDG/t5b3h9VyeXazJYj6+Kb5bM3WYYssrd7RvaWD7vUkWGXa7OjfLa+KZ/V42UxSmWP7K7SXl9IgZUftKtprtpmrswb2syZvt60092jo6NltVqVmZlZbvm+ffsUExNz2m1nz56tUaNG6bPPPjttQJekgIAABQRU/CbSz8/PY18AJ/KW/axKVdVmJaV2PfT5WuUVlapLfG2NuaSZxw4Wx+vMebSZczy9vTx532qKN/f33rCPVc0l2iw/3TFa+eG/JGuwLL0+lrXhVbKaW9UpuUSbnaj1/VLtNtKiG+RzcKl8frtYuuQHKbyF2ZWVcbk2cwOe3GbO7JdpicHf31+JiYlKTk4utzw5Obnc6e8n+/jjjzVy5EjNmjVLQ4YMqe4ygXP2+m9btTLtsMICfPXq0I4eG9ABAICTDq91DH52+C/H0d/L5ksNrzK7KvdTr5/UP0UKSZCObJfm9ZD2LTK7KuC8mZoaJkyYoHfeeUfvvvuuNmzYoPHjxystLU1jxoyRJE2cOFHDhw8vW//jjz/W8OHD9fLLL+vCCy9UZmamMjMzlZ2dbdYuAJVasfOg/vfLFknSs9e2VVwk10MCAABJmb9KyRdJ+buk8JbSgCVSVBezq3JftVpJA5dIUd2k4oPSr5dJO2ebXRVwXkwN6UOHDlVSUpImT56sjh07asGCBZo7d67i4+MlSRkZGeXmTH/rrbdUUlKie++9V7GxsWW3Bx980KxdACrILbTpwU9Wy25I13ZqoKs7NjC7JAAA4Ap2fCj9frlky5bqXOQ4ChzqWePVmCKwrtTvN6nhNZK9SPrjJmn9fyVzht4Czptp16QfM3bsWI0dO7bS382YMaPc/d9//736CwLOg2EYeuLrtdp9qEBxkUGafPUFZpcEAADMZhjS+inSX4877je6UerxvmQNNLcuT+IbLF30ubTqn9Km/yetfkw6kip1eU3yMT3yAE7hIlmgCn2ybJe+Xp0uH4uUNLSjwgI9c+ALAABwluwl0rIxxwN664elXh8T0KuDj1VKTJI6J0mySFvflBZcLdmOmFwY4BxCOlBF1u7J1pNz1kmSHh7YSonxkSZXBAAATGU74giJW9+WLD5Sl9elTi84/o3q0+pBqfeXkjVISp8r/dxHKsgwuyrgrPEXAqgC2QU23TtrpYpL7OrXqq7uvriJ2SUBAAAzFWQ6wmH6XEdY7P2l1OJes6vyHnHXSP1+lwLqSIdWOkZ+z15vclHA2SGkA+fJMAw9/Nlf2nkgXw0igvTyjR3k42MxuywAAGCW7A2OKdYOrXSExH6/SQ2vNrsq7xPdTRqwWAprLuXtlOb1kvbON7sq4IwI6cB5emfhDs1bv1f+Vh9Nu7WzIoL9zS4JAACYZd8CaV5PRygMa+4IidHdza7Ke4U1dYyiH91Tsh2WfhsgpX5sdlXAaRHSgfOwcMt+TflhgyTpiStaq33DCHMLAgAA5tk5W/q1vyMMRvdwhMOwpmZXhcBo6dKfpbjrJXuxlHILU7TBpRHSgXOUmpWn+2atkt2Q/pHYULdeGG92SQAAwAyGIa1/wTE/t71YirtOuvQXRziEa/ANki76VGo53nF/9WPS8nsdo+8DLoaQDpyD3EKbRs9cruwCmzo1itBz17aVxcJ16AAAeB17iSPsrX7Ucb/lg1KvTx2hEK7F4iMlvnJ8irYt06QF10oleWZXBpRDSAecZLcbGj97tbbuO6J64YF669ZEBfhazS4LAADUtJI8R8jbMk2SxRH+EpMc83XDdbV6UOr9hWOu+vTvpJ8vkQr2ml0VUIaQDjjphZ826ecN++Tv66O3bktU3fBAs0sCAAA1rWyKte8cYa/3F47wB/cQd61j1P2AaOngcsdo/Nkbza4KkERIB5zy4ZKdenP+NknSC9e3V4e4CHMLAgAANe/YFGsHVzhCXr/fHKEP7iX6Qsfo+6HNpLxUKbmntG+R2VUBhHTgbP2yYa8mfbNWkjShfwtd06mByRUBAIAad+IUa6HNjk6xdqHZVeFchTWTBqRIURdKxYekXy+Tdn5qdlXwcoR04Cz8vftw2UjuN3ZpqPsvbWZ2SQAAoKalflx+irUBix0hD+4tsI7U7xep4bWSvUj6Y6i04SWmaINpCOnAGew6mK87ZixXga1UvZtH67lr2zGSOwAA3sQwpLXPOubXthc75ttmijXP4hssXfSZ1OIBx/1VD0srHpDspebWBa9ESAdOY29OoYa9s1RZR4rUOjZcU4d1lp+Vtw0AAF6jtFBKuVX6+wnH/ZbjHfNtM8Wa5/GxSl3+n9T5FUkWafPr0qLrmaINNY60AZzCobxi3TZ9qdIO5qtRZLBm3N5VYYF+ZpcFAABqSkGmY3qunbMki6/U9U3HPNsWPkJ7tFZHv4jxCZB2fyMlXyTlpZldFbwIf2GASuQW2jTivT+1ee8RxYQH6KPR3RXDVGsAAHiPQ39JP3WTDiyV/GtLfX+Smt9tdlWoKY3+IfX7VQqsKx1aLf3UVdqfYnZV8BKEdOAkhbZSjX5/uf7ena3awX76cFR3xUUGm10WAACoKbvnSMm9pPxdUlgLacBSqd6lZleFmlanpzRwmRTRQSrcJ/3SV9o+w+yq4AUI6cAJikuluz5cpaU7Dio0wFcz7+iu5jFhZpcFAABqgmFI6/8rLbjGcR1yvcukgUuk8OZmVwazhDSSBvzhGCzQXiwtuV1a+U8GlEO1IqQDR+UXl+itjT5avP2gQvyteu/2rmrXsJbZZQEAgJpgO+KYemv1Y5IMqfk90iVzHae6w7v5hjiuUW/7pOP+xlek+VdIxYdNLQuei5AOSDpSVKJRM1dqa46PQgKsmjmqm7o2jjS7LAAAUBNytkjzuktpn0k+flLXqY6bDwPG4iiLj9T+KUdYtwZJGT86xiw4vNbsyuCBCOnwetkFNo18908t33lYgVZDM0YkKjGegA4AgFfY/a30Uxcpe70UFCv1+91xFB2oTKMbpP5/SMFxUu4W6afuUuoss6uChyGkw6vtyynU0LcWa/nOQwoP9NXYNqXqGBdhdlkAAKC6GXbp7yelBVdJthypTi/p8hWOwcKA04nsJF2+UqrXXyrNl1KGScvvl0qLza4MHoKQDq+180Ce/vHmYm3MzFV0aIA+uKOL4kPNrgoAAFS7ov3S70OktZMd91vcJ136q+NIOnA2AqOlS36Q2j7huL/5denni6W8XebWBY9ASIdXWrsnW9dPW6y0g/mKjwrWl/f0VJvYcLPLAgAA1SyqdI1853VxXFNsDZQunCF1eU2y+ptdGtyNj1VqP1nq853kFyEdWCr92FmWzGSzK4ObI6TD6/y2cZ9uenuJso4UqU1suD4b00ONopgHHQAAj2Yvlc+6Z9Sr8ElZCjOk8NaOObCbjDC7Mri7BkOkQSul2p2koiz5LhyiNsUzHFO2AeeAkA6vYRiG3lm4XaPeX6YjRSXq0SRKn9x9oeqGBZpdGgAAqE4FGdJv/WVd/4wsssveeIR0+TIpoq3ZlcFThCY4BpQ7Ouhgc9vXsv7axzFzAOAkQjq8gq3Urn99tUbPfr9BdkO6qWuc3r+jm8IDmVoFAACPtutraW4Hae9vMqwhWuH/oEq7/p9j7mugKvkGSV2nqqTnZypWqHwOrZB+7CxtnykZhtnVwY0Q0uHxDuYVa8S7f+rjP3fJYpH+PaS1plzXTv6+vPwBAPBYthxpye3SwmsdA8VFdFBJ/yXa7dfX7Mrg4YwGV+u3oCTZ61wslRyRloxwjABfdNDs0uAmSCnwaCvTDmnI/xYqZdsBhfhb9c7wLhrdu4ksFovZpQEAgOqy93dpbntp+wxJFqnNo9LApVJYS5MLg7co9IlWaZ+fpPbPSBartPNjaW5bac/3ZpcGN0BIh0cyDEMz/tihoW8tVkZ2oRKiQ/Tl2F7q1zrG7NIAAEB1KSmQVj4k/XKplLdTCkmQLlsgdfyPZA0wuzp4G4tVavtvqf8iKbylY2yE+VdIS+6Qig+bXR1cGCEdHudIUYke+GS1nvp2vWylhga3q6c59/VSy3phZpcGAACqy9750g8dpI0vSzKkpqOlwX9JdS8yuzJ4u+gLpctXSa3+KckibX9PmttOSv/J7MrgonzNLgCoSit2HtL42auVdjBfvj4WTRzcWnf0aszp7QAAeKriQ9KqR6Rt7zjuB8VKXd+SGl5pbl3AiXyDpM4vSXHXSotHSke2Sr9fLjW+Ter8shRYx+wK4UI4kg6PYCu165XkzbrhzRSlHcxXg4ggfXLXhRp1UQIBHQAAT2QYUtoX0ndtjgf0ZmOkIRsI6HBddXo5zvBo+aAki5T6gfRdS2nrO5JhN7s6uAiOpMPtbd9/ROM//Ut/7TosSbq2UwM9ffUFTK8GAICnytksrXhQyvjRcT+8pdTt/6S6vc2tCzgbvsFSYpIUf4u07G7p0GrpzzulHTOkrm9KEW1NLhBmI6TDbdlK7Xp7wXb9v1+2qLjErvBAXz13bTtd2aG+2aUBAIDqYMuV1j4rbXpVstskHz+p9SOOwbmsgWZXBzgnups0cJm0+TXp7yek/X9IP3SSWtwntZsk+dc2u0KYhJAOt/TXrsN69Iu/tTEzV5LUu3m0/nt9e9WPCDK5MgAAUOUMQ0qdJa1+RCpIdyyLHeQ4GhnewtTSgPPi4yu1Gi/F/UNa8YC0+2tpU5LjNPh2k6VmdznWgVfhfxxuJbvApqSfN+v9lFTZDal2sJ8mXdlG13RswLXnAAB4or2/OwaGO7jMcT+0idQ5SWpwhUTfD08REidd/JWUMU9aOUHKXictv1fa8obU+VUpdoDZFaIGEdLhFkrthmYv26WX5m3SwbxiSdI1HevriSvaKCqUeU8BAPA4h/6WVj8mZfzguO8bIrWZKLX+J6e2w3PFDpAGrZa2vi2tmSRlr5d+GyjF9JM6PCdFdze7QtQAQjpc3p87DuqpOeu0PiNHktSsbqgmXdFGF7dgqgoAADxO7jZp7WRpxweSDMniKzW7W2r7hBQUY3Z1QPXz8ZVajJUa3yytmew4mr73F2neL1LDq6X2z0gR7cyuEtWIkA6XtSkzVy/P26R56/dKksICfTX+sha6rUe8/KzMHggAgEfJ2SKte05K/VAySh3LGt3oOHoY1szc2gAz+NeWEl91TNe2drK0431p9zfS7jlS/E2OARNrtTG7SlQDQjpcTmpWnpJ+3qxv/kqXYUg+Fummbo30z/4tOLUdAABPk73REc53zjo+T3TsIKn901JUV3NrA1xBaGPpwncdMxmsmSSlfSbt/Nhxa3itdMFE3isehpAOl7HzQJ7enL9dny3fpRK7IUka0i5W4/s3V7O6YSZXBwAAqoxhOKab2viy48igHP2+6l/hmHqKwAFUVKuVdNGn0sGVjqkId391/BbTT7rgX1JMXwZU9ACEdJhufXqO3py/Td/9na6j2VyXtKyjhwa0VNsGtcwtDgAAVB27TUr7Qtr4yvHR2iXHdbZtJ0mRnc2rDXAXkZ2li7+UsjdI6/8rpX7kuGZ97y9S7c5Sy/sdp8MzwKLbIqTDFIZhaPH2A/q/Bdv126b9Zcv7tKij+y5tpq6NI02sDgAAVKmCvdKOGdLmN6T8XY5l1kApYbjUcpxUq7WZ1QHuqVZrqccMx6UhG16Str0jHVopLbldWvWwY4715vdIwQ3NrhROIqSjRh0pKtFXK3dr5uKd2rLviCTHNedD2tfXmD5NdEF9jpwDAOARDLuU+YtjKqndX0tGiWN5YF2p+X1S8zFSIDO1AOctJF7q8prU7ilHUN88VcpPk9Y97zjS3vBqqeloqd4AycdqdrU4C4R01IhNmbmatXSnvli5R0eKHJ10sL9V13VuoNEXNVHj6BCTKwQAAFUiL81x+u22d6Qj248vj7rQcWSv8c2chgtUh4Aoqc2jUqt/Snu+lTa/Ju39Tdr1peMWVF9KGCE1uV0Kb252tTgNQjqqzYEjRfpmdbq+WLlb69JzypY3qROi4RfG67rEhgoP9DOxQgAAUCWKDjpGnE79SNq/8Phyv3Cp8W2OcF67vXn1Ad7Ex1eKu9ZxO7zW8YVZ6odSQbq0forjVuciqfEwKe56zmhxQYR0VKmC4lL9vmmfvli5R79v2lc2Sruf1aJLW9XVbRc2Vq9mUbIw6iQAAO7NliPtmSulfSKlz3UMCidJskh1+ziuN4+/UfLlbDnANBFtpcQkqeN/pT3fSdvflTJ+lPYvctyW3yfFXCo1ulGKu04KYFwoV0BIx3nLKyrRrxv36ce1mfp14z4V2ErLfte+YS1d37mhruxQX5Eh/iZWCQAAzltBprRnjrTrK8dI0mXBXFJEB8eRufibpJA482oEUJE1QGp0veOWv0dKnSWlzZYOrpAykx23ZfdI9S5zXMNefwjvYxMR0nFOMrMLNX/zPv2yYZ/mb96vohJ72e8aRATpivaxuj6xoVrEML85AABuy7A75mTOnOc4Cpe1RGVzmktSWAvH6bKNb3EcsQPg+oIbSG0edtxytzouVdk5Wzr8l+Moe8aPjvVqd5TqXyE1uEKK6ipZfEwt25sQ0nFWbKV2LU89pPmb9+v3Tfu0MTO33O8bRwVrULtYDWpbT+0a1OJ0dgAA3FX+HiljniOYZyZLRQfK/z6qm9TwGqnhtVKtVqaUCKCKhDWTLpjouOVscgwwt+c7KWuxdGi147buWSkg2nFafL1+Ukw/KbSJxOf9akNIR6VspXb9vTtbS3cc0NLtB7U89aDyio+fxm6xSB0aRuiSlnU08IJ6alUvjGAOAIC7MQwpb6djsLd9Cx0/czaWX8c3TKp3qRQ7UGpwJXMuA54qvOXxwF6Y5RhrIv07Kf1HqShLSvvUcZOk4EZHA/ulUp1eUkhjQnsVIqRDkpRdYNOa3dlalXZIS3cc1Iqdh8pdWy5JkSH+6tOiji5pWUe9m9fhGnMAANxNabGUvUY68OfxUJ6/+6SVLFJkF0cojx0gRV8o+TAbC+BVAqOlJsMdt9Jix9+Mvb9Imb9IB5Y45mHf/p7jJkmB9aQ6PaXonlJ0DymyM1MtngdCuhcqKinVhoxc/bXrsP7adVirdx/W9v15FdaLCPZTt8aR6t4kSt0TItUmNlw+PnxDBgCAW7DbpOz10sHl0oHljp+H/5bsxeXXs/hKkYlS3d5Snd6OqZkY4RnAMVZ/qe5Fjlu7J6WSPMeXfHt/kfbOlw6tkgozj8/HLkk+/lJEe0dYr93JcYtoL/kGmbsvboKQ7sEMQ9p9qEDbsg5qY2aONmbmalNmrrZn5anUblRYPy4ySB0aRqhr40h1bxKpFnXDCOUAALg6w5Dy0hzzIWevPf4ze71kL6q4vn+kI5TX6eUI5dHdmSYNwNnzDZHqX+64SVJJgWOU+KyUo7fFUuE+xxeDB5cf385ilcJbHQ3s7aRabaTw1o5T5VEOId0D5BeXKDUrXzuy8pR6IE/b9+dp+/5crU+3qmjJwkq3qR3spw5xEerQMEId4yLUvmEtRYUG1HDlAADgrNlyHCMx526VcrfImrNZvQuWyvfr26SS3Mq38Qt3nLoe2UWKOvqTa0cBVCXfoONH2iXHF4dHtkuHVjpmhzi0yvGzaL+Uvc5xO5E1UL6hLZRYWEs+61dJtdtKoU2lsKaOv2FeiJDuBmyldmVmFyr9cIH2HC4o+7kjK0+pWfnKzCk8xZYW+VktalonVK3qhallvXC1ig1Tq3phqhceyEBvAAC4kpICKX+X41rPvDTHv4/skI44QrkK95Vb3UdSpCTZ5ThlPbyVYxq0Wm2P/wxNYNokADXLYnEE7LCmUqMbHMsMQypIPx7as9dLORscI8qXFsqS/bcaStK6kw4wBkRJIU0cjxXa5PgtOE4KauCxp88T0k1kGIYO59u0/0iRsnKLtP9IkfbnOm7px0L5oQLtzS2UUfHs9HJqB/upcXSIEqJDlBAVorjagcrYtFIjrr1cwYEcIQcAwDR2myNgF2ZKBXsdPwv3SgUZRwP50WBelHXmxwqoI4U1l8KaqzQ4QSu3HlHHPsPkV7uN47pRAHBFFotjfvbgBlLDK48vt5dKeTtUcnCNNv35tVrXN+RzZLPjSHzRfscUkEUHpIPLKn/cgCgpqKFj1ongho7gHtzQ8TyBMVJgXcf0cW42+CUhvYoYhqECW6kO5dt0OL9Y2fk2HS6w6VB+sQ7n25RdYNOhvGIdyCvW/twiZR1x3GylZ0jfR/lbfVQ/IlD1I4LKbo2jgh2hPDpEEcHlO2abzaa5uyQ/K9+eAwBQZewlUvFhqfig41Z0sOK/i7IcIbzwaCA/eZ7x0/ENkULiHUeJghtJIY2OhvJmUmgzyb/W8VJsNqWnzlXHWhdIVvf6AAoAkiQfqxTWTEZgvLb6+6hFt8Hy8Tv698yW6wjrZbdtR3/ucJxpVFpwPMQf/uv0z+MfeTSw13H8DKwrBdSVAus4fudf+/jPgEjJL8JRm0lMD+lTp07Viy++qIyMDF1wwQVKSkpS7969T7n+/PnzNWHCBK1bt07169fXI488ojFjxtRgxcd9sWK33py/TYcLbMrOt6m41H5OjxMR7Kc6oQGqExag6FDH7cRA3iAiSFEh/gziBgBATcvZJKUMOx7Abdnn9jgW69EPhvUcR3eCjv48FsSDG0khcY4PhlyOBgCSX5hUu4PjdjLDkGyHHVNI5u85+nO3VLDn+M/C/Y6j8Ubp8S9TtdGJ5691NLzXlrq9JUV1rao9OyNTQ/rs2bM1btw4TZ06Vb169dJbb72lQYMGaf369WrUqFGF9Xfs2KHBgwfrzjvv1Icffqg//vhDY8eOVZ06dXT99dfXeP35tlJt2Xek3DI/q0URwf6KCPJTRLCfagX5q3bwsX/7KfqEMF4nLEBRof4K8DXvWxoAAHAaFl/HqMUn8ws/etQl0nHU5eR/lwXxo2E8IIprwwGgqlgsxwN0RLtTr2fYpeJDR89s2nf8VnTs537H74sOOn4WHzo+EKct23HLS9UZrz2uYqaG9FdeeUWjRo3S6NGjJUlJSUn66aefNG3aNE2ZMqXC+m+++aYaNWqkpKQkSVLr1q21fPlyvfTSS6aE9H6t6qrp6O6qFexXFsyD/a0MyAYAgKcIbiD1+e6kMB7hdtc3AoBXsvg4viQNiHJM+XY27LYTLms6GtxrtarWMk9mWkgvLi7WihUr9Nhjj5VbPmDAAKWkpFS6zeLFizVgwIByywYOHKjp06fLZrPJz69ih1lUVKSiouNzhObk5EhyXLNts9nOax/qhPiqTkitE5YYKikpOa/HrCrH9u1899Gb0GbOo82cR5s5x1vay9P3ryZUX39vleqW/+yhUkml5v+fecv7oyrRZs6jzZxHmznP5drMGiEFRUgnDh5/nrU5s2+mhfSsrCyVlpYqJiam3PKYmBhlZmZWuk1mZmal65eUlCgrK0uxsbEVtpkyZYqefvrpCsvnzZun4ODg89gD95CcnGx2CW6HNnMebeY82sw5nt5e+fn5Zpfg9ry5v/f090d1oM2cR5s5jzZznie3mTN9vekDx518arhhGKc9Xbyy9StbfszEiRM1YcKEsvs5OTmKi4vTgAEDFB4efq5luzybzabk5GT179+/0jMMUBFt5jzazHm0mXO8pb2OHfXFufPG/t5b3h9ViTZzHm3mPNrMed7QZs709aaF9OjoaFmt1gpHzfft21fhaPkx9erVq3R9X19fRUVFVbpNQECAAgIqzhPu5+fnsS+AE3nLflYl2sx5tJnzaDPneHp7efK+1RRv7u+9YR+rGm3mPNrMebSZ8zy5zZzZL9OGGfX391diYmKFUxqSk5PVs2fPSrfp0aNHhfXnzZunLl26eOx/JgAAAADAe5g6F8iECRP0zjvv6N1339WGDRs0fvx4paWllc17PnHiRA0fPrxs/TFjxmjnzp2aMGGCNmzYoHfffVfTp0/XQw89ZNYuAAAAAABQZUy9Jn3o0KE6cOCAJk+erIyMDLVt21Zz585VfHy8JCkjI0NpaWll6yckJGju3LkaP3683njjDdWvX1//+9//TJl+DQAAAACAqmb6wHFjx47V2LFjK/3djBkzKizr06ePVq5cWc1VAQAAAABQ80w93R0AAAAAABxHSAcAAAAAwEUQ0gEAAAAAcBGEdAAAAAAAXAQhHQAAAAAAF0FIBwAAAADARRDSAQAAAABwEabPk17TDMOQJOXk5JhcSfWy2WzKz89XTk6O/Pz8zC7HLdBmzqPNnEebOcdb2utYn3Ssj8L584b+3lveH1WJNnMebeY82sx53tBmzvT1XhfSc3NzJUlxcXEmVwIAQHm5ubmqVauW2WV4BPp7AIArOpu+3mJ42df2drtd6enpCgsLk8ViMbucapOTk6O4uDjt2rVL4eHhZpfjFmgz59FmzqPNnOMt7WUYhnJzc1W/fn35+HAlWlXwhv7eW94fVYk2cx5t5jzazHne0GbO9PVedyTdx8dHDRs2NLuMGhMeHu6xL/TqQps5jzZzHm3mHG9oL46gVy1v6u+94f1R1Wgz59FmzqPNnOfpbXa2fT1f1wMAAAAA4CII6QAAAAAAuAhCuocKCAjQk08+qYCAALNLcRu0mfNoM+fRZs6hvYBT4/3hPNrMebSZ82gz59Fm5XndwHEAAAAAALgqjqQDAAAAAOAiCOkAAAAAALgIQjoAAAAAAC6CkA4AAAAAgIsgpHuRoqIidezYURaLRatXrza7HJeVmpqqUaNGKSEhQUFBQWratKmefPJJFRcXm12aS5k6daoSEhIUGBioxMRELVy40OySXNaUKVPUtWtXhYWFqW7durrmmmu0adMms8tyK1OmTJHFYtG4cePMLgVwefT3Z0Zff3bo688eff35o68/jpDuRR555BHVr1/f7DJc3saNG2W32/XWW29p3bp1evXVV/Xmm2/qX//6l9mluYzZs2dr3Lhxevzxx7Vq1Sr17t1bgwYNUlpamtmluaT58+fr3nvv1ZIlS5ScnKySkhINGDBAeXl5ZpfmFpYtW6a3335b7du3N7sUwC3Q358Zff2Z0dc7h77+/NDXn8SAV5g7d67RqlUrY926dYYkY9WqVWaX5FZeeOEFIyEhwewyXEa3bt2MMWPGlFvWqlUr47HHHjOpIveyb98+Q5Ixf/58s0txebm5uUbz5s2N5ORko0+fPsaDDz5odkmAS6O/P3f09eXR158f+vqzR19fEUfSvcDevXt155136oMPPlBwcLDZ5bil7OxsRUZGml2GSyguLtaKFSs0YMCAcssHDBiglJQUk6pyL9nZ2ZLEa+os3HvvvRoyZIguu+wys0sBXB79/fmhrz+Ovv780defPfr6inzNLgDVyzAMjRw5UmPGjFGXLl2UmppqdkluZ9u2bXrttdf08ssvm12KS8jKylJpaaliYmLKLY+JiVFmZqZJVbkPwzA0YcIEXXTRRWrbtq3Z5bi0Tz75RCtXrtSyZcvMLgVwefT354e+vjz6+vNDX3/26Osrx5F0N/XUU0/JYrGc9rZ8+XK99tprysnJ0cSJE80u2XRn22YnSk9P1+WXX64bbrhBo0ePNqly12SxWMrdNwyjwjJUdN999+nvv//Wxx9/bHYpLm3Xrl168MEH9eGHHyowMNDscgDT0N87h76+atHXnxv6+rNDX39qFsMwDLOLgPOysrKUlZV12nUaN26sm266Sd9++225P6ilpaWyWq0aNmyY3n///eou1WWcbZsd+yORnp6uvn37qnv37poxY4Z8fPhOS3KcAhccHKzPPvtM1157bdnyBx98UKtXr9b8+fNNrM613X///fr666+1YMECJSQkmF2OS/v666917bXXymq1li0rLS2VxWKRj4+PioqKyv0O8FT0986hr68a9PXnjr7+7NHXnxoh3cOlpaUpJyen7H56eroGDhyozz//XN27d1fDhg1NrM517dmzR3379lViYqI+/PBDr/0DcSrdu3dXYmKipk6dWrasTZs2uvrqqzVlyhQTK3NNhmHo/vvv11dffaXff/9dzZs3N7skl5ebm6udO3eWW3b77berVatWevTRRzl9EDgJ/b3z6OtPj77eOfT1zqOvPzWuSfdwjRo1Knc/NDRUktS0aVM67FNIT0/XJZdcokaNGumll17S/v37y35Xr149EytzHRMmTNBtt92mLl26qEePHnr77beVlpamMWPGmF2aS7r33ns1a9YsffPNNwoLCyu7nq9WrVoKCgoyuTrXFBYWVqFzDgkJUVRUlFd32sCp0N87h77+zOjrnUNf7zz6+lMjpAMnmTdvnrZu3aqtW7dW+GDDiScOQ4cO1YEDBzR58mRlZGSobdu2mjt3ruLj480uzSVNmzZNknTJJZeUW/7ee+9p5MiRNV8QAHg5+vozo693Dn09qhKnuwMAAAAA4CIYHQMAAAAAABdBSAcAAAAAwEUQ0gEAAAAAcBGEdAAAAAAAXAQhHQAAAAAAF0FIBwAAAADARRDSAQAAAABwEYR0AAAAAABcBCEdAAAAAAAXQUgHAAAAAMBFENIBAAAAAHARhHQATtm/f7/q1aun559/vmzZ0qVL5e/vr3nz5plYGQAAqAr09YC5LIZhGGYXAcC9zJ07V9dcc41SUlLUqlUrderUSUOGDFFSUpLZpQEAgCpAXw+Yh5AO4Jzce++9+vnnn9W1a1f99ddfWrZsmQIDA80uCwAAVBH6esAchHQA56SgoEBt27bVrl27tHz5crVv397skgAAQBWirwfMwTXpAM7J9u3blZ6eLrvdrp07d5pdDgAAqGL09YA5OJIOwGnFxcXq1q2bOnbsqFatWumVV17RmjVrFBMTY3ZpAACgCtDXA+YhpANw2sMPP6zPP/9cf/31l0JDQ9W3b1+FhYXpu+++M7s0AABQBejrAfNwujsAp/z+++9KSkrSBx98oPDwcPn4+OiDDz7QokWLNG3aNLPLAwAA54m+HjAXR9IBAAAAAHARHEkHAAAAAMBFENIBAAAAAHARhHQAAAAAAFwEIR0AAAAAABdBSAcAAAAAwEUQ0gEAAAAAcBGEdAAAAAAAXAQhHQAAAAAAF0FIBwAAAADARRDSAQAAAABwEYR0AAAAAABcxP8HFy0TcmeJDHIAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def sigmoid(x):\n", + " return 1 / (1 + np.exp(-x))\n", + "\n", + "def sigmoid_derivative(x):\n", + " x = sigmoid(x)\n", + " return x * (1 - x)\n", + "\n", + "plot_function_and_derivative(sigmoid, sigmoid_derivative, \"sigmoid\", (-5, 5))" + ] + }, + { + "cell_type": "markdown", + "id": "3c1a0491", + "metadata": {}, + "source": [ + "**Eigenschaften:**\n", + "\n", + "* Mappt auf den Bereich $(0,1)$\n", + "* Sieht wie ein $S$ aus, darum der Name *Sigmoid*" + ] + }, + { + "cell_type": "markdown", + "id": "39012064", + "metadata": {}, + "source": [ + "**Vorteile:**\n", + "* Gut geeignet für binäre Klassifikationsprobleme (wenn wir nur ein Output Neuron verwenden wollen, siehe Logistic Regression).\n", + "* Einfache Ableitung (welche aus dem Funktionswert berechnet werden kann)\n", + "\n", + "*Hinweis:* Wir werden später sehen, dass bei Klassifikation es mehr Sinn macht, wenn wir statt nur einem Output Neuron für jede Klasse ein eigenen Neuron verwenden." + ] + }, + { + "cell_type": "markdown", + "id": "365c9f87", + "metadata": {}, + "source": [ + "**Nachteile:**:\n", + "* Ableitung ist klein, genauer gesagt liegt sie im Bereich $(0,0.25]$\n", + "* Nicht verwendbar bei mehr als 2 Klassen, weil es auf jedes Neuron individuell angewendet wird (siehe Softmax)\n", + "* Für Werte, weit weg von der $0$ ist die Sigmoid Funktion sehr flach und somit die Ableitung quasi $0$ (Normalisieren!)" + ] + }, + { + "cell_type": "markdown", + "id": "09ff8376", + "metadata": {}, + "source": [ + "> **Übung:** Zeigen Sie, dass für die Ableitung der Sigmoid Funktion $\\sigma(x)=(1+e^{-x})^{-1}$, $\\sigma'(x)=\\sigma(x)\\cdot (1-\\sigma(x))$ gilt." + ] + }, + { + "cell_type": "markdown", + "id": "7db42c37", + "metadata": {}, + "source": [ + "### tanh" + ] + }, + { + "cell_type": "markdown", + "id": "018d0ca2", + "metadata": {}, + "source": [ + "\\begin{align*}\n", + " f(x) &= \\tanh(x) = \\frac{\\sinh(x)}{\\cosh(x)}\\\\\n", + " f'(x) &= 1-\\tanh^2(x)\n", + "\\end{align*}" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "5b51dcc2", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA/0AAAHUCAYAAABoA0i3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB880lEQVR4nO3dd3hUZfr/8c/MZNIISYCQAgQIiBTpIFUUFwmI3VWwsRbEZbEh6+6K5SvYWDv2sos/FCzouooFleAKghTp0kR6gCRAgCSEtMnM+f0xJBISIAMzOVPer+sa5syZZ87cz80kT+45zznHYhiGIQAAAAAAEHSsZgcAAAAAAAB8g6IfAAAAAIAgRdEPAAAAAECQougHAAAAACBIUfQDAAAAABCkKPoBAAAAAAhSFP0AAAAAAAQpin4AAAAAAIIURT8AAAAAAEGKoh/ACS1atEgTJ05UXl6eT99nx44dslgseu6552r9mttuu01Dhw71+L2+//57xcTEaM+ePR6/FgCAYOGPY/zEiRPVsmXLauvz8vKUkJCgjz76yOP3HzlypK688kqPXwcEE4p+ACe0aNEiTZo0yed/EHhq1apVevfdd/XEE094/NpBgwapV69eevDBB30QGQAAgcFfx/iaTJo0SU2aNNGIESM8fu3EiRP19ddf63//+58PIgMCA0U/gIDzz3/+U7169VLPnj1P6/V33nmn3n//fe3atcvLkQEAAG86ePCg3nrrLd15552yWCwev75169YaOnSo/vnPf/ogOiAwUPQDqNHEiRP1t7/9TZKUlpYmi8Uii8WiefPmaebMmUpPT1dKSoqioqLUvn17PfDAAzpy5EiVbdxyyy2KiYnRli1bNGzYMMXExCg1NVV//etfVVpaWuP7vvDCC0pLS1NMTIz69u2rJUuWVHl+7969+uyzzzRy5Mgq68eMGaPIyEitWLGicp3L5dKgQYOUlJSk7OzsyvWXXXaZYmJi9K9//euMcgQAQCDy1zG+JtOmTVN5eXmVvfy5ublKTU1Vv3795HA4Ktdv2LBB9erVq/Y3wsiRIzV37lxt3bq11jkCgglFP4Aa3X777br77rslSf/973+1ePFiLV68WN27d9fmzZs1bNgwTZ06Vd9++63GjRunjz/+WJdddlm17TgcDl1++eUaNGiQZs2apdtuu00vvviinn766WptX3vtNWVkZGjKlCl6//33deTIEQ0bNkz5+fmVbebMmSOHw6ELL7ywymunTJmi9u3ba/jw4ZVTFSdNmqR58+ZpxowZSklJqWwbHh6ufv366euvv/ZGqgAACCj+OsZPnDhRO3bsqPK6r7/+Wt26dVN8fHzluorj+5ctW6Z//OMfkqSioiJde+21at68ud58880q2xg4cKAMw9Ds2bNPN2VAYDMA4ASeffZZQ5Kxffv2E7ZxuVyGw+Ew5s+fb0gy1qxZU/nczTffbEgyPv744yqvGTZsmNG2bdvKx9u3bzckGZ06dTLKy8sr1//888+GJOPDDz+sXPeXv/zFiIqKMlwuV7VYNm/ebMTGxhpXXnmlMXfuXMNqtRoPP/xwjXE/9NBDhtVqNQoLC0+ZBwAAgo0/jvE1iY6ONsaMGVPjc08//bQhyfjss8+Mm2++2YiKijJ++eWXGts2bdrUGDFixEnfCwhW7OkH4LFt27bphhtuUHJysmw2m+x2uy644AJJ0saNG6u0tVgs1fYOdO7cWTt37qy23UsuuUQ2m61KO0lV2mZlZalx48Y1Htd31lln6V//+pc+//xzXXrppRowYIAmTpxYYx8SExPlcrmUk5NTu04DABACzBzjj5eXl6eioiIlJibW+Pzf/vY3XXLJJbr++uv17rvv6pVXXlGnTp1qbJuYmMiVexCywswOAEBgKSws1IABAxQZGaknnnhCZ599tqKjo7Vr1y5dffXVKi4urtI+OjpakZGRVdZFRESopKSk2rYbNWpUrZ2kKtssLi6utr1jXXLJJUpKStLevXs1fvz4Kn9gHKtiG8fHCwBAqDJ7jD9exXMnGvctFotuueUWff3110pOTq52LP+xIiMjGfMRsij6AXjkf//7n7KysjRv3rzKb/4l1dklfxISErRy5coTPj9mzBgdPnxY55xzju655x4NGDBADRo0qNbu4MGDldsDAADmj/HHq/iioGLMPl52drbuvPNOde3aVevXr9f999+vl19+uca2Bw8eVMuWLX0VKuDXmN4P4IRq+ha+Ylp9xXMV3nrrrTqJqV27djpw4ECVE/9U+Pe//60ZM2bo1Vdf1RdffKG8vDzdeuutNW5n27ZtatSokZKSknwdMgAAfscfx/jjhYeHq1WrVjWedd/pdOr666+XxWLRN998o8mTJ+uVV17Rf//732pty8vLtWvXLnXo0KEuwgb8DkU/gBOqOC7upZde0uLFi7V8+XJ17txZDRo00JgxY/TZZ5/pq6++0vXXX681a9bUSUwVZ+BdunRplfVr167VPffco5tvvlm33nqrWrVqpalTp2rWrFmaMmVKte0sWbJEF1xwwWld8xcAgEDnj2N8TQYOHFjjpf0effRRLViwQO+//76Sk5P117/+VZdddplGjRql7du3V2n7yy+/qKioqNqVf4BQQdEP4IQGDhyoCRMm6Msvv9R5552nc889V9u3b9fXX3+t6Oho3XTTTbrtttsUExOjmTNn1klM/fv3V8uWLTVr1qzKdUeOHNHw4cOVlpam119/vXL9H//4R9155536+9//rp9//rly/datW7V27VrdeOONdRIzAAD+xh/H+JrceOONys7O1rJlyyrXZWRkaPLkyXrkkUc0aNCgyvXTpk1TbGysRowYobKyssr1n3/+uRISEpSenl6nsQP+wmIYhmF2EADgieeff15PPvmk9uzZo6ioKI9f/8gjj+i9997T1q1bFRbGqU0AAPBnnTt3Vv/+/fXGG294/Fqn06mzzjpLN9xwg5588kkfRAf4P/b0Awg4d955p+Li4vTaa695/Nq8vDy99tpreuqppyj4AQAIAM8884ymTZum3bt3e/zaGTNmqLCwUH/72998EBkQGCj6AQScyMhITZ8+vdqJhmpj+/btmjBhgm644QYfRAYAALxt6NChevbZZ6sdq18bLpdL77//vuLj470fGBAgmN4PAAAAAECQYk8/AAAAAABBiqIfAAAAAIAgRdEPAAAAAECQ4tTVXuByuZSVlaX69evLYrGYHQ4AADIMQ4cPH1aTJk1ktfId/5lirAcA+JvajvUU/V6QlZWl1NRUs8MAAKCaXbt2qVmzZmaHEfAY6wEA/upUYz1FvxfUr19fkjvZsbGxJkfjOw6HQ3PmzFF6errsdrvZ4fg98uU5cuY5cua5UMlZQUGBUlNTK8conBnGepwIOfMcOfMM+fJcqOSstmM9Rb8XVEzzi42NDfo/BKKjoxUbGxvUPzzeQr48R848R848F2o5Yyq6dzDW40TImefImWfIl+dCLWenGus5yA8AAAAAgCBF0Q8AAAAAQJCi6AcAAAAAIEhxTH8dMQxD5eXlcjqdZody2hwOh8LCwlRSUhKw/bDZbAoLC+MYVwAAAOCoYKhVjhUMdYvkvdqFor8OlJWVKTs7W0VFRWaHckYMw1BycrJ27doV0EVzdHS0UlJSFB4ebnYoAAAAgKmCpVY5VrDULZJ3aheKfh9zuVzavn27bDabmjRpovDw8ID94LlcLhUWFiomJkZWa+AdGWIYhsrKyrR//35t375dbdq0Cch+AAAAAN4QTLXKsQK9bpG8W7tQ9PtYWVmZXC6XUlNTFR0dbXY4Z8TlcqmsrEyRkZEB+8MTFRUlu92unTt3VvYFAAAACEXBVKscKxjqFsl7tUvgZiDABPKHLdjwfwEAAAD8jr+P/Zc3/m/43wUAAAAAIEhR9AMAAAAAEKQCquj/8ccfddlll6lJkyayWCz6/PPPT/ma+fPnq0ePHoqMjFSrVq305ptvVmvz6aefqkOHDoqIiFCHDh302Wef+SB6VGjZsqWmTJlyynZTp05Venp6rbf71VdfqVu3bnK5XGcQHQAAAIBQVdta5XiPPPKI7rjjjlq3f/XVV3X55Zd7/D6nI6CK/iNHjqhLly569dVXa9V++/btGjZsmAYMGKBVq1bpwQcf1D333KNPP/20ss3ixYs1YsQIjRw5UmvWrNHIkSM1fPhwLV261FfdCBgDBw7UuHHjTHnv0tJS/d///Z8eeeSRWr/m0ksvlcVi0QcffODDyAAAAACYzcxaxWKxaMeOHZWP9+7dq5deekkPPvhgrbcxevRoLVu2TAsXLvRBhFUFVNF/8cUX64knntDVV19dq/ZvvvmmmjdvrilTpqh9+/a6/fbbddttt+m5556rbDNlyhQNHjxYEyZMULt27TRhwgQNGjTotL7dgfd8+umniomJ0YABAzx63a233qpXXnnFR1EBAAAAQFVTp05V37591bJly1q/JiIiQjfccEOd1C5Bfcm+xYsXV5sePmTIEE2dOlUOh0N2u12LFy/WfffdV63NyYr+0tJSlZaWVj4uKCiQJDkcDjkcjiptHQ6HDMOQy+WSy+WSYRgqdjjPsGenJ8puq/V1N2+99VbNnz9f8+fP10svvSRJ2rRpkx5//HEtXLhQOTk5at68uf7yl7/onnvuqfK6vLw8nXfeeXrhhRdUVlamESNG6MUXX5Tdbq9sd+TIEd166636z3/+owYNGujBBx+sMh3mww8/1GWXXVY5Vb+kpETnnnuu+vXrp7feekuSeyZH9+7d9cwzz2j06NGS3Hv777nnHm3ZskWtWrWqsW8V/w8Oh0M2m82DDHqm4rNw/GcCJ0bOPOfvOTMMQ2XlLpWUu1TicKq03KVSh0ul5S6VlDtV4nCptNypsnKXypyGnC6Xyp2GHC5DTpehcqdL5S5D5U5D5a7fl52uijYuOV2GXIbkMgwZhvs9qz52L7sMQ4Ykp9Olvfus+ix3hQyLRap83v1aQ1UfV+3P0ftj+lf1ccXzVRseu5Xj21Q+Pnp/a78WuqpbkzNNvd9+JgKFJ2N9MPH33yn+iJzVkrNE1h3TZcn5RtbiHPUuMWT8ulmOs0ZJYTFmR+fXfPkZO75WkeQekJxFXn+vWrFFS7WoV2qqVX777TdNnjxZP/zwg3JyctSsWTONHTtW9957b5XXeaNWkVQlZx999JHuuOOOysf79+9Xly5ddPfdd2vChAmSpKVLl+qCCy7QF198UVmjXnrppRo6dKiOHDmiqKioGvt6stqltp+JoC76c3JylJSUVGVdUlKSysvLlZubq5SUlBO2ycnJOeF2J0+erEmTJlVbP2fOnGrXtwwLC1NycrIKCwtVVlam4jKn+r6w5Ax6dfoWj++jqPDaFbmPPfaYNm7cqA4dOlR+UOPi4tSkSRNNnTpVjRo10tKlS3XfffcpLi5OV111lST3B++HH35Qo0aNNGvWLG3btk2jRo1S27ZtdfPNN0tyf3Cff/55Pfjgg7r77rs1a9Ys3XnnnerevbvOPvtsSdKCBQt01VVXVf6RJblnblx00UUaOHCghg4dqhtvvFHnnXeeRowYUdmuQYMGaty4sTIyMnT99dfX2LeysjIVFxfrxx9/VHl5+ekl0wMZGRk+f49gQ84858uclbukQodUWC4VOiyVy0ccFpU4VfVWXnVdmat2XzTWPat06IDZQdTopxVrFJG9+oy3U1Rk0h9sQcKTsT4Y8XvYc+TsxBo6N6hH6YuKNvZXrkuWpLUrVLLuKa0Kv1v7wrqbFl+g8MVn7PhaRZJUfkTxc5p5/b1qIy99txRW75TtaqpVYmNj1bhx42q1Snx8vNdrFUkqLCxUQUGB8vLytG7dOrVr166yJomIiNDLL7+sm266Sf369VObNm100003adSoUerTp09lu7PPPlsOh0Pz5s1T//79a+zryWqX2o71QV30S6q2Z7tir8yx62tqc7I94hMmTND48eMrHxcUFCg1NVXp6emKjY2t0rakpES7du1STEyMIiMjFVbm+yLzROrH1ld0eO3+y2NjYxUdHa24uDi1adNGkjsvEyZMUP369WWxWNSpUyetXr1aX331VeUPid1uV8OGDfXWW2/JZrOpZ8+e+vTTT7Vo0SLdfffdktzXmhw2bFhlDrt06aI333xTy5cvV8+ePZWXl6f8/Hy1bt26Sj779++vxx9/XOPGjdN1112nnTt36osvvqiW82bNmmnv3r3V1lcoKSlRVFSUzj//fEVGRnqWRA84HA5lZGRo8ODBVb45xImRM8+dac4Mw1BOQal2HihSdn6JsvJLlJ1fouz8YmXnlyinoFSHS7zze8tqkSLtNkWEWRURZv192W5VRJhNdptFdqtVNqtFYTaLwqwWhVmtstksslstR9dbj64/erNZZLW4n7PI/fvcapWsFvf6ivd1P3Y/b7hc2rhxgzqe00FhYWGVz1e+vuKx5bixouLeUvHYctzjqg0rXnv866quq9pGFimtUT01a1Dzt/2eOPZLU3jOk7E+mPB72HPk7OQsuz6RbemjshgOGVFN5Wo9RuX12mrzqm/UIWyeIou2q0/pE3Kd84pcrUebHa5f8uVn7PhaRZJU7ruZsKcSGxtbq6K/plpFcn9hK7n/vmnRooVPahVJcjp/n7m9bds2GYahNm3aVBkfrrnmGs2bN09jxoxRz549FR0dreeff75K/REbG6v4+Hjt27fvtGqX2o71QV30JycnV9tjv2/fPoWFhalRo0YnbXP83v9jRUREKCIiotp6u91e7QfR6XQe/SPUKqvVqnoRdm14bMjpdumMeDK9v0JF7JL7W6933nlHH3zwgXbu3Kni4mKVlZWpa9eulW0sFovOOeecKnlo0qSJ1q5dW9lGcv/wHPs4OTlZubm5slqtldMpo6Ojq7SRpPvvv19ffPGFXn31VX3zzTdKTEys3s+oKBUXF1d7bQWr1SqLxVLj/5cv1NX7BBNy5rlT5cwwDO0+VKx1e/L1295Cbd1fqG25hdq2/4iKyk59yJHNalGD6HAlxISrYT33rUF0uGKjwhQTYVdMZJhiI8MUE3H0Fhmm+hF2RYXbFGl3F/hhVovHv4N8weFwaPbB9RrWq0VQf86CuW91wZOxPhiFSj+9iZzVIOtbaenNklEuNb9Wlt7vyGaPkcvh0NZ14Wo75GVZ19wny9Z/y7byLtkiG0otrzM7ar/li8/Y8bWK+41ipOGFXn2f2rLWcnp/hWNrFck9M/jf//63T2uV452sdnn++efVsWNHffLJJ1q+fHmNM8WioqJUUlJyWrVLbT8PQV309+3bV19++WWVdXPmzFHPnj0rE9S3b19lZGRUOa5/zpw56tevn09islgstd7b7m8+/vhjPfTQQ3ruuefUr18/1a9fX88++2y1Kx0c/+GzWCzVLqN3sjaNGjWSxWLRoUOHqsWwb98+bdq0STabTZs3b9bQoUOrtTl48KAaN258Wn0EgkV+sUPLth/U8p2HtG5PvtZl5SuvqObjvsKsFqU2jFbT+Cg1iY9USlyUmsZHKSU+UilxkUqIiVBspF1Wq/kFOwAgQBzZJS26wV3wt7hB6jddshxX1NgipF5vS7ZI6bdXpaW3SfEd3TeYx2Kp1d52f/Pxxx/rvvvu0/PPP6/evXvLYrHozTff1M8//1yl3ZnWKsdLSEiQJB06dKhaDbJt2zZlZWXJ5XJp586d6ty5c7XX10XtElDVZ2FhobZs2VL5ePv27Vq9erUaNmyo5s2ba8KECdqzZ4/ee+89SdKYMWP06quvavz48Ro9erQWL16sqVOn6sMPP6zcxr333qvzzz9fTz/9tK644grNmjVLc+fOrZNLJ/i78PDwKlNXFi5cqF69eukvf/lL5TdRW7du9cn7dujQQRs2bKh2IsbbbrtNHTt21OjRozVq1CgNGjRIHTp0qHy+pKREW7duVbdu3bweF+DPSstd+mnbPi3cnKsl2w9ofVaBjjsHnew2i9om11f75Fi1ToxRq4R6ap0Yo+YNo2W3BdTFXAAA/szllBbdKJUdkhqeK/X5f9UL/goWi9TjJenwZin7O+mnEdLQVZItvG5jRsA5vlZZsGCB+vXrp7Fjx8rlcqmgoEDbtm3zeRwVhyRv2LChyjH/ZWVluvHGGzVixAi1a9dOo0aN0tq1a6vMKN+6datKSkp8XrsEVNG/fPlyXXjhhZWPK46zuPnmmzVt2jRlZ2crMzOz8vm0tDTNnj1b9913n1577TU1adJEL7/8sv74xz9WtunXr58++ugjPfzww3rkkUfUunVrzZw5U7179667jvmpli1baunSpdqxY4diYmJ01lln6b333tN3332n1q1ba/r06Vq2bJnS0tK8/t5DhgzRwoULq1x787XXXtPixYv1yy+/KDU1Vd98841uvPFGLV26VOHh7oFhyZIlioiIUN++fb0eE+Bv8osc+m5dlt7fZNWDK37QkeOm6bdKqKdeaQ3VuVm8OjWN09nJMYoIM+84PQBAiNjyprR/gRRWX+r/4akLeItV6jtdmt1Ryt8gbXxW6vhQ3cSKgHWyWqVFixaaOnWqz2qVY1mtVl100UVauHChrrzyysr1Dz30kPLz8/Xyyy8rJiZG33zzjUaNGqWvvvqqss2CBQvUqlUrtW7d2qcxBlTRP3DgwGqXTzrWtGnTqq274IILtHLlypNu95prrtE111xzpuEFnfvvv18333yzOnTooOLiYm3YsEHLli3T9ddfL4vFouuvv15jx47VN9984/X3Hj16tLp37678/HzFxcXp119/1d/+9jdNnTpVqampktxfAnTp0kWPPPKInn76aUnuS/3deOONIXFmZYQmh9Ol+Zv267+rdmvuhn0qc7okWSU5lRQboQvbJqpv60bq06qRkmJ9d6JKAABqVLJfWvOwe7nrZKl+LYuZyMZStxekxTdJ65+QWlxX+9ciJB1fq/z6669avXq1RowYIYvFoquvvlp/+ctf9O233/o8ljvuuEOjRo3SM888I6vVqnnz5mnKlCn64YcfKk/QN336dHXu3FlvvPGG/vKXv0hy1y4Vlx73JYtxsioatVJQUKC4uDjl5+fXePb+7du3Ky0tzadniq8LFdNkYmNjT3iiCW8aPny4unXrVnkZjlPZv3+/2rVrp+XLl5/0G726+j9xOByaPXu2hg0bxol9aomcndi+wyWasSRTHyzdqdzCssr1bRLrqZX9sP58WV91a9HIL06U5+9C5XN2srEJnguVfIbKz4c3kbPj/PwX957+Bl2lIcsla/UZZifMmWFIP6RLOXOl5sOl82bWXdx+zJefsWCqVY5V13WLYRjq06ePxo0bd8LLhh9v3bp1GjRokH777TfFxcWdsN3J/o9qOzZxECf81rPPPquYmJhat9++fbtef/11n0/hAerSrzkFGj9ztfr/8396+fvNyi0sU0JMhG4/L02z7xmg2Xf317DmLnVqGkfBDwAwV+F2aeu/3cs9Xq6x4D8pi0Xq9rwki5T5sXRwlddDBHzBYrHo7bffVnl57S9znJWVpffee++kBb+3BNT0foSWFi1aVF4vszZ69eqlXr16+TAioO5s2VeoKXN/09drsytPyNejRQPd1j9NQ85JUtjRE+85HDWfkR8AgDq37gn32fpThkiJA05vGw06u6f27/xQ+uURaeBXp34N4Ae6dOmiLl261Lr98Scs9yWKfgDwI7mFpXr22036ZMUuuY4W+5d0StEd57dSl9R4U2MDAOCECrdJ2991L3eaeGbb6jTJvac/62vp0Gr3oQIAThtFPwD4gXKnSzOW7NTzGb/pcIl7alh6hyTdN/hstU8J3uOHAQBB4teXJMMpJadLCX3ObFuxbaTm10o7P5I2Pi/1m+6dGIEQRdFfRzhfov/g/wL+5re9hzX+49Vat6dAktSxaawmXX6OerRoaHJkAADUgqNA2vb/3Mvt/+qdbbb7q7vo3/mR1OUpqV6qd7aLGvH3sf/yxv8NJ/LzsYozbBYVFZkcCSpU/F9whl2Yzeky9PaPW3XpKwu1bk+B4qLseuLKjpp153kU/ACAwLH1/0nlh6XY9lLyYO9ss1FPKXGg+xwBv73qnW2iGmoV/+eN2oU9/T5ms9kUHx+vffv2SZKio6MD9gzbLpdLZWVlKikpqZNLX3ibYRgqKirSvn37FB8fL5vNwzPKAl508EiZ7v1olRZszpUkXdi2sZ7+Y2clxgbP5XIAACHA5ZR+e9m93PZe9xn4vaXtvdK+ee5ZBJ0fl2zh3ts2JAVXrXKsQK9bJO/WLhT9dSA5OVmSKn+YApVhGCouLlZUVFRA/zKIj4+v/D8BzLBmV57Gvr9Se/KKFWW36dHLOmjEuakB/XMFAAhROXPcJ/ELbyCl3eTdbTe9VIpqIhVnSbs/k1qM8O72ISl4apVjBUvdInmndqHorwMWi0UpKSlKTEwM6MtrORwO/fjjjzr//PMDdmq83W5nDz9M9cWaLN3/8RqVOV1KS6inN2/qobbJ9c0OCwCA07P1Hfd9y5FSWD3vbtsaJrUeJa17XNryNkW/jwRLrXKsYKhbJO/VLhT9dchmswV0wWmz2VReXq7IyMiA/uEBzGAYhv61YJuemv2rJGlwhyQ9P7yLYiP5WQIABKjSA9KeL9zLrW/1zXu0vl1a/6S0939SwWb3mf3hE4FeqxyLuqWqwDzAAQACiGEYeuyrDZUF/639W+qtm3pQ8AMAAtuODyRXmdSgq/vmC/WaSykXu5e3veOb9wCCHEU/APiQYRj6v1nr9f9+2iFJeviS9nr0snNktQb28WUAAFRepq+Vj/byV2h1i/t+xweS4fLtewFBiKIfAHykouCfvmSnLBbp2Ws66/YBrcwOCwCAM3dotXRolWS1Sy1u8O17Nb1UssdKRZnS/oW+fS8gCFH0A4CPPPH1xmMK/i66tmeq2SEBAOAd295z3ze9XIpM8O172SKl1Gvcyzve9+17AUGIoh8AfODfC7Zp6sLtktwF/zU9mpkcEQAAXmK4pMyP3cstvXyZvhOpuBzgzo8lZ2ndvCcQJCj6AcDLvlyTpSe+3ihJemhYewp+AEBwyV0sFe9xT7lvMrRu3jPxAim6meTIk7Jm1817AkGCoh8AvGjFzoP668drJEm39Gup2wekmRwRAABetnOm+77pFe6p93XBYpVaXO9eZoo/4BGKfgDwkn2HS/SXGStV5nRpyDlJeuTSDrJYOEs/ACCIuJzSrv+4l1sMr9v3bnn0hIFZs6XyI3X73kAAo+gHAC9wOF266/1V2ne4VGclxuiF4V1l47J8AIBgs3+hVJwt2eOk5PS6fe/4LlJMK8lZLGV9U7fvDQQwin4A8IJ/fvOrft5xUDERYXprZA/ViwgzOyQAALyv4gR+qVdJtvC6fW+LRUr9o3t516d1+95AAKPoB4Az9MOv+yrP1P/ctV3UunGMyREBAOADx07tbz7CnBgqiv49X0nOEnNiAAIMRT8AnIEDhaX6239+kSTd2r+lhnZMNjkiAAB8ZP9CqWSfFN5ASh5kTgyNznWfxb+8UMrOMCcGIMBQ9APAaTIMQxP+u1a5haVqkxijfwxtZ3ZIAAD4TtZX7vsml0pWuzkxWKxSs6vdy0zxB2qFoh8ATtN/VuzWnA17ZbdZ9OKIroq028wOCQAA39nztfu+6SXmxtG8Yor/F5LLYW4sQACg6AeA07D/cKke/2qDJGncRWerY9M4kyMCAMCHCrdLBRsli01KGWJuLAn9pchEqeyQtPcHc2MBAgBFPwCchie/3qCCknKd0yRWfz6/ldnhAADgWxV7+Rv3l8LjTQ1FVpvU7Cr38u7PTQ0FCAQU/QDgoQWb9+vz1VmyWqTJV3dSmI1fpQCAIJd1tOhvYvLU/gpNL3Pf7/lKMgxzYwH8HH+pAoAHShxOPfL5OknSn/q2VOdm8eYGBACAr5Uf+X0avb8U/Ul/kGxRUtEuKW+N2dEAfo2iHwA88K8ft2nHgSIlxUbor+lnmx0OAAC+l/M/yVUq1WshxXUwOxq3sCgpebB7ec9X5sYC+DmKfgCopX2HS/TG/K2SpAeHtVf9SJMuVwQAQF06dmq/xWJuLMeqnOL/pblxAH6Ooh8AaunFjM0qKnOqS2q8Lu/SxOxwAADwPcP4vehveqm5sRyv4tKBB36WinPMjQXwYxT9AFALm3IOa+ayTEnSw5e0l8Wf9nQAAOAreb9IRbvdx88nDjQ7mqqiUqSGPd3LFV9MAKgm4Ir+119/XWlpaYqMjFSPHj20YMGCE7a95ZZbZLFYqt3OOeecyjbTpk2rsU1JSUlddAdAgJj8zUa5DGnoOck6t2VDs8MBAKBuVBTTSYPcx9H7G6b4A6cUUEX/zJkzNW7cOD300ENatWqVBgwYoIsvvliZmZk1tn/ppZeUnZ1dedu1a5caNmyoa6+9tkq72NjYKu2ys7MVGRlZF10CEACWbjugeZv2K8xq0QMXtzM7HAAA6s6eiqn9fnLW/uNVFP3ZGZKTnXZATQKq6H/hhRc0atQo3X777Wrfvr2mTJmi1NRUvfHGGzW2j4uLU3JycuVt+fLlOnTokG699dYq7SwWS5V2ycnJddEdAAHipe83S5JGnJuqlgn1TI4GAIA6UnpAOrDEvdxkmLmxnEiDrlJ0M8lZ5L7KAIBqwswOoLbKysq0YsUKPfDAA1XWp6ena9GiRbXaxtSpU3XRRRepRYsWVdYXFhaqRYsWcjqd6tq1qx5//HF169bthNspLS1VaWlp5eOCggJJksPhkMPhqG2XAk5F34K5j95Evjznjzn7ecdBLdp6QHabRXec18KvYpP8M2f+LlRyFuz98zXG+uDto7cFc84su75SmOGSEddR5eEpkpf66O2cWZOHybbtbTl3fyFX4mCvbNOfBPNnzFdCJWe17Z/FMAzDx7F4RVZWlpo2baqffvpJ/fr1q1z/1FNP6d1339WmTZtO+vrs7Gylpqbqgw8+0PDhwyvXL1myRFu2bFGnTp1UUFCgl156SbNnz9aaNWvUpk2bGrc1ceJETZo0qdr6Dz74QNHR0afZQwD+6NX1Vm0usKp/kkvDW7nMDgeotaKiIt1www3Kz89XbGys2eEEHMZ6QOpR8ryaORfoN/sftTF8pNnhnFBi+XL1LX1CxZZGmhP1b/+6rCDgQ7Ud6wOu6F+0aJH69u1buf7JJ5/U9OnT9euvv5709ZMnT9bzzz+vrKwshYeHn7Cdy+VS9+7ddf755+vll1+usU1N3/6npqYqNzc3qP+wcjgcysjI0ODBg2W3c33yUyFfnvO3nP2846BunLpcdptFc8edpybx/ncCI3/LWSAIlZwVFBQoISGBov80MdYH98+HNwVtzlzlCvuiqSyOQyq/8AcZCf29tmmv58xZrLBZybI4i+UYvFyK73zm2/QjQfsZ86FQyVltx/qAmd6fkJAgm82mnJyq1+Dct2+fkpKSTvpawzD0zjvvaOTIkSct+CXJarXq3HPP1ebNm0/YJiIiQhEREdXW2+32oP5QVQiVfnoL+fKcv+Ts9fnbJUnDe6aqRWP//iPfX3IWSII9Z8Hct7rAWB8a/fSmoMvZvqWS45AU3kBhSedJVu+XDV7Lmd3uvrpA1ley75sjNe5x5tv0Q0H3GasDwZ6z2vYtYE7kFx4erh49eigjI6PK+oyMjCrT/Wsyf/58bdmyRaNGjTrl+xiGodWrVyslJeWM4gUQ2NbtyddPWw7IZrXoLwNbmx0OAAB1q+JSfSlDfVLwe13F1QUq4gZQKQB+gn83fvx4jRw5Uj179lTfvn319ttvKzMzU2PGjJEkTZgwQXv27NF7771X5XVTp05V79691bFjx2rbnDRpkvr06aM2bdqooKBAL7/8slavXq3XXnutTvoEwD/9e8E2SdIlnVLUrAHH7wIAQkxF8dzETy/Vd7yKqwvkLnZfdSCikbnxAH4koIr+ESNG6MCBA3rssceUnZ2tjh07avbs2ZVn48/OzlZmZmaV1+Tn5+vTTz/VSy+9VOM28/LydMcddygnJ0dxcXHq1q2bfvzxR/Xq1cvn/QHgn7LyivXVL9mSpNEDWpkcDQAAdexIppS3VrJYpSZDzY6mduo1l+I7uePO/k5qeYPZEQF+I6CKfkkaO3asxo4dW+Nz06ZNq7YuLi5ORUVFJ9zeiy++qBdffNFb4QEIAtMW7VC5y1CfVg3VqVmc2eEAAFC3sma77xP6BtYe8yaXuIv+PV9T9APHCJhj+gGgLhwucejDpe4ZQ3ecz15+AEAI2hNgU/srVMSb/Y3kKjc3FsCPUPQDwDE+Xr5bh0vL1bpxPQ08O9HscAAAqFvlxdLe793LgVb0J/SRwhtIZYek3CVmRwP4DYp+ADjKMAy9v2SnJOnW/mmyWi0mRwQAQB3b+4PkLJaim7mPkQ8k1jD31QYkzuIPHIOiHwCOWrT1gLblHlFMRJiu6tbU7HAAAKh7x5613xKAX3434dJ9wPEo+gHgqBlH9/Jf1a2p6kUE3HlOAQA4M4YReJfqO16Toe6rDuStdV+FAABFPwBI0t6CEs3ZsFeSdGOf5iZHAwCACfI3SEd2StYIKfkPZkdzeiIaSY36uJcrrkIAhDiKfgCQ9NHPu+R0GerZooHaJceaHQ4AAHWvYi9/0oVSWD1zYzkTTY/OUtjDFH9AougHAJU7XfrwZ/cUwJv6tDA5GgAATBLoU/srVMS/93v31QiAEEfRDyDk/bh5v3IKStQg2q6LOyWbHQ4AAHWv7JC0/yf3ctMAL/rjO7uvPuAslvbNMzsawHQU/QBC3n9W7JYkXdmtqSLCbCZHAwCACbLnSIZTim0vxaSZHc2ZsVikJsPcy0zxByj6AYS2vKIyzd2wT5J0TY9mJkcDAIBJKorjppeaG4e3HHvpPsMwNxbAZBT9AELal2uyVOZ0qX1KrM5pEmd2OAAA1D2XU8r+xr0c6MfzV0ge5L4KwZEdUsFGs6MBTEXRDyCk/WflHknSH7s3NTkSAABMcnCZVJor2eOkxv3MjsY7wuq5r0IgMcUfIY+iH0DI2rLvsNbsylOY1aIru1H0AwBCVEVRnDJEstrNjcWbjp3iD4Qwin4AIes/K9x7+Qe2bayEmAiTowEAwCTBcqm+41VchWD/Qqksz9RQADNR9AMISS6Xoc9XVUzt5wR+AIAQVZQlHVolySI1udjsaLwrJs19NQLD6b46ARCiKPoBhKTlOw8pp6BE9SPD9If2iWaHAwCAObJmu+8b9ZIiG5sbiy80ZYo/QNEPICR9uSZLkjTknGRFhNlMjgYAAJME69T+CpXH9X8jGS5zYwFMQtEPIOSUO12avTZbknRZlyYmRwMAgEmcpVJOhnu5aZAW/Y37u69KULpfOrDM7GgAU1D0Awg5S7Yd1IEjZWoQbVe/1o3MDgcAAHPsmy+VH5GiUqQG3cyOxjesdikl3b3MFH+EKIp+ACGnYmr/xZ1SZLfxaxAAEKIqLtXXZJhksZgbiy9VTPHfQ9GP0MRfuwBCSlm5S9+sc0/tv7RzisnRAABgEsM45nj+S82NxdeaXCzJIh1aKRVnmx0NUOco+gGElIVb9qugpFyN60eodxpT+wEAIapgk1S4VbKGS8mDzI7GtyITpUbnupcrrlYAhBCKfgAh5etfciRJl3RKkc0axFMZAQA4mayv3PdJF0r2+ubGUheY4o8QRtEPIGSUO136/te9kqShHZNNjgYAABPt+dJ9H+xT+ytUXJ0gJ8N91QIghFD0AwgZP+84qLwihxrWC1fPFg3MDgcAAHOUHpT2/+RebhoiRX+DblJkslReKO1fYHY0QJ2i6AcQMuasd+/lH9QuUWGctR8AEKqyv5MMpxTXUYppaXY0dcNidV+lQGKKP0IOf/UCCAmGYWjOevfx/OnnMLUfABDCKqb2h8pe/goVU/yzKPoRWij6AYSE9VkFysovUZTdpgFtEswOBwAAc7jKpaxv3MtNLzM3lrqWPFiy2qXDm6WC38yOBqgzFP0AQsJ3R/fyX3B2Y0XabSZHAwCASfb/JDnypIgEqVFvs6OpW/b6UuPz3cvs7UcIoegHEBIqjucf0jHJ5EgAADBRxaX6mgyTrCH4JXhTLt2H0EPRDyDo7cg9ok17D8tmtegPbSn6AQAhbM/Roj/Ujuev0ORo0b//R8lx2NxYgDpC0Q8g6M3Z4J7a36dVQ8VF202OBgAAkxzeIhX8KlnCpOR0s6MxR+zZUsxZkssh5WSYHQ1QJwKu6H/99deVlpamyMhI9ejRQwsWnPg6m/PmzZPFYql2+/XXX6u0+/TTT9WhQwdFRESoQ4cO+uyzz3zdDQB1aO7GfZKk9A6ctR8AEMIq9vInXiCFx5kbi5mY4o8QE1BF/8yZMzVu3Dg99NBDWrVqlQYMGKCLL75YmZmZJ33dpk2blJ2dXXlr06ZN5XOLFy/WiBEjNHLkSK1Zs0YjR47U8OHDtXTpUl93B0AdyC92aMXOQ5KkP7RLNDkaAABMFKqX6jtexRT/rNmS4TI3FqAOBFTR/8ILL2jUqFG6/fbb1b59e02ZMkWpqal64403Tvq6xMREJScnV95stt9PWjJlyhQNHjxYEyZMULt27TRhwgQNGjRIU6ZM8XFvANSFhZtz5XQZat24nlIbRpsdDgAA5ig9IO2b715udrm5sZgt8XwprJ5UkiMdWmV2NIDPhZkdQG2VlZVpxYoVeuCBB6qsT09P16JFi0762m7duqmkpEQdOnTQww8/rAsvvLDyucWLF+u+++6r0n7IkCEnLfpLS0tVWlpa+bigoECS5HA45HA4atulgFPRt2DuozeRL8/5Imffbzx6qb42CUH5f8HnzHOhkrNg75+vMdYHbx+9LVByZsn8XGGGU0ZcZ5VHpEomxmt+zqyyJQ6SNesLOXd9IVf9zibFUTvm5yvwhErOatu/gCn6c3Nz5XQ6lZRU9czbSUlJysnJqfE1KSkpevvtt9WjRw+VlpZq+vTpGjRokObNm6fzz3dfozMnJ8ejbUrS5MmTNWnSpGrr58yZo+jo4N+TmJHBSU88Qb48562cuQwpY51NkkWReds0e/ZWr2zXH/E581yw56yoqMjsEAIaY31w/3z4gr/nrFfJ20qRtKmogzbNnm12OJLMzVlzR6q6SSrY+KF+3NHdtDg84e+fMX8U7Dmr7VgfMEV/BYvFUuWxYRjV1lVo27at2rZtW/m4b9++2rVrl5577rnKot/TbUrShAkTNH78+MrHBQUFSk1NVXp6umJjYz3qTyBxOBzKyMjQ4MGDZbdzBvRTIV+e83bO1u0p0OElS1Qv3Kax116kiLCAOqKpVviceS5UclaxZxqnh7E+uH8+vCkgclZeqLBZ10mSWg+8X63jzd2z7Rc5K+4qffWa4l1bNOwPPaVI/z3vj1/kK8CESs5qO9YHTNGfkJAgm81WbQ/8vn37qu2pP5k+ffpoxowZlY+Tk5M93mZERIQiIiKqrbfb7UH9oaoQKv30FvLlOW/lbOHWg5Kk/mclKCaq+s9sMOFz5rlgz1kw960uMNaHRj+9ya9zlv295CqRYlrJntBdOsnOrbpkas7sLaQG3WU5tFL2fXOk1reaE4cH/Poz5qeCPWe17VvA7PYKDw9Xjx49qk3RyMjIUL9+/Wq9nVWrViklJaXycd++fattc86cOR5tE4B/+mGT+1J9F3LWfgBAKNt19HLUqVf7TcHvF5pd4b7f9V9z4wB8LGD29EvS+PHjNXLkSPXs2VN9+/bV22+/rczMTI0ZM0aSeyrenj179N5770lyn5m/ZcuWOuecc1RWVqYZM2bo008/1aefflq5zXvvvVfnn3++nn76aV1xxRWaNWuW5s6dq4ULF5rSRwDecfBImVbtypMkDWzb2NxgAAAwi7NMyvrKvdzsKnNj8TfNr5HWPirlzJHK8qXwOLMjAnwioIr+ESNG6MCBA3rssceUnZ2tjh07avbs2WrRooUkKTs7W5mZmZXty8rKdP/992vPnj2KiorSOeeco6+//lrDhg2rbNOvXz999NFHevjhh/XII4+odevWmjlzpnr37l3n/QPgPQs275dhSO2S6yslLsrscAAAMMfe/0mOAikyWUroY3Y0/iWugxTbXirYKO35Skq70eyIAJ8IqKJfksaOHauxY8fW+Ny0adOqPP773/+uv//976fc5jXXXKNrrrnGG+EB8BM//MrUfgAAtPvo1P5mV0iWgDmyt+40v0Za97i06z8U/Qha/OQDCDoul6GFW3IlSReczdR+AECIcjml3bPcy6lXmxuLv0o9uuMv6xvJcdjcWAAfoegHEHQ27T2s3MIyRdlt6t68gdnhAABgjtxFUsleyR4nJQ40Oxr/FN9Jqt9GcpVKWbPNjgbwCYp+AEFn4Wb3Xv7erRoqPIxfcwCAELVzpvu+2RWSLdzcWPyVxfL73v7M/5gbC+Aj/DUMIOhUTO0/76wEkyMBAMAkrnJp1yfu5RbXmRuLv2teMcV/tlR+xNxYAB+g6AcQVErLnVq6/YAk6bw2FP0AgBC1b75Usk8KbyglX2R2NP6tQTepXprkLJKyvjU7GsDrKPoBBJWVO/NU4nApISZCbZPqmx0OAADm2PmR+z71j5LVbm4s/s5i+X1vf+Yn5sYC+ABFP4CgsnDLfknSeWc1ksViMTkaAABM4CyTdv3XvczU/tqpOK5/z5eSo9DcWAAvo+gHEFQWbnFP7e/P8fwAgFCVM1cqOyhFJkmJF5gdTWBodK4U09o9xb/iModAkKDoBxA08oscWrs7TxLH8wMAQljm0bP2N79WstrMjSVQWCxSyxvdyzveNzcWwMso+gEEjcXbcuUypNaN6yklLsrscAAAqHvOEmnXZ+7l5iPMjSXQVBT9OXPcJ0EEggRFP4CgsWAzl+oDAIS4rNlS+WEpupnUuJ/Z0QSW2LOlhj0lwynt/NjsaACvoegHEDR+2nK06G/T2ORIAAAwyfb33Pctrpcs/KnvMab4IwjxmwBAUNh9qEg7DhTJZrWod6uGZocDAEDdK9kv7fnavZx2s7mxBKoW17m/LDmwRDq81exoAK+g6AcQFBZvdZ+1v3OzOMVGcj1iAEAI2vmhZJRLDXtI8eeYHU1gikqWkga5l9nbjyBB0Q8gKCzZdlCS1KdVI5MjAQDAJBVT+9nLf2Za3uS+3/G+ZBjmxgJ4AUU/gKCwZJt7Tz9FPwAgJOWtlw6ukCxh7inqOH2pV0m2aOnwb1LuIrOjAc4YRT+AgLfrYJH25BXLZrWoZ4sGZocDAEDd2/6u+77pJVIkJ7Q9I/b6UoujlzvcOtXcWAAvoOgHEPCWbndP7e/cLE71IsJMjgYAgDrmKpd2zHAvM7XfO1qPct9nfiw5DpsbC3CGKPoBBDym9gMAQlrWbKk4W4pIkJoMMzua4JDQT4ptK5UfkXbONDsa4IxQ9AMIeBT9AICQtuUt932rWyRbhKmhBA2LRWp1dG8/U/wR4Cj6AQS0XQeLtPsQx/MDAELUkUwp6xv3cuvR5sYSbNL+5D4x4oElUv4Gs6MBThtFP4CAxvH8AICQtvXfkgwp6UIp9myzowkuUUlS00vdy1v+ZW4swBmg6AcQ0JYytR8AEKpc5b9PPT/rz+bGEqzOusN9v22a+/h+IABR9AMIaEu2U/QDAEJU1tdScZb7BH7NrjQ7muCUMkSKaS058qQdH5gdDXBaKPoBBKzdh4q06yDH8wMAQtTmN9z3rW7lBH6+YrFKbca6l397VTIMc+MBTgNFP4CAtXQbx/MDAEJU/kYp+ztJFqb2+1rrWyVblJT3i7T/J7OjATxG0Q8gYHGpPgBAyNr0kvu+2RVS/dbmxhLswhtILW9yL//2qrmxAKeBoh9AwFq+85AkqVfLhiZHAgBAHSo9IG1/z73cdpypoYSMs+903+/6VCrKMjcWwEMU/QAC0v7Dpdqee0QWi9Sd4/kBAKFky78kZ7HUoKuUeL7Z0YSGBl2kxgMko1z67RWzowE8QtEPICAt3+E+nr9tUn3FRdlNjgYAgDricvw+xbztOMliMTWckNL+fvf95jckR4G5sQAeoOgHEJCW7XBP7T+Xqf0AgFCy82OpeI8UmSS1uM7saEJL00ul2HaSI1/a8rbZ0QC1RtEPICAt3+ne09+zJVP7AQAhwnBJGya7l8++i8v01TWLVWr/N/fyry9KzjJz4wFqiaIfQMA5Ulqu9VnuaXXs6QcAhIzds6T89ZI91l30o+61vFGKaiIVZ0k7PzA7GqBWAq7of/3115WWlqbIyEj16NFDCxYsOGHb//73vxo8eLAaN26s2NhY9e3bV999912VNtOmTZPFYql2Kykp8XVXAJym1bvy5HQZahofpSbxUWaHAwCA7xmGtP5J9/LZd0nh8aaGE7JsEb9fMWHDM+7ZF4CfC6iif+bMmRo3bpweeughrVq1SgMGDNDFF1+szMzMGtv/+OOPGjx4sGbPnq0VK1bowgsv1GWXXaZVq1ZVaRcbG6vs7Owqt8jIyLroEoDTsOzoSfzOZWo/ACBUZM+RDq6QbNFcps9sbf4s2eOlgo3ucywAfi7M7AA88cILL2jUqFG6/fbbJUlTpkzRd999pzfeeEOTJ0+u1n7KlClVHj/11FOaNWuWvvzyS3Xr1q1yvcViUXJycq3jKC0tVWlpaeXjggL3NGOHwyGHw+FJlwJKRd+CuY/eRL48V9uc/bz9gCSpW2pcyOeXz5nnQiVnwd4/X2OsD94+eltd5cy27glZJTlbjZbLFi8F8P9R4H/OomQ9e5xs6yfKWPuoylOukKy+K6sCP191L1RyVtv+BUzRX1ZWphUrVuiBBx6osj49PV2LFi2q1TZcLpcOHz6shg2rHgNcWFioFi1ayOl0qmvXrnr88cerfClwvMmTJ2vSpEnV1s+ZM0fR0dG1iiWQZWRkmB1CQCFfnjtZzpyGtHy7TZJFRZlrNTt3bd0F5sf4nHku2HNWVFRkdggBjbE+uH8+fMGXOWvsXK1+JQvlVJjm7u6skqzZPnuvuhTIn7Mw42wNVn2FH/5Nv3w5QbvtF/r8PQM5X2YJ9pzVdqy3GIZh+DgWr8jKylLTpk31008/qV+/fpXrn3rqKb377rvatGnTKbfx7LPP6p///Kc2btyoxMRESdKSJUu0ZcsWderUSQUFBXrppZc0e/ZsrVmzRm3atKlxOzV9+5+amqrc3FzFxsaeYU/9l8PhUEZGhgYPHiy7neuinwr58lxtcrZ2T76ufnOpYiPDtGzChbJaQ/v6xHzOPBcqOSsoKFBCQoLy8/ODemzyFcb64P758Caf58xwyfZ9P1kPrZSzzd1ydX3e++9Rx4Llc2b99VnZ1j4ko15rlQ/9RbL6pi/Bkq+6FCo5q+1YHzB7+itYLFX/wDcMo9q6mnz44YeaOHGiZs2aVVnwS1KfPn3Up0+fysf9+/dX9+7d9corr+jll1+ucVsRERGKiKh+iRS73R7UH6oKodJPbyFfnjtZzlbtPixJ6tmyoSIiwusyLL/G58xzwZ6zYO5bXWCsD41+epPPcrbzY+nQSiksRrZOj8gWRP8vAf85a3eP9NsUWY5slX33h1LrUT59u4DPlwmCPWe17VvAnMgvISFBNptNOTk5Vdbv27dPSUlJJ33tzJkzNWrUKH388ce66KKLTtrWarXq3HPP1ebNm884ZgDet/zoSfx6chI/AECwczmkXx52L7e/X4psbG48qMoeI3U4eujxL49K5UfMjQc4gYAp+sPDw9WjR49qx2VkZGRUme5/vA8//FC33HKLPvjgA11yySWnfB/DMLR69WqlpKScccwAvMswDC3bcUiSdG7LhqdoDQBAgNv6jnR4sxTRWGo33uxoUJOzx0r1WkrFe6SNz5kdDVCjgCn6JWn8+PH697//rXfeeUcbN27Ufffdp8zMTI0ZM0aSNGHCBP3pT3+qbP/hhx/qT3/6k55//nn16dNHOTk5ysnJUX5+fmWbSZMm6bvvvtO2bdu0evVqjRo1SqtXr67cJgD/sfNAkXILSxVus6pT0zizwwEAwHfK8qW1/+de7viwZK9vbjyomS1S6vq0e3nDM1LRHnPjAWoQUEX/iBEjNGXKFD322GPq2rWrfvzxR82ePVstWrSQJGVnZyszM7Oy/VtvvaXy8nLdeeedSklJqbzde++9lW3y8vJ0xx13qH379kpPT9eePXv0448/qlevXnXePwAn9/PRqf1dUuMUabeZHA0AAD60dqJUsk+KbSudxc4ov9b8Wqlxf8lZJK150OxogGoC7kR+Y8eO1dixY2t8btq0aVUez5s375Tbe/HFF/Xiiy96ITIAvvb78fxM7QcABLG8ddJvr7iXe7ws2ThxrV+zWKTuL0rf9ZK2vyedfZfU6FyzowIqBdSefgChbXnl8fycxA8AEKQMQ1p+t2Q4pdSrpZR0syNCbTQ6V2o50r38858lV7m58QDHoOgHEBAOHSnTtlz3WXG7N6foBwAEqZ0fSfvmuY8V7/a82dHAE92elcIbSIdWSZteMjsaoBJFP4CAsGqXey9/q8b1FB/NNEcAQBAq2SetuMe93OFBKaalqeHAQ1FJ7sJfkn75P6lwh6nhABUo+gEEhJU78ySxlx8AEMSW3yWV5krxnaUO/zA7GpyOVrdJiRe4T+q3bKz7cA3AZBT9AALCykz3nn6KfgBAUMr8VMr8RLLYpD7TOHlfoLJYpF5vSdZwKfsbaetUsyMCKPoB+D+ny9CaXXmSpO4t4k2NBQAAryvZJy0/enWqDhOkht3MjQdnJrat1OVJ9/KKe6WCzebGg5BH0Q/A723KOawjZU7FRISpTWJ9s8MBAMB7DJe0aKS78I/rKHV82OyI4A3txktJF7qn+S+6UXI5zI4IIYyiH4Dfq5ja3yU1TjarxeRoAADwog3PSDlzJFuUdN5MyRZhdkTwBotV6vOuZI+XDi6T1k40OyKEMIp+AH6P4/kBAEFp/0/SL0f37Pd8TYrrYG488K56qe7j+yVp/VPS7i/MjQchi6IfgN9blZkniaIfABBEirKkhcMlwym1vElqdYvZEcEXWgyXzr7Lvbx4pFSwydx4EJIo+gH4tUNHyrQ994gkqVvzeHODAQDAG8qLpB+vkIqz3Hv3z33dfdZ3BKfuL0iNz5McBdKPV0mOw2ZHhBBD0Q/Ar63a5Z7a36pxPcVHc/kiAECAMwxpya3SweVSRCPpgi8lOyepDWpWu3TeJ1JUE6lgo7TwWk7shzpF0Q/Ar63cmSeJqf0AgCCx5kEp82N3ITjgv1JMK7MjQl2ISpYGfCbZoqXs76Slo91fAAF1gKIfgF/jJH4AgKCx4Wlpwz/dy+e+JSWeb248qFsJvaTzPpYsNmn7u7+fxBHwMYp+AH7L6TK0ZleeJKl7i3hTYwEA4Iz89rq0+gH3ctdnpNa3mhsPzNH0EqnX2+7l9U9J654wNx6EBIp+AH5rU85hHSlzKiYiTG0SOd4RABCgNr8hLT96BvdzHpI6/M3ceGCu1rdJXY/O+PjlEWntJHPjQdCj6Afgtyqm9ndJjZPNylmNAQABaP0/pWVjJRlS23ulzo+bHRH8QYd//F74r50orXmIY/zhMxT9APwWx/MDAAKW4ZJW/V1aM8H9+JyHpe4vcmk+/K7DP9yHekjuqf5LbpWcZebGhKBE0Q/Ab63KzJNE0Q8ACDCOQmnB1dLGZ92Puz0rdXmcgh/Vdfib1Out30/uN2+YVJZvdlQIMhT9APzSoSNl2p57RJLUrXm8ucEAAFBbhTukjP7S7lmSNVzq+57U/n6zo4I/O+sO6fwvpLB60t7vpW97Sod+MTsqBBGKfgB+adUu99T+Vo3rKT463ORoAACohcxPpG+6Snm/SJFJ0kXzpbSRZkeFQNB0mHTRj1J0c6lwizSnj7TtXbOjQpCg6Afgl1buzJPE1H4AgP8LM4plW/4XaeFwyZEvNeojDVkmJfQxOzQEkobdpYtXSilDJWextOQW92eqdL/ZkSHAUfQD8EucxA8AEAgsWV/rwuK7Zd0+VZJFOudBafCPUr1Us0NDIIpoJA382n2VB0uYlPmJwr7rqpTyRWZHhgBG0Q/A7zhdhtbsypMkdW8Rb2osAADU6MguaeEIhf10laKNXBn10qQ/zJW6PClZ7WZHh0BmsUodH5aGLJXiOspSul+9Sp+RbcHlUv6vZkeHAETRD8Dv/La3UEfKnIqJCFObxPpmhwMAwO/K8qXVE6SvzpYyP5Yhqzbbr1R5+kop+Q9mR4dg0rC7NHS5nO3+IZfCZM35VprdSVoxXirJNTs6BBCKfgB+Z9XRvfxdUuNks3J5IwCAHyjLc19L/cvW0oZ/Ss4SqfEAlV+0WBvCb3GfeR3wNluEXJ0e1/+iXpIrZZhklEubXpS+SJNWP0jxj1qh6Afgd1ZXTO3neH4AgNmO7JJW/UP6vLm05iGp9IAU2959ibWL5ksNupkdIULAEWtTOc/7XBr4jdSgq1ReKG2Y7C7+l98tFWwyO0T4sTCzAwCA463alS+Joh8AYBJXuZT1jbTlLSn7G8lwudfHdZQ6PCC1GCFZ+TMaJmgyVEoZIu35Ulo7UTq0SvrtVfctebDU5i9Sk2GSLcLsSOFH+G0FwK8UOqQdB4okSd2ax5sbDAAgdLicUu5PUuYnUuZ/pJKc359LulBqe5/U9BL3SdYAM1ksUrPLpaaXSXu/lza94v4SICfDfbPHSy2GSy1ukBqfJ1ltZkcMk1H0A/ArOwvdx/C3alxP8dHhJkcDAAhqZYeknP+5C6U9X0jF2b8/F9FISrtFOusOKfZs00IETshikZIvct8Kt0ub35R2zJCKs6Qtb7tvEY2klGFS00ullHQpPN7sqGECin4AfmXHYXfRz9R+AIDXleyXDiyVche7i/2DP/8+dV+S7HFSsyul5sPdhZSNL58RIGLSpG5PS12ekvbNdxf/uz5zn4Nix3T3zWJ1n4Oi8flS4vnuWQCRCWZHjjpA0Q/Ar2w/7L6n6AcAnDbDkIp2S/nrpPz10qHVUu4SqXBr9bax7aTkdPdx0smDOBYagc1qc186MvkPUq+3pdxF0p6v3NP/C36VDq5w3za96G5fL819acAG3dz38Z2lqCbuWQQIGgFX9L/++ut69tlnlZ2drXPOOUdTpkzRgAEDTth+/vz5Gj9+vNavX68mTZro73//u8aMGVOlzaeffqpHHnlEW7duVevWrfXkk0/qqquu8nVXABzH6TKUeXR6f/cW8eYGAwDwby6nexrzke1S4Q7pyA73cv6vUsEGyVFQ8+ti20sJfaTG/d3Ffr3UuowaqDvWMPce/cTzpW7PuL8I27dA2vejtP9HKX+D+2fmyHZp16e/vy4sRqp/tvuwlvptpfptpHotpOhmUnRTyWo3r084LQFV9M+cOVPjxo3T66+/rv79++utt97SxRdfrA0bNqh58+bV2m/fvl3Dhg3T6NGjNWPGDP30008aO3asGjdurD/+8Y+SpMWLF2vEiBF6/PHHddVVV+mzzz7T8OHDtXDhQvXu3buuuwiEtN/2FqrUZVG9CJvaJNY3OxwAQF1yOaXyw5IjXyrLlxx5UsleqXiv+/74W9Fu9zXLT8QS5i5a4jq6bwm9pUa9OKYZoSu6mdTyevdNcp/T4tBq6eBK9+3QSunwZvflAA8dfVyNRYpKlqJT3duLTJIiEo7eGv++HNnYfULBsHrMGvADHhf9t9xyi2677Tadf/75vojnpF544QWNGjVKt99+uyRpypQp+u677/TGG29o8uTJ1dq/+eabat68uaZMmSJJat++vZYvX67nnnuusuifMmWKBg8erAkTJkiSJkyYoPnz52vKlCn68MMP66ZjACRJq3blSZK6NI2TzcoAAQA+YxiSjKP3Lslwui9TV1ascCPffUK7Mou7qHY5jt6XS4bj6H0Nj52lkrO46q28uPo6Z7FUXuTeE+/I/73ILz/seT8sYVK95u4pyjEtpXotpZizpPiO7j2VHJMPnFh4A/eVKZIu/H2ds0wq3CYd3iQV/Oa+P7xFKtrl/qLNVeb+/VCcLR34+dTvYbG6Zw6E1ZfssZK9ftVlW7RkizzuFuW+tx633hru/pm3hlW9r2md01CYUej+AsMa5V4nizueEPwSwuOi//Dhw0pPT1dqaqpuvfVW3XzzzWratKkvYquirKxMK1as0AMPPFBlfXp6uhYtWlTjaxYvXqz09PQq64YMGaKpU6fK4XDIbrdr8eLFuu+++6q1qfiioCalpaUqLS2tfFxQ4J4+5nA45HA4POlWQKnoWzD30ZvIl+dW7jwkSerctD55qyU+Z54LlZwFe/98zZdjvXX132TNnq3fi+7jC/Bj17mqtqt4fML2p96GRcYJY7NLuliSvjqjLp4RwxrhPqGePU5GZKIU0VhGZJIUkShFJh1dlyQjutnRY49PcDkyl9xfWPhYqPxO8SZy5pm6zZdFim7tviUNq/qU4ZJK90vFe2Qp2iVL8R7349IDslTcl+VKpe6bxSh3v8ZR4L4V76mD+N3ski6RpM9qft6QRZVfAnjzXjrmSwVLDeuOvn9CfznP/dcZ97O2nwmPi/5PP/1UBw4c0IwZMzRt2jQ9+uijuuiiizRq1ChdccUVstt9c4xHbm6unE6nkpKSqqxPSkpSTk5Oja/JycmpsX15eblyc3OVkpJywjYn2qYkTZ48WZMmTaq2fs6cOYqOjq5tlwJWRkaG2SEEFPJVe4t+s0myyLV/m2bPruFkSzghPmeeC/acFRUVmR1CQPPlWN+jZKWaOTef0TZ8zZBVLlllyCZDNrlkk2E5ZrnG9WFyKlxOS7hcR++dipBT4XJZwiufcypcLkXIYYmSw1JP5aonhyVaDku0ylVPLou9Igip+OitmkNHb2vrKiWnFOy/U3yBnHnGv/Jll9Ty6O04NklRhmwqVZhRrDAVHb0vlv3ofZjhXmdTmaxyyGYcvVeZrIajcr3VKJNNDtlUKovhlFVOWeSURS5Z5JTVqHjsvrl/c53ksJ+jLFW+FHUv1qX9RfW0eP/sM95Obcf60zqmv1GjRrr33nt17733atWqVXrnnXc0cuRIxcTE6KabbtLYsWPVpk2b09n0KVmO/5bEMKqtO1X749d7us0JEyZo/PjxlY8LCgqUmpqq9PR0xcbGnroTAcrhcCgjI0ODBw/22Zc7wYR8eebgkTLtXzxPknTLZReocWzwf4HmDXzOPBcqOavYM43T49OxvqCVyktzj9lDdNweJ1lkVK6zVK6rqZ1n66w1bO/oOotNstjlcBrKmPuDBqcPqfLzYT2zHge1UPmd4k3kzDPkq3YMSc6jN0dZqeZmfKuL/jBQ9jDL0Vk/x89+OtP7o7OnDNcxl/40qt4bRo3rG9jjNCyu4xn3ubZj/RmdyC87O1tz5szRnDlzZLPZNGzYMK1fv14dOnTQM888U23a/JlISEiQzWartgd+37591fbUV0hOTq6xfVhYmBo1anTSNifapiRFREQoIqL65VzsdntI/CCGSj+9hXzVzvqcg5KkxEhDjWOjyZmH+Jx5LthzFsx9qws+HesbdTqz1/uSwyFZrEH/8+EL5Mxz5Mwz5MszhsUue1RcUOestn3z+Itbh8OhTz/9VJdeeqlatGihTz75RPfdd5+ys7P17rvvas6cOZo+fboee+wxj4M+mfDwcPXo0aPatJaMjAz169evxtf07du3Wvs5c+aoZ8+elQk6UZsTbROAb6zcmSdJalm/judXAQAAAEHM4z39KSkpcrlcuv766/Xzzz+ra9eu1doMGTJE8fHxXgivqvHjx2vkyJHq2bOn+vbtq7fffluZmZkaM2aMJPdUvD179ui9996TJI0ZM0avvvqqxo8fr9GjR2vx4sWaOnVqlbPy33vvvTr//PP19NNP64orrtCsWbM0d+5cLVy40OvxAzixlZnuk/ilUfQDAAAAXuNx0f/iiy/q2muvVWRk5AnbNGjQQNu3bz+jwGoyYsQIHThwQI899piys7PVsWNHzZ49Wy1atJDkPtwgMzOzsn1aWppmz56t++67T6+99pqaNGmil19+ufJyfZLUr18/ffTRR3r44Yf1yCOPqHXr1po5c6Z69+7t9fgB1MzpMrTm6OX6WsZQ9AMAAADe4nHRP3LkSF/EUWtjx47V2LFja3xu2rRp1dZdcMEFWrly5Um3ec011+iaa67xRngATsOmnMM6UuZUvXCbkqNPfcZVAAAAALXDyVgBmK5ian+XZnGynvjCGQAAAAA8RNEPwHQVRX/X1HhzAwEAAACCDEU/ANOtysyTJHVrHmduIAAAAECQoegHYKqDR8q0PfeIJKlrs3hzgwEAAACCDEU/AFOtOjq1v1XjeoqPtpscDQAAABBcKPoBmKrieP7uzRuYHAkAAAAQfCj6AZhq5c48SRT9AAAAgC9Q9AMwTbnTpTW78yRJ3VvEmxoLAAAAEIwo+gGYZtPewyoqcyomIkxtEuubHQ4AAAAQdCj6AZhmZeWl+uJls1rMDQYAAAAIQhT9AEyzaqf7JH7dOJ4fAAAA8AmKfgCm+f3M/fHmBgIAAAAEKYp+AKY4UFiqHQeKJEndUtnTDwAAAPgCRT8AU6w6ejz/WYkxiou2mxsMAAAAEKQo+gGYgqn9AAAAgO9R9AMwxe9FP1P7AQAAAF+h6AdQ58qdLq3ZlS9J6t6Coh8AAADwFYp+AHXu15zDKnY4VT8yTGc1jjE7HAAAACBoUfQDqHOrjk7t75oaL6vVYnI0AAAAQPCi6AdQ51YePXM/x/MDAAAAvkXRD6DOVZ7Ej+P5AQAAAJ+i6AdQp3ILS7XzQJEk9/R+AAAAAL5D0Q+gTq06OrW/TWKM4qLs5gYDAAAABDmKfgB1qnJqP8fzAwAAAD5H0Q+gTq3cWXE8f7y5gQAAAAAhgKIfQJ1xOF1asztPEnv6AQAAgLpA0Q+gzqzPKlCJw6X4aLtaN44xOxwAAAAg6FH0A6gzy3cclCT1aN5AVqvF5GgAAACA4EfRD6DOLN/hPp6/Z8uGJkcCAAAAhAaKfgB1wjAMLd9ZUfRzPD8AAABQFyj6AdSJnQeKlFtYqnCbVZ2axpkdDgAAABASKPoB1ImKvfydm8Up0m4zORoAAAAgNFD0A6gTlSfxY2o/AAAAUGcCpug/dOiQRo4cqbi4OMXFxWnkyJHKy8s7YXuHw6F//OMf6tSpk+rVq6cmTZroT3/6k7Kysqq0GzhwoCwWS5Xbdddd5+PeAKGnYk//uS04iR8AAABQVwKm6L/hhhu0evVqffvtt/r222+1evVqjRw58oTti4qKtHLlSj3yyCNauXKl/vvf/+q3337T5ZdfXq3t6NGjlZ2dXXl76623fNkVIOQcOlKmLfsKJUk9WrCnHwAAAKgrYWYHUBsbN27Ut99+qyVLlqh3796SpH/961/q27evNm3apLZt21Z7TVxcnDIyMqqse+WVV9SrVy9lZmaqefPmleujo6OVnJzs204AIWzF0b38ZyXGqEG9cJOjAQAAAEJHQBT9ixcvVlxcXGXBL0l9+vRRXFycFi1aVGPRX5P8/HxZLBbFx8dXWf/+++9rxowZSkpK0sUXX6xHH31U9evXP+F2SktLVVpaWvm4oKBAkvuQAofD4UHPAktF34K5j95Evn63dFuuJKl7atxJ80HOPEfOPBcqOQv2/vkaY33w9tHbyJnnyJlnyJfnQiVnte1fQBT9OTk5SkxMrLY+MTFROTk5tdpGSUmJHnjgAd1www2KjY2tXH/jjTcqLS1NycnJWrdunSZMmKA1a9ZUmyVwrMmTJ2vSpEnV1s+ZM0fR0dG1iieQnSw3qI58SXPX2SRZZM/L1OzZO0/Znpx5jpx5LthzVlRUZHYIAY2xPrh/PnyBnHmOnHmGfHku2HNW27HeYhiG4eNYTmjixIk1DqjHWrZsmebMmaN3331XmzZtqvJcmzZtNGrUKD3wwAMn3YbD4dC1116rzMxMzZs3r0rRf7wVK1aoZ8+eWrFihbp3715jm5q+/U9NTVVubu5Jtx3oHA6HMjIyNHjwYNntdrPD8Xvky63U4VS3J/8nh9PQ3PvOU4uGJ/5jmZx5jpx5LlRyVlBQoISEBOXn5wf12OQrjPXB/fPhTeTMc+TMM+TLc6GSs9qO9abu6b/rrrtOeab8li1b6pdfftHevXurPbd//34lJSWd9PUOh0PDhw/X9u3b9b///e+UA3X37t1lt9u1efPmExb9ERERioiIqLbebrcH9YeqQqj001tCPV+r9xyWw2koISZCrRNjZbFYTvmaUM/Z6SBnngv2nAVz3+oCY31o9NObyJnnyJlnyJfngj1nte2bqUV/QkKCEhISTtmub9++ys/P188//6xevXpJkpYuXar8/Hz169fvhK+rKPg3b96sH374QY0aNTrle61fv14Oh0MpKSm17wiAE1q+4+il+lo2qFXBDwAAAMB7AuKSfe3bt9fQoUM1evRoLVmyREuWLNHo0aN16aWXVjmJX7t27fTZZ59JksrLy3XNNddo+fLlev/99+V0OpWTk6OcnByVlZVJkrZu3arHHntMy5cv144dOzR79mxde+216tatm/r3729KX4Fgs3zHQUlcqg8AAAAwQ0AU/ZL7DPudOnVSenq60tPT1blzZ02fPr1Km02bNik/P1+StHv3bn3xxRfavXu3unbtqpSUlMrbokWLJEnh4eH6/vvvNWTIELVt21b33HOP0tPTNXfuXNlstjrvIxBsXC5DKzIr9vQ3NDkaAAAAIPQExNn7Jalhw4aaMWPGSdsce07Cli1b6lTnKExNTdX8+fO9Eh+A6rbsL1RekUNRdps6NAneE18BAAAA/ipg9vQDCDxLtx2QJHVvES+7jV83AAAAQF3jr3AAPrNku/t4/t5ppz6JJgAAAADvo+gH4BOGYWjptoqin+P5AQAAADNQ9APwiW25R5RbWKrwMKu6pMabHQ4AAAAQkij6AfhExV7+bqnxirRzNQwAAADADBT9AHxi6Xb3Sfx6t+J4fgAAAMAsFP0AvO7Y4/n7cDw/AAAAYBqKfgBel3mwSDkFJbLbLOrWvIHZ4QAAAAAhi6IfgNdV7OXvmhqvqHCO5wcAAADMQtEPwOuWbDt6PH8ax/MDAAAAZqLoB+B1S7e79/T3bsXx/AAAAICZKPoBeNWug0Xak1esMKtFPVpwPD8AAABgJop+AF5VsZe/U7M4RYeHmRwNAAAAENoo+gF4FcfzAwAAAP6Doh+A1xiGoUVbciVJfVtT9AMAAABmo+gH4DU7DhQpK79E4Tarzm3J8fwAAACA2Sj6AXjNT0f38ndrHs/x/AAAAIAfoOgH4DWLtrqL/v5nJZgcCQAAAACJoh+Al7hchhZvdZ/Er/9ZHM8PAAAA+AOKfgBesTGnQIeKHKoXblPnZvFmhwMAAABAFP0AvGTRlqOX6mvVSHYbv1oAAAAAf8Bf5gC84qejx/P341J9AAAAgN+g6AdwxsrKXfp5+0FJnMQPAAAA8CcU/QDO2JrdeSoqc6pRvXC1TapvdjgAAAAAjqLoB3DGftrintrft3UjWa0Wk6MBAAAAUIGiH8AZW7jZXfQztR8AAADwLxT9AM5IfpFDKzMPSZLOP7uxydEAAAAAOBZFP4Az8tPWXLkMqU1ijJrGR5kdDgAAAIBjUPQDOCPzN+2XJF3AXn4AAADA71D0AzhthmFo/m/uop+p/QAAAID/oegHcNp+21uonIISRdqt6pXW0OxwAAAAAByHoh/AaZv/2z5JUp9WjRRpt5kcDQAAAIDjUfQDOG0VU/s5nh8AAADwTwFT9B86dEgjR45UXFyc4uLiNHLkSOXl5Z30NbfccossFkuVW58+faq0KS0t1d13362EhATVq1dPl19+uXbv3u3DngDB4UhpuZZtd1+qj6IfAAAA8E8BU/TfcMMNWr16tb799lt9++23Wr16tUaOHHnK1w0dOlTZ2dmVt9mzZ1d5fty4cfrss8/00UcfaeHChSosLNSll14qp9Ppq64AQWHJtgMqc7qU2jBKaQn1zA4HAAAAQA3CzA6gNjZu3Khvv/1WS5YsUe/evSVJ//rXv9S3b19t2rRJbdu2PeFrIyIilJycXONz+fn5mjp1qqZPn66LLrpIkjRjxgylpqZq7ty5GjJkiPc7AwSJyrP2t2ksi8VicjQAAAAAahIQRf/ixYsVFxdXWfBLUp8+fRQXF6dFixadtOifN2+eEhMTFR8frwsuuEBPPvmkEhMTJUkrVqyQw+FQenp6ZfsmTZqoY8eOWrRo0QmL/tLSUpWWllY+LigokCQ5HA45HI4z6qs/q+hbMPfRm4I5X4Zh6H8b90qSzmvd0Gt9DOac+Qo581yo5CzY++drjPXB20dvI2eeI2eeIV+eC5Wc1bZ/AVH05+TkVBbqx0pMTFROTs4JX3fxxRfr2muvVYsWLbR9+3Y98sgj+sMf/qAVK1YoIiJCOTk5Cg8PV4MGDaq8Likp6aTbnTx5siZNmlRt/Zw5cxQdHe1BzwJTRkaG2SEElGDMV1aRtDsvTGEWQ4e3LNfs7d7dfjDmzNfImeeCPWdFRUVmhxDQGOuD++fDF8iZ58iZZ8iX54I9Z7Ud600t+idOnFjjgHqsZcuWSVKN04cNwzjptOIRI0ZULnfs2FE9e/ZUixYt9PXXX+vqq68+4etOtd0JEyZo/PjxlY8LCgqUmpqq9PR0xcbGnrQ/gczhcCgjI0ODBw+W3W43Oxy/F8z5emP+NklbdF6bxrrqsu5e224w58xXyJnnQiVnFXumcXoY64P758ObyJnnyJlnyJfnQiVntR3rTS3677rrLl133XUnbdOyZUv98ssv2rt3b7Xn9u/fr6SkpFq/X0pKilq0aKHNmzdLkpKTk1VWVqZDhw5V2du/b98+9evX74TbiYiIUERERLX1drs9qD9UFUKln94SjPn64bdcSdLgc5J90rdgzJmvkTPPBXvOgrlvdYGxPjT66U3kzHPkzDPky3PBnrPa9s3Uoj8hIUEJCQmnbNe3b1/l5+fr559/Vq9evSRJS5cuVX5+/kmL8+MdOHBAu3btUkpKiiSpR48estvtysjI0PDhwyVJ2dnZWrdunZ555pnT6BEQ/PYfLtXqXXmSpEHtav+lGwAAAIC6FxCX7Gvfvr2GDh2q0aNHa8mSJVqyZIlGjx6tSy+9tMpJ/Nq1a6fPPvtMklRYWKj7779fixcv1o4dOzRv3jxddtllSkhI0FVXXSVJiouL06hRo/TXv/5V33//vVatWqWbbrpJnTp1qjybP4Cqfvh1nwxD6tQ0TslxkWaHAwAAAOAkAuJEfpL0/vvv65577qk80/7ll1+uV199tUqbTZs2KT8/X5Jks9m0du1avffee8rLy1NKSoouvPBCzZw5U/Xr1698zYsvvqiwsDANHz5cxcXFGjRokKZNmyabzVZ3nQMCyNyjZ+0f1L76yTUBAAAA+JeAKfobNmyoGTNmnLSNYRiVy1FRUfruu+9Oud3IyEi98soreuWVV844RiDYlTicWrDZfTz/Re2Z2g8AAAD4u4CY3g/APyzeekDFDqdS4iJ1TpPgPXs1AAAAECwo+gHU2rfrciS5p/af7LKWAAAAAPwDRT+AWil3ujRng7vov7hjisnRAAAAAKgNin4AtbJ0+0EdKnKoQbRdvdMamh0OAAAAgFqg6AdQK9+sy5YkpXdIVpiNXx0AAABAIOAvdwCn5HQZ+nad+1J9F3dKNjkaAAAAALVF0Q/glFbsPKTcwlLVjwxTv9YJZocDAAAAoJYo+gGc0uy17qn9gzskKTyMXxsAAABAoOCvdwAn5XIZ+m49Z+0HAAAAAhFFP4CTWpl5SNn5JaoXbtOANkztBwAAAAIJRT+Ak/p89R5J0pCOyYq020yOBgAAAIAnKPoBnJDD6dLXv7iP57+ya1OTowEAAADgKYp+ACf042/7dajIoYSYCPVr3cjscAAAAAB4iKIfwAnNWp0lSbqsS4rCbPy6AAAAAAINf8UDqNGR0nJlbNgrSbqCqf0AAABAQKLoB1CjORtyVOxwqmWjaHVpFmd2OAAAAABOA0U/gBp9vso9tf+Krk1lsVhMjgYAAADA6aDoB1BNdn6xFmzeL0m6shtT+wEAAIBARdEPoJr/LN8tlyH1TmuotIR6ZocDAAAA4DRR9AOowuUyNHP5LknSiHNTTY4GAAAAwJmg6AdQxeJtB7T7ULHqR4Tp4o4pZocDAAAA4AxQ9AOoYuYy917+y7s2UVS4zeRoAAAAAJwJin4AlfKKyvTt+hxJ0nXnNjc5GgAAAABniqIfQKXPVu1RWblL7VNi1bFprNnhAAAAADhDFP0AJLlP4Dd98U5J0vW9UmWxWEyOCAAAAMCZougHIElasCVX23KPKCYiTFd3b2Z2OAAAAAC8gKIfgCTp3UU7JEnX9GimmIgwc4MBAAAA4BUU/QC0I/eIfti0T5J0c7+W5gYDAAAAwGso+gHovcU7ZRjSwLaNlZZQz+xwAAAAAHgJRT8Q4o6UluuT5bsksZcfAAAACDYU/UCI+/DnTB0uLVdaQj1d0Kax2eEAAAAA8CKKfiCElZY79e8F2yVJd5zfSlYrl+kDAAAAgglFPxDCPl+1RzkFJUqKjdDV3ZuaHQ4AAAAALwuYov/QoUMaOXKk4uLiFBcXp5EjRyovL++kr7FYLDXenn322co2AwcOrPb8dddd5+PeAOZzugy9NX+bJOn281opIsxmckQAAAAAvC1gLsZ9ww03aPfu3fr2228lSXfccYdGjhypL7/88oSvyc7OrvL4m2++0ahRo/THP/6xyvrRo0frscceq3wcFRXlxcgB//Td+hxtyz2iuCi7ru/d3OxwAAAAAPhAQBT9Gzdu1LfffqslS5aod+/ekqR//etf6tu3rzZt2qS2bdvW+Lrk5OQqj2fNmqULL7xQrVq1qrI+Ojq6WtuTKS0tVWlpaeXjgoICSZLD4ZDD4aj1dgJNRd+CuY/e5M/5MgxDr/2wWZJ0U+9URVgNv4jTn3Pmr8iZ50IlZ8HeP19jrA/ePnobOfMcOfMM+fJcqOSstv2zGIZh+DiWM/bOO+9o/Pjx1abzx8fH68UXX9Stt956ym3s3btXzZo107vvvqsbbrihcv3AgQO1fv16GYahpKQkXXzxxXr00UdVv379E25r4sSJmjRpUrX1H3zwgaKjo2vfMcAkaw5Y9M5vNkVYDf1fd6di7GZHBMDbioqKdMMNNyg/P1+xsbFmhxNwGOsBAP6utmN9QOzpz8nJUWJiYrX1iYmJysnJqdU23n33XdWvX19XX311lfU33nij0tLSlJycrHXr1mnChAlas2aNMjIyTritCRMmaPz48ZWPCwoKlJqaqvT09KD+w8rhcCgjI0ODBw+W3U6VeCr+mi+ny9Arry6SdESjBrTW8IvOMjukSv6aM39GzjwXKjmr2DON08NYH9w/H95EzjxHzjxDvjwXKjmr7VhvatF/om/Rj7Vs2TJJ7pPyHc8wjBrX1+Sdd97RjTfeqMjIyCrrR48eXbncsWNHtWnTRj179tTKlSvVvXv3GrcVERGhiIiIauvtdntQf6gqhEo/vcXf8vXVqt3asv+IYiPD9OeBZ/lVbBX8LWeBgJx5LthzFsx9qwuM9aHRT28iZ54jZ54hX54L9pzVtm+mFv133XXXKc+U37JlS/3yyy/au3dvtef279+vpKSkU77PggULtGnTJs2cOfOUbbt37y673a7NmzefsOgHApXD6dKUue5j+f98QWvFRQXvL0EAAAAAJhf9CQkJSkhIOGW7vn37Kj8/Xz///LN69eolSVq6dKny8/PVr1+/U75+6tSp6tGjh7p06XLKtuvXr5fD4VBKSsqpOwAEmBlLdmrngSIlxITr1v4tzQ4HAAAAgI9ZzQ6gNtq3b6+hQ4dq9OjRWrJkiZYsWaLRo0fr0ksvrXLm/nbt2umzzz6r8tqCggJ98sknuv3226ttd+vWrXrssce0fPly7dixQ7Nnz9a1116rbt26qX///j7vF1CXDh0pq9zLP35wW0WHB8QpPQAAAACcgYAo+iXp/fffV6dOnZSenq709HR17txZ06dPr9Jm06ZNys/Pr7Luo48+kmEYuv7666ttMzw8XN9//72GDBmitm3b6p577lF6errmzp0rm83m0/4Ade2l7zcrv9ihdsn1NeLcVLPDAQAAAFAHAmZXX8OGDTVjxoyTtqnp6oN33HGH7rjjjhrbp6amav78+V6JD/BnW/Yd1vQlOyVJ/3dpB9mstTsBJgAAAIDAFjB7+gGcHsMw9Mjn6+V0GbqofZL6nXXq82gAAAAACA4U/UCQ+3TlHi3edkCRdqsevayD2eEAAAAAqEMU/UAQO3ikTE9+vUGSdO+gs5XaMNrkiAAAAADUJYp+IIg98fUGHSpyn7zv9gFpZocDAAAAoI5R9ANB6rv1Ofrvyj2yWKSnru4ku40fdwAAACDUUAUAQWj/4VJN+O9aSdKfz2+t7s0bmBwRAAAAADNQ9ANBxjAMTfjvLzp4pEztkuvrvsFtzA4JAAAAgEko+oEgM3Xhds3duE/hNqumXNdVEWE2s0MCAAAAYBKKfiCILNtxUP/85ldJ0kOXtFe75FiTIwIAAABgJop+IEjsP1yqO99fqXKXocu6NNGf+rYwOyQAAAAAJqPoB4JAablTd36wUvsOl+qsxBj98+pOslgsZocFAAAAwGQU/UCAMwxDf//PL/p5+0HFRITpzZu6q15EmNlhAQAAAPADFP1AgHt+zm+atTpLNqtFr9/YXWcl1jc7JAAAAAB+gqIfCGDTl+zUqz9skSRNvqqTzj+7sckRAQAAAPAnFP1AgPrw50w98vk6SdLdfzhLw89NNTkiAAAAAP6Goh8IQB8v36UHP1srSbqtf5rGDz7b5IgAAAAA+COKfiDAvLtoh/7x6S8yDOnmvi30yKXtOVM/AAAAgBpxim8gQBiGoefmbNJrP2yV5C74J15+DgU/AAAAgBOi6AcCQInDqQc/W6v/rtwjSbo//WzdeeFZFPwAAAAAToqiH/Bze/KKNWb6Cq3dky+b1aKnruqoEec2NzssAAAAAAGAoh/wY/M27dP4j9fo4JEyNYi269Ubuqv/WQlmhwUAAAAgQFD0A36ouMypyd9s1HuLd0qSOjaN1Zs39VCzBtEmRwYAAAAgkFD0A35m8dYDeujztdq2/4gk6ZZ+LfXAxe0UabeZHBkAAACAQEPRD/iJfYdL9NTXG/X56ixJUlJshJ69povOP7uxyZEBAAAACFQU/YDJCkvL9f8WbtfbP27T4dJyWSzSjb2b62/p7RQXbTc7PAAAAAABjKIfMElRWbk+WJqpN+Zt1YEjZZKkzs3i9MSVHdW5Wby5wQEAAAAIChT9QB3bW1Cidxft0PtLM5Vf7JAktWwUrfsGn63LOjeR1WoxOUIAAAAAwYKiH6gDTpehhVty9Z8Vu/Xtumw5nIYkqXnDaP1lYGtd06OZ7DaryVECAAAACDYU/YCPGIahnYXSs3N+0xdrcpRTUFL5XK+WDXXbeWka3CFJNvbsAwAAAPARin7Ai4rKyrVsxyHN3bBXczbkaG9BmKQdkqT4aLuu6NJE1/ZMVcemcabGCQAAACA0UPQDZ+BwiUNrd+dr8bYDWrz1gNbszqucui9JEVZDF7ZP1uVdm2pQ+0RFhNlMjBYAAABAqKHoB2rBMAwdOFKmLfsKtSGrQGv35OuX3XnalntEhlG1bdP4KJ1/doL+0DZB+b8t0xWXdpHdzqX3AAAAANS9gCn6n3zySX399ddavXq1wsPDlZeXd8rXGIahSZMm6e2339ahQ4fUu3dvvfbaazrnnHMq25SWlur+++/Xhx9+qOLiYg0aNEivv/66mjVr5sPewB+5XIb2F5ZqT16xsvKKtftQsbbtL9TW/Ue0ZV9h5Zn2j9c0Pkrntmygvq0bqW+rBKU2jJLFYpHD4dDsLXXcCQAAAAA4RsAU/WVlZbr22mvVt29fTZ06tVaveeaZZ/TCCy9o2rRpOvvss/XEE09o8ODB2rRpk+rXry9JGjdunL788kt99NFHatSokf7617/q0ksv1YoVK2SzMRU7kBmGoRKHS4dLHSooduhAYZkOHHHfDhaW6cCRUh04Uqbcw6XKyi9WTn5Jlan5x7NYpGYNotQ2qb46N4tXp2Zx6tQ0TgkxEXXYKwAAAACovYAp+idNmiRJmjZtWq3aG4ahKVOm6KGHHtLVV18tSXr33XeVlJSkDz74QH/+85+Vn5+vqVOnavr06broooskSTNmzFBqaqrmzp2rIUOG+KQvwczlMuQ0DDldhspd7nv3sksul1TuclWuq97GfV9W7lJpuVOlFfcOV7XlEsfvzx8pc+pIabkKS8pVWFquw0fvC0vL5XSduIivic1qUXJspJrER6pJfJTSEuqpdeMYnZUYo7SEeoq080UQAAAAgMARMEW/p7Zv366cnBylp6dXrouIiNAFF1ygRYsW6c9//rNWrFghh8NRpU2TJk3UsWNHLVq06IRFf2lpqUpLSysfFxQUSJIcDoccjpqngNfW9f/+WQXF5TJkyDCkipLVfdz47+sqjiOv0s4wqrR3tzu6rrL97+vcbYxjtv/7OqNK+9/XlZfb9MDy71XxymPfp9xlVDu+3R9YLFL9iDA1qheuhlVudjWqF65G9cKVEucu8hvHhCvMZj3BllxyOFy1ft+Kz8KZfiZCCTnzHDnzXKjkLNj752u+HOv9Waj8fHgTOfMcOfMM+fJcqOSstv0L2qI/JydHkpSUlFRlfVJSknbu3FnZJjw8XA0aNKjWpuL1NZk8eXLlzINjzZkzR9HR0WcU94Y9NhWV+/N12y2Sy3karzJktch9k35fPnqzWSSL3PdhVslulcIskt1q/P7YKtmPfd4qhVkMRdikyCo3Q5Fhvz+2WyWrpVxSSdWgyo7eDkl7d0t7zzw5NcrIyPDRloMXOfMcOfNcsOesqKjI7BACmi/H+kAQ7D8fvkDOPEfOPEO+PBfsOavtWG9q0T9x4sQaB9RjLVu2TD179jzt97BYqhbQhmFUW3e8U7WZMGGCxo8fX/m4oKBAqampSk9PV2xs7GnHKkmN2h+U02XIYpEssujYMI5dZ9HvfbMc/adinaWG9hWP3U2P3Yb7sWrYxrGvs8ii8vJy/fTTQg047zyF2cOqbMdqtSjMapHV4r63HXuzWGS1+vMXGb7hcDiUkZGhwYMHc/b+WiJnniNnnguVnFXsmcbp8eVY789C5efDm8iZ58iZZ8iX50IlZ7Ud600t+u+66y5dd911J23TsmXL09p2cnKyJPfe/JSUlMr1+/btq9z7n5ycrLKyMh06dKjK3v59+/apX79+J9x2RESEIiKqn7zNbref8YfqvLOTTt3IJA6HQ5sipbTE2KD+4fE2b3wuQg058xw581yw5yyY+1YXfDnWB4JQ6ac3kTPPkTPPkC/PBXvOats3U4v+hIQEJSQk+GTbaWlpSk5OVkZGhrp16ybJfQWA+fPn6+mnn5Yk9ejRQ3a7XRkZGRo+fLgkKTs7W+vWrdMzzzzjk7gAAAAAAKgrAXNMf2Zmpg4ePKjMzEw5nU6tXr1aknTWWWcpJiZGktSuXTtNnjxZV111lSwWi8aNG6ennnpKbdq0UZs2bfTUU08pOjpaN9xwgyQpLi5Oo0aN0l//+lc1atRIDRs21P33369OnTpVns0fAAAAAIBAFTBF///93//p3XffrXxcsff+hx9+0MCBAyVJmzZtUn5+fmWbv//97youLtbYsWN16NAh9e7dW3PmzFH9+vUr27z44osKCwvT8OHDVVxcrEGDBmnatGmy2bg0GwAAAAAgsAVM0T9t2jRNmzbtpG2M464XZ7FYNHHiRE2cOPGEr4mMjNQrr7yiV155xQtRAgAAAADgP050QXIAAAAAABDgKPoBAAAAAAhSFP0AAAAAAAQpin4AAAAAAIIURT8AAAAAAEGKoh8AAAAAgCBF0Q8AAAAAQJCi6AcAAAAAIEhR9AMAAAAAEKQo+gEAAAAACFJhZgcQDAzDkCQVFBSYHIlvORwOFRUVqaCgQHa73exw/B758hw58xw581yo5KxiTKoYo3BmGOtxIuTMc+TMM+TLc6GSs9qO9RT9XnD48GFJUmpqqsmRAABQ1eHDhxUXF2d2GAGPsR4A4K9ONdZbDHYBnDGXy6WsrCzVr19fFovF7HB8pqCgQKmpqdq1a5diY2PNDsfvkS/PkTPPkTPPhUrODMPQ4cOH1aRJE1mtHM13phjrcSLkzHPkzDPky3OhkrPajvXs6fcCq9WqZs2amR1GnYmNjQ3qHx5vI1+eI2eeI2eeC4WcsYffexjrcSrkzHPkzDPky3OhkLPajPV89Q8AAAAAQJCi6AcAAAAAIEhR9KPWIiIi9OijjyoiIsLsUAIC+fIcOfMcOfMcOQNOjJ8Pz5Ezz5Ezz5Avz5GzqjiRHwAAAAAAQYo9/QAAAAAABCmKfgAAAAAAghRFPwAAAAAAQYqiHwAAAACAIEXRjzNSWlqqrl27ymKxaPXq1WaH47d27NihUaNGKS0tTVFRUWrdurUeffRRlZWVmR2aX3n99deVlpamyMhI9ejRQwsWLDA7JL81efJknXvuuapfv74SExN15ZVXatOmTWaHFTAmT54si8WicePGmR0K4PcY62uHsb52GOtrj7H+zDHeu1H044z8/e9/V5MmTcwOw+/9+uuvcrlceuutt7R+/Xq9+OKLevPNN/Xggw+aHZrfmDlzpsaNG6eHHnpIq1at0oABA3TxxRcrMzPT7ND80vz583XnnXdqyZIlysjIUHl5udLT03XkyBGzQ/N7y5Yt09tvv63OnTubHQoQEBjra4ex/tQY6z3DWH9mGO+PYQCnafbs2Ua7du2M9evXG5KMVatWmR1SQHnmmWeMtLQ0s8PwG7169TLGjBlTZV27du2MBx54wKSIAsu+ffsMScb8+fPNDsWvHT582GjTpo2RkZFhXHDBBca9995rdkiAX2OsPzOM9VUx1p8ZxvraY7yvij39OC179+7V6NGjNX36dEVHR5sdTkDKz89Xw4YNzQ7DL5SVlWnFihVKT0+vsj49PV2LFi0yKarAkp+fL0l8pk7hzjvv1CWXXKKLLrrI7FAAv8dYf+YY63/HWH/mGOtrj/G+qjCzA0DgMQxDt9xyi8aMGaOePXtqx44dZocUcLZu3apXXnlFzz//vNmh+IXc3Fw5nU4lJSVVWZ+UlKScnByTogochmFo/PjxOu+889SxY0ezw/FbH330kVauXKlly5aZHQrg9xjrzxxjfVWM9WeGsb72GO+rY08/Kk2cOFEWi+Wkt+XLl+uVV15RQUGBJkyYYHbIpqttzo6VlZWloUOH6tprr9Xtt99uUuT+yWKxVHlsGEa1dajurrvu0i+//KIPP/zQ7FD81q5du3TvvfdqxowZioyMNDscwDSM9Z5jrPcuxvrTw1hfO4z3NbMYhmGYHQT8Q25urnJzc0/apmXLlrruuuv05ZdfVvkF7XQ6ZbPZdOONN+rdd9/1dah+o7Y5q/ilk5WVpQsvvFC9e/fWtGnTZLXyvZvknvIXHR2tTz75RFdddVXl+nvvvVerV6/W/PnzTYzOv9199936/PPP9eOPPyotLc3scPzW559/rquuuko2m61yndPplMVikdVqVWlpaZXngGDFWO85xnrvYKw/fYz1tcd4XzOKfngsMzNTBQUFlY+zsrI0ZMgQ/ec//1Hv3r3VrFkzE6PzX3v27NGFF16oHj16aMaMGSH5C+dkevfurR49euj111+vXNehQwddccUVmjx5somR+SfDMHT33Xfrs88+07x589SmTRuzQ/Jrhw8f1s6dO6usu/XWW9WuXTv94x//YKokcBzG+tPDWH9yjPWeYaz3HON9zTimHx5r3rx5lccxMTGSpNatW/NHwAlkZWVp4MCBat68uZ577jnt37+/8rnk5GQTI/Mf48eP18iRI9WzZ0/17dtXb7/9tjIzMzVmzBizQ/NLd955pz744APNmjVL9evXrzweMi4uTlFRUSZH53/q169fbaCvV6+eGjVqFLJ/AAAnw1jvOcb6U2Os9wxjvecY72tG0Q/UgTlz5mjLli3asmVLtT+WmGzjNmLECB04cECPPfaYsrOz1bFjR82ePVstWrQwOzS/9MYbb0iSBg4cWGX9//t//0+33HJL3QcEACGOsf7UGOs9w1gPb2F6PwAAAAAAQYoziwAAAAAAEKQo+gEAAAAACFIU/QAAAAAABCmKfgAAAAAAghRFPwAAAAAAQYqiHwAAAACAIEXRDwAAAABAkKLoBwAAAAAgSFH0AwAAAAAQpCj6AQAAAAAIUhT9AAAAAAAEKYp+AKbbv3+/kpOT9dRTT1WuW7p0qcLDwzVnzhwTIwMAAN7AWA+Yx2IYhmF2EAAwe/ZsXXnllVq0aJHatWunbt266ZJLLtGUKVPMDg0AAHgBYz1gDop+AH7jzjvv1Ny5c3XuuedqzZo1WrZsmSIjI80OCwAAeAljPVD3KPoB+I3i4mJ17NhRu3bt0vLly9W5c2ezQwIAAF7EWA/UPY7pB+A3tm3bpqysLLlcLu3cudPscAAAgJcx1gN1jz39APxCWVmZevXqpa5du6pdu3Z64YUXtHbtWiUlJZkdGgAA8ALGesAcFP0A/MLf/vY3/ec//9GaNWsUExOjCy+8UPXr19dXX31ldmgAAMALGOsBczC9H4Dp5s2bpylTpmj69OmKjY2V1WrV9OnTtXDhQr3xxhtmhwcAAM4QYz1gHvb0AwAAAAAQpNjTDwAAAABAkKLoBwAAAAAgSFH0AwAAAAAQpCj6AQAAAAAIUhT9AAAAAAAEKYp+AAAAAACCFEU/AAAAAABBiqIfAAAAAIAgRdEPAAAAAECQougHAAAAACBIUfQDAAAAABCk/j+/Xmgqyt+aegAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def tanh(x):\n", + " return np.tanh(x) \n", + "\n", + "def tanh_derivative(x):\n", + " return 1-tanh(x)**2\n", + "\n", + "plot_function_and_derivative(tanh, tanh_derivative, \"tanh\", (-5, 5))" + ] + }, + { + "cell_type": "markdown", + "id": "9357658a", + "metadata": {}, + "source": [ + "**Eigenschaften:**\n", + "\n", + "* Mappt auf den Bereich $(-1,1)$\n", + "* Ist eine skalierte Version der Sigmoid Funktion $\\sigma(x)$. Es gilt $\\tanh(x)=2\\sigma(2x)-1$" + ] + }, + { + "cell_type": "markdown", + "id": "42f3fd83", + "metadata": {}, + "source": [ + "**Vorteile:**\n", + "\n", + "* Einfache Ableitung (welche aus dem Funktionswert berechnet werden kann)\n", + "* Ableitung ist nicht mehr beschränkt durch $0.25$ sondern liegt jetzt im Bereich $(0,1]$" + ] + }, + { + "cell_type": "markdown", + "id": "65ca5812", + "metadata": {}, + "source": [ + "**Nachteile:**\n", + "\n", + "* Nach wie vor die meisten Werte der Ableitung kleiner 1, somit zu klein\n", + "* Auch hier ist die Ableitung relativ schnell $0$ für Werte, die nicht im Bereich von $0$ liegen (Normalisieren!)" + ] + }, + { + "cell_type": "markdown", + "id": "1bdd02bc", + "metadata": {}, + "source": [ + "> **Übung:** Zeigen Sie, dass für die Ableitung der *tanh* Funktion $f(x)=\\tanh(x)$, $f'(x)=1-\\tanh^2(x)$ gilt." + ] + }, + { + "cell_type": "markdown", + "id": "cbb673a3", + "metadata": {}, + "source": [ + "### ReLU" + ] + }, + { + "cell_type": "markdown", + "id": "7aa1ebd7", + "metadata": {}, + "source": [ + "\\begin{align*}\n", + " f(x) &= \\max(0, x) = \\begin{cases}x & x>0, \\\\ 0 & \\text{sonst}.\\end{cases}\\\\\n", + " f'(x) &= \\begin{cases} 1 & x > 0, \\\\ 0 & \\text{sonst}. \\end{cases}\n", + "\\end{align*}" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "fb27ee69", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA9wAAAHUCAYAAADInCBZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABRAElEQVR4nO3deXxU9b3/8fdkMknIxh7WEBACQYGCoIgIBpQgeG313loVpWLdqIggKiD8rmKroKjgdsWtFatFqWhd2cKuKMiiqDQEAsGwGsKShKyTmfP7g5IaEiBDZvKd5fV8PPBhTk4m7/PJ8p13zswZm2VZlgAAAAAAgFeFmQ4AAAAAAEAwonADAAAAAOADFG4AAAAAAHyAwg0AAAAAgA9QuAEAAAAA8AEKNwAAAAAAPkDhBgAAAADAByjcAAAAAAD4AIUbAAAAAAAfoHADfmbu3Lmy2WyV/8LDw9WqVSvdeOON2rFjh8e3t2rVKtlsNi1YsOC0+9hsNt177701vm/BggWy2WxatWpVtfddccUVGj16tMeZ/vKXv6hNmzYqKiry+GMBAAhUgbDGjxo1SqmpqdX23blzpyIjI/X11197nHPgwIEaP368xx8HBAMKN+Cn3nzzTX399ddatmyZ7r33Xn3yySe67LLLdPToUdPRJEkff/yx1q5dq//93//1+GNvvfVWxcTEaObMmT5IBgCAf/P3Nb4mDz74oIYMGaJ+/fp5/LF//vOf9fLLLyszM9MHyQD/RuEG/FS3bt10ySWXKDU1VVOnTtXkyZOVm5urjz76yHQ0SdL06dN13XXXqU2bNh5/bHh4uO6++249//zzKi4u9kE6AAD8l7+v8afKyMjQRx99pLFjx57Tx19++eXq0qWLnn32WS8nA/wfhRsIEH369JEk/fzzz5XbNm7cqF//+tdq0qSJoqKi1KtXL/3jH//weZZvv/1W33zzjUaOHFm5zbIsDR8+XE2bNlVOTk7l9uLiYl1wwQXq2rVrlYeQ33zzzSooKNB7773n87wAAPgzf1rjazJnzhy1bNlSQ4YMqdy2Y8cOxcfH6/rrr6+y74oVK2S326s9Am7kyJGaN2+eCgsL6yUz4C8o3ECAyM7OliR17txZkrRy5Ur1799fx44d0yuvvKKPP/5YPXv21A033KC5c+f6NMtnn30mu92ugQMHVm6z2Wx6++23FR0drd/97ndyOp2SpHvuuUfZ2dn6xz/+oZiYmMr9W7ZsqZSUFH3++ec+zQoAgL/zpzV+7ty51a7b8vnnn2vgwIEKC/tPdUhOTtbrr7+uBQsW6IUXXpAkHTx4UCNGjNCAAQM0bdq0KreRmpqqoqKiGq8JAwSzcNMBANTM5XKpoqJCpaWlWrt2rR5//HENHDhQv/71ryWdKLIXXHCBVqxYofDwEz/KQ4cOVV5enqZMmaLf//73VRZGb/r666+VnJys2NjYKtubNm2q9957T6mpqZo4caJ69Oiht956S2+88Ya6d+9e7XYuvPBCLVu2zCcZAQDwV/68xp8qNzdXu3bt0l133VXtfTfccINWr16thx56SBdffLGmTp0qy7L07rvvym63V9m3V69estlsWrt2ra655pp6yQ74Awo34KcuueSSKm937dpVH3/8scLDw5WVlaVt27bpmWeekSRVVFRU7jd8+HB99tlnyszMVNeuXX2Sbf/+/UpISKjxff3799cTTzyhSZMmKTIyUrfccotuv/32GvdNSEhQbm6uKioqKu9QAAAQ7Px5jT/V/v37Jem06/7s2bO1bt06DRo0SOXl5Vq8eLFatWpVbT+Hw6FGjRpp3759Ps0L+BseUg74qb/97W/asGGDVqxYobvvvlsZGRm66aabJP3nOV4PPvigHA5HlX/33HOPJCkvL6/Wn8tut8vlctX4vpMLvcPhqNxWUlKiqKio097ezTffrIiICJWVlemhhx467X5RUVGyLEulpaW1zgoAQKDz5zX+VCUlJZJ02nU/MjJSI0aMUGlpqXr27Fnled6nioqKqrw9IFRwSgnwU127dq28iMqgQYPkcrn0xhtvaMGCBZUPz3744Yf13//93zV+fJcuXWr9uVq0aHHavzif3N6iRYvKbc2aNdORI0dq3N/lcunmm29W48aNFRkZqdtvv11r165VREREtX2PHDmiyMjIag9NBwAgmPnzGn+qZs2aSdJp1/0ff/xRjzzyiC666CJt2LBBs2bN0oQJE2rc9+jRo5W3B4QKCjcQIGbOnKkPPvhAjzzyiH788UclJydry5Ytmj59ep1v+8orr9SHH36oQ4cOqXnz5pXbLcvS+++/r/bt26tTp06V21NSUk770iWPPvqovvjiCy1dulQxMTEaOHCgHnroIT3//PPV9t21a5fOP//8OucHACCQ+dMaf6qkpCQ1aNBAO3furPa+oqIiXX/99Wrfvr1WrlypyZMna/Lkyerfv7/69u1bZd/9+/ertLSUdR8hh8INBIjGjRvr4Ycf1sSJEzVv3jy9+uqrGjZsmIYOHapRo0apTZs2OnLkiDIyMrR582a9//77VT5+3bp1Nd7u5ZdfrkceeUSffvqp+vbtq8mTJys5OVkHDx7U66+/rg0bNlR7GZLU1FT99a9/1fbt2yuvqCpJ6enpmjFjhv73f/9XV1xxhSRpxowZevDBB5Wamqrrrruucl+3261vvvnmtM/vBgAgVPjTGn+qiIgI9evXr8bPMXr0aOXk5Oibb75RTEyMnn32WX399de68cYb9e2336pRo0bVMg4aNMjD6QABzgLgV958801LkrVhw4Zq7yspKbHatWtnJScnWxUVFdaWLVus3/3ud1ZCQoLlcDisli1bWoMHD7ZeeeWVyo9ZuXKlJem0/1auXGlZlmXt2LHDuuWWW6xWrVpZ4eHhVqNGjay0tDRr+fLl1XLk5+dbsbGx1syZMyu37d+/30pISLAGDx5suVyuyu1ut9u65pprrEaNGlnZ2dmV25cvX25JsjZt2uSFqQEA4P8CYY2vyV/+8hfLbrdb+/fvr9z2+uuvW5KsN998s8q+WVlZVnx8vHXttddW2T5y5Eire/futZwUEDxslmVZ9V3yAQS+sWPHavny5dq6datsNpvHHz9y5Ejt2rVLa9eu9UE6AADgLaWlpWrXrp0eeOABTZo0yeOPLygoUOvWrTV79mzdeeedPkgI+C+uUg7gnPy///f/tG/fPn3wwQcef+zOnTs1f/58PfXUUz5IBgAAvCkqKkqPPfaYZs2apaKiIo8/fvbs2WrXrp1uu+02H6QD/BvP4QZwTlq0aKG///3vOnr0qMcfm5OTo5deekmXXXaZD5IBAABvu+uuu3Ts2DHt2rWr8krqtRUfH6+5c+cqPJzqgdDDQ8oBAAAAAPABHlIOAAAAAIAPULgBAAAAAPABCjcAAAAAAD4Q0FcucLvd2r9/v+Li4s7pZYkAAPA2y7JUWFio1q1bKyyMv2vXFWs9AMDfeLLWB3Th3r9/vxITE03HAACgmj179qht27amYwQ81noAgL+qzVof0IU7Li5O0okDjY+PN5zGd5xOp5YuXaq0tDQ5HA7TcQICM/McM/MM8/JcqMysoKBAiYmJlWsU6oa1HqfDzDzHzDzHzDwXCjPzZK0P6MJ98qFl8fHxQb8IR0dHKz4+Pmi/ab2NmXmOmXmGeXku1GbGw5+9g7Uep8PMPMfMPMfMPBdKM6vNWs+TywAAAAAA8AEKNwAAAAAAPkDhBgAAAADABwL6Ody1YVmWKioq5HK5TEc5Z06nU+Hh4SotLQ3o4zjJbrcrPDyc5zcCAAAg5ARDPzmTYOgu3uwrQV24y8vLdeDAARUXF5uOUieWZally5bas2dP0JTU6OhotWrVShEREaajAAAAAPUiWPrJmQRLd/FWXwnawu12u5WdnS273a7WrVsrIiIiYL/gbrdbx48fV2xs7FlfWN3fWZal8vJyHTp0SNnZ2UpOTg74YwIAAADOJpj6yZkEenfxdl8J2sJdXl4ut9utxMRERUdHm45TJ263W+Xl5YqKigrIb9pTNWjQQA6HQz/99FPlcQEAAADBLJj6yZkEQ3fxZl8JzAl4IFC/yMGOrwsAAABCEfeDA4O3vk58tQEAAAAA8AEKNwAAAAAAPmC0cE+bNk02m63Kv5YtW5qMhFNkZmaqZcuWKiwsrNX+ubm5at68ufbt2+fjZAAAAABC3YoVK5SSkiK3212r/X/44Qe1bdtWRUVFPk52gvEz3BdccIEOHDhQ+e+HH34wHcm4UaNGVf4BIjw8XO3bt9eECRN09OjRWt+GzWbTRx99VG377t27ZbPZ9N1331V737XXXqtRo0ZV2TZ16lSNGTNGcXFxtfq8CQkJGjlypB599NFaZwUAAADgv07tJ+3atdMf//jHeu8nqampmjt3bpV9Jk6cqKlTp9b6Odfdu3fXxRdfrNmzZ9c6e10YL9zh4eFq2bJl5b/mzZubjuQXrrrqKh04cEC7d+/Wa6+9piVLlmjMmDH1mmHv3r365JNPdNttt3n0cbfddpv+/ve/e/QDCAAmFZVVmI4AAIBf+2U/eeONN/Tpp5/qnnvuMZrpq6++0o4dO3T99dd79HG33Xab5syZI5fL5aNk/2H8ZcF27Nih1q1bKzIyUn379tX06dN13nnn1bhvWVmZysrKKt8uKCiQJDmdTjmdzir7Op1OWZYlt9td+fACy7JU4vT9UE/VwGH36DX2LMtSRESEEhISJEmtWrXSddddp3fffbfyWN58800988wzys7OVvv27TV27Fj98Y9/rHI7vzz2X2473fssy6qcmSTNnz9fv/rVr9S6devKbbfffrs2bdqk9evXKzIyUk6nU5deeqm6dOmid955R9KJRy20bNlSH3zwgf7whz/UeIxut1uWZcnpdMput9d6NrV18vvh1O8LnB4z8wzz8py/ziw7r0j/8+p6jezbTuMGd1RYWN1eE9Xfji/QeLLWBxN//fnwZ8zMc8zMc96cWU39RJYluYrrfNvnxB4t1bKjnNpPWrdurd/97nd66623auwn7dq103333VetkNe1n5y637vvvqshQ4YoIiKisl8MHTpUdrtdCxculM1m07Fjx9SzZ0/dcsstevzxxyVJQ4YM0eHDh7Vy5UoNHjy4xmM+U1/x5PvBaOHu27ev/va3v6lz5876+eef9fjjj+vSSy/V1q1b1bRp02r7z5gxQ4899li17UuXLq32WnYnz5wfP35c5eXlkqSScpf6zVrnm4M5g68nXKIGEbUvlU6nUxUVFZV3Mnbv3q3ly5crPDxcBQUFeuutt/Tkk09q5syZ6tGjh77//nuNGzdOYWFhuummmypvp6SkpPI2Tjp+/LgkqaioqNr7Kioq5HQ6K7evWLFC3bt3r7Lfn//8Zw0YMEAPPPCApk+frmnTpik3N1cffvhhlf169eqllStX6re//W2Nx1heXq6SkhKtWbNGFRW+O7OUnp7us9sOVszMM8zLc/42s79khqmwNEyrv89Sl/Ltdb694mJDd5yChCdrfTDyt5+PQMDMPMfMPOeNmdXUT1RRpEZL29b5ts/FsbS9UnhMrfatqZ8sWrTorP3Ebrd7tZ9UVFSotLS08u1Vq1bpf/7nf6p83AsvvKD+/fvr6aef1ujRo3XHHXeoWbNmuv/++6vs161bNy1fvlx9+vSp8ZjP1Fc8WeuNFu5hw4ZV/n/37t3Vr18/dezYUW+99ZYmTJhQbf+HH364yvaCggIlJiYqLS1N8fHxVfYtLS3Vnj17FBsbW/lC5eHlZh4yGBcfp+iI2o/a4XBoyZIlatu2rVwul0pLSyVJzzzzjOLj4/Xss8/qmWeeqfzm7d69u3bv3q23335bd999d+XtNGjQoNpcYmNjJUkxMTHV3hceHi6Hw1G5fd++ferbt2+V/eLj4/XOO+9o0KBBatq0qf7v//5P6enpSkxMrHJbSUlJ+u6776p9jpNKS0vVoEEDDRw4sE4vJH86TqdT6enpGjJkiBwOh9dvPxgxM88wL8/548w2/XRU33+9QWE26elbLlNyQmydb/PUOwvwjCdrfTDxx58Pf8fMPMfMPOfNmdXUT1Th/Ud61lZ8fHytC/fp+smzzz5brZ9YlqWkpCRlZ2d7vZ+sWbOmyvv37NmjDh06VOsrr7zyim699Vbl5+dryZIl2rRpU7UTuomJiTpw4MA59RVP1nrjDyn/pZiYGHXv3l07duyo8f2RkZGKjIystt3hcFT7AXC5XLLZbAoLC6t8An1MpEP/+tNQ7wc/C08fUm6z2TRo0CDNmTNHxcXFev3115WRkaGxY8fq8OHD2rNnj+68884q37wVFRVq2LBhlYsF/PLYf7ntdO87eSGEk9tLSkrUoEGDavv1799fDz74oB5//HFNmjRJqamp1Y4hOjpaxcXFp714QVhYmGw2W41fO2/y9e0HI2bmGeblOX+ZmWVZmrn0xHpzw0WJOr9NY6/crj8cWyDzZK0PRqFynN7EzDzHzDznjZnV1E/kiJV+d9wLCT0X5sFDyk/tJ2+88Ya2b9+u++67r177yalKSkoUHR1d7f033HCDPv74Yz355JOaM2eOUlJSqn1sdHS0SkpKzqmvePK94FeFu6ysTBkZGRowYIBPbt9ms3l0ptmkmJgYderUSZL0/PPP6/LLL9ef/vQnjR07VpL0+uuvq2/fvlU+pjbPhW7YsKEkKT8/v9r7jh07pqSkpMq3mzVrVuOFz9xut9auXSu73X7aP44cOXKEC+AB8GuLfzyozTnHFB1h1/1XdjYdBwAQimy2Wp9lNu2X/eSFF17QoEGD9Nhjj+nee++V9J9+4na7dfz4ccXGxtaqmHrST051ur5SXFysTZs2nbWvdOzY8az56sroVcoffPBBrV69WtnZ2Vq/fr1++9vfqqCgQLfeeqvJWH5p0qRJevbZZ+VyudSmTRvt2rVLnTp1qvKvQ4cOZ72dxo0bq3nz5tqwYUOV7SUlJdq6dau6dOlSua1Xr17617/+Ve02nn76aWVkZGj16tVasmSJ3nzzzWr7/Pjjj+rVq9c5HCkA+F55hVtPLd4mSbpzwHlKiPf+U1sAAAhmjz76qJ555pka+8l5553nk35yqtP1lQceeEBhYWFatGiRXnjhBa1YsaLaPvXVV4ye7t27d69uuukm5eXlqXnz5rrkkku0bt26M/4VI1RddtlluuCCCyovVHbfffcpPj5ew4YNU1lZmTZu3KijR49Wed5bdnZ2tdez69Spkx588EFNnz5dLVq00KWXXqqjR4/qqaeeUnh4uG655ZbKfYcOHao77rhDLper8uz5d999p0ceeUQLFixQ//799fzzz2vcuHG6/PLLK68uf/IvStOnT/f9YADgHLz7TY52Hy5Ws9hI3TWw5lfGAAAAp5eamlpjPxk6dKgOHz6sbdu2KT8/36v95FRDhw7VW2+9VWXb559/rr/+9a/6+uuvdeGFF2ry5Mm69dZb9f3336tx4xNPH9u9e7f27dunK6+80nsDOQ2jhfu9994z+ekDzvjx43X77bcrKytLb7zxhp5++mlNnDix8rnv48ePr7J/TReeW7lypR588EHFxsbqmWee0c6dO9WoUSNdcskl+uKLL6pcNGD48OFyOBxatmyZhg4dqtLSUt18880aNWqUrrnmGkknXibs888/18iRI7VmzRrZ7XZ9/PHHateunc+eGgAAdVFQ6tTzy088vOz+IcmKiQyMpxoBAOBvJkyYoNtuu61aP4mOjlaPHj283k9Odcstt2jSpEnKzMxUly5ddOjQId1+++2aNm2aLrzwQkknzsQvXbpUo0eP1vz58yWdeDmxtLS0ejnRy70MPzR37twat48YMaLyLzwjRozQiBEjTnsblmWd8XPcc889Z32hervdrilTpmjWrFkaOnSooqKitHXr1mr7ffjhh1Xenj17th555JEz3jYAmPLq6p06UlSujs1jdEOfxLN/AAAAIe5M/eRkJzn5/263WwUFBYqPj69yQTJv9JNTNW7cWPfee69mzZqlV199Vc2bN9fBgwer7BMeHq7169dXvl1WVqY5c+bo3Xff9ehznSujz+GG/7vrrrs0cOBAFRYW1mr/3Nxc/fa3v63yensA4C8O5JfojS+yJUmTh3VVuJ1lEACAQDZ16lQlJSXJ5XLVav+ffvpJU6dOVf/+/X2c7ATOcOOMwsPDNXXq1Frvn5CQoIkTJ/owEQCcu1lLt6uswq2L2zfRlV0TTMcBAAB11LBhQ02ZMqXW+3fu3FmdO9ffq5Pwp30AQEjIOFCgBZv3SpKmXN1Vtlq+9igAAMC5onADAELCk4u2ybKkq3u0Us/ERqbjAACAEBD0hftsT86HGXxdANSnL3fkafX2Q3LYbZo49PSv5wkAgK9xPzgweOvrFLSF2+FwSDrxmtDwPye/Lie/TgDgK263pekLMyRJt1ySpKSmMYYTAQBCEf0ksHirrwTtRdPsdrsaNWqk3NxcSVJ0dHTAPl/P7XarvLxcpaWlVS6tH4gsy1JxcbFyc3PVqFEj2e1205EABLmPvtunfx0oUFxUuO4bnGw6DgAgRAVTPzmTQO8u3u4rQVu4Jally5aSVPlNHagsy1JJSYkaNGgQND+UjRo1qvz6AICvlDpdemZJpiRpzKBOahwTYTgRACCUBUs/OZNg6S7e6itBXbhtNptatWqlhIQEOZ1O03HOmdPp1Jo1azRw4MCgeAi2w+HgzDaAejH3q93an1+q1g2jNOrS9qbjAABCXLD0kzMJhu7izb4S1IX7JLvdHtAFz263q6KiQlFRUQH7TQsA9e1oUbn+b2WWJOnBoV0U5QjcdQAAEFwCvZ+cCd2lqsB7UD0AALXw4oosFZZW6PxW8bq2ZxvTcQAAQAiicAMAgs5Ph4v09rrdkqQpw7sqLCxwn0MGAAACF4UbABB0nl6SKafL0sDOzXVZcjPTcQAAQIiicAMAgsp3e47ps+8PyGaTHh6WYjoOAAAIYRRuAEDQsCxL0xdmSJL+58K26toq3nAiAAAQyijcAICgsSwjV99kH1FkeJgeSOtsOg4AAAhxFG4AQFCocLn15KITZ7dvv6yDWjVsYDgRAAAIdRRuAEBQmL9xj3YeKlKTmAiNTu1oOg4AAACFGwAQ+I6XVWh2+g5J0n2DOyk+ymE4EQAAAIUbABAEXl+zS3nHy9S+abRG9E0yHQcAAEAShRsAEOByC0r12ppdkqRJV6UoIpylDQAA+AfulQAAAtrsZTtU4nTpwnaNdFW3lqbjAAAAVKJwAwAC1o6fCzV/Q44kacrwrrLZbIYTAQAA/AeFGwAQsJ5avE1uSxp6QQv1ad/EdBwAAIAqKNwAgIC0btdhLcvIlT3MpklXpZiOAwAAUA2FGwAQcNxuS9MXZkiSRlzcTuc1jzWcCAAAoDoKNwAg4Hz2wwF9vzdfMRF2jbsy2XQcAACAGlG4AQABpazCpaeXbJMkjb68o5rFRhpOBAAAUDMKNwAgoLz99U/ac6RELeIjdceA80zHAQAAOC0KNwAgYOQXO/XiiixJ0oQhndUgwm44EQAAwOlRuAEAAePlVVnKL3GqS4s4/bZ3ouk4AAAAZ0ThBgAEhL1Hi/XmV7slSZOHp8geZjMbCAAA4Cwo3ACAgPDs0u0qr3Dr0o5Nldq5uek4AAAAZ0XhBgD4vR/35euf3+6TJE0Z3lU2G2e3AQCA/6NwAwD8mmVZmr4wQ5J0bc/W6tamoeFEAAAAtUPhBgD4tVXbD+mrnYcVYQ/Tg0O7mI4DAABQaxRuAIDfcrktPblwmyRpVP/2ats42nAiAACA2qNwAwD81geb9irz50I1bODQmNROpuMAAAB4hMINAPBLJeUuPZueKUkaO7iTGkY7DCcCAADwDIUbAOCX/vLlLv1cUKa2jRtoZL8k03EAAAA8RuEGAPidvONlemX1LknSQ0O7KDLcbjgRAACA5yjcAAC/88LyHTpeVqEebRvqmh6tTccBAAA4JxRuAIBf2XXouOatz5EkPTysq8LCbIYTAQAAnBsKNwDAr8xcnKkKt6UrUhLUr2NT03EAAADOGYUbAOA3Nu4+osVbDyrMJk0elmI6DgAAQJ1QuAEAfsGyLE1fmCFJuuGiRCW3iDOcCAAAoG4o3AAAv7D4x4PanHNMDRx23X9lZ9NxAAAA6ozCDQAwzuly66nF2yRJdw48TwnxUYYTAQAA1B2FGwBg3Lz1Odp9uFjNYiN118DzTMcBAADwCgo3AMCowlKnnl++Q5I0/spkxUaGG04EAADgHRRuAIBRr6zeqSNF5TqveYxuvCjRdBwAAACvoXADAIw5kF+iN77IliRNvipF4XaWJQAAEDy4ZwMAMGbW0u0qq3Dr4vZNNOT8FqbjAAAAeBWFGwBgRMaBAi3YvFeS9PDwFNlsNsOJAAAAvIvCDQAw4slF22RZ0tU9WqlXu8am4wAAAHgdhRsAUO++3JGn1dsPyWG3aeLQLqbjAAAA+ASFGwBQr9xuS9MXZkiSbrkkSUlNYwwnAgAA8A0KNwCgXn3y/QH960CB4qLCdd/gZNNxAAAAfMZvCveMGTNks9k0fvx401EAAD7idEuzlmVJku5J7aTGMRGGEwEAAPiOXxTuDRs26LXXXlOPHj1MRwEA+NCaAzYdyC9V64ZRuq1/e9NxAAAAfMp44T5+/Lhuvvlmvf7662rcmKvUAkCwOlpcrvR9J5adB9K6KMphN5wIAADAt8JNBxgzZoyuvvpqXXnllXr88cfPuG9ZWZnKysoq3y4oKJAkOZ1OOZ1On+Y06eSxBfMxehsz8xwz8wzz8txLK7JU4rKpS4tY/Ve3hKCdXbAeV31hrQ/eY/Q2ZuY5ZuY5Zua5UJiZJ8dmsyzL8mGWM3rvvff0xBNPaMOGDYqKilJqaqp69uyp5557rsb9p02bpscee6za9nnz5ik6OtrHaQEA5yqvVJr+nV0uy6Y/dnUppZGxpcfniouLNWLECOXn5ys+Pt50nIDDWg8A8HeerPXGCveePXvUp08fLV26VL/61a8k6ayFu6a/eicmJiovLy+o79Q4nU6lp6dryJAhcjgcpuMEBGbmOWbmGeblmfHzv9fnPx5USkO3Phx3RVDPrKCgQM2aNaNwnyPWen6n1BYz8xwz8xwz81wozMyTtd7YQ8o3bdqk3Nxc9e7du3Kby+XSmjVr9NJLL6msrEx2e9Xn90VGRioyMrLabTkcjqD9Yv5SqBynNzEzzzEzzzCvs/tuzzF9/uNB2WzSr5PcQT+zYD62+sBaHxrH6U3MzHPMzHPMzHPBPDNPjstY4b7iiiv0ww8/VNl22223KSUlRZMmTapWtgEAgceyLE1fmCFJuq5na7WJyjGcCAAAoP4YK9xxcXHq1q1blW0xMTFq2rRpte0AgMC0LCNX32QfUWR4mMZf0UnfrqVwAwCA0GH8ZcEAAMGpwuXWk4tOnN2+/bIOatUwynAiAACA+mX8ZcF+adWqVaYjAAC8ZP7GPdp5qEhNYiI0OrWj6TgAAAD1jjPcAACvKyqr0Oz0HZKk+wZ3UnxUcF40BQAA4Ewo3AAAr3ttzS7lHS9T+6bRGtE3yXQcAAAAIyjcAACvyi0o1etf7JIkTbwqRRHhLDUAACA0cS8IAOBVs5ftUHG5S73aNdKwbi1NxwEAADCGwg0A8Jqs3ELN33Dipb+mDu8qm81mOBEAAIA5FG4AgNc8uWib3JY09IIW6tO+iek4AAAARlG4AQBesW7XYS3LyJU9zKaJV6WYjgMAAGAchRsAUGdut6XpCzMkSSMubqeOzWMNJwIAADCPwg0AqLPPfjig7/fmKybCrvuuSDYdBwAAwC9QuAEAdVJW4dLTS7ZJkkZf3lHN4yINJwIAAPAPFG4AQJ28/fVP2nOkRAlxkbp9QAfTcQAAAPwGhRsAcM7yi516cUWWJOmBtM6Kjgg3nAgAAMB/ULgBAOfs5VVZyi9xqnOLWP22d6LpOAAAAH6Fwg0AOCd7jxbrza92S5IeHtZV9jCb2UAAAAB+hsINADgnzy7drvIKty7t2FSpXZqbjgMAAOB3KNwAAI/9uC9f//x2n6QTZ7dtNs5uAwAAnIrCDQDwiGVZmrEoQ5J0bc/W6t62oeFEAAAA/onCDQDwyOrth7Q267Ai7GF6IK2L6TgAAAB+i8INAKg1l9vSjIXbJEmj+rdXYpNow4kAAAD8F4UbAFBrH2zaq8yfC9WwgUNjUjuZjgMAAODXKNwAgFopKXfp2fRMSdLYwZ3UMNphOBEAAIB/o3ADAGrlL1/u0s8FZWrbuIFG9ksyHQcAAMDvUbgBAGeVd7xMr6zeJUl6aGgXRYbbDScCAADwfxRuAMBZvbB8h46XVahH24a6pkdr03EAAAACAoUbAHBGuw4d17z1OZKkh4d1VViYzXAiAACAwEDhBgCc0czFmapwW7oiJUH9OjY1HQcAACBgULgBAKe16acjWrz1oMJs0uRhKabjAAAABBQKNwCgRpZl6YnPMyRJN1yUqOQWcYYTAQAABBYKNwCgRku2HtTmnGNq4LDr/is7m44DAAAQcCjcAIBqnC63nlqcKUm6c+B5SoiPMpwIAAAg8FC4AQDVzFufo+y8IjWLjdBdA88zHQcAACAgUbgBAFUUljr1/PIdkqTxV3ZWbGS44UQAAACBicINAKjildU7daSoXOc1j9ENFyWajgMAABCwKNwAgEoH8kv0xhfZkqTJV6XIYWeZAAAAOFfckwIAVJq1dLvKKty6qH1jDTm/hek4AAAAAY3CDQCQJG07WKAFm/dKkqYM7yqbzWY4EQAAQGCjcAMAJEkzFm6TZUlX92ilXu0am44DAAAQ8CjcAAB9uSNPq7cfksNu08ShXUzHAQAACAoUbgAIcW63pRmLMiRJt1ySpKSmMYYTAQAABAcKNwCEuI+37NPW/QWKiwzX2MHJpuMAAAAEDQo3AISwUqdLzyzZLkm6Z1AnNYmJMJwIAAAgeFC4ASCEzf1qt/YdK1HrhlG6rX9703EAAACCCoUbAELU0aJy/d/KLEnSA2ldFOWwG04EAAAQXCjcABCiXlyRpcLSCnVtFa9re7UxHQcAACDoULgBIATlHC7W2+t2S5KmDE+RPcxmNhAAAEAQonADQAiauWSbnC5LA5KbaUByc9NxAAAAghKFGwBCzHd7jumz7w/IZpOmDO9qOg4AAEDQonADQAixLEvTF2ZIkv7nwrbq2irecCIAAIDgReEGgBCyPCNX32QfUWR4mB5I62w6DgAAQFCjcANAiKhwuTVj0Ymz27df1kGtGjYwnAgAACC4UbgBIET8Y+Ne7TxUpCYxERqd2tF0HAAAgKBH4QaAEFBUVqFZ6dslSfcN7qT4KIfhRAAAAMGPwg0AIeD1L3Yp73iZ2jeN1oi+SabjAAAAhAQKNwAEudzCUr22ZpckaeJVKYoI51c/AABAfeBeFwAEudnpO1Rc7lKvdo00rFtL03EAAABCBoUbAIJYVm6h5m/IkSRNGd5VNpvNcCIAAIDQQeEGgCD25KJtcltS2vktdFH7JqbjAAAAhBQKNwAEqXW7DmtZRq7sYTZNGpZiOg4AAEDIoXADQBByuy3NWJghSbrp4kR1bB5rOBEAAEDoMVq458yZox49eig+Pl7x8fHq16+fFi1aZDISAASFz384oC178xUTYde4KzqbjgMAABCSjBbutm3b6sknn9TGjRu1ceNGDR48WL/5zW+0detWk7EAIKCVVbg0c8k2SdLoyzuqeVyk4UQAAAChKdzkJ7/mmmuqvP3EE09ozpw5WrdunS644AJDqQAgsL2zLkd7jpQoIS5Stw/oYDoOAABAyDJauH/J5XLp/fffV1FRkfr161fjPmVlZSorK6t8u6CgQJLkdDrldDrrJacJJ48tmI/R25iZ55iZZ/x1XgUlTr24fIckafwVHeWwWX6T0V9n5m3Bfny+xlofvMfobczMc8zMc8zMc6EwM0+OzWZZluXDLGf1ww8/qF+/fiotLVVsbKzmzZun4cOH17jvtGnT9Nhjj1XbPm/ePEVHR/s6KgD4vU9+CtPy/WFq2cDSpF+5FMbLbte74uJijRgxQvn5+YqPjzcdJ+Cw1gMA/J0na73xwl1eXq6cnBwdO3ZMH3zwgd544w2tXr1a559/frV9a/qrd2JiovLy8oL6To3T6VR6erqGDBkih8NhOk5AYGaeY2ae8cd57TtWorTn16q8wq3XR/ZSaufmpiNV4Y8z84WCggI1a9aMwn2OWOuD++fDm5iZ55iZ55iZ50JhZp6s9cYfUh4REaFOnTpJkvr06aMNGzbo+eef16uvvlpt38jISEVGVr/4j8PhCNov5i+FynF6EzPzHDPzjD/N6/kVW1Ve4Va/85rqyvNbyWbzz9Pb/jQzXwjmY6sPrPWhcZzexMw8x8w8x8w8F8wz8+S4/O51uC3LqvKXbQDA2f24L1///HafJGnK8K5+W7YBAABCidEz3FOmTNGwYcOUmJiowsJCvffee1q1apUWL15sMhYABBTLsjRjUYYk6Tc9W6t724aGEwEAAEAyXLh//vlnjRw5UgcOHFDDhg3Vo0cPLV68WEOGDDEZCwACyurth7Q267Ai7GF6MK2L6TgAAAD4N6OF+y9/+YvJTw8AAc/ltjRj4TZJ0q2XJimxCVdxBgAA8Bd+9xxuAEDtfbB5rzJ/LlTDBg7dOyjZdBwAAAD8AoUbAAJUSblLzy7NlCTdO6iTGkYH55VAAQAAAhWFGwAC1F/XZuvngjK1bdxAv780yXQcAAAAnILCDQABKO94meas2ilJemhoF0WG2w0nAgAAwKko3AAQgF5cvkPHyyrUvU1DXdOjtek4AAAAqAGFGwACzK5Dx/X39TmSpIeHpygszGY4EQAAAGpC4QaAADNzcaYq3JYGpyTo0o7NTMcBAADAaVC4ASCAbPrpiBZvPagwmzR5WIrpOAAAADgDCjcABAjLsvTE5xmSpN/1SVTnFnGGEwEAAOBMKNwAECCWbD2ozTnH1MBh1/1DOpuOAwAAgLOgcANAAHC63HpqcaYk6c4BHdQiPspwIgAAAJwNhRsAAsC73+QoO69IzWIjdNflHU3HAQAAQC1QuAHAzxWWOvX8sh2SpHFXdlZsZLjhRAAAAKgNCjcA+LlXV+/S4aJyndc8RjdelGg6DgAAAGqJwg0Afuxgfqne+HKXJGnyVSly2Pm1DQAAECi45wYAfmxWeqZKnW5d1L6xhpzfwnQcAAAAeIDCDQB+atvBAr2/aa8kacrwrrLZbIYTAQAAwBMUbgDwU08u2ibLkq7u3kq92jU2HQcAAAAeonADgB9am5WnVZmH5LDbNPGqLqbjAAAA4BxQuAHAz7jdlqYvzJAk3dw3SUlNYwwnAgAAwLmgcAOAn/l4yz5t3V+guMhw3XdFsuk4AAAAOEcUbgDwI6VOl55Zsl2S9MdBHdUkJsJwIgAAAJwrCjcA+JG3vtqtfcdK1KphlP7Qv4PpOAAAAKgDCjcA+ImjReV6aWWWJOmBtC6KctgNJwIAAEBdULgBwE+8tDJLhaUV6toqXtf1amM6DgAAAOqIwg0AfiDncLH+9vVuSdLDw1JkD7OZDQQAAIA687hwjxo1SmvWrPFFFgAIWU8vzZTTZWlAcjMN7NzcdBwAAAB4gceFu7CwUGlpaUpOTtb06dO1b98+X+QCgJCxZc8xfbplv2w26eFhXU3HAQAAgJd4XLg/+OAD7du3T/fee6/ef/99tW/fXsOGDdOCBQvkdDp9kREAgpZlWZq+MEOS9N+92ur81vGGEwEAAMBbzuk53E2bNtW4ceP07bff6ptvvlGnTp00cuRItW7dWvfff7927Njh7ZwAEJSWZ+RqffYRRYaH6YG0zqbjAAAAwIvqdNG0AwcOaOnSpVq6dKnsdruGDx+urVu36vzzz9fs2bO9lREAglKFy60Zi06c3f7DZR3UulEDw4kAAADgTR4XbqfTqQ8++ED/9V//paSkJL3//vu6//77deDAAb311ltaunSp3n77bf3pT3/yRV4ACBr/2LhXOw8VqXG0Q39M7Wg6DgAAALws3NMPaNWqldxut2666SZ988036tmzZ7V9hg4dqkaNGnkhHgAEp6KyCs1K3y5Juu+KZMVHOQwnAgAAgLd5XLhnz56t66+/XlFRUafdp3HjxsrOzq5TMAAIZq9/sUt5x8uU1DRaN/dNMh0HAAAAPuBx4R45cqQvcgBAyMgtLNVra3ZJkiYOTVFEeJ0upwEAAAA/xb08AKhnzy3boeJyl3omNtLw7i1NxwEAAICPULgBoB5l5RZq/oY9kqSpV3eVzWYznAgAAAC+QuEGgHr05KJMudyW0s5voYvaNzEdBwAAAD5E4QaAerJ+12Ety/hZ9jCbJg1LMR0HAAAAPkbhBoB6YFmWpi/MkCTddHGiOjaPNZwIAAAAvkbhBoB68Nn3B7Rlb75iIuwad0Vn03EAAABQDyjcAOBjZRUuzVyyTZJ09+Ud1Twu0nAiAAAA1AcKNwD42DvrcrTnSIkS4iJ1x4AOpuMAAACgnlC4AcCH8kucenHFDknShCGdFR0RbjgRAAAA6guFGwB86OVVWTpW7FTnFrH6be+2puMAAACgHlG4AcBH9h4t1ptrd0uSJg9LUbidX7kAAAChhHt/AOAjs5ZuV3mFW/3Oa6pBXRJMxwEAAEA9o3ADgA/8uC9f//xunyRpyvCustlshhMBAACgvlG4AcDLLMvSjEUZsizpNz1bq3vbhqYjAQAAwAAKNwB42erth7Q267Ai7GF6MK2L6TgAAAAwhMINAF7kclt6ctE2SdKtlyYpsUm04UQAAAAwhcINAF70wea92nawUA0bOHTvoGTTcQAAAGAQhRsAvKSk3KVZS7dLku4d1EkNox2GEwEAAMAkCjcAeMlf12brYEGp2jZuoN9fmmQ6DgAAAAyjcAOAF+QdL9OcVTslSQ8N7aLIcLvhRAAAADCNwg0AXvDi8h06Xlah7m0a6poerU3HAQAAgB+gcANAHe06dFx/X58jSXp4eIrCwmyGEwEAAMAfULgBoI6eXpKpCrelwSkJurRjM9NxAAAA4Cco3ABQB5tzjmnRjwcVZpMmD0sxHQcAAAB+xGjhnjFjhi666CLFxcUpISFB1157rTIzM01GAoBasyzpqSUnXgbsd30S1blFnOFEAAAA8CdGC/fq1as1ZswYrVu3Tunp6aqoqFBaWpqKiopMxgKAWvn+iE2bc46pgcOu+4d0Nh0HAAAAfibc5CdfvHhxlbfffPNNJSQkaNOmTRo4cKChVABwdk6XW5/mnPib5Z0DOqhFfJThRAAAAPA3Rgv3qfLz8yVJTZo0qfH9ZWVlKisrq3y7oKBAkuR0OuV0On0f0JCTxxbMx+htzMxzzMwz89b/pEOlNjWNcei2S9sxt1oIle+xYD8+X2OtD95j9DZm5jlm5jlm5rlQmJknx2azLMvyYZZasyxLv/nNb3T06FF98cUXNe4zbdo0PfbYY9W2z5s3T9HR0b6OCACSpNIK6c/f2nW8wqbrO7h0WUu/+DUKP1FcXKwRI0YoPz9f8fHxpuMEHNZ6AIC/82St95vCPWbMGH3++ef68ssv1bZt2xr3qemv3omJicrLywvqOzVOp1Pp6ekaMmSIHA6H6TgBgZl5jpnV3qxlOzRndbYSoiylP5Cq6KhI05ECQqh8jxUUFKhZs2YU7nPEWh/cPx/exMw8x8w8x8w8Fwoz82St94uHlI8dO1affPKJ1qxZc9qyLUmRkZGKjKx+x9bhcATtF/OXQuU4vYmZeY6ZndnB/FK9+dVPkqRrktyKjopkXh4K9u+xYD62+sBaHxrH6U3MzHPMzHPMzHPBPDNPjsvoVcoty9K9996rDz/8UCtWrFCHDh1MxgGAs5qVnqlSp1t9khqpe2O/eIAQAAAA/JTRwj1mzBi98847mjdvnuLi4nTw4EEdPHhQJSUlJmMBQI22HSzQ+5v2SpImDu0sm81wIAAAAPg1o4V7zpw5ys/PV2pqqlq1alX5b/78+SZjAUCNnly0TZYlXd29lXolNjIdBwAAAH7O6HO4/eR6bQBwVmuz8rQq85AcdpseGtrFdBwAAAAEAKNnuAEgELjdlqYvzJAk3dw3Se2bxRhOBAAAgEBA4QaAs/h4yz5t3V+guMhw3XdFsuk4AAAACBAUbgA4g1KnS88s2S5J+uOgjmoSE2E4EQAAAAIFhRsAzuCtr3Zr37EStWoYpT/056ULAQAAUHsUbgA4jaNF5XppZZYk6YG0Lopy2A0nAgAAQCChcAPAaby0MkuFpRXq2ipe1/VqYzoOAAAAAgyFGwBqkHO4WH/7erck6eFhKbKH2cwGAgAAQMChcANADZ5emimny9KA5GYa2Lm56TgAAAAIQBRuADjFlj3H9OmW/bLZpIeHdTUdBwAAAAGKwg0Av2BZlqYvzJAk/Xevtjq/dbzhRAAAAAhUFG4A+IXlGblan31EkeFheiCts+k4AAAACGAUbgD4twqXW08u3iZJ+sNlHdS6UQPDiQAAABDIKNwA8G//2LhXWbnH1TjaoT+mdjQdBwAAAAGOwg0AkorKKjR72XZJ0n1XJCs+ymE4EQAAAAIdhRsAJL3+xS4dKixTUtNo3dw3yXQcAAAABAEKN4CQl1tYqtfW7JIkTRyaoohwfjUCAACg7rhXCSDkPbdsh4rLXeqZ2EjDu7c0HQcAAABBgsINIKRl5RZq/oY9kqSpV3eVzWYznAgAAADBgsINIKQ9uShTLreltPNb6KL2TUzHAQAAQBChcAMIWet3HdayjJ9lD7Np0rAU03EAAAAQZCjcAEKSZVmavjBDknTTxYnq2DzWcCIAAAAEGwo3gJD02fcHtGVvvmIi7Bp3RWfTcQAAABCEKNwAQk5ZhUszl2yTJN19eUc1j4s0nAgAAADBiMINIOS8sy5He46UKCEuUncM6GA6DgAAAIIUhRtASMkvcerFFTskSROGdFZ0RLjhRAAAAAhWFG4AIeXlVVk6VuxUckKsftu7rek4AAAACGIUbgAhY9+xEr25drck6eHhKQq38ysQAAAAvsO9TQAh49klmSqvcOuS85poUJcE03EAAAAQ5CjcAELCj/vy9c/v9kmSpg4/XzabzXAiAAAABDsKN4CgZ1mWnly0TZYl/aZna3Vv29B0JAAAAIQACjeAoLd6+yF9mZWnCHuYHkzrYjoOAAAAQgSFG0BQc7lPnN2WpFsvTVJik2jDiQAAABAqKNwAgtoHm/dq28FCxUeFa8ygTqbjAAAAIIRQuAEErZJyl2Yt3S5JGjs4WY2iIwwnAgAAQCihcAMIWn9dm62DBaVq06iBRvZLMh0HAAAAIYbCDSAoHT5epjmrdkqSJl7VRVEOu+FEAAAACDUUbgBB6YXlO3S8rELd2sTrmh6tTccBAABACKJwAwg62XlF+vv6HEnSlOFdFRZmM5wIAAAAoYjCDSDozFy8TRVuS4O6NNelHZuZjgMAAIAQReEGEFQ2/XRUi348qDCbNHlYV9NxAAAAEMIo3ACChmVZmr4wQ5J0fe9EdWkZZzgRAAAAQhmFG0DQWLL1oDb9dFQNHHZNSOtsOg4AAABCHIUbQFBwutx6anGmJOnOAR3UIj7KcCIAAACEOgo3gKDw7jc5ys4rUrPYCN11eUfTcQAAAAAKN4DAV1jq1PPLdkiSxl3ZWbGR4YYTAQAAABRuAEHg1dW7dLioXOc1i9GNFyWajgMAAABIonADCHAH80v1xpe7JEmThqXIYefXGgAAAPwD90wBBLRZ6ZkqdbrVJ6mx0s5vYToOAAAAUInCDSBgbTtYoAWb9kqSplzdVTabzXAiAAAA4D8o3AAC1pOLtsltScO7t9SF7RqbjgMAAABUQeEGEJDWZuVpVeYhhYfZNHFoiuk4AAAAQDUUbgABx+22NH1hhiTplkuS1L5ZjOFEAAAAQHUUbgAB55Mt+7V1f4HiIsN13xXJpuMAAAAANaJwAwgopU6Xnl6SKUn646COahITYTgRAAAAUDMKN4CA8tZXu7XvWIlaNYzSH/p3MB0HAAAAOC0KN4CAcbSoXC+tzJIkPZDWRVEOu+FEAAAAwOlRuAEEjJdWZqmwtEIpLeN0Xa82puMAAAAAZ0ThBhAQ9hwp1t++3i1JmjK8q+xhNrOBAAAAgLOgcAMICDOXZMrpsjQguZkGdm5uOg4AAABwVkYL95o1a3TNNdeodevWstls+uijj0zGAeCntuw5pk+37JfNJk0elmI6DgAAAFArRgt3UVGRfvWrX+mll14yGQOAH7MsS9MXZkiSruvVRhe0bmg4EQAAAFA74SY/+bBhwzRs2DCTEQD4uRXbcrU++4giwsP0YFoX03EAAACAWjNauD1VVlamsrKyyrcLCgokSU6nU06n01Qsnzt5bMF8jN7GzDznjzOrcLkrz26P6tdOzWPC/SafP87L34XKzIL9+HyNtT54j9HbmNk52DROqcULZV8yVZaNi4/Wht2ylFp8nJl5wK9nZgtXxZBv6nwznvzesVmWZdX5M3qBzWbTP//5T1177bWn3WfatGl67LHHqm2fN2+eoqOjfZgOgAlf/WzT/F12xYRb+t9eLjUIqD8RIlQVFxdrxIgRys/PV3x8vOk4AYe1HvCNcKtYVxePMB0DMMotuz6N+aDOt+PJWh9Qhbumv3onJiYqLy8vqO/UOJ1Opaena8iQIXI4HKbjBARm5jl/m1lRWYWGPPelDh0v19ThXTSqX5LpSFX427wCQajMrKCgQM2aNaNwnyPW+uD++fAmZuah8iNyfNxSklTa7yOFOyINBwoMFRUV2rx5sy688EKFh/OX/9rw75nZZLUYXOdb8WSt97cJnFFkZKQiI6v/cnA4HCHxizZUjtObmJnn/GVmc1dn69DxciU1jdatl54nR7h/voqhv8wrkAT7zIL52OoDa31oHKc3MbNacv/nbr+9zVUKj6Bw14bldOrQFpfsbYYqnO+zWgmFmXnyO8c/78ECCGm5haV6bc0uSdLEoSmK8NOyDQBAwKjyoFY/e14tEMSMnuE+fvy4srKyKt/Ozs7Wd999pyZNmqhdu3YGkwEw6bllO1Rc7lLPxEYa3r2l6TgAAAQB93/+198uZAUEMaOFe+PGjRo0aFDl2xMmTJAk3XrrrZo7d66hVABMysot1PwNeyRJU4Z3lY07BQAA1N2/z3BbnN0G6pXRwp2amio/uWYbAD/x5KJMudyWhpzfQhd3aGI6DgAAQeLEGW4KN1C/eGIkAL+xftdhLcv4WfYwmyZdlWI6DgAAwaPyJBeFG6hPFG4AfsGyLE1ftE2SdONFieqUEGs4EQAAwYTCDZhA4QbgFz7/4YC27Dmm6Ai7xl/Z2XQcAACCi8VDygETKNwAjCurcGnm4kxJ0t0DO6p5HK8NCgCAd3GGGzCBwg3AuL+vy1HOkWI1j4vUnQM7mI4DAEDw4Qw3YASFG4BR+SVOvbBihyRpwpDOio4w+uIJAAAEKc5wAyZQuAEYNWfVTh0rdio5IVbX925rOg4AAEGKl+IFTKBwAzBm37ES/XVttiRp8rAUhdv5lQQAgE9UPqSctRaoT/zEATDm2SWZKq9w65LzmmhwSoLpOAAABC+LM9yACRRuAEb8uC9f//xunyRpyvCustl4ThkAAL7DGW7ABH7iANQ7y7L05KJtsizp179qrR5tG5mOBABAcLO4aBpgAoUbQL1bsyNPX2blKcIepoeGdjEdBwCAEHDyDDeA+kThBlCvXG5LMxZmSJJ+3y9JiU2iDScCACAEVJ7h5u4/UJ/4iQNQrz7cvFfbDhYqPipc9w7uZDoOAAAhwvrFfwHUFwo3gHpTUu7Ss0u3S5LuHdxJjaIjDCcCACBE/PtlwWTj7j9Qn/iJA1Bv/ro2WwcLStWmUQP9vl9703EAAAghnNsGTKBwA6gXh4+Xac6qnZKkh4Z2UZTDbjgRAAAhxDp50TSuUg7UJwo3gHrx4oosHS+rULc28fr1r1qbjgMAQIjhZcEAEyjcAHwuO69I76z7SZI0ZVhXhYWx2AMAUL9OXjSNNRioTxRuAD43c/E2VbgtDerSXJd2amY6DgAAoefkRdMo3EC9onAD8KlNPx3Voh8PKswmTR7W1XQcAABCk8UZbsAECjcAn7EsS9MXZkiSru+dqC4t4wwnAgAgVHGGGzCBwg3AZ5Zs/VmbfjqqKEeYJqR1Nh0HAIDQxRluwAgKNwCfcLrcemrxNknSnQPOU4v4KMOJAAAIZVylHDCBwg3AJ977JkfZeUVqGhOhuy/vaDoOAAChjYumAUZQuAF4XWGpU88t2yFJGn9lsmIjww0nAgAg1PGQcsAECjcAr3ttzS4dLirXec1idOPF7UzHAQAAnOEGjKBwA/Cqg/mlev2LXZKkiVelyGHn1wwAAOZxhhswgXvCALxqdvp2lTrd6pPUWEMvaGE6DgAAkMRF0wAzKNwAvCbzYKHe37RHkvTw8K6y2VjUAQDwC/9+SDlnuIH6ReEG4DUzFmXIbUnDu7dU76TGpuMAAIBK/z7DzR/DgXpF4QbgFWuz8rQq85DCw2x6aGiK6TgAAOCXOMMNGEHhBlBnbrel6QszJEm3XJKkDs1iDCcCAABVWDyHGzCBwg2gzj7Zsl9b9xcoLjJcYwd3Mh0HAABUY519FwBeR+EGUCelTpeeXpIpSRqd2lFNYyMNJwIAANVUPqScu/9AfeInDkCd/O3r3dp3rEStGkbp9ss6mI4DAABqxEPKARMo3ADO2bHicr20IkuSNGFIZ0U57IYTAQCAGnHRNMAICjeAc/bSiiwVlFYopWWc/vvCtqbjAACA0+IMN2AChRvAOdlzpFh/+/onSdLDw7vKHsYCDgCA/+KiaYAJFG4A5+TpJZkqd7k1ILmZLu/c3HQcAABwJlw0DTCCnzgAHvt+7zF9smW/bDZp8rAU03EAAMBZcYYbMIHCDcAjlmXpic8zJEnX9WqjC1o3NJwIAACcFWe4ASP4iQPgkRXbcrU++4giwsP0QFoX03EAAECtcIYbMIHCDaDWKlxuzVi0TZL0h/4d1KZRA8OJAABArfCyYIARFG4Atfb+pr3Kyj2uxtEO3TOoo+k4AACgtixeFgwwgcINoFaKyio0K327JGns4GTFRzkMJwIAALVH4QZMoHADqJU3vsjWocIytWsSrVsuSTIdBwAAeIKHlANGULgBnFVuYaleXbNTkjTxqi6KCOdXBwAAgeXfZ7htFG6gPnGvGcBZPb9sh4rLXfpVYiNd3b2V6TgAAMBTnOEGjKBwAzijrNzjem/DHknS1OFdZeMv4wAABCCeww2YQOEGcEZPLd4ml9vSkPNb6OIOTUzHAQAA58T6938p3EB9onADOK1vso8o/V8/yx5m06SrUkzHAQAA5+rfDynnDDdQvyjcAGpkWZaeWJghSbrxokR1Sog1nAgAAJw7znADJlC4AdTo8x8OaMueY4qOsGvclcmm4wAAgLrgDDdgBIUbQDVlFS7NXJwpSbp7YEclxEUZTgQAAOqGi6YBJlC4AVTz93U5yjlSrOZxkbpjQAfTcQAAQF1ZPKQcMIHCDaCKghKnXlixQ5I0YUhnxUSGG04EAADqjoeUAyZQuAFU8eoX2TpW7FRyQqyu793WdBwAAOANnOEGjODUFYBKR8qkuRtyJEmTh6Uo3M7f5AAACApcNA0wwvi96ZdfflkdOnRQVFSUevfurS+++MJ0JCBkLdwTpvIKty45r4kGpySYjgMAALyGM9yACUYL9/z58zV+/HhNnTpV3377rQYMGKBhw4YpJyfHZCwgJP3rQIE2HjqxCE8Z3lU2GwsyAADBg6uUAyYYLdyzZs3S7bffrjvuuENdu3bVc889p8TERM2ZM8dkLCAkzVyyQ5Zs+q/uLdWjbSPTcQAAgDf9+yHlnOEG6pex53CXl5dr06ZNmjx5cpXtaWlp+uqrr2r8mLKyMpWVlVW+XVBQIElyOp1yOp11yjN9Uaa+2nm4TrfhK5ZlqfC4Xf+3cy1nHWuJmXmmqKxCe4+Vym6zdF9q+zr/PIWCkzNiVrUXKjML9uPzNV+u9bY9C2TPmF6n2/AVu2Uptfi47EumymLdqhVm5qGyvH9XbRu/pzwQKmuXN4XCzDw5NmOFOy8vTy6XSy1atKiyvUWLFjp48GCNHzNjxgw99thj1bYvXbpU0dHRdcqzKTNMmUeMP6X9DGw6UFxkOkSAYWaeGtrWrYyNXyrDdJAAkp6ebjpCwAn2mRUXF5uOENB8udYnOb9Qz/If63QbvmKT1FCSCgwHCSDM7NwUhyVoc5D/HvaFYF+7fCGYZ+bJWm+zLMs6+27et3//frVp00ZfffWV+vXrV7n9iSee0Ntvv61t27ZV+5ia/uqdmJiovLw8xcfH1ynPtoOFOlxUXqfb8JWKigpt3rRZF/a+UOHhXFi+NpiZ5xpGhinn+681ZMgQORwO03H8ntPpVHp6OvPyQKjMrKCgQM2aNVN+fn6d16ZQ5Mu1XsV7ZSvMrGNC36ioqNDmzZt14YWsW7XFzDxXYYVryaYCDUm7Kqh/D3tTqKxd3hQKM/NkrTf226lZs2ay2+3Vzmbn5uZWO+t9UmRkpCIjI6ttdzgcdf5idk9sUqeP9yWn06minZYu79IiaL9pvY2Zec7pdCrne+/8PIUS5uW5YJ9ZMB9bffDlWq+GHU7880OW06lDW1yytxmqcL6HaoWZec5yOiXbwqD/PewLzMxzwTwzT47L2GOoIyIi1Lt372oPNUhPT9ell15qKBUAAAAAAN5h9PE3EyZM0MiRI9WnTx/169dPr732mnJycjR69GiTsQAAAAAAqDOjhfuGG27Q4cOH9ac//UkHDhxQt27dtHDhQiUlJZmMBQAAAABAnRm/wsQ999yje+65x3QMAAAAAAC8yp9fBwsAAAAAgIBF4QYAAAAAwAco3AAAAAAA+ACFGwAAAAAAH6BwAwAAAADgAxRuAAAAAAB8gMINAAAAAIAPULgBAAAAAPABCjcAAAAAAD5A4QYAAAAAwAfCTQeoC8uyJEkFBQWGk/iW0+lUcXGxCgoK5HA4TMcJCMzMc8zMM8zLc6Eys5Nr0sk1CnXDWo/TYWaeY2aeY2aeC4WZebLWB3ThLiwslCQlJiYaTgIAQFWFhYVq2LCh6RgBj7UeAOCvarPW26wA/hO82+3W/v37FRcXJ5vNZjqOzxQUFCgxMVF79uxRfHy86TgBgZl5jpl5hnl5LlRmZlmWCgsL1bp1a4WF8cytumKtx+kwM88xM88xM8+Fwsw8WesD+gx3WFiY2rZtazpGvYmPjw/ab1pfYWaeY2aeYV6eC4WZcWbbe1jrcTbMzHPMzHPMzHPBPrParvX86R0AAAAAAB+gcAMAAAAA4AMU7gAQGRmpRx99VJGRkaajBAxm5jlm5hnm5TlmBpwePx+eY2aeY2aeY2aeY2ZVBfRF0wAAAAAA8Fec4QYAAAAAwAco3AAAAAAA+ACFGwAAAAAAH6BwAwAAAADgAxTuAFVWVqaePXvKZrPpu+++Mx3Hb+3evVu33367OnTooAYNGqhjx4569NFHVV5ebjqaX3n55ZfVoUMHRUVFqXfv3vriiy9MR/JbM2bM0EUXXaS4uDglJCTo2muvVWZmpulYAWPGjBmy2WwaP3686SiA32Otrx3W+tphra891vq6Y73/Dwp3gJo4caJat25tOobf27Ztm9xut1599VVt3bpVs2fP1iuvvKIpU6aYjuY35s+fr/Hjx2vq1Kn69ttvNWDAAA0bNkw5OTmmo/ml1atXa8yYMVq3bp3S09NVUVGhtLQ0FRUVmY7m9zZs2KDXXntNPXr0MB0FCAis9bXDWn92rPWeYa2vG9b7U1gIOAsXLrRSUlKsrVu3WpKsb7/91nSkgDJz5kyrQ4cOpmP4jYsvvtgaPXp0lW0pKSnW5MmTDSUKLLm5uZYka/Xq1aaj+LXCwkIrOTnZSk9Pty6//HJr3LhxpiMBfo21vm5Y66tira8b1vraY72vjjPcAebnn3/WnXfeqbffflvR0dGm4wSk/Px8NWnSxHQMv1BeXq5NmzYpLS2tyva0tDR99dVXhlIFlvz8fEnie+osxowZo6uvvlpXXnml6SiA32OtrzvW+v9gra871vraY72vLtx0ANSeZVkaNWqURo8erT59+mj37t2mIwWcnTt36sUXX9Szzz5rOopfyMvLk8vlUosWLapsb9GihQ4ePGgoVeCwLEsTJkzQZZddpm7dupmO47fee+89bd68WRs2bDAdBfB7rPV1x1pfFWt93bDW1x7rfc04w+0Hpk2bJpvNdsZ/Gzdu1IsvvqiCggI9/PDDpiMbV9uZ/dL+/ft11VVX6frrr9cdd9xhKLl/stlsVd62LKvaNlR377336vvvv9e7775rOorf2rNnj8aNG6d33nlHUVFRpuMAxrDWe4613rtY688Na33tsN6fns2yLMt0iFCXl5envLy8M+7Tvn173Xjjjfr000+r/HJ0uVyy2+26+eab9dZbb/k6qt+o7cxO/sDv379fgwYNUt++fTV37lyFhfG3JunEw8yio6P1/vvv67rrrqvcPm7cOH333XdavXq1wXT+bezYsfroo4+0Zs0adejQwXQcv/XRRx/puuuuk91ur9zmcrlks9kUFhamsrKyKu8DghVrvedY672Dtf7csdbXHuv96VG4A0hOTo4KCgoq396/f7+GDh2qBQsWqG/fvmrbtq3BdP5r3759GjRokHr37q133nknZH/YT6dv377q3bu3Xn755cpt559/vn7zm99oxowZBpP5J8uyNHbsWP3zn//UqlWrlJycbDqSXyssLNRPP/1UZdttt92mlJQUTZo0iYfnAadgrT83rPVnxlrvGdZ6z7Henx7P4Q4g7dq1q/J2bGysJKljx44swKexf/9+paamql27dnrmmWd06NChyve1bNnSYDL/MWHCBI0cOVJ9+vRRv3799NprryknJ0ejR482Hc0vjRkzRvPmzdPHH3+suLi4yue/NWzYUA0aNDCczv/ExcVVW2RjYmLUtGnTkF58gdNhrfcca/3ZsdZ7hrXec6z3p0fhRlBbunSpsrKylJWVVe2OCg/uOOGGG27Q4cOH9ac//UkHDhxQt27dtHDhQiUlJZmO5pfmzJkjSUpNTa2y/c0339SoUaPqPxAAhDjW+rNjrfcMaz28iYeUAwAAAADgA1xNAgAAAAAAH6BwAwAAAADgAxRuAAAAAAB8gMINAAAAAIAPULgBAAAAAPABCjcAAAAAAD5A4QYAAAAAwAco3AAAAAAA+ACFGwAAAAAAH6BwAwAAAADgAxRuAAAAAAB8gMINhLBDhw6pZcuWmj59euW29evXKyIiQkuXLjWYDAAAeANrPWCWzbIsy3QIAOYsXLhQ1157rb766iulpKSoV69euvrqq/Xcc8+ZjgYAALyAtR4wh8INQGPGjNGyZct00UUXacuWLdqwYYOioqJMxwIAAF7CWg+YQeEGoJKSEnXr1k179uzRxo0b1aNHD9ORAACAF7HWA2bwHG4A2rVrl/bv3y+3262ffvrJdBwAAOBlrPWAGZzhBkJceXm5Lr74YvXs2VMpKSmaNWuWfvjhB7Vo0cJ0NAAA4AWs9YA5FG4gxD300ENasGCBtmzZotjYWA0aNEhxcXH67LPPTEcDAABewFoPmMNDyoEQtmrVKj333HN6++23FR8fr7CwML399tv68ssvNWfOHNPxAABAHbHWA2ZxhhsAAAAAAB/gDDcAAAAAAD5A4QYAAAAAwAco3AAAAAAA+ACFGwAAAAAAH6BwAwAAAADgAxRuAAAAAAB8gMINAAAAAIAPULgBAAAAAPABCjcAAAAAAD5A4QYAAAAAwAco3AAAAAAA+MD/B5SmbU3rawyiAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def relu(x):\n", + " return np.maximum(0, x)\n", + "\n", + "def relu_derivative(x):\n", + " return np.where(x > 0, 1, 0)\n", + "\n", + "plot_function_and_derivative(relu, relu_derivative, \"ReLU\", (-5, 5))" + ] + }, + { + "cell_type": "markdown", + "id": "63fbea04", + "metadata": {}, + "source": [ + "**Eigenschaften:**\n", + "\n", + "* **Re**ctified **L**inear **U**nit\n", + "* Gibt es noch in weiteren Versionen" + ] + }, + { + "cell_type": "markdown", + "id": "65dcb9af", + "metadata": {}, + "source": [ + "**Vorteile:**\n", + "\n", + "* Einfache Ableitung (nur eine If-Abfrage)\n", + "* Ableitung für die Hälfte der Werte $1$" + ] + }, + { + "cell_type": "markdown", + "id": "653c2926", + "metadata": {}, + "source": [ + "**Nachteile:**\n", + "\n", + "* \"Dying ReLU Problem\": Neuronen können inaktiv werden, weil sie aufgrund der ReLU Funktion einerseits keinen Output liefern können und auch die Ableitung ist $0$ in diesem Punkt, somit kann sich deren Zustand auch nicht mehr ändern" + ] + }, + { + "cell_type": "markdown", + "id": "e717a3ff", + "metadata": {}, + "source": [ + "**Weitere ReLU Versionen:**" + ] + }, + { + "cell_type": "markdown", + "id": "e0379942", + "metadata": {}, + "source": [ + "#### Leaky ReLU:\n", + "\n", + "\\begin{equation*}\n", + " f(x) = \\begin{cases} x & x>0, \\\\ \\alpha x & x\\leq 0 \\end{cases}\n", + "\\end{equation*}\n", + "\n", + "mit $\\alpha\\in\\mathbb R$ meistens im Bereich $\\alpha = 0.01$." + ] + }, + { + "cell_type": "markdown", + "id": "49b87e8f", + "metadata": {}, + "source": [ + "#### Exponential Linear Unit (ELU):\n", + "\n", + "\\begin{equation*}\n", + " f(x) = \\begin{cases} x & x>0, \\\\ \\alpha (e^x-1) & x\\leq 0 \\end{cases}\n", + "\\end{equation*}\n", + "\n", + "mit $\\alpha>0$." + ] + }, + { + "cell_type": "markdown", + "id": "5e9ac32e", + "metadata": {}, + "source": [ + "> **Übung:** Berechnen Sie die Ableitungen der einzelnen Funktionen und zeichnen Sie die Funktionen! " + ] + }, + { + "cell_type": "markdown", + "id": "877ef6e7", + "metadata": {}, + "source": [ + "### Swish" + ] + }, + { + "cell_type": "markdown", + "id": "a15524d6", + "metadata": {}, + "source": [ + "\\begin{align*}\n", + " f(x) &= x \\cdot \\sigma(x)\\\\\n", + " f'(x) &= \\sigma(x) + x\\cdot \\sigma'(x) = \\sigma(x) + x\\cdot \\sigma(x)\\cdot (1-\\sigma(x))\n", + "\\end{align*}" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "467503e3", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA9wAAAHUCAYAAADInCBZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABkFklEQVR4nO3dd3hUZeL28XtmMpn0AiHU0EvoVRBQijSBtbcVdcV13XVFBbFg2VX0p+JrAytrW3VlWetiWRUJKsUCIoKC0gktAUKA9DblvH8MiYQEzJAMZ8r3c11zTebkZHKfJ4En95wyFsMwDAEAAAAAgAZlNTsAAAAAAAChiMINAAAAAIAfULgBAAAAAPADCjcAAAAAAH5A4QYAAAAAwA8o3AAAAAAA+AGFGwAAAAAAP6BwAwAAAADgBxRuAAAAAAD8gMINBICVK1fqggsuUOvWreVwONS0aVMNHjxYt956a72ed8eOHbJYLHrttdd8+roRI0aoR48edV5/+fLlcjgc2rlzp0/fx+l0qkOHDpozZ45PXwcAQLAI1jm+8vmXLFlS43N//OMfdfbZZ/v0fSXp888/V1xcnLKysnz+WiBYUbgBk3388ccaMmSICgoK9Oijj2rRokV66qmnNHToUL311lv1eu7mzZvr22+/1cSJExsobU2GYWjatGm67rrr1KZNG5++1m63695779UDDzyggwcP+ikhAADmCPY5vjZr1qzR66+/rgcffNDnrx01apQGDhyou+++2w/JgMBkMQzDMDsEEM6GDx+urKwsbdy4UREREdU+5/F4ZLWe+tfFRowYodzcXK1fv/431/300081YcIEbdy4UV26dPH5e1VUVKh58+a69dZbmYABACElmOf4HTt2qF27dvryyy81YsSIquWXXXaZdu3apW+//fakvv97772nyy67TJmZmUpLSzup5wCCCXu4AZMdPHhQKSkpNSZiSVUT8e23367ExES53e6qz910002yWCx67LHHqj2X1WrVM888I6n2w80OHDigP//5z0pLS5PD4VCTJk00dOhQLV68uMb3X7Vqlc4880zFxMSoffv2euSRR+TxeKqtM3fuXJ122mnVyvZXX30lu92u2267rdq6r732miwWi1555ZWqZZGRkbrsssv04osvitf/AAChJNjn+GPt379fCxYs0FVXXVVt+fXXX6+oqCitXr26apnH49GoUaPUtGlT7d27t2r5Oeeco7i4OL300ksn/F5AqKBwAyYbPHiwVq5cqZtvvlkrV66U0+mssc7o0aNVUFCg7777rmrZ4sWLFR0drYyMjKpln3/+uQzD0OjRo4/7/a666iq9//77uvfee7Vo0SK9/PLLGj16dI1Duvft26crrrhCV155pT788EONHz9ed911l+bNm1e1TkVFhRYvXqyRI0dW+9ozzjhDDz74oJ544gl9+OGHkqSff/5ZU6ZM0ZVXXqlrr7222vojRozQzp0767RHHQCAYBHMc3zbtm1lGEa1vduLFi2S0+msMe/PmTNHXbt21aWXXqq8vDxJ0v33368lS5Zo3rx5at68edW6kZGRGjJkiD7++OMTDx4QKgwApsrNzTXOOOMMQ5IhybDb7caQIUOMWbNmGYWFhYZhGEZxcbERGRlpPPDAA4ZhGMaePXsMScaMGTOM6Ohoo6yszDAMw7juuuuMFi1aVD13ZmamIcl49dVXq5bFxcUZ06ZNO2Gm4cOHG5KMlStXVlverVs3Y9y4cVWPV65caUgy3nzzzRrP4fF4jAkTJhhJSUnG+vXrjW7duhnp6elGUVFRjXW3bNliSDLmzp37G6MFAEDwCOY5vjZ//etfjejoaMPj8dT43JYtW4yEhATj/PPPNxYvXmxYrVbjb3/7W63Pc8899xhWq7XWvwmAUMMebsBkjRs31vLly7Vq1So98sgjOu+887R582bddddd6tmzp3JzcxUTE6PBgwdXHRKWkZGhpKQk3X777aqoqNBXX30lyfuK+Ile+ZakgQMH6rXXXtODDz6oFStW1PpquyQ1a9ZMAwcOrLasV69e1a5Enp2dLUlKTU2t8fUWi0X/+te/FB8frwEDBigzM1Nvv/22YmNja6xb+fVctRQAEEqCeY6vTXZ2tpo0aSKLxVLjcx07dtRLL72k999/X7/73e905plnaubMmbU+T2pqqjwej/bt23fC7weEAgo3ECAGDBigGTNm6J133lF2drZuueUW7dixQ48++qgk7yFnK1asUHFxsRYvXqyzzjpLjRs3Vv/+/bV48WJlZmYqMzPzNyfjt956S1dffbVefvllDR48WI0aNdIf/vCHGpNe48aNa3ytw+FQaWlp1ePKj6Oiomr9Xo0bN9a5556rsrIynX322erZs2et61V+/dHPDQBAqAjGOb42paWlx53zJWnixIlq2rSpysrKNH36dNlstlrXY95HOKFwAwHIbrfrvvvuk6Sq85pHjRqliooKLVu2TJ9//rnGjBlTtTwjI6PqPK9Ro0ad8LlTUlI0Z84c7dixQzt37tSsWbP03//+V5MnT/Y5Z0pKiiTp0KFDtX4+IyNDc+fO1cCBA7VgwQK99957ta5X+fWVzwcAQKgKljn+eM9/vDlf8l48rbCwUN27d9fNN9+sw4cP17oe8z7CCYUbMNnRV+482oYNGyRJLVq0kOQ9TCwhIUFz5szRvn37qibj0aNHa82aNXr77bfVrVu3qvXronXr1rrxxhs1ZswY/fDDDz5n79q1qyRp27ZtNT63d+9eXXnllRo+fLi++eYbnXvuubr22muVmZlZY93t27dLkrp16+ZzBgAAAlUwz/G1SU9P18GDB5Wfn1/jcy+//LLmzZunZ599Vh9++KHy8vJ0zTXX1Po827dvV+PGjdW0adMGyQUEsprvUQDglBo3bpxatWqlc845R+np6fJ4PFq7dq2eeOIJxcXFaerUqZIkm82m4cOH66OPPlK7du3UoUMHSdLQoUPlcDj0+eef6+abbz7h98rPz9fIkSM1adIkpaenKz4+XqtWrdLChQt14YUX+py9VatWat++vVasWFHte7vdbl1++eWyWCyaP3++bDabXnvtNfXp00eXXXaZvvrqK0VGRlatv2LFCtlsNg0bNsznDAAABKpgnuNrM2LECBmGoZUrV2rs2LFVy9etW6ebb75ZV199dVXJfuWVV3TxxRdrzpw5mjZtWrXnWbFihYYPH17rueBAqGEPN2Cyv/3tb0pOTtbs2bN17rnnavz48Xr66ac1evRofffdd9XOe648d+voc7gcDofOOOOMGstrExUVpUGDBumNN97QFVdcofHjx+vll1/WjBkzTvr9MK+44gotXLhQ5eXlVcvuu+8+LV++XPPnz1ezZs0kScnJyXrzzTe1Zs0a3XHHHdWe4/3339eECROUlJR0UhkAAAhEwT7HH2vo0KFq27atPvjgg6plxcXFuvTSS9WuXTs9//zzVcsvuugiTZkyRXfccUe1tzzbtm2b1q1bpyuuuKJBMgGBzmIYhmF2CADBKzs7W+3atdO//vUvXXbZZT5//bZt29SpUyd99tlnVYfQAQCAwPTEE0/ooYceUlZWlqKjo33++r///e/617/+pW3btikigoNtEfoo3ADqbcaMGfr000+1du1aWa2+HThzzTXXaM+ePVUXhAEAAIGrrKxMXbt21ZQpU3Tbbbf59LV5eXlq3769nnnmGfZwI2xwSDmAevvb3/6miy66yOf30Xa5XOrQoYOee+45PyUDAAANKSoqSm+88YYcDofPX5uZmam77rpLkyZN8kMyIDCxhxsAAAAAAD9gDzcAAAAAAH5A4QYAAAAAwA8o3AAAAAAA+EFQX4vf4/EoOztb8fHxslgsZscBAECGYaiwsFAtWrTw+ar9qIm5HgAQaHyZ64O6cGdnZystLc3sGAAA1LB79261atXK7BhBj7keABCo6jLXB3Xhjo+Pl+Td0ISEBJPT+I/T6dSiRYs0duxY2e12s+MEBcbMd4yZbxgv34XLmBUUFCgtLa1qjkL9MNfjeBgz3zFmvmPMfBcOY+bLXB/Uhbvy0LKEhISQn4RjYmKUkJAQsr+0DY0x8x1j5hvGy3fhNmYc/twwmOtxPIyZ7xgz3zFmvgunMavLXM/JZQAAAAAA+AGFGwAAAAAAP6BwAwAAAADgB0F9DnddGIYhl8slt9ttdpST5nQ6FRERobKysqDejhOx2WyKiIjgnEcAAACEjFDoIr4Khe7SkN0kpAt3RUWF9u7dq5KSErOj1IthGGrWrJl2794d0oU0JiZGzZs3V2RkpNlRAAAAgHoJlS7iq1DpLg3VTUK2cHs8HmVmZspms6lFixaKjIwM2h+4x+NRUVGR4uLifvON1YORYRiqqKjQgQMHlJmZqU6dOoXkdgIAACA8hFIX8VWwd5eG7iYhW7grKirk8XiUlpammJgYs+PUi8fjUUVFhaKiooLyl7YuoqOjZbfbtXPnzqptBQAAAIJRKHURX4VCd2nIbhKcI+CDYP0hhyN+VgAAAAgl/H0bvBrqZ2fqb8DMmTNlsViq3Zo1a2ZmJAAAAAAAGoTph5R3795dixcvrnpss9lMTAMAAAAAQMMw/RiHiIgINWvWrOrWpEkTsyOFlMmTJ+v8889v0HU3bdqkZs2aqbCwsE7Pm5OToyZNmigrK6tO6wMAAAAIDf7oI8f64osvlJ6eLo/HU6f1161bp1atWqm4uNjn7+Ur0/dwb9myRS1atJDD4dCgQYP08MMPq3379rWuW15ervLy8qrHBQUFkrzv9eZ0Oqut63Q6ZRiGPB5PnQc+UOTk5Ojee+/VwoULtX//fiUnJ6t79+66//77NWTIEJ+ea/bs2VXj8FsMw6jTunfffbduuOEGxcbG1ul5U1JSdOWVV+ree+/VSy+9dNz1PB6PDMOQ0+ms95EOlb8Px/5e4PgYM98wXr4LlzEL9e3zN1/m+lASLv8+GhJj5jvGzHcnO2bB3EWk2vtIr169dN9992nw4MEn/FrDMKruPR5Pg/eRs846S3/4wx80efLkqmV33HGH7rrrLkmq0/fp3r27TjvtND355JO65557al3nRN3El98HUwv3oEGD9K9//UudO3fW/v379eCDD2rIkCH6+eef1bhx4xrrz5o1S/fff3+N5YsWLapx9b/KPedFRUWqqKjw2zb4wwUXXCCXy6XnnntObdq00YEDB7R06VJlZWVV/eFRV5Xnxtfl65xOp1wu1wnXzcrK0kcffaQHHnjApywXX3yxRo8erb///e9KSkqqdZ2KigqVlpZq2bJlcrlcdX7uE8nIyGiQ5wknjJlvGC/fBeKYlbslRwOd0RRu77fa0HyZ60NRIP77CHSMme8YM9/5OmbB3EWk4/eRPXv21LkDVB4N29B9xOVyqaysrGqdlStXavPmzRo3bpxP/eTSSy/VrbfeqhtuuKHWnX0n6ia+zPUWo/IliABQXFysDh066I477tD06dNrfL62V73T0tKUm5urhISEauuWlZVp9+7datu2bdVl3A3DUKnT7d+NqEW03Vbn993Ly8tT48aN9cUXX2j48OGSvLkLCwsVHx+v22+/XZs3b9aHH34oSXrqqac0ffp0ffjhh5o4caIkqWvXrpo2bZr+8pe/6JprrlFeXp4WLFggSXr33Xf1f//3f9q6datiYmLUt29fLViwQLGxsVXrnnHGGXryySdVUVGhyy67TLNnz5bdbpfk3WP+5ptvauXKlVWZr732Wq1evVorV66Uw+GQ0+nUkCFD1KVLF82bN69qvQ4dOuiee+7RH//4x1q3vaysTDt27FBaWlq93xbM6XQqIyNDY8aMqcqOE2PMfMN4+S5Qxywzt1gX/mOlJg9urZtGdpDVWr/3SS0oKFBKSory8/NrzE34bb7M9aEkUP99BDLGzHeMme9Odsxq6yIyDMlt0ouythipHn3kaLfddtsJ+4hhGEpPT9f06dP90keO3cM9depU7d27V2+//bYkb3caN26cbDabPvnkE1ksFuXl5alPnz668sor9eCDD0ryFuqkpCT973//01lnnVVjO0/UTXyZ600/pPxosbGx6tmzp7Zs2VLr5x0OhxwOR43ldru9xj8At9sti8Uiq9VadUn3kgqXesw89a/o/fLAOMVE1m3XSUJCguLi4vThhx9qyJAhcjgcVYdFWCwWjRw5Uv/85z8leS9Vv2zZMqWkpGj58uU655xztG/fPm3evFkjR46U1WqtekXJarVq7969uuKKK/Too4/qggsuUGFhoZYvX171eYvFoiVLlqhFixb68ssvtXXrVl122WXq27evrrvuOknS8uXLNWDAgGqXyX/mmWfUu3dv3X333Zo9e7buu+8+5ebm6osvvqi23sCBA/X111/rT3/6U63bXpmhtp/nyWrI5woXjJlvGC/fBdqYPZaxVUXlLv2yr0gOR2S9ny+Qti0Y+TLXh6Jw2c6GxJj5jjHzna9jVlsXkatYetekFw4vLZJssXVatbY+crTf6iPZ2dnaunWrhg8f7pc+Uvl9K8d1+fLluvzyy6v1jtdff109e/bUs88+q6lTp+qGG25Q06ZNdf/991etFxUVpd69e+vrr7/W6NGja4zDibqJL78LAVW4y8vLtWHDBp155plmRzFNRESEXnvtNV133XX6xz/+oX79+mnYsGGaOHGihgwZomHDhqmwsFBr1qxRv379tHz5ct12223673//K0n68ssv1bRpU6Wnp9d47r1798rlcunCCy9UmzZtJEk9e/astk5ycrKeffZZ2Ww2paena+LEifr888+rfsF37Nih/v37V/uauLg4zZs3T8OHD1d8fLyeeOIJff7550pMTKy2XsuWLbVmzZoGGysAqK9vtx1Uxi/7ZbNadPeEmv9vAgAQbmrrI8OHD9fvf/979erVq059JDU11W99ZMmSJdXW37Fjh1q0aFFtWcuWLfXCCy/oqquu0v79+/XRRx9pzZo1NYpyy5YttWPHjvoM128ytXDfdtttOuecc9S6dWvl5OTowQcfVEFBga6++mq/fL9ou02/PDDOL8/9W9/XFxdddJEmTpyo5cuX69tvv9XChQv12GOP6cUXX9Qf//hH9enTR0uWLJHdbpfVatVf/vIX3XfffSosLNSSJUtqPfRDknr37q1Ro0apZ8+eGjdunMaOHauLL75YycnJVet079692jkMzZs317p166oel5aW1nq49+DBg3Xbbbfp//7v/zRjxgwNGzas5jhER3NuI4CA4fEYeuiTXyRJkwa2VsfUeJMTAQBCmi3Gu6fZrO/tg9r6yKOPPqqXX35ZkydPPmEfWbp0qYYOHVrr8zZEHznW8frJJZdcogULFmjWrFmaO3euOnfuXGOdU9FPTH1bsD179ujyyy9Xly5ddOGFFyoyMlIrVqyoerWjoVksFsVERpzyW13P3z5aVFSUxowZo3vvvVdfffWVJk2aVHURmREjRmjJkiVaunSphg8fXnUV86+//lpLlizRiBEjan1Om82mjIwMffrpp+rWrZueeeYZdenSRZmZmVXrHPuqj8ViqXalv5SUFB0+fLjGc3s8Hn399dey2WzHPSXg0KFDvO0bgICxYE2W1mcVKN4RoWmjO5kdBwAQ6iwWKSLWnFs9+8g333yjyZMn67777pN04j5yosLdEH3kWMfrJyUlJVq9erXp/cTUwv3mm28qOztbFRUVysrK0nvvvadu3bqZGSlgdenSpep94kaMGKHly5friy++qCrXw4cP15tvvqnNmzcfdw+35P2FHTp0qO6//36tWbNGkZGRVRcwqIu+ffvql19+qbH8scce04YNG7R06VJ99tlnevXVV2uss379evXt27fO3wsA/KW0wq3HPtskSbphZEc1jqt5zjAAAPhVt27d6txHjle4pfr3kWMdr5/ceuutslqt+vTTT/X000/riy++qLHOqegnphZu1HTw4EGdddZZmjdvnn766SdlZmbqnXfe0dNPP61zzz1XkqrOm/joo4+qfsFHjBihefPmqUmTJsd90WLlypV6+OGH9f3332vXrl3673//qwMHDqhr1651zjdu3Dh9++23crt/vdr72rVrde+99+qVV17R0KFD9dRTT2nq1Knavn171TqVrzCNHTv2JEYFABrWS8u3a19BmVomReuaoW3NjgMAQMA4Xh959NFHdd5550n67T5S2/nbUsP0kWONGzdOX331VbVlH3/8sf75z3/q3//+t8aMGaM777xTV199dbU94Tt27FBWVlatF0xrSAF10TR4L0A2aNAgzZ49W9u2bZPT6VRaWpr+8Ic/aObMmZKkxMRE9e3bV7t27aoq12eeeaY8Hs8J924nJCRo2bJlmjNnjgoKCtSmTRs98cQTGj9+fJ3zTZgwQXa7XYsXL9a4ceNUVlamK664QpMnT9Y555wjyfs2YR9//LGuuuoqLVu2TDabTR988IFat24d1hfEAxAYcgrK9I+l2yRJM8anK8rH62wAABDKjtdHrrvuOt19992STtxHaruWU6WG6CPHuvLKKzVjxgxt2rRJXbp00YEDB3Tttddq5syZ6tevnyTpvvvu06JFi3T99dfrrbfekiT95z//0dixY/12OnMlCneAcTgcmjVrlmbNmlW1zOPxqKCgQNHR0VXLvv/++2pf16hRo1rPbXjttdeqPu7atasWLlx43O999LqV5syZU+2xzWbT3XffrSeffFLjxo1TVFSUfv755xpfV3mVwkqzZ8/Wvffee9zvDQCnypMZm1VS4VaftCSd06u52XEAAAgotfWR2hyvj1R2l0oN3UeOlZycrBtvvFFPPvmkXnjhBTVp0kT79u2rtk5ERIRWrlxZ9bi8vFxz587Vf/7znxM+d0PgkHL47M9//nPVYSR1kZOTo4svvliXX365n5MBwIlt2Fugt77fLUn6+++6ntRFLQEAQGC555571KZNm2qnvZ7Izp07dc8995zwXPOGwh5u+CwiIkL33HNPnddPTU3VHXfc4cdEAPDbDMPQQx9vkGFIE3s2V/82jcyOBAAAGkBiYmLV4e510blz51rfJswf2MMNAAgLSzYd0FdbcxVps2rG2bVfzAUAAKAhUbgBACHP5fbooU82SJImD22r1o1jTE4EAADCQcgXbsMwzI6AOuJnBcBf3ly1W1tzipQcY9eUkR3NjgMACBP8fRu8GupnF7KF2263S/K+/zOCQ+XPqvJnBwANobDMqTmLN0uSbh7VSYnR/B8DAPAvukjwa6huErIXTbPZbEpKSlJOTo4kKSYmJmivRuvxeFRRUaGysjJZraH3GolhGCopKVFOTo6SkpJks/GeuAAazgtLtyu3qELtUmJ1xSD/vtcmAABSaHURXwV7d2nobhKyhVuSmjVrJklVv+jByjAMlZaWKjo6OqT/oSYlJVX9zACgIezNL9VLy7dLku4cn67IiOCb+AEAwSlUuoivQqW7NFQ3CenCbbFY1Lx5c6WmpsrpdJod56Q5nU4tW7ZMw4YNC9nDre12O3u2ATS4xz7bpHKXRwPbNtLYbk3NjgMACCOh0kV8FQrdpSG7SUgX7ko2my2oy5zNZpPL5VJUVFTQ/tICwKm2PitfC9ZkSZLuntg1qF9lBwAEr2DvIr6iu1THsXUAgJBjGIYe+niDDEM6t3cL9UlLMjsSAAAIQxRuAEDI+XJTjr7dflCREVbdPq6L2XEAAECYonADAEKKy+3Rw59slCRdM7St0hrFmJwIAACEKwo3ACCkvLlqt7bmFCk5xq4bRnQ0Ow4AAAhjFG4AQMgoLHNqzuLNkqSpozopMZqLtQAAAPNQuAEAIeOFpduVW1ShdimxuuL0NmbHAQAAYY7CDQAICdl5pXpp+XZJ0p3j02W3McUBAABz8dcIACAkPL5ok8pdHg1s20hjuzU1Ow4AAACFGwAQ/NZn5WvBmixJ0j0Tu8pisZicCAAAgMINAAhyhmHooY83yDCk8/q0UO+0JLMjAQAASKJwAwCC3Bcbc/Tt9oOKjLDqtrFdzI4DAABQhcINAAhaLrdHD3+yQZJ0zdC2SmsUY3IiAACAX1G4AQBB681Vu7XtQLGSY+y6YURHs+MAAABUQ+EGAASlwjKn5izeLEmaNrqzEqPtJicCAACojsINAAhK/1i6TblFFWqXEqtJg1qbHQcAAKAGCjcAIOhk55Xq5eWZkqQ7x6fLbmM6AwAAgYe/UAAAQefxRZtU7vJoYNtGGtutqdlxAAAAakXhBgAElfVZ+VqwJkuSdM/ErrJYLCYnAgAAqB2FGwAQNAzD0EMfb5BhSOf1aaHeaUlmRwIAADguCjcAIGh8sTFH324/qMgIq24b28XsOAAAACdE4QYABAWX26OHP9kgSbpmaFulNYoxOREAAMCJUbgBAEHhzVW7te1AsZJj7JoysqPZcQAAAH4ThRsAEPAKy5yas3izJGna6M5KiLKbnAgAAOC3UbgBAAHvH0u3KbeoQu1SYjVpUGuz4wAAANQJhRsAENCy80r18vJMSdKd49NltzF1AQCA4MBfLQCAgPb4ok0qd3k0sF0jje3W1Ow4AAAAdUbhBgAErPVZ+VqwJkuSdM+ErrJYLCYnAgAAqDsKNwAgIBmGoYc+3iDDkM7r00K905LMjgQAAOATCjcAICB9sTFH324/qMgIq24f18XsOAAAAD6jcAMAAo7L7dGsTzdKkv44tJ1aJceYnAgAAMB3FG4AQMB5+/s92ppTpOQYu/46ooPZcQAAAE4KhRsAEFCKy116MmOzJOnmUZ2UGG03OREAAMDJoXADAALKS8u3K7eoXG0ax+iKQW3MjgMAAHDSKNwAgICRU1imF5dtlyTdMS5dkRFMUwAAIHjxlwwAIGDMWbxFJRVu9UlL0oSezcyOAwAAUC8UbgBAQNiaU6i3Vu2WJN09oassFovJiQAAAOqHwg0ACAiPfLpJbo+hMd2aamC7RmbHAQAAqDcKNwDAdCu3H9TiDftls1o04+x0s+MAAAA0CAo3AMBUhmHo4U83SpJ+f1qaOqbGmZwIAACgYVC4AQCm+njdXv24O08xkTZNHd3J7DgAAAANhsINADBNhcujRxdukiT9ZVgHpcZHmZwIAACg4VC4AQCmmbdip3YdKlGTeIf+dGY7s+MAAAA0KAo3AMAU+aVOPfPFFknS9DGdFeuIMDkRAABAwwqYwj1r1ixZLBZNmzbN7CgAgFNg7pJtOlziVMfUOF3Sv5XZcQAAABpcQBTuVatW6cUXX1SvXr3MjgIAOAWy8kr1z68zJUl3np2uCFtATEcAAAANyvS/cIqKinTFFVfopZdeUnJystlxAACnwJOLNqvC5dGgdo00qmuq2XEAAAD8wvQT5qZMmaKJEydq9OjRevDBB0+4bnl5ucrLy6seFxQUSJKcTqecTqdfc5qpcttCeRsbGmPmO8bMN4yX7yrHat3uw/rvmj2SpDvGdpLL5TIzVoPjd6J+mOtDdxsbGmPmO8bMd4yZ78JhzHzZNothGIYfs5zQm2++qYceekirVq1SVFSURowYoT59+mjOnDm1rj9z5kzdf//9NZbPnz9fMTExfk4LAGgIc3+xamO+Vf0ae3R1Z4/ZcRpcSUmJJk2apPz8fCUkJJgdJ+gw1wMAAp0vc71phXv37t0aMGCAFi1apN69e0vSbxbu2l71TktLU25ubkj/UeN0OpWRkaExY8bIbrebHScoMGa+Y8x8w3j5zul06pl3FmvuBpvsNosW3jxUrRuFXoEqKChQSkoKhfskMdfzf0pdMWa+Y8x8x5j5LhzGzJe53rRDylevXq2cnBz179+/apnb7dayZcv07LPPqry8XDabrdrXOBwOORyOGs9lt9tD9od5tHDZzobEmPmOMfMN41V3bo+hD3d6Lx1y1elt1aFposmJ/IPfh/phrg+P7WxIjJnvGDPfMWa+C+Ux82W7TCvco0aN0rp166otu+aaa5Senq4ZM2bUKNsAgOD24Y97lVViUXxUhG46q6PZcQAAAPzOtMIdHx+vHj16VFsWGxurxo0b11gOAAhuZU63Zn++VZJ0/bB2So6NNDkRAACA/5n+tmAAgND36tc7tDe/TMmRhq4+vbXZcQAAAE4J098W7GhLliwxOwIAoIEdKq7Q8196925PbO2Rw84pQwAAIDywhxsA4FfPfLFFheUudW0Wr/4ppr0TJQAAwClH4QYA+M3Og8Wat2KnJGnG2Z1ltZgcCAAA4BSicAMA/ObRzzbJ6TY0rHMTDe3Q2Ow4AAAApxSFGwDgF2t2HdbHP+2VxSLdNT7d7DgAAACnHIUbANDgDMPQrE83SpIu6tdKXZsnmJwIAADg1KNwAwAa3OINOfou85AcEVbdOraz2XEAAABMQeEGADQol9ujRz7dIEm69ox2ap4YbXIiAAAAc1C4AQAN6q3vd2vbgWI1io3U9SM6mB0HAADANBRuAECDKalwac7iLZKkm87qqIQou8mJAAAAzEPhBgA0mFeWZ+pAYblaN4rRFYPamB0HAADAVBRuAECDOFhUrheWbZck3TauiyIjmGIAAEB4468hAECDePbLrSoqd6lny0T9rmdzs+MAAACYjsINAKi3XQdLNG/FTknSnePTZbVaTE4EAABgPgo3AKDensjYJKfb0JmdUjS0Y4rZcQAAAAIChRsAUC/rs/L1wdpsSdKMs9NNTgMAABA4KNwAgHr5fws3SpLO79NCPVommpwGAAAgcFC4AQAnbfmWA1q+JVd2m0W3ju1idhwAAICAQuEGAJwUj8eo2rt95eltlNYoxuREAAAAgYXCDQA4KR/9lK31WQWKc0ToprM6mR0HAAAg4FC4AQA+q3B59PiiTZKk64e3V6PYSJMTAQAABB4KNwDAZ/NX7tTuQ6VKjXfoj2e0MzsOAABAQKJwAwB8Uljm1NNfbJUkTRvdWTGRESYnAgAACEwUbgCAT15atl2HiivUPiVWlw5oZXYcAACAgEXhBgDUWU5hmV5anilJuuPsLoqwMY0AAAAcD38pAQDq7KnFW1TqdKtv6ySN697M7DgAAAABjcINAKiT7QeK9Oaq3ZKkO89Ol8ViMTkRAABAYKNwAwDq5PFFm+T2GBqVnqpB7RubHQcAACDgUbgBAL9pza7D+mTdPlkt0h1np5sdBwAAIChQuAEAJ2QYhmZ9ulGSdFG/VurSLN7kRAAAAMGBwg0AOKEvN+Xou8xDckRYdcuYzmbHAQAACBoUbgDAcbk9hv7fp5skSZOHtlWLpGiTEwEAAAQPCjcA4LgWrMnSpv2FSoy264bhHc2OAwAAEFQo3ACAWpU53XpykXfv9pSRHZQYYzc5EQAAQHChcAMAavWvb3coO79MLRKj9IfBbc2OAwAAEHQo3ACAGvJLnHruy22SpFvGdFaU3WZyIgAAgOBD4QYA1PD80q3KL3WqS9N4XdivldlxAAAAghKFGwBQTXZeqV79eockacb4LrJZLeYGAgAACFIUbgBANXMWb1aFy6OB7RppZJdUs+MAAAAELQo3AKDK5v2Fenf1HknSXePTZbGwdxsAAOBkUbgBAFUeXbhJHkM6u3sz9W2dbHYcAACAoEbhBgBIklbvPKzFG/bLapFuG9fF7DgAAABBj8INAJBhGHrss42SpIv7t1LH1DiTEwEAAAQ/CjcAQMu35GrF9kOKtFk1dXRns+MAAACEBAo3AIQ5797tTZKkK09vo5ZJ0SYnAgAACA0UbgAIc5+u36d1WfmKjbRpysgOZscBAAAIGRRuAAhjLrdHjy/y7t2+9sz2ahznMDkRAABA6KBwA0AY++8PWdp+oFjJMXZdd2Y7s+MAAACEFAo3AISpMqdbcxZvliRNGdlR8VF2kxMBAACEFgo3AISpf6/cpez8MjVPjNKVp7cxOw4AAEDIoXADQBgqKnfpuS+3SpKmjuqkKLvN5EQAAAChh8INAGHoleWZOlRcofYpsbq4fyuz4wAAAIQkCjcAhJlDxRV6afl2SdL0sZ0VYWMqAAAA8Af+ygKAMDN3yVYVlbvUvUWCJvRobnYcAACAkEXhBoAwsje/VK9/u1OSdPu4LrJaLSYnAgAACF0UbgAII09/vkUVLo8Gtmuk4Z2bmB0HAAAgpFG4ASBMbD9QpLe/3yNJmnF2F1ks7N0GAADwJ1ML99y5c9WrVy8lJCQoISFBgwcP1qeffmpmJAAIWU9mbJbbY2hUeqr6t2lkdhwAAICQZ2rhbtWqlR555BF9//33+v7773XWWWfpvPPO088//2xmLAAIOeuz8vW/n/bKYpFuG9fF7DgAAABhIcLMb37OOedUe/zQQw9p7ty5WrFihbp3715j/fLycpWXl1c9LigokCQ5nU45nU7/hjVR5baF8jY2NMbMd4yZb4JtvB5buFGS9LuezdQxJdqU3ME2Zicr1LfP35jrQ3cbGxpj5jvGzHeMme/CYcx82TaLYRiGH7PUmdvt1jvvvKOrr75aa9asUbdu3WqsM3PmTN1///01ls+fP18xMTGnIiYABJ2tBdIzP0fIajF0Tx+3UqLMThTaSkpKNGnSJOXn5yshIcHsOEGHuR4AEOh8metNL9zr1q3T4MGDVVZWpri4OM2fP18TJkyodd3aXvVOS0tTbm5uSP9R43Q6lZGRoTFjxshut5sdJygwZr5jzHwTLONlGIZ+//Iq/bArT5ef1koPnFvzxcxTJVjGrL4KCgqUkpJC4T5JzPWh/e+jITFmvmPMfMeY+S4cxsyXud7UQ8olqUuXLlq7dq3y8vL03nvv6eqrr9bSpUtr3cPtcDjkcDhqLLfb7SH7wzxauGxnQ2LMfMeY+SbQx+vzDfv1w648RdmtmjamS0BkDfQxq69Q3rZTgbk+PLazITFmvmPMfMeY+S6Ux8yX7TK9cEdGRqpjx46SpAEDBmjVqlV66qmn9MILL5icDACCm8dj6LHPNkmSrh7SVk0TOJYcAADgVAq49+E2DKPaoWQAgJPz0U/Z2rivUPFREfrr8A5mxwEAAAg7pu7hvvvuuzV+/HilpaWpsLBQb775ppYsWaKFCxeaGQsAgp7T7dGTGZslSX8Z1l5JMZEmJwIAAAg/phbu/fv366qrrtLevXuVmJioXr16aeHChRozZoyZsQAg6L21ard2HixRSlykrhnazuw4AAAAYcnUwv3KK6+Y+e0BICSVVrj19OdbJEk3juyoWIfpl+sAAAAISwF3DjcAoH5e/3aHcgrL1TIpWpcPam12HAAAgLBF4QaAEJJf6tTcJdskSbeM6SxHhM3kRAAAAOGLwg0AIeSlZduVX+pUp9Q4XdC3pdlxAAAAwhqFGwBCxIHCcv3z60xJ0m3jushmtZicCAAAILxRuAEgRDz35VaVVLjVJy1JY7s1NTsOAABA2KNwA0AI2H2oRP9euVOSdMe4LrJY2LsNAABgNgo3AISAOYu3yOk2dEbHFA3pmGJ2HAAAAIjCDQBBb8v+Qi1Ys0eSdPu4LianAQAAQCUKNwAEuccXbZLHkM7u3ky905LMjgMAAIAjKNwAEMTW7s7TZz/vl9Ui3Taus9lxAAAAcBQKNwAEscc+2yhJurBfK3VMjTc5DQAAAI5G4QaAIPX11lx9vfWgIm1WTRvdyew4AAAAOAaFGwCCkGEYevSzTZKkSYNaq1VyjMmJAAAAcCwKNwAEoc9+3q8fd+cpJtKmG8/qaHYcAAAA1ILCDQBBxu0x9Pgi797ta89op5Q4h8mJAAAAUBsKNwAEmf/+sEdbc4qUFGPXdcPamx0HAAAAx0HhBoAgUu5ya87iLZKkvw7voIQou8mJAAAAcDwUbgAIIvNX7lJWXqmaJjh09ZC2ZscBAADACVC4ASBIFJe79OwXWyVJN4/qpCi7zeREAAAAOBEKNwAEiX9+lamDxRVq2zhGlw5IMzsOAAAAfgOFGwCCwOHiCr24bLsk6ZYxnWW38d83AABAoOMvNgAIAv9Yuk2F5S51bZ6gc3q1MDsOAAAA6oDCDQABbl9+mV77Zock6fZxnWW1WswNBAAAgDqhcANAgHv6iy0qd3k0oE2yRnZJNTsOAAAA6ojCDQABbEdusd5etVuSdMfZ6bJY2LsNAAAQLCjcABDAnszYLJfH0IguTTSwXSOz4wAAAMAHFG4ACFC/ZBfowx+zJUm3je1ichoAAAD4isINAAHq8UWbJEnn9G6hHi0TTU4DAAAAX1G4ASAAfb/jkL7YmCOb1aLpYzqbHQcAAAAngcINAAHGMAw9utC7d/vSAWlqlxJrciIAAACcDAo3AASYJZsP6Lsdh+SIsGrqqE5mxwEAAMBJonADQADxeAw9dmTv9tVD2qpZYpTJiQAAAHCyKNwAEEA+XrdXv+wtULwjQn8d3sHsOAAAAKgHCjcABAin26MnMzZLkq4b1l7JsZEmJwIAAEB9+Fy4J0+erGXLlvkjCwCEtXdX71FmbrEax0bqj2e0MzsOAAAA6snnwl1YWKixY8eqU6dOevjhh5WVleWPXAAQVsqcbs1Z7N27PWVkR8U5IkxOBAAAgPryuXC/9957ysrK0o033qh33nlHbdu21fjx4/Xuu+/K6XT6IyMAhLx/fbtD+wvK1TIpWlec3trsOAAAAGgAJ3UOd+PGjTV16lStWbNG3333nTp27KirrrpKLVq00C233KItW7Y0dE4ACFkFZU49v2SbJGnq6E5yRNhMTgQAAICGUK+Lpu3du1eLFi3SokWLZLPZNGHCBP3888/q1q2bZs+e3VAZASCkvbxsu/JKnOrQJFYX9m1pdhwAAAA0EJ8Lt9Pp1Hvvvaff/e53atOmjd555x3dcsst2rt3r15//XUtWrRIb7zxhh544AF/5AWAkJJbVK6Xv8qUJN02tosibLx5BAAAQKjw+ao8zZs3l8fj0eWXX67vvvtOffr0qbHOuHHjlJSU1ADxACC0PfflVpVUuNWrVaLO7tHM7DgAAABoQD4X7tmzZ+uSSy5RVFTUcddJTk5WZmZmvYIBQKjbc7hE/16xS5J0+7guslgsJicCAABAQ/K5cF911VX+yAEAYeepxVtU4fZocPvGOqNjitlxAAAA0MA4WRAATLA1p1Dv/bBHknT72ezdBgAACEUUbgAwwROLNstjSGO6NVW/1slmxwEAAIAfULgB4BT7aU+ePl2/TxaL98rkAAAACE0UbgA4xR77bJMk6YI+LdWlWbzJaQAAAOAvFG4AOIW+2Zar5VtyZbdZdMuYzmbHAQAAgB9RuAHgFDEMQ48u9O7dvnxga6U1ijE5EQAAAPyJwg0Ap0jGL/u1dneeou023XhWR7PjAAAAwM8o3ABwCrg9hh5f5N27fc3QtkqNjzI5EQAAAPyNwg0Ap8AHa7O0eX+REqPt+svwDmbHAQAAwClA4QYAP6tweTR78WZJ0vXDOygx2m5yIgAAAJwKFG4A8LM3V+3S7kOlSo13aPKQtmbHAQAAwClC4QYAPyqpcOnpz7dKkm4a1UnRkTaTEwEAAOBUMbVwz5o1S6eddpri4+OVmpqq888/X5s2bTIzEgA0qFe/3qHconK1bhSjywakmR0HAAAAp5CphXvp0qWaMmWKVqxYoYyMDLlcLo0dO1bFxcVmxgKABpFXUqF/LN0mSZo+prMiIzioCAAAIJxEmPnNFy5cWO3xq6++qtTUVK1evVrDhg0zKRUANIx/LN2uwjKX0pvF69zeLcyOAwAAgFPM1MJ9rPz8fElSo0aNav18eXm5ysvLqx4XFBRIkpxOp5xOp/8DmqRy20J5GxsaY+Y7xsw3vzVeOYXleu2bTEnStFEd5Ha75HafsngBKVx+x0J9+/yNuT50t7GhMWa+Y8x8x5j5LhzGzJdtsxiGYfgxS50ZhqHzzjtPhw8f1vLly2tdZ+bMmbr//vtrLJ8/f75iYmL8HREA6uzt7VZ9vd+qtnGGpvVwy2IxOxFOlZKSEk2aNEn5+flKSEgwO07QYa4HAAQ6X+b6gCncU6ZM0ccff6yvvvpKrVq1qnWd2l71TktLU25ubkj/UeN0OpWRkaExY8bIbuf9e+uCMfMdY+abE43XzkMlOvupr+XyGJr3xwEa1K72o3bCTbj8jhUUFCglJYXCfZKY60P730dDYsx8x5j5jjHzXTiMmS9zfUAcUn7TTTfpww8/1LJly45btiXJ4XDI4XDUWG6320P2h3m0cNnOhsSY+Y4x801t4/Xsl9vl8hga1rmJzujc1KRkgSvUf8dCedtOBeb68NjOhsSY+Y4x8x1j5rtQHjNftsvUwm0Yhm666SYtWLBAS5YsUbt27cyMAwD1tnFfgT74MVuSdMe4LianAQAAgJlMLdxTpkzR/Pnz9cEHHyg+Pl779u2TJCUmJio6OtrMaABwUh7/bJMMQ5rYs7l6tEw0Ow4AAABMZOqbws6dO1f5+fkaMWKEmjdvXnV76623zIwFACdl9c5DWrwhRzarRdPHdjY7DgAAAExm+iHlABAKDMPQows3SZIu7tdKHZrEmZwIAAAAZjN1DzcAhIplW3K1MvOQIiOsmjq6k9lxAAAAEAAo3ABQTx6Pocc+2yhJuur0NmqRxDUoAAAAQOEGgHr7dP0+rc8qUGykTTeM6GB2HAAAAAQICjcA1IPL7dETi7znbv/pzPZqHFfz/YMBAAAQnijcAFAP/12Tre25xWoUG6k/ndnO7DgAAAAIIBRuADhJFW7p6S+3SZKmjOyo+Ci7yYkAAAAQSCjcAHCSvtpv0f6CcrVMitYVg1qbHQcAAAABhsINACehsMypjCzvf6FTR3dSlN1mciIAAAAEGgo3AJyEl7/aqRKXRR2axOrCvi3NjgMAAIAAROEGAB8dKCzXq9/skCRNH91RETb+KwUAAEBN/JUIAD569ostKnV61CbO0JiuqWbHAQAAQICicAOAD3YfKtH873ZJks5p7ZHFYjE5EQAAAAIVhRsAfDA7Y7OcbkNDOzRWp0TD7DgAAAAIYBRuAKijjfsKtGBtliTp1jEdTU4DAACAQEfhBoA6evyzTTIMaWLP5urZMtHsOAAAAAhwFG4AqIPvdxzS4g05slktmj62s9lxAAAAEAQo3ADwGwzD0KMLN0mSLh3QSh2axJmcCAAAAMGAwg0Av2HJ5gP6bschRUZYdfOoTmbHAQAAQJCgcAPACXg8v+7dnjykrZonRpucCAAAAMGCwg0AJ/DRT9nasLdA8Y4I/XV4B7PjAAAAIIhQuAHgOJxuj57M2CxJ+svw9kqOjTQ5EQAAAIIJhRsAjuOtVbu182CJUuIidc3QdmbHAQAAQJChcANALUor3Hrq8y2SpJvO6qRYR4TJiQAAABBsKNwAUItXv8nUgcJytUqO1uUDW5sdBwAAAEGIwg0Ax8gvceofS7ZJkm4d21mREfxXCQAAAN/xVyQAHGPu0m0qKHMpvVm8zu3d0uw4AAAACFIUbgA4SnZeqV79OlOSdNvYLrJZLSYnAgAAQLCicAPAUWZnbFa5y6OBbRtpVNdUs+MAAAAgiFG4AeCITfsK9d4PeyRJd05Il8XC3m0AAACcPAo3ABzx6MKN8hjS+B7N1K91stlxAAAAEOQo3AAgaeX2g/p8Y45sVotuH9fF7DgAAAAIARRuAGHPMAzN+nSjJOnygWlq3yTO5EQAAAAIBRRuAGHv0/X7tHZ3nmIibZo6qrPZcQAAABAiKNwAwprT7dFjn22SJF13Zns1iXeYnAgAAAChgsINIKy9+d0uZeYWKyUuUtcNa292HAAAAIQQCjeAsFVU7tJTn2+RJN08qpPiHBEmJwIAAEAooXADCFsvLduu3KIKtW0co8sHtjY7DgAAAEIMhRtAWMopLNNLy7dLkm4fly67jf8OAQAA0LD4CxNAWHrm860qqXCrd1qSJvRsZnYcAAAAhCAKN4Cwk5lbrP98t0uSdNf4dFksFpMTAQAAIBRRuAGEncc+2yiXx9BZ6ak6vX1js+MAAAAgRFG4AYSVNbsO65N1+2S1SDPOTjc7DgAAAEIYhRtA2DAMQw99vEGSdFG/VurSLN7kRAAAAAhlFG4AYWPh+n36fudhRdttunVsF7PjAAAAIMRRuAGEhQqXR48s3ChJum5YezVLjDI5EQAAAEIdhRtAWPjXtzu082CJmsQ79Jdh7c2OAwAAgDBA4QYQ8vJKKvTMF1slSbeO6axYR4TJiQAAABAOKNwAQt7Tn29VfqlT6c3idcmANLPjAAAAIExQuAGEtB25xXpjxQ5J0t0TuspmtZgbCAAAAGGDwg0gpD3y6UY53YaGd26iYZ2bmB0HAAAAYYTCDSBkfZd5SAt/3ierRbpnYlez4wAAACDMULgBhCSPx9BDH/8iSbrstNbq3DTe5EQAAAAIN1yqF0BI+uinbP24J1+xkTZNH9PZ7DgAACCYeNxSxSGpOFuN3T/LsqdMcuV5l7mKJFex5Crx3ruPfGy4JcPjvZfnyMdHbtYIyWqXLHbvvdUuWSN/vbdFSxGxv95ssdUfR8RKEXGSPUGyJ0qRiZItyuxRQh1QuAGEnDKnW48u3CRJ+uuIDmoS7zA5EQAACDjOIqlgg5T/i1S0TSreKRXv8t6X7JYMl+ySzpCkb03OWhtrpLd82xOlyKRfi7j9qFtkomRP+nV55XqVN1ukyRsR+ijcAELOP7/OVFZeqZonRunaM9qbHQcAAJjJMKSSPdLB77y3vJ+8Jbtk129/qT1Zxa5oxTRqI2tUE8nRSIqIP7IXOuaovc8x3r3XFutRN5skq2SxSB6XZDglj1PyVBy5P+qx+8je8qNvNZYVSc4C702G9+vKD3hvJ8sWfVQJTzqmoCcdVeCP87mIOO+24rgo3ABCSk5hmZ7/cpsk6fZxXRQdaTM5EQAAOKUMQ8pbJ+3/XMpZ6i3ZpXtrXzeqmZTYTYrvJMW28d5iWnvvo5vJ5ZY+/+QTTThrgqx2+6ndjuMxPJKzUHLme28V+Ud9nFeH5Xne8i5J7lKptPT44/NbLFYpIqHaHnZbRIL6lRfJuiZDciQf9bkj9/Z4b1GvvNnjJKvD+8JECDK1cC9btkyPPfaYVq9erb1792rBggU6//zzzYwEIMg9/tkmFZW71DstSef3aWl2HAAAcCqUHZCy/ifty/AW7bKc6p+32KSknlLjQVKjflJidymhq3eP9Ym4nf7LfLIsVu/e5cjEk38Oj+vI3vJjCnld75153r3zhsf7sTOv6qmtktIkaesSH7bJVr2AR8TVLOW1LbPFeI8usEVX//joZbYoU8u8qYW7uLhYvXv31jXXXKOLLrrIzCgAQsBPe/L0zuo9kqT7zukmqzU0XykFAADynmu9+31pz3+lA195y18lW4yUOkxqepaUMthbsiNiTIsacKwR3hcbfusFh+MxDMld5i3aFfnV7l1lh7Tpp5VK79BMNnfRMevkHbno3JGbu+zI87l/Lf+lDbKF1VWV8BjpvB2n9DB4Uwv3+PHjNX78eDMjAAgRhmFo5oc/yzCkC/u2VL/WyWZHAgAADa0iX9r1jpT5urdkHy25n9RyotRstNT4dC4I5k8WixQR7b1FN6/2KcPp1NaNLdS55wTZfuswfI/r1/PTK2/OohM/PnqZu9R7hXh3yVEfl3ofe446OsFd6r1Zi075OedBdQ53eXm5ysvLqx4XFBRIkpxOp5zOADzco4FUblsob2NDY8x8F+xj9uGPe/XDrjzFRNp0y+gOft+OYB8vM4TLmIX69vkbc33obmNDY8x8F7RjZhiyHFgq6/ZXZMn6QBaPd6+oIauMJmfIaHmePC3O9Z53Xcmj6oXrJAXtmJnI5zGzxEj2GMme2rBBPK5fy3fVfbnUAD9LX34fLIZhGPX+jg3AYrH85jncM2fO1P33319j+fz58xUTwyEiQLgqd0sPrbUpv8KiiWlujW0VEP+tIUyVlJRo0qRJys/PV0JCgtlxgg5zPYBKNqNcrVxL1N75sRKMX68oXmBJ0+6IkdoTMUJl1pM8JBqoB1/m+qAq3LW96p2Wlqbc3NyQ/qPG6XQqIyNDY8aMkT1Qro4Y4Bgz3wXzmM35fKueW7JdrZKitPDmoXLY/X9l8mAeL7OEy5gVFBQoJSWFwn2SmOtD+99HQ2LMfBc0Y1aWI+uWp2Td9rIszsOSJMMWI0+bK2W0u0ZGcr9TdhGsoBmzABIOY+bLXB9Uh5Q7HA45HI4ay+12e8j+MI8WLtvZkBgz3wXbmO0+VKKXv9ohSfrb77opLibqlH7/YBuvQBDqYxbK23YqMNeHx3Y2JMbMdwE7ZiVZ0obHpa0veA8BlqS49lLnG2Vpf41skUmmRQvYMQtgoTxmvmxXUBVuADjWI59uVLnLo8HtG2tc92ZmxwEAAL4q3Set/z9p28uSp8K7rNFpUo97pBa/k6z+P3IN8BdTC3dRUZG2bt1a9TgzM1Nr165Vo0aN1Lp1axOTAQgGK7Yf1Mfr9spqke49p5ssJr7HIgAA8JGzyLtHe+Pj3itVS1KToVL3v0vNx5r63slAQzG1cH///fcaOXJk1ePp06dLkq6++mq99tprJqUCEAzcHkP3f/SLJGnSoNbq2jx0z+0EACCkeFzevdnrZkpl+73LGg+U+jwipY6gaCOkmFq4R4wYoQC5ZhuAIPOf73Zpw94CJURFaPqYLmbHAQAAdXHgG2nVDVLej97HcR2lPrOktIso2ghJnMMNIOgcLCrXY59tkiRNH9NZjWIjTU4EAABOqOyAtHaGtP1V7+PIZKnn/VLHv0g25nGELgo3gKDz/xZuVH6pU92aJ+jK09uYHQcAAByP4fEePr72TqnC+xZf6nCt1PsRKSrF3GzAKUDhBhBUVu88rLe/3yNJ+r/zuyvCZjU5EQAAqFXRDmnltdL+L7yPk3pLpz0vNRliaizgVKJwAwgabo+hv7+/XpJ0Sf9W6t+mkcmJAABADYbH+17aa273Xn3cFi31fkjqfJNkpX4gvPAbDyBo/HvlTv1y5EJpM8anmx0HAAAcq3intOKPv+7VbnKmdPo/pfiO5uYCTELhBhAUDhT+eqG0289OV0qcw+REAACgmp1vS9/9WXLmH9mrPUvqcpNk4fQvhC8KN4Cg8MinG1VY5lKPlgmaNLC12XEAAEAlV7G0eqq07RXv48anS0PeYK82IAo3gCCwaschvffDkQulnddDNivv0wkAQEA4vFb6+vdSwSZJFqn73VLP+ySr3exkQECgcAMIaC63p+pCab8/LU19WyebnAgAAMgwpC3PSz9MlzwVUnQLacg8qelIs5MBAYXCDSCg/fPrTG3cV6ikGLvuOJsLpQEAYDpXsfTdX6Qd//Y+bnmuNOgV3lcbqAWFG0DA2n2oRE9mbJYk3T2+qxrFRpqcCACAMFe4VVp+oZS3TrLYpD6PSum3SBZO9wJqQ+EGEJAMw9Df3l+vMqdHp7dvpEsGtDI7EgAA4W3Ph9K3f/BehTyqqXTG21LqMLNTAQGNwg0gIH30014t3XxAkTarHrqgpyy8cg4AgDk8bmndfdLPD3kfNxkqDX1bimlhbi4gCFC4AQSc/BKnHvjoZ0nSlJEd1aFJnMmJAAAIU84i6ZtJUtZH3sddpkp9H+Mq5EAdUbgBBJxHFm5QblGFOjSJ1fUj2psdBwCA8FSyR1p6jvetv2xR3gujtZ1kdiogqFC4AQSU7zIP6T/f7ZYkzbqwlxwRNpMTAQAQhg794C3bpdlSVKo07AMp5XSzUwFBh8INIGCUu9y6678/SZIuH5imge0amZwIAIAwtOcD6etJkrtESuwuDf+fFNfW7FRAULKaHQAAKj335TZtO1CslDiH7jy7q9lxAAAIL4YhbXhSWnaBt2w3GyuN+ZqyDdQDe7gBBISfs/P1/JdbJUkzz+2mxBguxgIAwCnjcUrf3yRtfcH7uOP10oBnJCt1AagP/gUBMJ3T7dFt7/wkl8fQ+B7NNLFnc7MjAQAQPirypa8ukfZlSLJI/Z70Xo2ct+QE6o3CDcB0z3+5TRv2Fig5xq4HzuvBe24DAHCqFO2Qlk6U8n+RbDHS0P9Irc41OxUQMijcAEy1YW+BnvliiyRp5rnd1STeYXIiAADCRO4Kadl5UlmOFN1CGv6R1Kif2amAkELhBmAap9uj29/9US6PobHdmurc3i3MjgQAQHjY+Zb07dWSp1xK7ust2zEtzU4FhByuUg7ANC8s3ab1WQVKjLbrwQs4lBwAAL8zDFk3zJK+/r23bLc8Rxq9jLIN+Al7uAGYYtO+Qj31ufdQ8vvP7a7U+CiTEwEAEOLc5epb8bRs67/0Pu5yi9T3MclqMzcXEMIo3ABOuXKXW9PeWiun29Dorqk6rw+HkgMA4FflB2VbdoFau5bLsNhkGfCM1OmvZqcCQh6FG8Ap9+Sizdqwt0CNYiP18IU9OZQcAAB/KtgsLZkoa9FWORUjyxlvKyJtotmpgLDAOdwATqlvtx3Ui8u3S5IeubAnh5IDAOBP+5dKiwZLRVtlxLTR8uhZMpqNNTsVEDYo3ABOmfxSp259e60MQ/r9aWka272Z2ZEAAAhd21+XvhwjVRySGg+Sa9RXKrS2MTsVEFYo3ABOmfs+WK/s/DK1aRyjv/+um9lxAAAITYZH+vFv0orJkscptb5EGvWlFNXU7GRA2OEcbgCnxAdrs/T+2mzZrBbNvqyPYh389wMAQINzlUorrpF2veV93P1uqdf/SRar5HSamw0IQ/zFC8Dv9hwu0d/fXy9JmjKyo/q1TjY5EQAAIah0n7TsPOngd5LVLg18UWo/2exUQFijcAPwK6fbo5v+s0YFZS71SUvSTWd1NDsSAACh5/CP0tJzpJLdUmQj6cz3pKYjzE4FhD0KNwC/evyzTVqzK0/xURF65vK+stu4dAQAAA1qz0fSN5dLrmIpoYs0/H9SPC9wA4GAv3wB+M2XG3P0wjLvW4A9dnFvpTWKMTkRAAAhxDCkDY97DyN3FUvNRktjv6VsAwGEPdwA/GJvfqmmv71WkjR5SFud3YO3AAMAoMG4K6Tvb5C2veJ93PF6acDT3nO3AQQMCjeABudyezT1P2t1uMSpHi0TdNeEdLMjAQAQOsoOSF9dKuUs8V59vN9sqfNNksVidjIAx6BwA2hwj322Sd/tOKQ4R4SevbyfHBE2syMBABAaDv0gLbtAKtklRcRLZ7wltRhvdioAx0HhBtCg/vdTdtV52//vol5qmxJrciIAAEJE5hvSd3+W3GVSfCdp2PtSYjezUwE4AQo3gAazaV+h7nj3J0nSX4a318RezU1OBABACPA4pR9ukzY/7X3c4nfSkDekyCRTYwH4bRRuAA0iv9Spv7zxvUoq3DqjY4puH9vF7EgAAAS/shzpq0uknGXexz3ulXre5z13G0DAo3ADqDePx9Atb63VjoMlapkUrWcu76sI3m8bAID6yVkmfX25VJrtPV97yBtSq/PMTgXABxRuAPX2ZMZmfbExR44Iq164qr+SYyPNjgQAQPAyPNLPs6R193o/TkiXzlwgJfKuH0CwoXADqJf3Vu/Rs19ulSTNurCnerRMNDkRAABBrHS/9O1V0r4M7+N2f5AGPCfZ48zNBeCkULgBnLTvMg/pzv96L5J2w4gOurBfK5MTAQAQxPZ9IX1zhVS2T7LFSKc9J7WfbHYqAPVA4QZwUnbkFusvb3wvp9vQhJ7NdBsXSQMA4OS4SqUf75E2zfY+TuwunfE2b/kFhAAKNwCf5Zc49cfXV+lwiVO9WyXqiUv6yGq1mB0LAIDgc+gH7yHk+b94H3f8s9RvthQRY24uAA2Cwg3AJ6UVbv3x9VXafqBYLRKj9NLVAxQdaTM7FgAAwcXjkn55RFp3v2S4pKim0qBXpJYTzU4GoAFRuAHUmdPt0ZT5P2j1zsNKiIrQP685TanxUWbHAgAguBxaLa28Tjq8xvs47WLptLlSVIq5uQA0OAo3gDoxDEN3vreu6u2/Xpl8mtKbJZgdCwCA4OEskn66V9r8lPftviKTpf5PS22vkCycmgWEIgo3gDp55NONeu+HPbJZLXpuUj+d1raR2ZEAAAgeWR9Lq26QSnZ5H7e53HuudnRTc3MB8CsKN4DfNGfxZr2wbLsk6f9d1Euju/HHAQAAdZK/UfphurT3U+/j2Lbew8dbnG1qLACnBoUbwAk98/kWzVm8RZL0t4lddXF/3msbAIDfVH7Ie0G0Lc97L4pmtUtdpkk975MiYs1OB+AUoXADOK7nvtyqJzI2S5LuHJ+uP53Z3uREAAAEOFextPk56Zf/J1Uc8i5rea7U93EpoZO52QCcchRuALX6x9JteuyzTZKk28d10fXDO5icCACAAOYuk7a8IP0ySyrb712W2EPq96TUfIy52QCYhsINoBrDMPTEok165outkqRbx3TWlJEdTU4FAECAchZJ216RNjwmlWZ5l8W28x463vYKycqf20A4s5od4Pnnn1e7du0UFRWl/v37a/ny5WZHAsKWx5Ae+HhjVdm+fVwX3TSKw98AAKihdK+09m7p/TTph2nesh3TShr4gnTOJqn91ZRtAObu4X7rrbc0bdo0Pf/88xo6dKheeOEFjR8/Xr/88otat25tZjQg7DjdHs3batXq3N2yWKQHzuuhq05vY3YsAAACh2FIud9IW1+Udr4peSq8y+M7SenTpfaTJVuUqREBBBZTC/eTTz6pa6+9Vn/6058kSXPmzNFnn32muXPnatasWWZGA8JKfqlTf533g1bnWhVhteiJS3vrvD4tzY4FAEBgKD8oZb4hbXtJyv/l1+VNhkrpt0mtzpUsph84CiAAmVa4KyoqtHr1at15553Vlo8dO1bffPNNrV9TXl6u8vLyqscFBQWSJKfTKafT6b+wJqvctlDexobGmNXdrkMl+vO8Ndp2oFiRVkNPXdZLo7ulMna/gd8x34XLmIX69vkbc33obmND8/uYOQtkyf5Q1l3vyLI/QxbDJUkybDEy0i6Vp/2fZDQe6F3X5Zbk9k+OBsTvme8YM9+Fw5j5sm0WwzAMP2Y5ruzsbLVs2VJff/21hgwZUrX84Ycf1uuvv65NmzbV+JqZM2fq/vvvr7F8/vz5iomJqVee3DLJ5ZEcNinK5r23Wur1lEDAyyyUXt5oU5HLosRIQ39Od6sVbw0K1EtJSYkmTZqk/Px8JSQkmB0n6Phzrm/sXqc2zsVyWyLllkNuS5TccshlcXgfK1Jui0NuRR25d8htcch19HJFsiczhEV5DinVvVrN3N8r1f2DbPr1j+o8a3vtjBijPRHD5LIwWQLhzJe53vTC/c0332jw4MFVyx966CG98cYb2rhxY42vqe1V77S0NOXm5tb7j5ppb/2kj9fvq7YsJtKmOEeE4hw2xToijnxc83Gsw7teQrRdiVERSoy2KynGroQouyIj6j8pO51OZWRkaMyYMbLb7fV+vnDAmJ2YYRiav2qPHvpko5xuQ92ax+u5y3rqp5XLGLM64nfMd+EyZgUFBUpJSaFwnyR/zvXWbS/K9sON9Y0owxolRcRIthgpIkZGRLwUES/Z46SIeO9je+WyI48j4qqWGZWfq1zPGhk2/z4aUoOMmatIloMrZMlZKuu+z2TJW1vt00Z8F3nSLpEn7RIpoWv9Q5uM3zPfMWa+C4cx82WuN+2Q8pSUFNlsNu3bV73k5uTkqGnTprV+jcPhkMPhqLHcbrfX+4cZ7YhQcoxdhWUuuTze1yBKKtwqqXArp/Dknzc20qbEaLsSYyKVdKSIJ8XYlRgdWVXMG8dGqnGcQylxkUqJcygm0iaLpebu9YbYznDDmNVUUuHSPQt+1oI13rcuObt7Mz15WW/ZLYZ+EmPmK8bLd6E+ZqG8baeCP+d6NT1D6vuE5C6RXCWSq/jXj6vui495XHlfWvU0Fk+ZVFEm6ZD3cf1SSdZIRUTEa7QzQtFLmsviSJbsSVJk0lH3id772pbZE8J6r3udfzcMj1SUKR1eIx34Rjqw3PuxcfSh4Bap8WlSiwlSq/NlSeolm8Uim9/SmyPU/x/2B8bMd6E8Zr5sl2mFOzIyUv3791dGRoYuuOCCquUZGRk677zzTnmexy/pLcm756/c5VFxuUtF5S4VlrmqPq66lVX/uLjCu15BqVN5pU7llThVUOaUYUjFFW4VV7iVnV9W5yxRdqtS4hxqHOdQk7hIJcfYdXifVTnf7lRqQrSaxDvUNCFKzRKiFOvg7SZQd1tzCjXl32u0aX+hbFaLZpzdRded2V4WiyWkz7MBAElScm/v7WQYHm/pPraMO4skV6HkLPz1vurjgurLj72vLPGeClkqDipWkvL2n0Q4i7d0VyvhSUfdJx51f9Tt6Me2KKmWF/uDksctleyWCrd4bwUbpMNrpcM/esf+WLFtpCZnSM3HeW9Rqac8MoDQZWpbmz59uq666ioNGDBAgwcP1osvvqhdu3bp+uuvNy2TxWJRlN2mKLtNjeNqvsJeV26PoaIyl/JKK5RXUlnEK5R/pJBX3ueVVCi3uEIHi8qVW1SuMqdHZU6P9hwu1Z7DpUc9o1WLs2qe1x7viFDTRG/5bnbkvupxQpSaJjqUEuuQlRPSw5rHY+j1b3fokU83qtzlUUqcQ89O6qvT2zc2OxoABAeLVYqI9d4aisdVVb6dpYf07bLPNGRAN0V4iiVnnlSRd9R9vve+2rI8yV0myfB+3pl/8lms9uOX8WMfVxZ4e8KRQ+ujjxxef+TeGtnw5d3jPjJW+VKFd1stxdlq51ws67pvpPL93vfFLtklFW3/9e26amxnpJTUU2p0mpR6ptTkTCk2rWGzAsBRTC3cl112mQ4ePKgHHnhAe/fuVY8ePfTJJ5+oTZvgf+9fm9WixBi7EmPsauNDpykud+lgUYUOFJXrYFG5DhZXaH9+qX74ebPiUprrUIlTBwrLtb+g3LsHvtylwpwibc0pOu5z2m0WtUyKVqvkGLVKjj5yi6m6T42nkIeyfflluv3dH7V8S64k6cxOKXrikt5KTeB9QgHAVNYIKTL5yK25Dtt2ymh+tuTLIZju8pplvLZyXlnaK28VlR8XSDIkj1Mqz/Xe6sti/bWE26K9Rdxilyy2IzfrUR8feWy4vRk8FUduR31ceVTAMSIk9ZKkmpf98RbruPbe98eO7yQl9/HeEtK9Ly4AwCli+vHIN9xwg2644QazYwSMWEeEYh0Rat341yuxOp1OfVKyURMm9K52vkBhmVP7C8q0L79c+wrKjnxcVu3jA0XlcroN7ThYoh0HS2r9npE2q1okRalVcozSGkUrrVGM2jWOVduUWLVtHKvoyFA7cyk8uD2G/r1ypx5buEmF5S5F2a26e0JXXXV6m1qvEQAACEI2h2RLPfnDoA2P5Co6qoBXlvG8Y4r5MR9XlvWjD7M3PEc9Z7H31tCsjqq97Z7IFO3Lk5q26ydbbCspurkU01KK6yjFpElW/n4BYD7TCzdOXnyUXfFRdnVMjT/uOk63R/sLypR1uFS7D5dqz+GSI4ere+/35pepwu05YSFvlhCltikxanekgLdNiVW7lFi1bhSjKDuTWSBan5Wvexas0497vIcX9k5L0hOX9FbH1DiTkwEAAorF6j003J4gqR6HVhtH9pK7S44q4UeXcZe3iBvuIzdP9XuLzbtX2mo/cl/5sd17NffKQ9ptv57u53Y6teqTTzSh7wTZQvTCTACCH4U7xNlt1iOHj8doUC2fd7k92ldQVnXO+O5DJdp5sFiZB0u0I7dY+aVO7Svw7jVfsf1Qta+1WKQWidFq3yRWnVLj1alpnDqmxqlTapySYiJPzQaimv0FZZqzeLPeWrVbHsN7jv8dZ3fRpEFtZOO0AQCAv1gski3Se1OS2WkAIGBQuMNcxFGFvDaHiyuUebBYO3K9t8oiviO3WIXlLmXllSorr7Tq/OBKKXGR6phaWcDj1enIx03iHRzO7AeFZU69uGy7Xlq+XWVO7yF9v+vVXPf+rhvnagMAAAAmoXDjhJJjI5UcG6l+rZOrLTcMQweLK5SZW6xtOUXacuTCbVtzipSVV6rcogrlFh2qsVc8ISri1xLeNE6dmnrLePPEKIr4SThUXKHXvs7U69/uVH6p9229+rdJ1l3j0zWgbSOT0wEAAADhjcKNk2KxWJQS51BKnEOnHVPsistd2nbAW7635BRpy/4ibTtQpJ0Hi1VQ5tIPu/L0w668al8T5/AW8c5NvWW8Y9M4dW4arxYU8Vpl5hbrjW936j/f7VKp0y1Jat8kVneMS9e47k0ZMwAAACAAULjR4GIdEerVKkm9WiVVW17mdGvHwWJt2V+5R7xQm/cXaUdusYrKXVq7O09rd+dVf65Imzoe2QteWcY7NY1Ti8TosHsrM6fbo8837Ne8Fbv01dZfD+Hv3iJBU0Z21LjuzThPGwAAAAggFG6cMlF2m9KbJSi9WUK15RUuz1FFvLDqfvuBYhVXuPXj7jz9eEwRj4m0VTs0vbKMt0wKrSLu9hhasf2g/vdTthau36fDJd7Dxi0WaUTnJpo8tJ2GdUphjzYAAAAQgCjcMF1khFWdm8arc9N4Sc2rljvdHu08WKzN+72HpW/OKdTW/UXanlukkgq3ftqTr5+OvO1VpWi7t4h3bBIj1yGLojYdUNfmSWqVHDxF/EBhuZZvOaBlmw9o+ZZcHSyuqPpcSpxDl53WSr8/rbXSGtV+oTsAAAAAgYHCjYBlt1nVMTXe+z7jPX9d7i3iJdqyv9B7jnhOkbbs9+4RL3W6tS4rX+uy8iXZ9NG8NZKkKLu1ao94WqMYpSVHH7k6e7SaJ0YpwmY1ZRudbo+27C/S2iN78dfuztOm/YXV1kmKsWt8j2b6Xa8WGtSukWlZAQAAAPiGwo2g4y3i3rcZG3/Ucpfbo52HSrRlf5E27s3XsrWbVRKRoO25JSpzerQ+q0DrswpqPJ/NalHzxKgj5TtaKXGRVReES4l3qEmcQ0kxdsU6IhQbaatz4TUMQ+Uujw6XVGh/QblyCsqUU1iurLxSbcupvJBciVweo8bX9miZoGGdmmhY5ybq3yZZdko2AAAAEHQo3AgZETarOjSJU4cmcRrVpbHalWzUhAlDZLHatOtQSdVbl+05XKo9h0u053Cpsg6XqsLtObKstE7fJ8puVWxkhKIjbbJaLLJavFdtt0gyJJVUuFRS7lZxhUu1dOka4h0R6pWWqN6tktQnLUn92iQrJc5Rr7EAAAAAYD4KN0JehM2q9k3i1L5JnMZ1r/45j8fQgaJy7T7kLeD7C8qUW1R+5H3Ey3WgsFy5ReXKL3XK6fa25zKnR2XOCqm4jt/falGTeIdSE6KUGu9Q88QotU+JVYdU74sDzRKigub8cgAAAAB1R+FGWLNaLWqaEKWmCVEa0PbE65a73Coud6u43KWicpdKKtySDBmGd8+2YXivHh5tt1Udfh4daVNsZASFGgAAAAhDFG6gjhwRNjkibGoUG2l2FAAAAABBgCsxAQAAAADgBxRuAAAAAAD8gMINAAAAAIAfULgBAAAAAPADCjcAAAAAAH5A4QYAAAAAwA8o3AAAAAAA+AGFGwAAAAAAP6BwAwAAAADgBxRuAAAAAAD8gMINAAAAAIAfULgBAAAAAPADCjcAAAAAAH5A4QYAAAAAwA8izA5QH4ZhSJIKCgpMTuJfTqdTJSUlKigokN1uNztOUGDMfMeY+Ybx8l24jFnlnFQ5R6F+mOtxPIyZ7xgz3zFmvguHMfNlrg/qwl1YWChJSktLMzkJAADVFRYWKjEx0ewYQY+5HgAQqOoy11uMIH4J3uPxKDs7W/Hx8bJYLGbH8ZuCggKlpaVp9+7dSkhIMDtOUGDMfMeY+Ybx8l24jJlhGCosLFSLFi1ktXLmVn0x1+N4GDPfMWa+Y8x8Fw5j5stcH9R7uK1Wq1q1amV2jFMmISEhZH9p/YUx8x1j5hvGy3fhMGbs2W44zPX4LYyZ7xgz3zFmvgv1MavrXM9L7wAAAAAA+AGFGwAAAAAAP6BwBwGHw6H77rtPDofD7ChBgzHzHWPmG8bLd4wZcHz8+/AdY+Y7xsx3jJnvGLPqgvqiaQAAAAAABCr2cAMAAAAA4AcUbgAAAAAA/IDCDQAAAACAH1C4AQAAAADwAwp3kCovL1efPn1ksVi0du1as+MErB07dujaa69Vu3btFB0drQ4dOui+++5TRUWF2dECyvPPP6927dopKipK/fv31/Lly82OFLBmzZql0047TfHx8UpNTdX555+vTZs2mR0raMyaNUsWi0XTpk0zOwoQ8Jjr64a5vm6Y6+uOub7+mO9/ReEOUnfccYdatGhhdoyAt3HjRnk8Hr3wwgv6+eefNXv2bP3jH//Q3XffbXa0gPHWW29p2rRpuueee7RmzRqdeeaZGj9+vHbt2mV2tIC0dOlSTZkyRStWrFBGRoZcLpfGjh2r4uJis6MFvFWrVunFF19Ur169zI4CBAXm+rphrv9tzPW+Ya6vH+b7YxgIOp988omRnp5u/Pzzz4YkY82aNWZHCiqPPvqo0a5dO7NjBIyBAwca119/fbVl6enpxp133mlSouCSk5NjSDKWLl1qdpSAVlhYaHTq1MnIyMgwhg8fbkydOtXsSEBAY66vH+b66pjr64e5vu6Y72tiD3eQ2b9/v6677jq98cYbiomJMTtOUMrPz1ejRo3MjhEQKioqtHr1ao0dO7ba8rFjx+qbb74xKVVwyc/PlyR+p37DlClTNHHiRI0ePdrsKEDAY66vP+b6XzHX1x9zfd0x39cUYXYA1J1hGJo8ebKuv/56DRgwQDt27DA7UtDZtm2bnnnmGT3xxBNmRwkIubm5crvdatq0abXlTZs21b59+0xKFTwMw9D06dN1xhlnqEePHmbHCVhvvvmmfvjhB61atcrsKEDAY66vP+b66pjr64e5vu6Y72vHHu4AMHPmTFkslhPevv/+ez3zzDMqKCjQXXfdZXZk09V1zI6WnZ2ts88+W5dccon+9Kc/mZQ8MFkslmqPDcOosQw13Xjjjfrpp5/0n//8x+woAWv37t2aOnWq5s2bp6ioKLPjAKZhrvcdc33DYq4/Ocz1dcN8f3wWwzAMs0OEu9zcXOXm5p5wnbZt2+r3v/+9Pvroo2r/ObrdbtlsNl1xxRV6/fXX/R01YNR1zCr/wWdnZ2vkyJEaNGiQXnvtNVmtvNYkeQ8zi4mJ0TvvvKMLLrigavnUqVO1du1aLV261MR0ge2mm27S+++/r2XLlqldu3ZmxwlY77//vi644ALZbLaqZW63WxaLRVarVeXl5dU+B4Qq5nrfMdc3DOb6k8dcX3fM98dH4Q4iu3btUkFBQdXj7OxsjRs3Tu+++64GDRqkVq1amZgucGVlZWnkyJHq37+/5s2bF7b/2I9n0KBB6t+/v55//vmqZd26ddN5552nWbNmmZgsMBmGoZtuukkLFizQkiVL1KlTJ7MjBbTCwkLt3Lmz2rJrrrlG6enpmjFjBofnAcdgrj85zPUnxlzvG+Z63zHfHx/ncAeR1q1bV3scFxcnSerQoQMT8HFkZ2drxIgRat26tR5//HEdOHCg6nPNmjUzMVngmD59uq666ioNGDBAgwcP1osvvqhdu3bp+uuvNztaQJoyZYrmz5+vDz74QPHx8VXnvyUmJio6OtrkdIEnPj6+xiQbGxurxo0bh/XkCxwPc73vmOt/G3O9b5jrfcd8f3wUboS0RYsWaevWrdq6dWuNP1Q4uMPrsssu08GDB/XAAw9o79696tGjhz755BO1adPG7GgBae7cuZKkESNGVFv+6quvavLkyac+EACEOeb638Zc7xvmejQkDikHAAAAAMAPuJoEAAAAAAB+QOEGAAAAAMAPKNwAAAAAAPgBhRsAAAAAAD+gcAMAAAAA4AcUbgAAAAAA/IDCDQAAAACAH1C4AQAAAADwAwo3AAAAAAB+QOEGAAAAAMAPKNwAAAAAAPgBhRsIYwcOHFCzZs308MMPVy1buXKlIiMjtWjRIhOTAQCAhsBcD5jLYhiGYXYIAOb55JNPdP755+ubb75Renq6+vbtq4kTJ2rOnDlmRwMAAA2AuR4wD4UbgKZMmaLFixfrtNNO048//qhVq1YpKirK7FgAAKCBMNcD5qBwA1Bpaal69Oih3bt36/vvv1evXr3MjgQAABoQcz1gDs7hBqDt27crOztbHo9HO3fuNDsOAABoYMz1gDnYww2EuYqKCg0cOFB9+vRRenq6nnzySa1bt05NmzY1OxoAAGgAzPWAeSjcQJi7/fbb9e677+rHH39UXFycRo4cqfj4eP3vf/8zOxoAAGgAzPWAeTikHAhjS5Ys0Zw5c/TGG28oISFBVqtVb7zxhr766ivNnTvX7HgAAKCemOsBc7GHGwAAAAAAP2APNwAAAAAAfkDhBgAAAADADyjcAAAAAAD4AYUbAAAAAAA/oHADAAAAAOAHFG4AAAAAAPyAwg0AAAAAgB9QuAEAAAAA8AMKNwAAAAAAfkDhBgAAAADADyjcAAAAAAD4wf8HrGHGgw5iSEoAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def swish(x):\n", + " return x * sigmoid(x)\n", + "\n", + "def swish_derivative(x):\n", + " s = sigmoid(x)\n", + " return s + x * s * (1 - s)\n", + "\n", + "plot_function_and_derivative(swish, swish_derivative, \"Swish\", (-5, 5))" + ] + }, + { + "cell_type": "markdown", + "id": "31fabc45", + "metadata": {}, + "source": [ + "**Eigenschaften:**\n", + "\n", + "* Nicht monoton\n", + "* Wird auch SiLU (**Si**gmoid **L**inear **U**nit) genannt.\n", + "* Mehr oder weniger Kombination von ReLU und Sigmoid" + ] + }, + { + "cell_type": "markdown", + "id": "9b7e013b", + "metadata": {}, + "source": [ + "**Vorteile:**\n", + "\n", + "* Ableitungswerte in einem guten Bereich (nicht zu klein)\n", + "* In manchen Machine-Learning Tasks besser als ReLU. " + ] + }, + { + "cell_type": "markdown", + "id": "99a5e699", + "metadata": {}, + "source": [ + "**Nachteile:**\n", + "\n", + "* Aufwendiger zum Berechnen\n", + "* Nicht-Monotonität macht es schwerer zu Optimieren." + ] + }, + { + "cell_type": "markdown", + "id": "b543bddd", + "metadata": {}, + "source": [ + "> **Übung:** Zeigen Sie, dass $\\sigma(x) + x\\cdot \\sigma(x)\\cdot (1-\\sigma(x))$ tatsächlich die Ableitung von $\\sigma(x)\\cdot x$ ist." + ] + }, + { + "cell_type": "markdown", + "id": "9a6c705a", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "4ab34c48", + "metadata": {}, + "source": [ + "Wir kommen nun zu einer speziellen *Aktivierungsfunktion*, der sogenannten Softmax-Funktion. Sie wird auf ein ganzes Layer angewendet (es werden also auch die Werte des ganzen Layers zur Berechnung herangezogen). Außerdem wird diese Funktion normalerweise nur im letzten Layer (beim Output) verwendet. Grund dafür ist, dass wir mit dieser Funktion die Outputs der einzelnen Neuronen in eine Wahrscheinlichkeitsverteilung umwandeln können. Sprich wir bekommen dann am Ende einen Vektor, der in Summe $1.0$ (=100%) ergibt. Diese Einträge spiegeln dann die Vorhersage des Models wieder. Ist der Output also $[0.1, 0.1, 0.8, 0]$, dann glaubt das Modell, dass wir zu 80% in Klasse 3 sind und zu je 10% in Klasse 1 oder 2." + ] + }, + { + "cell_type": "markdown", + "id": "4008bf04", + "metadata": {}, + "source": [ + "## Softmax" + ] + }, + { + "cell_type": "markdown", + "id": "8d12d8e4", + "metadata": {}, + "source": [ + "\\begin{equation*}\n", + " \\text{softmax}(x)_i = \\frac{e^{z_i}}{\\sum_{j=1}^{n}e^{z_j}}.\\\\\n", + "\\end{equation*}" + ] + }, + { + "cell_type": "markdown", + "id": "3b652b0c", + "metadata": {}, + "source": [ + "**Eigenschaften:**\n", + "\n", + "* Erzeugt eine Verteilung mit Summe $1.0$\n", + "* Gewichtet die Einträge exponentiell, also kleine Unterschiede können sich groß auswirken\n", + "* Alle Einträge sind im Anschluss positiv" + ] + }, + { + "cell_type": "markdown", + "id": "e71dd8c8", + "metadata": {}, + "source": [ + "**Hinweis:** Prinzipiell wäre es für die reine Klassifizierung ausreichend, die Logits zu verwenden. Sprich wir verwenden einfach den Eintrag mit dem maximalen Wert. Die exponentielle Gewichtung zu einer Wahrscheinlichkeitsverteilung ist aber zur Loss-Berechnung vorteilhaft bzw. notwendig." + ] + }, + { + "cell_type": "markdown", + "id": "5036865f", + "metadata": {}, + "source": [ + "**Wichtig:** Wir verwenden also im Fall einer Klassifikation in Zukunft nicht mehr die Sigmoid Funktion und erhalten dann eine Wahrschenlichkeit für Klasse 1, sondern wir verwenden einfach so viele Output-Neuronen, wie es Klassen gibt. Im Anschluss verwenden wir die Softmax Funktion und erhalten die Wahrscheinlichkeiten." + ] + }, + { + "cell_type": "markdown", + "id": "19249d9a", + "metadata": {}, + "source": [ + "Die folgenden beiden Beispiele zeigen das exemplarische Verhalten der Softmax Funktion." + ] + }, + { + "cell_type": "markdown", + "id": "71ed521d", + "metadata": {}, + "source": [ + "![Softmax_Example_1](../resources/softmax_example1.png)\n", + "\n", + "(von https://mriquestions.com/softmax.html)" + ] + }, + { + "cell_type": "markdown", + "id": "d0db8789", + "metadata": {}, + "source": [ + "![Softmax_Example_2](../resources/softmax_example2.png)\n", + "\n", + "(von https://ogunlao.github.io/2020/04/26/you_dont_really_know_softmax.html)" + ] + }, + { + "cell_type": "markdown", + "id": "2f876838", + "metadata": {}, + "source": [ + "Haben wir nun ein Dataset mit den Labels *Katze, Hund, Pferd* (egal ob jetzt Bilder davon oder ob wir einfach ein paar Eigenschaften in Tabellen gesammelt haben wie zbsp Gewicht, Farbe, etc.), so müssen wir jetzt einen **OneHot-Encoder** verwenden.\n", + "\n", + "Im obigen Beispiel wären dann die Labels (in diesem Fall dann eine Verteilung (ein Vektor mit Summe 1)): **Katze=[1,0,0], Hund=[0,1,0], Pferd=[0,0,1]** und unser Modell würde versuchen, diese zu lernen." + ] + }, + { + "cell_type": "markdown", + "id": "d9d9b429", + "metadata": {}, + "source": [ + "Jetzt wissen wir auch, warum genau ein *OneHot-Encoder* so wichtig ist. Er erzeugt uns die Ziel-Verteilung (in diesem Fall halt eben eine One-Hot Verteilung)\n", + "\n", + "Für andere Anwendungen kann es sein, dass es vielleicht mehrere Labels gibt, in so einem Fall würde man:\n", + "\n", + "1. Als Ziel-Verteilung keine One-Hot Verteilung verwenden sondern zum Beispiel $[1,1,0,1,0]$ (bzw. modifizierte/normalisierte Versionen davon),\n", + "2. und außerdem dementsprechend die Softmax-Verteilung anpassen." + ] + }, + { + "cell_type": "markdown", + "id": "3cd6f4a5", + "metadata": {}, + "source": [ + "**Wichtig:** Für die Regression verwenden wir nach wie vor nur ein Output Neuron. In so einem Fall würde man keine Aktivierungsfunktion (auch kein Softmax) auf den Output setzen. (Warum?)" + ] + }, + { + "cell_type": "markdown", + "id": "5d4f384b", + "metadata": {}, + "source": [ + "## Aktivierungsfunktionen in PyTorch" + ] + }, + { + "cell_type": "markdown", + "id": "a144be2a", + "metadata": {}, + "source": [ + "Betrachten wir wieder das Beispiel von vorher. Wir adaptieren dieses Beispiel und ergänzen nun Aktivierungsfunktionen. Dafür verwenden wir das `torch.nn.functional` Paket." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "1c2c9b3e", + "metadata": {}, + "outputs": [], + "source": [ + "import torch.nn.functional as F" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "bc527880", + "metadata": {}, + "outputs": [], + "source": [ + "class MyActivatedFirstNeuralNetwork(nn.Module):\n", + " \"\"\"A simple feedforward neural network, that expects n_input_features input features and produces 1 Output by using 3 Hidden Layers with 4 Features each.\"\"\"\n", + " def __init__(self, n_input_features: int):\n", + " super().__init__()\n", + " self.hidden_layer_1 = nn.Linear(n_input_features, 4) \n", + " self.hidden_layer_2 = nn.Linear(4, 4, bias=False) \n", + " self.hidden_layer_3 = nn.Linear(4, 4, bias=False) \n", + " self.output_layer = nn.Linear(4, 1, bias=False) \n", + "\n", + " def forward(self, x):\n", + " \"\"\"\n", + " :param x: Input tensor with shape (batch_size, n_input_features)\n", + " :return: Output tensor with shape (batch_size, 1)\n", + " \"\"\"\n", + " x = self.hidden_layer_1(x)\n", + " x = F.silu(x)\n", + " x = self.hidden_layer_2(x)\n", + " x = F.relu(x)\n", + " x = self.hidden_layer_3(x)\n", + " x = F.sigmoid(x)\n", + " x = self.output_layer(x)\n", + " return x" + ] + }, + { + "cell_type": "markdown", + "id": "1808d2b3", + "metadata": {}, + "source": [ + "Wir könnten diese auch in der Klasse speichern, es macht aber keinen Unterschied in diesem Fall." + ] + }, + { + "cell_type": "markdown", + "id": "41c71530", + "metadata": {}, + "source": [ + "Der Rest funktioniert wieder gleich, i.e.:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "cebfe78a", + "metadata": {}, + "outputs": [], + "source": [ + "model = MyActivatedFirstNeuralNetwork(3)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "02bf1618", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MyActivatedFirstNeuralNetwork(\n", + " (hidden_layer_1): Linear(in_features=3, out_features=4, bias=True)\n", + " (hidden_layer_2): Linear(in_features=4, out_features=4, bias=False)\n", + " (hidden_layer_3): Linear(in_features=4, out_features=4, bias=False)\n", + " (output_layer): Linear(in_features=4, out_features=1, bias=False)\n", + ")\n" + ] + } + ], + "source": [ + "print(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "749bc616", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "MyActivatedFirstNeuralNetwork(\n", + " (hidden_layer_1): Linear(in_features=3, out_features=4, bias=True)\n", + " (hidden_layer_2): Linear(in_features=4, out_features=4, bias=False)\n", + " (hidden_layer_3): Linear(in_features=4, out_features=4, bias=False)\n", + " (output_layer): Linear(in_features=4, out_features=1, bias=False)\n", + ")" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "53c84955", + "metadata": {}, + "outputs": [], + "source": [ + "input_data = torch.randn(5, 3) # was macht dieser Code?" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "844222eb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[ 0.7376, -0.6442, 1.3565],\n", + " [ 1.3303, 0.1905, 0.7533],\n", + " [ 1.3503, -0.5184, -1.8554],\n", + " [ 0.8644, 0.0134, 0.7739],\n", + " [ 0.8419, -1.3249, -1.0933]])\n" + ] + } + ], + "source": [ + "print(input_data)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "da98940e", + "metadata": {}, + "outputs": [], + "source": [ + "output = model(input_data)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "e20f08b6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[-0.4523],\n", + " [-0.4492],\n", + " [-0.4411],\n", + " [-0.4495],\n", + " [-0.4422]], grad_fn=)\n" + ] + } + ], + "source": [ + "print(output)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Abschließend halten wir noch die bisherigen (wir werden in den anderen Notebooks noch weitere Hyperparameter kennenlernen) Hyperparameter von einem Neuronalen Netz fest." + ] + }, + { + "cell_type": "markdown", + "id": "f262177b", + "metadata": {}, + "source": [ + "## Hyperparameter eines Neural Networks\n", + "\n", + "* Anzahl und Aufbau der Layers\n", + "* Aktivierungsfunktionen zwischen den Layers" + ] + }, + { + "cell_type": "markdown", + "id": "33f75162", + "metadata": {}, + "source": [ + "### Aufgabe\n", + "\n", + "Entwerfen Sie ein neuronales Netzwerk, welches das Titanic-Dataset (``titanic.csv``) verwendet um dann zu vorhersagen, ob eine Person überlebt hat oder nicht.\n", + "\n", + "Dabei:\n", + "* Dürfen Sie den Aufbau selber wählen\n", + "* Muss natürlich nur der Forward-Pass implementiert werden\n", + "* Die Daten müssen vorverarbeitet werden\n", + "* Die GPU soll verwendet werden (falls vorhanden)" + ] + }, + { + "cell_type": "markdown", + "id": "2ba8cd2b", + "metadata": {}, + "source": [ + "### Lösung" + ] + }, + { + "cell_type": "markdown", + "id": "4f83156d", + "metadata": {}, + "source": [ + "**WICHTIG:** Das obige Neuronale Netz ist noch **nicht trainiert**. Es ist der Output also derzeit noch ein zufälliges Ergebnis!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dsai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.23" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/06_NN/code/nn_3_optimierung.ipynb b/06_NN/code/nn_3_optimierung.ipynb new file mode 100644 index 0000000..607e95c --- /dev/null +++ b/06_NN/code/nn_3_optimierung.ipynb @@ -0,0 +1,1364 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cd13588c", + "metadata": {}, + "source": [ + "
\n", + "\n", + "

Neural Networks: Trainieren von Neuronalen Netzwerken

\n", + "

DSAI

\n", + "

Jakob Eggl

\n", + "\n", + "
\n", + " \"Logo\"\n", + "
\n", + " © 2025/26 Jakob Eggl. Nutzung oder Verbreitung nur mit ausdrücklicher Genehmigung des Autors.\n", + "
\n", + "
\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "65f25f8a", + "metadata": {}, + "source": [ + "In diesem Notebook wollen wir nun besprechen, wie ein Neuronales Netzwerk lernt.\n", + "\n", + "Dazu können wir uns zuerst einmal fragen, was eigentlich **Lernen** bedeutet?" + ] + }, + { + "cell_type": "markdown", + "id": "66bf621f", + "metadata": {}, + "source": [ + "> A computer program is said to **learn** from **experience E**\n", + "with respect to some class of **tasks T** and\n", + "performance **measure P**, if its performance at tasks in T,\n", + "as measured by P, improves with experience E.\n", + "\n", + "(Mitchell 1997)" + ] + }, + { + "cell_type": "markdown", + "id": "f0de04ed", + "metadata": {}, + "source": [ + "Was bedeutet das in unserem Fall?\n", + "\n", + "* Wir sagen, dass unser Modell lernt, falls es bei einer gegebenen Aufgabe *besser wird*:\n", + " * Was ist die Aufgabe bei uns?\n", + " * Wie messen wir, ob es besser geworden ist?" + ] + }, + { + "cell_type": "markdown", + "id": "9b8f0f43", + "metadata": {}, + "source": [ + "## Die Problemstellung" + ] + }, + { + "cell_type": "markdown", + "id": "078d9fcf", + "metadata": {}, + "source": [ + "### Die Ausgangslage" + ] + }, + { + "cell_type": "markdown", + "id": "886dca89", + "metadata": {}, + "source": [ + "Wir betrachten jetzt nochmal im Detail unsere Problemstellung.\n", + "\n", + "Wir haben Daten für ein Supervised Machine Learning Setting, sprich wir haben eine Menge $\\mathcal Z$, welche aus den Paaren $(X_i, y_i)$ besteht, wobei $X_i$ der Feature Vektor für den $i$-ten Datenpunkt und $y_i$ das dazugehörige $i$-te Label (Regression oder Klassifikation) ist.\n", + "\n", + "Ziel ist es, eine Funtion $f$ zu finden, welche uns diesen Zusammenhang abbildet." + ] + }, + { + "cell_type": "markdown", + "id": "d89c746a", + "metadata": {}, + "source": [ + "![Neural_Network_Function_Approximation](../resources/NN_Function_Approximation.png)\n", + "\n", + "(von https://stackoverflow.com/questions/13897316/approximating-the-sine-function-with-a-neural-network)" + ] + }, + { + "cell_type": "markdown", + "id": "198251b5", + "metadata": {}, + "source": [ + "Was wird in unserem Fall in Zukunft die Funktion $f$ sein?" + ] + }, + { + "cell_type": "markdown", + "id": "42e15ecc", + "metadata": {}, + "source": [ + "### Der Task eines Neuronalen Netzwerkes" + ] + }, + { + "cell_type": "markdown", + "id": "996f0170", + "metadata": {}, + "source": [ + "Was ist nun die Aufgabe von unserem Neuronalen Netz?\n", + "\n", + "Wir wollen die gegebenen Datenpaare gut mit unserem neuronalen Netz abbilden können. Sprich unser Neuronales Netzwerk soll überall ähnliche (bzw. die gleichen Werte) ausspucken." + ] + }, + { + "cell_type": "markdown", + "id": "1e300848", + "metadata": {}, + "source": [ + "Wie können wir das überprüfen?\n", + "\n", + "Händisch können wir natürlich die Ergebnisse mit den originalen Ergebnisse gut vergleichen und die Qualität beurteilen. Dies ist aber mühsam, somit stellt sich uns die Frage, wie wir sonst vorgehen können? Für eindimensionale Daten können wir auch eine Kurve (so wie oben) plotten, auch das ist einfach. Dies ist aber auch in der Praxis meistens nicht der Fall." + ] + }, + { + "cell_type": "markdown", + "id": "dc1ba5a1", + "metadata": {}, + "source": [ + "Um den Fehler quantifizieren zu können, widmen wir uns jetzt den sogenannten **Loss**-Funktionen. Sie sind uns schon von der linearen (bzw. logistischen) Regression bekannt und geben uns den Fehler, den das Modell macht." + ] + }, + { + "cell_type": "markdown", + "id": "e9353e7f", + "metadata": {}, + "source": [ + "### Die Loss Funktion" + ] + }, + { + "cell_type": "markdown", + "id": "fc0b1508", + "metadata": {}, + "source": [ + "Die Loss Funktion gibt uns den Fehler zurück, den das Netzwerk aktuell für einen Input $X_i$ im Vergleich zum Ziel Output $y_i$, macht.\n", + "\n", + "Wir schreiben von nun an für den Loss:\n", + "$$L(\\hat{y}_i, y_i)=L(\\hat f(X_i), y_i),$$\n", + "\n", + "wobei sämtliche Größen mit $\\mathbf{\\hat{}}$-Symbol immer als die von uns approximierten Größen (Predictions) bezeichnet werden." + ] + }, + { + "cell_type": "markdown", + "id": "b9c9cdf7", + "metadata": {}, + "source": [ + "Kommen wir nun zu den gängigsten Loss-Funktionen. Wir starten mit den Loss-Funktionen für die **Regression**. Diese sind die gleichen Loss-Funktionen wie für die lineare (logistische) Regression und werden hier kurz aufgezählt." + ] + }, + { + "cell_type": "markdown", + "id": "8c0cffd0", + "metadata": {}, + "source": [ + "#### Gängige Loss-Funktionen für Regression" + ] + }, + { + "cell_type": "markdown", + "id": "cf94e0b3", + "metadata": {}, + "source": [ + "**MSE:** Mean-Squared Error: $$L_{\\textrm{MSE}}(\\hat{y}, y)=\\frac{1}{n}\\sum_{i=1}^{n}(\\hat{y}_i - y_i)^2,$$\n", + "\n", + "mit $n$ als Anzahl der Datenpunkte." + ] + }, + { + "cell_type": "markdown", + "id": "82735f2f", + "metadata": {}, + "source": [ + "**MAE:** Mean Absolute Error: $$L_{\\mathrm{MAE}}(\\hat{y}, y)=\\frac{1}{n}\\sum_{i=1}^{n}\\lvert \\hat{y}_i - y_i\\rvert.$$" + ] + }, + { + "cell_type": "markdown", + "id": "cecb982a", + "metadata": {}, + "source": [ + "**Hinweis:** Nachdem wir hier die Regressionsloss-Funktionen betrachten, nehmen wir an, dass es jeweils nur ein Output Neuron gibt. Ansonsten müssten wir die Loss-Funktionen leicht adaptieren hier." + ] + }, + { + "cell_type": "markdown", + "id": "b893a8d7", + "metadata": {}, + "source": [ + "#### Gängige Loss-Funktionen für Klassifikation" + ] + }, + { + "cell_type": "markdown", + "id": "65e773ed", + "metadata": {}, + "source": [ + "Hier ist der Output immer ein Vektor, welchen wir mit dem Zielvektor vergleichen wollen. Siehe folgendes Beispiel, wo das Label nun offensichtlich die Klasse **Auto** ist. Mit einem OneHot-Encoder erhalten wir das gewünschte Format für die Loss-Funktion, sprich $\\text{Auto}=[1,0,0,0]$." + ] + }, + { + "cell_type": "markdown", + "id": "9f0b028f", + "metadata": {}, + "source": [ + "![Loss_Example_Car](../resources/loss_car_example.jpeg)\n", + "\n", + "(von https://www.shopdev.co/blog/cross-entropy-in-machine-learning)" + ] + }, + { + "cell_type": "markdown", + "id": "c0867e68", + "metadata": {}, + "source": [ + "![Loss_Example_Car](../resources/loss_car_example_2.jpeg)\n", + "\n", + "(von https://www.shopdev.co/blog/cross-entropy-in-machine-learning)" + ] + }, + { + "cell_type": "markdown", + "id": "b612f965", + "metadata": {}, + "source": [ + "**Categorical Cross-Entropy Loss**: Berchnet, wie sehr sich die Verteilungen $y$ und $\\hat{y}$ ähnlich sind. Er ist folgendermaßen definiert:\n", + "$$L_{\\mathrm{CE}}(\\hat{y}, y)=-\\frac{1}{n}\\sum_{i=1}^{n}\\sum_{k=1}^{K}y_{ik}\\log(\\hat{y}_{ik}),$$\n", + "\n", + "wobei $K$ die Anzahl der Klassen ist, also im obigen Beispiel zum Beispiel 4 und $n$ wieder die Anzahl der Datenpunkte." + ] + }, + { + "cell_type": "markdown", + "id": "395d0adf", + "metadata": {}, + "source": [ + "**Binary Cross-Entropy Loss** Spezialfall für nur 2 Klassen vom Categorical Cross-Entropy Loss ($K=2$). Es gilt automatisch, weil die Summe vom Vektor $1$ ergeben muss (repräsentiert ja die Wahrscheinlichkeiten bzw. Predictions), dass $y_{i2} = 1-y_{i1}$. Somit ergibt sich für den Loss:\n", + "$$L_{\\mathrm{BCE}}(\\hat{y}, y)=-\\frac{1}{n}\\sum_{i=1}^{n}[y_i \\log (\\hat{y}_i)+ (1-y_i)\\log(1-\\hat{y}_i)].$$" + ] + }, + { + "cell_type": "markdown", + "id": "53b31676", + "metadata": {}, + "source": [ + "**Wichtig:** Hier ist $y$ die Ziel-Verteilung und $\\hat y$ die Verteilung der Vorhersage (Prediction). " + ] + }, + { + "cell_type": "markdown", + "id": "068fc731", + "metadata": {}, + "source": [ + "> **Übung:** Warum sind diese beiden Loss-Funktionen mit einem negativen Vorzeichen behaftet?" + ] + }, + { + "cell_type": "markdown", + "id": "eb96de62", + "metadata": {}, + "source": [ + "> **Übung:** Wann ist der Loss 0 für den *Binary Cross-Entropy Loss*? Wann für den *Categorical Cross-Entropy Loss*?" + ] + }, + { + "cell_type": "markdown", + "id": "9fb3bc69", + "metadata": {}, + "source": [ + "**>Übung:** Der Logarithmus $\\log(x)$ ist für $x\\leq 0$ nicht definiert. Wieso macht uns das hier keine Probleme? *Tipp:* Softmax." + ] + }, + { + "cell_type": "markdown", + "id": "fb2e3b5f", + "metadata": {}, + "source": [ + "### Zusammenfassung vom Task eines Neuronalen Netzwerkes:" + ] + }, + { + "cell_type": "markdown", + "id": "5f1eeacc", + "metadata": {}, + "source": [ + "Folgende Punkte fassen nun unsere Ausgangssituation zusammen.\n", + "\n", + "* Wir haben ein Neuronales Netzwerk (beliebig \"komplex\", beliebige Aktivierungsfunktionen etc.). Dieses stellt unsere Funktion $\\hat{f}$ dar\n", + "* Wir haben die Datenpaare mit Features $X_i$ und Label (Wert oder Klasse) $y_i$\n", + "* Wir haben die Daten in Trainings- und Testsplit aufgeteilt (zBsp im Verhältnis: 80/20)\n", + "* Wir haben eine Loss-Funktion $L(\\hat{f}(X_i), \\mathbf y_i)$ definiert\n", + "* Diese Loss Funktion wollen wir auf den Trainingsdaten minimieren." + ] + }, + { + "cell_type": "markdown", + "id": "6e3a49b7", + "metadata": {}, + "source": [ + "**Wichtig:** Auch wenn wir den Loss auf den Trainingsdaten minimieren, wollen wir in Summe natürlich dann ein Modell, bei dem der Loss am *Testset* niedrig ist. (Nur so können wir überprüfen, dass wir nicht overfitten.)" + ] + }, + { + "cell_type": "markdown", + "id": "62d6bab0", + "metadata": {}, + "source": [ + "**Hinweis:** Wir können im Anschluss dann auch noch Metriken berechnen wie zum Beispiel Accuracy, Confusion Matrix, Anzahl der True-Positive samples, etc. Solche Metriken sind für uns Anwender oft von großem Interesse, da sie meistens dann die Brücke bilden zu den Praxisproblemen (zbsp.: Anzahl der Fehlklassifikationen bei Krebs Früherkennung usw.). Der Loss selber wird jedoch benötigt zum Optimieren (=Minimieren), also das Modell soll optimiert werden, sodass wir einen kleinen Loss (Fehler) haben. Der Grund dafür ist, dass wir unsere Loss-Funktion ableiten müssen. Jene Funktionen, die uns die Metriken liefern sind oft nicht differenzierbar." + ] + }, + { + "cell_type": "markdown", + "id": "6504c924", + "metadata": {}, + "source": [ + "**Hinweis:** Wir können uns auch eine eigene Loss-Funktionen basteln, welche auf besondere Wünsche/Anforderungen abgestimmt ist. Man muss dabei aber auf ein paar Dinge acht geben (werden wir uns eventuell zu einem späteren Zeitpunkt ansehen)." + ] + }, + { + "cell_type": "markdown", + "id": "5b3bf26e", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "47b15bc1", + "metadata": {}, + "source": [ + "### Finden eines Minimums (der Lossfunktion)" + ] + }, + { + "cell_type": "markdown", + "id": "43a87dfb", + "metadata": {}, + "source": [ + "Nun bleibt nur mehr die Frage, wie wir das Minimum einer Funktion finden?" + ] + }, + { + "cell_type": "markdown", + "id": "899c4878", + "metadata": {}, + "source": [ + "**Erinnerung:** Wie finden wir das Minimum einer Funktion?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc09ad5e", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "d840a068", + "metadata": {}, + "source": [ + "> **Übung:** Wie müssen die Parameter $w_1, w_2 \\in \\mathbb R$ gewählt werden, um ein die Funktion $$L(w_1, w_2)=(4w_1+7w_2-20)^2$$ zu minimieren?" + ] + }, + { + "cell_type": "markdown", + "id": "0d1031a0", + "metadata": {}, + "source": [ + "> **Übung:** Wie müssen die Parameter $w_1, w_2, w_3 \\in \\mathbb R$ gewählt werden, um ein die Funktion $$L(w_1, w_2, w_3)=\\max (0, 2w_1+3w_2-4w_3)^2$$ zu minimieren?" + ] + }, + { + "cell_type": "markdown", + "id": "b1e18b34", + "metadata": {}, + "source": [ + "> **Übung:** Wie muss der Parameter $w_1 \\in \\mathbb R$ gewählt werden, um ein die Funktion $$L(w_1)=\\lvert w_1 - e^{-w_1} \\rvert$$ zu minimieren?" + ] + }, + { + "cell_type": "markdown", + "id": "5913e518", + "metadata": {}, + "source": [ + "**Allgemein gilt natürlich:**\n", + "\n", + "Bei einem Extrempunkt ist die Ableitung $0$. Somit berechnen wir hier einfach die (partielle) Ableitung und wir erhalten alle Kandidaten. Im Anschluss können wir dann leicht prüfen, ob es ein Maximum oder Minimum ist (im 1d mit der 2. Ableitung, ansonsten zum Beispiel auch mit Einsetzen und Vergleichen der Werte)." + ] + }, + { + "cell_type": "markdown", + "id": "7d31eacf", + "metadata": {}, + "source": [ + "Aber können wir immer so leicht die Ableitung $0$ setzen?\n", + "\n", + "**Nein**, weil unsere Funktionen (neuronalen Netze) sind sehr kompliziert (und besitzen viele Parameter), weswegen das Berechnen der Nullstelle der Ableitung sich als schwierig/unmöglich gestaltet. Dies ist zum Beispiel beim letzten der drei Beispiele auch schon sehr schwer/unmöglich. " + ] + }, + { + "cell_type": "markdown", + "id": "f4cb3760", + "metadata": {}, + "source": [ + "Tatsächlich ist dies in der Praxis oft (bei Neuronalen Netzen quasi immer) der Fall, wie die folgenden beiden Grafiken zeigen. " + ] + }, + { + "cell_type": "markdown", + "id": "d60cd011", + "metadata": {}, + "source": [ + "![Loss_Landscape](../resources/Loss_Landscape.png)\n", + "\n", + "(von https://www.cs.umd.edu/~tomg/projects/landscapes/)" + ] + }, + { + "cell_type": "markdown", + "id": "3e956bce", + "metadata": {}, + "source": [ + "![Loss_Landscape_incl_Path](../resources/Loss_Landscape_Path.jpeg)\n", + "\n", + "(von https://discuss.pytorch.org/t/looking-for-the-lost-function-generating-the-lost-landscape-shown-in-this-article-on-sceince/130626)" + ] + }, + { + "cell_type": "markdown", + "id": "2c98dca5", + "metadata": {}, + "source": [ + "Wie wir oben sehen (diese Art von Bilder nennt man *Loss Landscape*), haben wir bei solchen Funktionen ein riesen Problem, das **globale** Minimum zu finden. Wir haben aber auch schon Probleme überhaupt ein Minimum zu berechnen." + ] + }, + { + "cell_type": "markdown", + "id": "af53f16d", + "metadata": {}, + "source": [ + "**Hinweis:** Obige Modelle haben nur 2 Parameter (zum Beispiel hat die Funktion $f(x)=kx+d$ auch nur 2 Parameter ($k, d$)), welche auf den $x$- und $y$-Achsen positioniert sind. Auf der $z$-Achse wird der Loss aufgetragen.\n", + "\n", + "Im Vergleich: Ein Language Model von Meta (llama4) hat etwa 405B Parameter, dass sind 405 Milliarden solcher Parameter, wo wir das gemeinsame Optimimum finden wollen." + ] + }, + { + "cell_type": "markdown", + "id": "6d65aefa", + "metadata": {}, + "source": [ + "Was ist, wenn wir das Minimum nicht analytisch berechnen können?" + ] + }, + { + "cell_type": "markdown", + "id": "ae47f39b", + "metadata": {}, + "source": [ + "In so einem Fall müssen wir uns einer numerischen/iterativen Methode bedienen, um unser Minimum zu finden. Eine bekannte Version davon ist **Gradient Descent**." + ] + }, + { + "cell_type": "markdown", + "id": "3c708f12", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "f73093b8", + "metadata": {}, + "source": [ + "## Gradient Descent" + ] + }, + { + "cell_type": "markdown", + "id": "921a88b7", + "metadata": {}, + "source": [ + "![Gradien_Descent](../resources/gradient_descent_mountain.png)\n", + "\n", + "(von https://ryanwingate.com/intro-to-machine-learning/deep-learning-with-pytorch/training-neural-networks-with-pytorch/)" + ] + }, + { + "cell_type": "markdown", + "id": "63527461", + "metadata": {}, + "source": [ + "Zuerst wollen wir uns einmal ansehen, woher der Begriff **Gradient Descent** eigentlich kommt." + ] + }, + { + "cell_type": "markdown", + "id": "9eb0066e", + "metadata": {}, + "source": [ + "**Gradient:** Eine mehrdimensionale Ableitung wird *Gradient* genannt. Wir verwenden dafür das Symbol $\\nabla$. Zum Beispiel hat die Funktion $f(x,y)=x^2+y^3+xy$ als Gradient den folgenden *Vektor*: $\\nabla f(x,y) = (\\frac{\\partial f}{\\partial x}, \\frac{\\partial f}{\\partial y}) = (2x+y,3y^2+x)$. Im Punkt $(1,1)$ ist somit der Gradient $\\nabla f(1,1) = (3,4)$. Er gibt also an, in welche Richtung die Funktion wie stark ansteigt (oder fällt bei einem negativen Vorzeichen). In unserem Fall steigt die Funktion am Punkt $(1,1)$ mit Steigung $3$ in Richtung $x$ und Steigung $4$ in Richtung $y$." + ] + }, + { + "cell_type": "markdown", + "id": "c5c80d03", + "metadata": {}, + "source": [ + "**Descent:** Englisches Wort für *Abstieg*." + ] + }, + { + "cell_type": "markdown", + "id": "8b185096", + "metadata": {}, + "source": [ + "Wir wollen also die besten Parameter für unser Modell finden, indem wir bei der Loss-Landscape immer in jene Richtung gehen, wo es am steilsten nach unten geht! Dabei wollen wir die Information vom Gradienten verwenden, also von der (mehrdimensionalen) Ableitung." + ] + }, + { + "cell_type": "markdown", + "id": "f3d604cf", + "metadata": {}, + "source": [ + "> **Übung:** Wie können wir diese Richtung finden?" + ] + }, + { + "cell_type": "markdown", + "id": "d449d2f4", + "metadata": {}, + "source": [ + "Der Gradient zeigt immer in jene Richtung, wo der Anstieg am größten ist. Somit geht es in die genau andere Richtung am steilsten nach unten!" + ] + }, + { + "cell_type": "markdown", + "id": "22002fbb", + "metadata": {}, + "source": [ + "Somit gehen wir bei dem Gradient Descent Algorithmus immer iterativ (also Schritt für Schritt) in die negative Richtung vom Gradienten an dem jeweiligen Punkt. Wie groß der Schritt in die jeweilige Richtung ist, wird über die sogenannte **Learning-Rate** $\\eta\\in \\mathbb R$ definiert. Sie ist in diesem Fall die Schrittweite und ein Hyperparameter, der vorher festgelegt werden muss. Wir formulieren nun die *Update-Rule*, also die Formel, mit der wir unsere Parameter $w$ vom neuronalen Netzwerk anpassen." + ] + }, + { + "cell_type": "markdown", + "id": "55b02e2a", + "metadata": {}, + "source": [ + "$$w_t = w_{t-1} - \\eta \\cdot \\nabla L(w_{t-1}).$$" + ] + }, + { + "cell_type": "markdown", + "id": "2cac2c81", + "metadata": {}, + "source": [ + "In Worten bedeutet das folgendes:\n", + "\n", + "Die Gewichte (Weights=Parameter) $w_t$ unseres Netzwerkes zum Zeitpunkt $t$ sind die bisherigen Gewichte $w_{t-1}$ minus dem Produkt aus Learning-Rate $\\eta$ und Gradienten (der Ableitung) von der Lossfunktion $L$ an der Stelle der bisherigen Parameter $w_{t-1}$. " + ] + }, + { + "cell_type": "markdown", + "id": "00e72fb7", + "metadata": {}, + "source": [ + "Vorstellen kann man sich das ganze folgendermaßen:\n", + "\n", + "* Man befindet sich irgendwo (zufällige Gewichte und somit Position am Anfang) in einem Gebirge (Loss-Landscape)\n", + "* Man möchte ins Tal (das *globale* Minimum finden)\n", + "* Es ist stark nebelig und man sieht nur, wie steil es an der aktuellen Position ist (Gradient bzw. negativer Gradient)\n", + "* Man bewegt sich einen kleinen Schritt (Learning Rate) in die Richtung des steilsten Abstiegs (Descent)\n", + "* Man wiederholt das ganze so lange, bis sich die Position nicht mehr wirklich ändert (Konvergenz)" + ] + }, + { + "cell_type": "markdown", + "id": "ee2bf385", + "metadata": {}, + "source": [ + "![Gradient_Descent_Mountain_2](../resources/gradient_descent_mountain_2.png)\n", + "\n", + "(von https://krishparekh.hashnode.dev/gradient-descent)" + ] + }, + { + "cell_type": "markdown", + "id": "0bbb95b6", + "metadata": {}, + "source": [ + "Man kann sich das auch vorstellen, wie wenn man einen Ball die Loss Landscape herunterrollen lässt (ohne Trägheit etc.), wie das folgende Beispiel zeigt." + ] + }, + { + "cell_type": "markdown", + "id": "f054ceb0", + "metadata": {}, + "source": [ + "![Gradient_Descent_Ball](../resources/gradient_descent_ball.jpg)\n", + "\n", + "(von https://datamites.com/blog/what-is-a-gradient-descent/?srsltid=AfmBOoozS1Fi9nBr4PlpiG-m1LHiooWr6KzNQ6IWiiJ6rrMqSuGxWfAz)" + ] + }, + { + "cell_type": "markdown", + "id": "6c98cee7", + "metadata": {}, + "source": [ + "> **Übung:** Warum ändert sich (bei passender Learning Rate) auf einmal die Position nicht mehr wirklich?" + ] + }, + { + "cell_type": "markdown", + "id": "b0e56a44", + "metadata": {}, + "source": [ + "![Intuition_Gradient_Descent_Derivative](../resources/Gradient_Descent_Intiution_Derivative.png)\n", + "\n", + "(von https://krishparekh.hashnode.dev/gradient-descent)" + ] + }, + { + "cell_type": "markdown", + "id": "9fbf2595", + "metadata": {}, + "source": [ + "> **Übung:** Überlege, was bei so einer numerischen Methode schief gehen kann." + ] + }, + { + "cell_type": "markdown", + "id": "dcf834fa", + "metadata": {}, + "source": [ + "Es gibt 2 (bzw. 3) Dinge, die bei Optimieren mit Gradient Descent schief gehen können:\n", + "1. Wir landen in einem lokalen Minimum\n", + "2. Die Learning Rate passt nicht:\n", + " * Die Learning Rate ist zu klein\n", + " * Die Learning Rate ist zu groß" + ] + }, + { + "cell_type": "markdown", + "id": "ac08507a", + "metadata": {}, + "source": [ + "Erster Fall ist hier dargestellt." + ] + }, + { + "cell_type": "markdown", + "id": "baae3f41", + "metadata": {}, + "source": [ + "![Gradient_Descent_Local_Minima](../resources/Gradient_Descent_Local_Minima.png)\n", + "\n", + "(von https://nvsyashwanth.github.io/machinelearningmaster/understanding-gradient-descent/)" + ] + }, + { + "cell_type": "markdown", + "id": "89cafc90", + "metadata": {}, + "source": [ + "Es hängt also auch von der Startposition (wir starten mit zufälligen Gewichten) ab, wie gut das Netzwerk lernen kann. Wenn wir Pech haben, dann können wir keine gute Lösung finden." + ] + }, + { + "cell_type": "markdown", + "id": "5a084754", + "metadata": {}, + "source": [ + "Auch für die Learning Rates zeigen wir nun zwei mögliche (ungünstige) Fälle." + ] + }, + { + "cell_type": "markdown", + "id": "08e0dfeb", + "metadata": {}, + "source": [ + "![Too_Small_Learning_Rate](../resources/Gradient_Descent_Too_Small_LR.png)\n", + "\n", + "(von https://krishparekh.hashnode.dev/gradient-descent)" + ] + }, + { + "cell_type": "markdown", + "id": "00347660", + "metadata": {}, + "source": [ + "![Too_Large_Learning_Rate](../resources/Gradient_Descent_Too_Big_LR.png)\n", + "\n", + "(von https://krishparekh.hashnode.dev/gradient-descent)" + ] + }, + { + "cell_type": "markdown", + "id": "301b48b6", + "metadata": {}, + "source": [ + "**Wie können wir die eben erwähnten Probleme lösen?**" + ] + }, + { + "cell_type": "markdown", + "id": "701f7fa0", + "metadata": {}, + "source": [ + "Für eine zu kleine oder zu große Learning Rate ist die Lösung recht einfach: Wir müssen die Learning Rate verändern.\n", + "\n", + "Sprich sollten wir das Gefühl haben, die Performance vom Modell schwankt sehr stark, dann sollten wir die Learning Rate reduzieren. Genauso sollten wir, falls wir das Gefühl haben, dass unser Modell zu langsam lernt und der Loss nach wie vor jede Iteration weniger wird, die Learning Rate (etwas) erhöhen.\n", + "\n", + "Meistens liegt die Learning Rate im Hundertstel oder Tausendstel Bereich, sprich eine Standard Learning-Rate liegt oft im Bereich $0.001-0.01$." + ] + }, + { + "cell_type": "markdown", + "id": "869ee2a0", + "metadata": {}, + "source": [ + "**Hinweis:** Wir werden zu einem späteren Zeitpunkt sehen, wie wir den Lernprozess beobachten können (Spoiler: Wir lassen uns laufend aktuelle Werte vom Loss/Accuracy/etc. ausgeben) und somit beurteilen können, ob das Modell vernünftig lernt. Insbesondere werden wir uns das im Notebook bzgl. der *Trainingsmethode* ansehen." + ] + }, + { + "cell_type": "markdown", + "id": "a8199ada", + "metadata": {}, + "source": [ + "**Frage:** Was, wenn wir in einem lokalem Minimum landen?" + ] + }, + { + "cell_type": "markdown", + "id": "d6e67465", + "metadata": {}, + "source": [ + "So ein Fall tritt wahrscheinlich bei jedem der praktischen Machine Learning Problemen auf. Auch, wenn es natürlich erwünscht wäre, in einem globalen Minimum zu landen, müssen wir uns meistens mit einem lokalen Minimum zufrieden geben. Dies hat folgende Gründe:\n", + "* Wir können nicht wissen, ob wir in einem lokalen oder globalen Optimum sind\n", + "* Sofern die Performance (Loss oder andere Metriken) gut genug ist, sind wir zufrieden\n", + "* Es gibt ein paar theoretische Resultate, bei denen gezeigt wird, dass in vielen Fällen bei komplizierten Machine Learning Tasks alle (lokalen) Minima gleich gut sind." + ] + }, + { + "cell_type": "markdown", + "id": "f9425292", + "metadata": {}, + "source": [ + "Falls wir doch das Gefühl haben, dass unser Modell viel schlechter performt, als es sollte, so können wir:\n", + "* Hyperparameter ändern (Learning Rate, Modellarchitektur)\n", + "* Mehr Daten verwenden (nicht immer verfügbar)\n", + "* Die Gewichte anders zufällig initialisieren (unüblich, nur \"Experten\" verändern diese Initialisierung, bringt nur in Spezialfällen was)\n", + "* Einen anderen Optimierungsalgorithmus, sprich einen anderen Optimizer verwenden." + ] + }, + { + "cell_type": "markdown", + "id": "ba79b08f", + "metadata": {}, + "source": [ + "**Hinweis:** Als Optimierer wird jener Algorithmus bezeichnet, der verwendet wird um zu lernen. In unserem Fall zBsp. (eine Variante von) Gradient Descent. Eine sehr bekannte Variante davon ist **Stochastic Gradient Descent** (SGD), welchen wir uns nun genauer ansehen wollen." + ] + }, + { + "cell_type": "markdown", + "id": "42c03208", + "metadata": {}, + "source": [ + "## Stochastic Gradient Descent" + ] + }, + { + "cell_type": "markdown", + "id": "0b4380eb", + "metadata": {}, + "source": [ + "Der erste Optimizer den wir betrachten wollen ist der sogenannte *Stochastic Gradient Descent* (**SGD**) Algorithmus. " + ] + }, + { + "cell_type": "markdown", + "id": "f249d352", + "metadata": {}, + "source": [ + "Er ist quasi der \"Standard\" Algorithmus in PyTorch für die Optimierung, und adaptiert den bisher besprochenen (Vanilla) Gradient Descent Algorithmus nur leicht." + ] + }, + { + "cell_type": "markdown", + "id": "e2147ca5", + "metadata": {}, + "source": [ + "Sehen wir uns zuerst ein Bild an, welches den Stochastic Gradient Descent mit dem (Vanilla) Gradient Descent vergleicht." + ] + }, + { + "cell_type": "markdown", + "id": "539515e6", + "metadata": {}, + "source": [ + "![SGD_vs_GD](../resources/SGD_vs_GD.png)\n", + "\n", + "(von https://www.geeksforgeeks.org/machine-learning/ml-stochastic-gradient-descent-sgd/)" + ] + }, + { + "cell_type": "markdown", + "id": "1c1969da", + "metadata": {}, + "source": [ + "Dieses Bild sieht jetzt vermutlich unintuitiv aus, weil die einzelnen Schritte des (Vanilla) Gradient Descent Algorithmus ja wesentlich besser aussehen. Diese Sichtweise ist teilweise richtig. Sehen wir uns mal die Details vom Stochastic Gradient Descent Algorithmus an." + ] + }, + { + "cell_type": "markdown", + "id": "1254a0bd", + "metadata": {}, + "source": [ + "Beim *SGD* ist die Update Rule leicht angepasst:\n", + "$$w_t = w_{t-1} - \\eta \\cdot \\nabla L_I(w_{t-1}).$$" + ] + }, + { + "cell_type": "markdown", + "id": "3f504b8a", + "metadata": {}, + "source": [ + "Der einzige Unterschied ist also, dass wir jetzt die Ableitung (den Gradient) von $L_I$ nehmen. Dabei bezeichnen wir mit $L_I$ die Loss-Funktion auf eine Teilmenge der Daten.\n", + "\n", + "Sprich einfach gesagt macht Stochastic Gradient Descent nichts anderes als Gradient Descent, nur wird der Fehler und somit auch die Ableitung nur für ein paar wenige Datenpaare $(\\hat y, y)$ berechnet anstatt für alle. Dabei werden die Datenpaare zufällig ausgewählt (stochastisch)." + ] + }, + { + "cell_type": "markdown", + "id": "b905eb8f", + "metadata": {}, + "source": [ + "**Vorteile von SGD:**\n", + "* Schneller, weil die Ableitung für weniger Punkte berechnet werden muss\n", + "* Aufgrund von zufälliger Wahl der Datenpaare wird dem Optimierungsprozess etwas Stochastik (\"Zufall\") hinzugefügt, dadurch besteht die Chance, aus lokalen Minima auszubrechen\n", + "* Garantie, dass wir in Erwartung (im Mittel) das gleiche machen wie der (Vanilla) Gradient Descent Algorithmus (Sprich der Algorithmus macht quasi im Mittel Sinn)." + ] + }, + { + "cell_type": "markdown", + "id": "05a133e0", + "metadata": {}, + "source": [ + "**Nachteile von SGD:**\n", + "* Stochastik führt zu \"instabilerem\" Training (der Effekt ist aber nicht so drastisch, wie im obigen Bild dargestellt.)\n", + "* Schlechte Nachvollziehbarkeit/Reproduzierbarkeit" + ] + }, + { + "cell_type": "markdown", + "id": "effb0d45", + "metadata": {}, + "source": [ + "**Hinweis:** Den (Vanilla) Gradient Descent gibt es in PyTorch eigentlich nicht direkt. Es wird eigentlich immer *SGD* verwendeten, falls von Gradient Descent gesprochen wird." + ] + }, + { + "cell_type": "markdown", + "id": "90b29550", + "metadata": {}, + "source": [ + "**Wichtig:** Es gibt auch andere Optimierungsverfahren, bei denen die Formeln für die Updates aufwendiger sind, jedoch in manchen Fällen bessere Konvergenzverhalten liefern. Ein Beispiel dafür ist zum Beispiel **Adam** oder **Adagrad**, diese können genauso verwendet werden und funktionieren für viele Probleme besser (bzw. meistens nicht schlechter). Wir werden diese aber nicht in der Theorie behandeln." + ] + }, + { + "cell_type": "markdown", + "id": "f929e1df", + "metadata": {}, + "source": [ + "![Meme_SGD](../resources/SGD_Local_Minima_Medal.jpg)\n", + "\n", + "(von https://x.com/drob/status/1425468713017425923)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Optimizer in PyTorch" + ] + }, + { + "cell_type": "markdown", + "id": "4bbe53ce", + "metadata": {}, + "source": [ + "Befassen wir uns nun damit, wie wir in Python (PyTorch) die Optimizer verwenden können und was sie bewirken." + ] + }, + { + "cell_type": "markdown", + "id": "5aaf4494", + "metadata": {}, + "source": [ + "Annahme, wir wollen die Funktion $f(w)=(w-3)^2$ minimieren. Wir sehen zwar relativ schnell, dass diese Funktion bei $x=3$ ihr Minimum erreicht, jedoch wollen wir das nun auch mit dem Gradient Descent Algorithmus zeigen." + ] + }, + { + "cell_type": "markdown", + "id": "26c3f80b", + "metadata": {}, + "source": [ + "**Hinweis:** Obiges Beispiel könnte man sich vorstellen, wie wenn wir die Funktion $f(x)=x$ haben mit Loss Funktion $g(x)=x^2$, sprich dem Mean Squarred Error. Das Label wär nun 3." + ] + }, + { + "cell_type": "markdown", + "id": "f5bcd79d", + "metadata": {}, + "source": [ + "> **Übung:** Berechne die Werte von $w$ für die ersten 3 Iterationen mit (Vanilla) Gradient Descent, wobei als Startwert $w=0$ verwendet werden soll und als Learning Rate $\\eta = 0.25$.\n", + "\n", + "Als Hilfestellung wird hier die Update-Rule nochmal dargestellt:\n", + "$$w_t = w_{t-1} - \\eta \\cdot \\nabla L(w_{t-1}).$$\n", + "Anders geschrieben (\"Programmierschreibweise\") heißt das\n", + "$$w \\mathrel{-}= \\eta \\cdot \\nabla L(w).$$\n", + "In unserem Fall (weil 1d) dann:\n", + "$$w \\mathrel{-}= \\eta \\cdot f'(w).$$" + ] + }, + { + "cell_type": "markdown", + "id": "8b97bf3c", + "metadata": {}, + "source": [ + "**Dieses Beispiel in PyTorch:**" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "250bec9a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Schritt 0: x = [0.0], f(x) = 9.0000\n", + "Schritt 1: x = [1.5], f(x) = 9.0000\n", + "Schritt 2: x = [2.25], f(x) = 2.2500\n", + "Schritt 3: x = [2.625], f(x) = 0.5625\n", + "Schritt 4: x = [2.8125], f(x) = 0.1406\n", + "Schritt 5: x = [2.90625], f(x) = 0.0352\n", + "Schritt 6: x = [2.953125], f(x) = 0.0088\n", + "Schritt 7: x = [2.9765625], f(x) = 0.0022\n", + "Schritt 8: x = [2.98828125], f(x) = 0.0005\n", + "Schritt 9: x = [2.994140625], f(x) = 0.0001\n", + "Schritt 10: x = [2.9970703125], f(x) = 0.0000\n" + ] + } + ], + "source": [ + "import torch\n", + "\n", + "learning_rate = 0.25\n", + "\n", + "# Parameter als Tensor mit Gradienten\n", + "x = torch.tensor([0.0], requires_grad=True)\n", + "optimizer = torch.optim.SGD([x], lr=learning_rate)\n", + "\n", + "def f(x):\n", + " return (x - 3)**2\n", + "\n", + "loss = f(x)\n", + "print(f\"Schritt 0: x = {x.tolist()}, f(x) = {loss.item():.4f}\")\n", + "for i in range(10):\n", + " optimizer.zero_grad() # Gradienten zurücksetzen\n", + " loss = f(x)\n", + " loss.backward() # Gradient berechnen\n", + " optimizer.step() # Update-Schritt\n", + " print(f\"Schritt {i+1}: x = {x.tolist()}, f(x) = {loss.item():.4f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "85e80b0d", + "metadata": {}, + "source": [ + "Hierbei sind folgende Punkte sehr wichtig in unserer Methode oben:\n", + "\n", + "* `x = torch.tensor([0.0], requires_grad=True)` $x$ muss ein Tensor sein, bei dem der Gradient gespeichert wird. Startwert ist in unserem Fall $0.0$.\n", + "* `optimizer = torch.optim.SGD([x], lr=learning_rate)` Der Optimizer muss die Parameter des Modells kennen (in unserem Fall ist unser Modell nur $x$)\n", + "* `optimizer.zero_grad()` Am Anfang jeder Iteration muss der Gradient wieder aus dem Optimizer gelöscht werden (wir wollen ja wieder einen neuen Gradienten berechnen)\n", + "* `loss = f(x)` Diese Zeile würde bei einem neuronalen Netz dann die Prediction mit dem True Label vergleichen\n", + "* `loss.backward()` Berechnet die Gradienten bzgl. **aller** Parameter, die vorher dem Optimizer übergeben wurden\n", + "* `optimizer.step()` Führt das Update $w_t = w_{t-1} - \\eta \\cdot \\nabla L(w_{t-1})$ durch" + ] + }, + { + "cell_type": "markdown", + "id": "137947aa", + "metadata": {}, + "source": [ + "> **Übung:** Verändere den obigen Code so, dass wir für die drei vorigen Aufgaben vom Beginn das (bzw. ein) Minimum finden. Ändere dazu die Dimension (und ggf. den Startwert) von $x$.\n", + "\n", + "Die verwendeten Funktionen waren:" + ] + }, + { + "cell_type": "markdown", + "id": "f67a2dd4", + "metadata": {}, + "source": [ + "$$L(w_1, w_2)=(4w_1+7w_2-20)^2$$\n", + "$$L(w_1, w_2, w_3)=\\max (0, 2w_1+3w_2-4w_3)^2$$\n", + "$$L(w_1)=\\lvert w_1 - e^{-w_1} \\rvert$$" + ] + }, + { + "cell_type": "markdown", + "id": "aa4adea8", + "metadata": {}, + "source": [ + "> **Übung:** Probiere auch die Optimizer *Adam* und *Adagrad* aus, indem du sie mit den Befehlen `from torch.optim import Adagrad, Adam` importierst und im Anschluss statt `SGD` verwendest." + ] + }, + { + "cell_type": "markdown", + "id": "069dc97d", + "metadata": {}, + "source": [ + "## Loss Funktionen in PyTorch" + ] + }, + { + "cell_type": "markdown", + "id": "605742b2", + "metadata": {}, + "source": [ + "Last but not least wollen wir uns noch die Loss-Funktionen in PyTorch ansehen." + ] + }, + { + "cell_type": "markdown", + "id": "0e909c5e", + "metadata": {}, + "source": [ + "Prinzipiell ist das Auswählen einer Loss-Funktion eine sehr kritische Sache, da wir im Laufe des Trainings versuchen, den Loss des Neuronalen Netzwerks zu reduzieren, idealerweise das (globale) Minimum davon zu finden. Es gibt jedoch für die Standard Probleme gängige Loss-Funktionen, die in vielen Fällen tadellos funktionieren. \n", + "\n", + "Diese beiden sind die bereits bekannten Funktionen:\n", + "* MSE (Mean Squared Error) für Regression\n", + "* (Binary) Cross-Entropy Loss für Klassifikation" + ] + }, + { + "cell_type": "markdown", + "id": "80220b45", + "metadata": {}, + "source": [ + "Verwendet kann dies folgendermaßen werden." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d070bf0a", + "metadata": {}, + "outputs": [], + "source": [ + "from torch.nn import CrossEntropyLoss, MSELoss" + ] + }, + { + "cell_type": "markdown", + "id": "3d1019f4", + "metadata": {}, + "source": [ + "Dies sind jetzt noch Klassen, welche noch instanziert werden müssen." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5661912", + "metadata": {}, + "outputs": [], + "source": [ + "loss_fn = MSELoss()\n", + "loss_fn = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "id": "b3cd8487", + "metadata": {}, + "source": [ + "Beides sind Funktionen, welche sich dann aufrufen lassen mit `loss_fn(input, target)`." + ] + }, + { + "cell_type": "markdown", + "id": "1c4ffa00", + "metadata": {}, + "source": [ + "Auch, wenn der Output gleich ist wie oben beschrieben gibt es ein paar implementierungsspezifische Details, auf die man bei der Verwendung achten muss. Ein paar Dinge werden wir hier aufzählen. Allgemein findet man natürlich die Infos in der Dokumentation. So zum Beispiel auch für den [Cross-Entropy-Loss](https://docs.pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html)." + ] + }, + { + "cell_type": "markdown", + "id": "5c9990d9", + "metadata": {}, + "source": [ + "Besonderheiten in der Implementierung:\n", + "* Wir erwarten uns beim Output Layer für eine Klassifikation normalerweise einen \"Wahrscheinlichkeitsvektor\", sprich die Summe muss 1 ergeben. Dies erreichen wir mit der Softmax Funktion am Schluss vom Netzwerk. In der Implementierung von PyTorch werden aber die Werte **vor der Softmax** Funktion erwartet\n", + "* Wir haben bisher gelernt, dass wir für die Klassifikation **One-Hot**-Vektoren für die Labels brauchen. Das stimmt in der Theorie, jedoch für die Implementierung reicht ein Integer Label aus." + ] + }, + { + "cell_type": "markdown", + "id": "c9162a3e", + "metadata": {}, + "source": [ + "> **Übung:** Warum reicht ein Integer Label aus, um den Loss zu berechnen? (Sprich warum kann sich PyTorch das Transformieren zu einem One-Hot Vektor sparen?)" + ] + }, + { + "cell_type": "markdown", + "id": "5be53daa", + "metadata": {}, + "source": [ + "**Hinweis:** (Nicht recht relevant für Test): Für eine Multiclass Klassifikation (wird bei uns nicht behandelt) reicht ein Index natürlich nicht mehr aus." + ] + }, + { + "cell_type": "markdown", + "id": "6dc09db0", + "metadata": {}, + "source": [ + "**Wichtig:** Solche Implementierungsdetails sollen beim Implementieren bekannt sein (oder beim Auftreten der Probleme zumindest nicht überraschen), jedoch ist es wichtiger, dass die zu Grunde liegende Theorie verstanden wird! " + ] + }, + { + "cell_type": "markdown", + "id": "1150531f", + "metadata": {}, + "source": [ + "## Vanishing and Exploding Gradient" + ] + }, + { + "cell_type": "markdown", + "id": "abd88832", + "metadata": {}, + "source": [ + "Ok, nachdem wir jetzt wissen, wie ein Neuronales Netzwerk lernt, stellt man sich vielleicht die Frage, warum man nicht einfach ein riesiges (tiefes) Netzwerk verwendet. Immerhin kann man dann mit Gradient Descent oder verwandte/ähnliche Optimierer die (hoffentlich) beste Lösung finden und man hat eine gute Performance." + ] + }, + { + "cell_type": "markdown", + "id": "1a2f407d", + "metadata": {}, + "source": [ + "Die Idee ist zwar prinzipiell nicht falsch, jedoch funktioniert das in der Praxis nicht. Grund dafür ist der sogenannte **Vanishing Gradient** Effekt. " + ] + }, + { + "cell_type": "markdown", + "id": "e0ab7dae", + "metadata": {}, + "source": [ + "### Vanishing Gradient" + ] + }, + { + "cell_type": "markdown", + "id": "3bca56ea", + "metadata": {}, + "source": [ + "(Erstmals formalisiert/festgestellt von Prof. Sepp Hochreiter 1991)." + ] + }, + { + "cell_type": "markdown", + "id": "361a82e2", + "metadata": {}, + "source": [ + "Nachdem beim Gradient Descent die Ableitungen bezüglich der Gewichte berechnet werden, kommt natürlich auch die Kettenregel ins Spiel. Das wird bei zu tiefen Modellen oft ein Problem, da zum Beispiel\n", + "\n", + "$$\\frac{\\partial L}{\\partial w_1} = \\frac{\\partial L}{\\partial a_n}\\underbrace{\\frac{\\partial a_n}{\\partial z_n}}_{=f'(a_n)} \\frac{\\partial z_n}{\\partial a_{n-1}}\\cdots \\frac{\\partial z_3}{\\partial a_2}\\underbrace{\\frac{\\partial a_2}{\\partial z_2}}_{=f'(a_2)}\\frac{\\partial z_2}{\\partial a_1}\\underbrace{\\frac{\\partial a_1}{\\partial z_1}}_{=f'(a_1)}\\frac{\\partial z_1}{\\partial w_1}.$$" + ] + }, + { + "cell_type": "markdown", + "id": "219b46bc", + "metadata": {}, + "source": [ + "> **Übung:** Wie nennt man obige Regel beim Ableiten?" + ] + }, + { + "cell_type": "markdown", + "id": "a2c10f0a", + "metadata": {}, + "source": [ + "Hier stellt $f$ die Aktivierungsfunktion, somit $f'$ die Ableitung davon dar." + ] + }, + { + "cell_type": "markdown", + "id": "d0b0edd6", + "metadata": {}, + "source": [ + "Warum ist das ein Problem?" + ] + }, + { + "cell_type": "markdown", + "id": "872f81c3", + "metadata": {}, + "source": [ + "Wenn wir uns die Ableitung von der Sigmoid Funktion $\\sigma(x) = (1+e^{-x})^{-1}$ ansehen, dann erhalten wir\n", + "$$\\sigma'(x) = \\sigma(x) \\cdot (1-\\sigma(x)).$$\n", + "\n", + "Diese Funktion hat als Maximalwert $0.25$. Mit der Kettenregel wird dieser Wert aber je nach Tiefe oft multipliziert." + ] + }, + { + "cell_type": "markdown", + "id": "a66201e6", + "metadata": {}, + "source": [ + "![Sigmoid_Gradient](../resources/Sigmoid_plus_Derivative.png)\n", + "\n", + "(eigene Abbildung)" + ] + }, + { + "cell_type": "markdown", + "id": "98ce7ce3", + "metadata": {}, + "source": [ + "Das bedeutet, wenn wir die Sigmoid Funktion als Aktivierungsfunktion verwenden und ein sehr tiefes Netzwerk verwenden, dann wird die Ableitung bezüglich der Gewichte am Anfang sehr klein (sie verschwindet - it vanishes)." + ] + }, + { + "cell_type": "markdown", + "id": "0ecd3b3c", + "metadata": {}, + "source": [ + "Somit werden **extrem tiefe Modelle nicht mehr lernbar**, wenn die Sigmoid (oder vergleichbare) Aktivierungsfunktion verwendet wird. " + ] + }, + { + "cell_type": "markdown", + "id": "50dd44dd", + "metadata": {}, + "source": [ + "**Hinweis:** Die Aktivierungsfunktionen müssen natürlich nicht gleich sein bei jeder Schicht, aber mit jedem zusätzlichen Layer wird das Netzwerk tiefer und somit mit der Kettenregel ein weiterer Term (bzw. 2 weitere Terme) dazu multipliziert." + ] + }, + { + "cell_type": "markdown", + "id": "d2b16ad3", + "metadata": {}, + "source": [ + "### Exploding Gradient" + ] + }, + { + "cell_type": "markdown", + "id": "28d6b444", + "metadata": {}, + "source": [ + "Das gleiche Problem kann passieren, wenn wir Aktivierungsfunktionen verwenden, bei denen die Ableitung größer als 1 ist. In solchen Fällen wird der Gradient sehr groß (er explodiert). Auch solche Netzwerke sind nicht lernbar." + ] + }, + { + "cell_type": "markdown", + "id": "d0cd639f", + "metadata": {}, + "source": [ + "> **Übung:** Warum sind Netzwerke, welche vom *Exploding-Gradient* betroffen sind, nicht lernbar?" + ] + }, + { + "cell_type": "markdown", + "id": "752db18f", + "metadata": {}, + "source": [ + "### Gradient $\\approx$ 1" + ] + }, + { + "cell_type": "markdown", + "id": "9aec786d", + "metadata": {}, + "source": [ + "Ideal ist natürlich eine Aktivierungsfunktion mit Werte der Ableitung in der Größenordnung 1. Solche Netzwerke können viel tiefer gemacht werden und leiden somit nicht unter den beiden oben genannten Problemen.\n", + "\n", + "Ein sehr guter Kandidat: $\\mathrm{ReLU}(x)$. Jedoch ist hier in vielen Fällen die Ableitung $0$." + ] + }, + { + "cell_type": "markdown", + "id": "329cc9ca", + "metadata": {}, + "source": [ + "**Hinweis:** In der Theorie kann natürlich ein Netzwerk bei genügend Iterationen/Epochen trainiert werden. Hier können die Gradienten dann zwar sehr sehr sehr klein werden, jedoch sind sie nicht $0$. In der Praxis sind die sehr kleinen Gradienten ein Problem, weil sie erstens den Trainingsprozess verlangsamen und andererseits ab einer gewissen Größe der Computer sie nicht mehr von der $0$ unterscheiden kann!" + ] + }, + { + "cell_type": "markdown", + "id": "c1854d33", + "metadata": {}, + "source": [ + "![Wait_Has_Been_Meme](../resources/Wait_Always_Has_Been.jpg)\n", + "\n", + "(Eigene Grafik mit imgflip.com)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dsai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/06_NN/code/nn_4_datasets_dataloader.ipynb b/06_NN/code/nn_4_datasets_dataloader.ipynb new file mode 100644 index 0000000..106b011 --- /dev/null +++ b/06_NN/code/nn_4_datasets_dataloader.ipynb @@ -0,0 +1,1031 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1a6c98c5", + "metadata": {}, + "source": [ + "
\n", + "\n", + "

Neural Networks: Datasets und Dataloader

\n", + "

DSAI

\n", + "

Jakob Eggl

\n", + "\n", + "
\n", + " \"Logo\"\n", + "
\n", + " © 2025/26 Jakob Eggl. Nutzung oder Verbreitung nur mit ausdrücklicher Genehmigung des Autors.\n", + "
\n", + "
\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "391f52bc", + "metadata": {}, + "source": [ + "# Datasets und Dataloader (in PyTorch)" + ] + }, + { + "cell_type": "markdown", + "id": "851d24f7", + "metadata": {}, + "source": [ + "In diesem Notebook wollen wir uns den sogenannten Dataloadern und den dazugehörigen Datasets widmen. Wir werden dabei zuerst die Notwendigkeit solcher Objekte besprechen und uns im Anschluss deren Erstellung betrachten. Am Ende werden wir noch Vorteile und Nachteile besprechen." + ] + }, + { + "cell_type": "markdown", + "id": "f6f2e229", + "metadata": {}, + "source": [ + "### Warum benötigen wir diese Objekte?" + ] + }, + { + "cell_type": "markdown", + "id": "0a37b0c4", + "metadata": {}, + "source": [ + "Der Grund, warum wir `torch.utils.data.Dataset` und `torch.utils.data.DataLoader` verwenden werden, bezieht sich auf folgende Punkte.\n", + "\n", + "1. Zu großes Dataset:\n", + " * Falls unser Dataset zu groß für die CPU (also für den RAM) bzw. für die GPU (also für den VRAM) ist, dann müssen wir das Dataset aufteilen in kleinere Teile\n", + " * Es kann natürlich auch händisch geteilt werden, wir werden aber eine elegantere Lösung (mit *Datasets* und/oder *DataLoader*) kennen lernen.\n", + "\n", + "2. Vorverarbeitungsschritte sind kompliziert bzw. nur temporär:\n", + " * Falls unsere Daten in ein bestimmtes Format gebracht werden müssen, dann wollen wir das oft nicht permament machen. Beispiele: Data-Augmentation, Downscaling bei Bildern oder Text in Embedding-Vektoren umwandeln.\n", + " * Wir wollen also nicht unser Änderungen auch abspeichern bzw. sogar unsere Daten mit den Änderungen ersetzen.\n", + "\n", + "3. Art der Daten lassen das \"gesamte Laden\" nicht zu:\n", + " * Für Bilder/Videos/Text ist es schwierig, diese überhaupt sinnvoll gleichzeitig zu öffnen. Natürlich ist es möglich, diese als einen großen Tensor/Array zu speichern, jedoch wird so die Vorverarbeitung und weitere Verwendung auch etwas kompliziert." + ] + }, + { + "cell_type": "markdown", + "id": "e45e8a97", + "metadata": {}, + "source": [ + "Alle diese Punkte schließen nicht aus, dass wir einfach wie bisher einen Teil unserer Daten laden, diese vorverarbeiten und im Anschluss an unser Modell übergeben, bis wir wieder die nächsten Daten laden usw.\n", + "\n", + "Jedoch ist dies einerseits umständlich und andererseits langsam. Das Training eines neuronalen Netzes nimmt nämlich einige Zeit in Anspruch und wir wollen so wenig Zeit wie möglich unsere GPU's \"unbeschäftigt\" lassen. Wir werden sehen, dass wir sowohl Speicher als auch Runtime mit diesen Methoden minimieren können." + ] + }, + { + "cell_type": "markdown", + "id": "13735986", + "metadata": {}, + "source": [ + "**Kurz gesagt:** Die Kombination aus **Dataset** und **Dataloader** erlaubt es uns, die Datenbereitstellung sauber von der Model-Klasse bzw. dem späteren Trainingsvorgang zu separieren und somit Speicher- und Zeiteffizient zu arbeiten." + ] + }, + { + "cell_type": "markdown", + "id": "b99c456a", + "metadata": {}, + "source": [ + "## Datasets" + ] + }, + { + "cell_type": "markdown", + "id": "35807b3c", + "metadata": {}, + "source": [ + "[PyTorch_Dataset_Documentation](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.Dataset)" + ] + }, + { + "cell_type": "markdown", + "id": "ca3c242d", + "metadata": {}, + "source": [ + "Ein Dataset ist eine Python Klasse, die von `torch.utils.data.Dataset` erbt und 3 Methoden implementiert. Wir starten mit einem Beispiel:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "1e9327d3", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from torch.utils.data import Dataset, DataLoader\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "01a2d3f3", + "metadata": {}, + "outputs": [], + "source": [ + "class MyDataset(Dataset):\n", + " def __init__(self, data, targets):\n", + " self.data = torch.from_numpy(data).float()\n", + " self.targets = torch.from_numpy(targets).long()\n", + "\n", + " def __len__(self):\n", + " return len(self.data)\n", + "\n", + " def __getitem__(self, idx):\n", + " return self.data[idx], self.targets[idx]" + ] + }, + { + "cell_type": "markdown", + "id": "d1c3c29f", + "metadata": {}, + "source": [ + "Wir können nun unser Dataset initialisieren und die Funktionsweise testen." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "d9ea68ff", + "metadata": {}, + "outputs": [], + "source": [ + "data = np.random.randn(100, 5)\n", + "targets = np.random.randint(0, 3, size=(100,))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b09bb0be", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = MyDataset(data, targets)\n", + "loader = DataLoader(dataset, batch_size=10, shuffle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "29f02dfe", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[ 0.5069, 0.1092, -0.2384, -0.2577, 0.8052],\n", + " [-0.3121, 0.2231, 0.5598, -1.4180, 0.1089],\n", + " [ 1.2742, -1.1464, -0.4507, -0.4376, -2.4784],\n", + " [ 0.2624, -1.1176, 0.5151, 2.4745, -1.0527],\n", + " [ 2.0679, -0.5308, -0.6077, 1.0965, -1.0772],\n", + " [-0.2201, 0.7058, -0.1247, 0.7160, -0.3400],\n", + " [-2.1532, 1.6777, 1.1952, -0.4793, 1.0680],\n", + " [ 0.3298, -0.8381, -0.3389, -0.1819, 0.7724],\n", + " [-0.7027, 0.2146, -0.8227, -0.0369, 0.3882],\n", + " [ 0.1065, 0.4170, -1.2056, 0.1908, 1.0696]])\n", + "tensor([0, 2, 2, 1, 0, 1, 0, 2, 0, 2])\n", + "tensor([[ 0.9685, -2.3074, 0.3342, -0.0494, -1.1488],\n", + " [-0.5176, -0.1098, 1.6226, 1.5894, 0.8609],\n", + " [-0.0244, 0.1765, -0.3933, -0.1908, -0.2194],\n", + " [ 0.0610, 1.4928, 1.0841, -0.2602, 0.8757],\n", + " [ 0.3410, -0.3119, 1.4237, 0.7171, -0.8697],\n", + " [-0.1329, 0.0656, -1.3961, 1.7001, 0.5371],\n", + " [-0.9989, -0.5213, -0.7923, -0.4502, 0.6730],\n", + " [ 0.0975, -2.0438, 1.7340, -0.3228, 0.1989],\n", + " [-0.0171, -2.5123, 1.7726, 2.0372, -0.1390],\n", + " [ 1.0011, 0.5684, 1.4757, 0.4278, -0.1118]])\n", + "tensor([1, 1, 2, 2, 1, 1, 2, 1, 2, 2])\n", + "tensor([[ 1.5740, 0.4927, -0.4670, 0.9567, 1.3826],\n", + " [-0.1016, -0.0583, -0.6909, 1.1347, 0.2186],\n", + " [-0.3474, 1.1566, 0.7346, -0.6135, 0.3152],\n", + " [ 1.0799, 1.6222, 0.2068, -1.4269, -0.2880],\n", + " [-0.8751, -1.8200, 0.9509, 1.0376, -2.3234],\n", + " [-1.3424, 0.3162, 1.3053, 1.2186, 0.6587],\n", + " [ 1.1749, 0.0887, 1.2147, 1.3284, 0.3012],\n", + " [ 0.0191, -1.3697, 0.5889, -0.1592, -0.2759],\n", + " [-1.1337, -1.0487, 0.5384, 0.2060, -0.9789],\n", + " [ 0.2615, -0.4887, -0.1512, 0.6128, 0.3353]])\n", + "tensor([0, 1, 1, 0, 0, 1, 2, 2, 2, 1])\n", + "tensor([[-0.2666, 1.0328, 0.4132, -0.3333, -0.0195],\n", + " [-0.2145, 0.9221, -0.4469, -1.2847, -0.3797],\n", + " [-0.9704, 0.5636, 0.2236, -0.5538, -0.6366],\n", + " [-1.0516, -0.0524, 1.3040, 0.4670, 1.2107],\n", + " [ 0.4674, 0.7978, -0.7506, -2.0194, 1.1103],\n", + " [ 0.3870, 0.5645, -1.7483, 0.4005, -1.4716],\n", + " [ 1.5839, -0.4092, 0.7452, 0.2300, 0.6801],\n", + " [ 0.1627, 1.2529, 0.8862, -0.0962, -0.2865],\n", + " [ 0.1322, 0.1244, -1.7163, 0.2510, 0.8945],\n", + " [-1.9448, 0.4907, -0.2076, 0.4113, -0.6400]])\n", + "tensor([0, 1, 1, 0, 0, 1, 0, 2, 0, 1])\n", + "tensor([[-0.8499, 2.5727, 0.1797, -0.9507, 0.0995],\n", + " [ 1.0104, -0.3346, 1.1222, 0.2464, 1.3047],\n", + " [-0.4544, -0.0389, -0.5465, -1.5053, -0.9029],\n", + " [ 0.5324, -0.2656, -1.1016, 1.0078, -0.2578],\n", + " [-0.0508, -0.3780, -0.2380, -0.3371, -0.2502],\n", + " [ 1.0109, 0.1582, 1.3518, 1.2197, 0.4554],\n", + " [ 1.3356, -0.5515, -2.4314, -1.0432, -1.3512],\n", + " [-1.2996, 0.0575, -0.5404, 0.4505, 1.7020],\n", + " [-1.8341, -0.3733, 0.7171, -1.4176, 0.9022],\n", + " [-0.3161, -0.6970, 0.4414, 0.1965, 0.3764]])\n", + "tensor([0, 1, 1, 1, 0, 2, 1, 2, 1, 2])\n", + "tensor([[ 2.2886, 0.7842, -0.5470, 0.0253, 0.8413],\n", + " [-0.6462, 1.0103, -0.3321, -1.0588, 0.8414],\n", + " [-0.8248, 0.4411, -1.2213, 0.2718, -0.2400],\n", + " [ 2.2413, 0.1434, 0.2953, -1.8516, -1.0865],\n", + " [ 0.0360, 1.0103, 0.3064, -0.1483, -0.1876],\n", + " [ 0.9376, -1.3014, -0.6922, 0.4957, 1.1379],\n", + " [ 0.3778, -0.4550, -0.1885, 1.6360, -0.0478],\n", + " [-0.6091, 0.4593, -1.0795, 1.0445, 1.4456],\n", + " [-0.8640, -1.1908, -0.5895, 0.7801, 0.4134],\n", + " [-2.0441, 1.1808, -0.5463, -1.5396, 1.2662]])\n", + "tensor([2, 0, 0, 2, 0, 1, 0, 0, 1, 1])\n", + "tensor([[-0.6330, -2.1510, -0.8983, -0.1312, 1.1575],\n", + " [-1.5544, 1.3828, 0.2993, 1.1653, -0.3601],\n", + " [-2.2274, -1.0692, 1.3987, -2.2117, -1.3616],\n", + " [ 1.1790, -0.8240, 0.2870, 0.5272, -0.1788],\n", + " [ 2.4232, 1.4392, -0.1076, 1.3713, 0.8555],\n", + " [ 0.1287, 1.3900, -0.6812, 0.3302, 1.2172],\n", + " [ 2.3943, -0.7160, 0.5549, 1.5188, -1.7078],\n", + " [ 0.2735, 1.4099, -0.0380, -1.1581, -1.4514],\n", + " [ 0.2183, 0.8333, 0.3393, 1.4346, 1.1063],\n", + " [ 0.6508, 0.3358, 0.3620, 0.2518, -1.4232]])\n", + "tensor([2, 0, 0, 2, 1, 1, 0, 2, 1, 0])\n", + "tensor([[ 0.6291, 0.0391, -0.3817, 1.2282, -1.9145],\n", + " [-1.1365, -0.3839, -0.3610, -1.8804, 0.6841],\n", + " [-1.3119, -1.0822, 1.5917, -0.6910, -0.0602],\n", + " [ 0.7270, -1.4085, 1.3205, 0.1780, 0.3625],\n", + " [ 0.2534, -1.3888, 0.2618, -0.2694, 0.4607],\n", + " [-0.7799, 0.7737, 0.3365, 1.0458, 0.2953],\n", + " [ 0.5289, 1.0393, -0.5213, -0.1095, -1.2833],\n", + " [-0.3864, -0.7646, -1.6940, -1.8911, -0.6540],\n", + " [-0.6470, -0.6383, -1.1541, 1.3606, -0.2598],\n", + " [ 0.3753, -0.8425, -0.3253, -0.7869, -1.0851]])\n", + "tensor([2, 2, 1, 2, 2, 2, 1, 2, 0, 0])\n", + "tensor([[-0.0050, -1.9399, 0.3255, -1.4550, -0.5262],\n", + " [ 0.5969, -0.4763, 0.3675, 0.1272, -0.0277],\n", + " [ 1.1594, -0.9677, 0.2382, 0.3479, 0.7002],\n", + " [-0.1447, 0.4655, -1.5942, 2.0188, -1.1407],\n", + " [-1.6292, -0.3160, 0.0407, 1.4238, 1.1886],\n", + " [-0.4782, 0.2491, -1.6189, 1.1369, 0.0859],\n", + " [-0.7143, -1.4070, -1.2773, -1.4295, 0.4830],\n", + " [-0.8684, -1.2333, -1.0899, -1.8206, 0.9641],\n", + " [-0.6433, -0.2733, -1.1265, -0.7918, -1.2049],\n", + " [ 1.1337, 0.3181, -0.1788, -0.7318, 0.5277]])\n", + "tensor([0, 0, 1, 1, 2, 2, 1, 2, 2, 1])\n", + "tensor([[ 9.1756e-01, -1.6051e+00, -3.7274e-01, 3.7328e-01, 2.4529e-01],\n", + " [ 9.9791e-01, 2.5327e+00, 8.4713e-01, -9.1185e-02, -5.1878e-01],\n", + " [ 1.4338e+00, -2.5058e-01, -9.5984e-01, -1.4446e+00, 5.3632e-01],\n", + " [-4.1472e-01, -5.3044e-01, 3.1832e-01, 1.7581e-01, 8.3517e-01],\n", + " [-1.8925e+00, 8.6463e-01, 1.7321e+00, 2.7823e+00, -2.0044e+00],\n", + " [ 6.8021e-01, -8.6837e-01, -1.5156e+00, -5.4434e-01, -2.3947e-01],\n", + " [ 6.6447e-01, -2.0667e+00, 1.8755e-01, 6.1421e-01, -3.4289e-01],\n", + " [-1.6799e-01, -3.6753e-01, -9.2703e-01, 8.1852e-01, -7.8784e-01],\n", + " [-4.6175e-01, -8.2761e-02, -2.5252e-01, 4.4818e-01, -8.2673e-01],\n", + " [-8.3406e-01, 4.4200e-01, -1.7149e-03, 7.3712e-01, 8.1555e-01]])\n", + "tensor([0, 2, 2, 2, 2, 1, 0, 0, 1, 0])\n" + ] + } + ], + "source": [ + "for x, y in loader:\n", + " print(x)\n", + " print(y)" + ] + }, + { + "cell_type": "markdown", + "id": "4f45426f", + "metadata": {}, + "source": [ + "Hier haben wir schon etwas vorgegriffen und bereits einen `DataLoader` genutzt, dieser wird später noch genauer erklärt." + ] + }, + { + "cell_type": "markdown", + "id": "124756d0", + "metadata": {}, + "source": [ + "Von uns gibt es also folgende Methoden zu implementieren:\n", + "* `__init__(self, data, targets)` (Konstruktor): Bekommt die Daten übergeben (in welcher Form auch immer)\n", + "* `__len__(self)`: Muss die Anzahl der Daten in dem Dataset liefern. Meist ein einfacher `return len(self.data)` Befehl.\n", + "* `__getitem__(self, idx)`: Für einen gegebenen Index `idx` soll *ein* Datenpunkt returned werden. Dieser kann aber auch ein Tupel sein (zum Beispiel werden wir immer gleich $X$ (Daten) und Label $y$ übergeben)." + ] + }, + { + "cell_type": "markdown", + "id": "4b976b2d", + "metadata": {}, + "source": [ + "Hier könnte man dann statt dem obigen Testbeispiel von uns ein Pandas Dataframe übergeben.\n", + "\n", + "Sehen wir uns nun noch weitere Beispiele an." + ] + }, + { + "cell_type": "markdown", + "id": "1cd45025", + "metadata": {}, + "source": [ + "**Anderes Beispiel:**\n", + "\n", + "Was, wenn wir ein Bilder Dataset haben? Sprich einen wir haben einen Ordner, in welchem unsere Daten in der Form von Bildern abgespeichert sind." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "20918f81", + "metadata": {}, + "outputs": [], + "source": [ + "from PIL import Image\n", + "from torchvision import transforms\n", + "import os\n", + "import glob\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "eb0bdcfe", + "metadata": {}, + "outputs": [], + "source": [ + "class MyImageFolderDataset(Dataset):\n", + " def __init__(self, image_paths, labels, transform=None):\n", + " self.image_paths = image_paths\n", + " self.labels = labels\n", + " self.transform = transform or transforms.ToTensor()\n", + "\n", + " def __len__(self):\n", + " return len(self.image_paths)\n", + "\n", + " def __getitem__(self, idx):\n", + " img = Image.open(self.image_paths[idx]).convert('RGB')\n", + " label = torch.tensor(self.labels[idx])\n", + " if self.transform:\n", + " img = self.transform(img)\n", + " return img, label" + ] + }, + { + "cell_type": "markdown", + "id": "bff2605d", + "metadata": {}, + "source": [ + "Hier übergeben wir in der `__init__` Methode 2 Argumente für die Daten und noch ein zusätzliches Argument für etwaige Transformationen. Wir sehen also, dass wir hier im Dataset auch die Daten dementsprechend *transformieren* können. Dazu zählt zum Beispiel das Downscalen (zum Beispiel auf die Größe $100\\times 100$, das Transformieren in Schwarz-Weiß Bilder oder das Data-Augmentieren.)" + ] + }, + { + "cell_type": "markdown", + "id": "10fd2a64", + "metadata": {}, + "source": [ + "Wir können das jetzt selber testen. Übergebt dafür einfach einen Ordner mit dem entsprechenden Pfad. " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "34653e2f", + "metadata": {}, + "outputs": [ + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] No such file or directory: '../private/katzen/labels.txt'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 6\u001b[0m\n\u001b[1;32m 2\u001b[0m label_path \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m..\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprivate\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mkatzen\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlabels.txt\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 4\u001b[0m image_paths \u001b[38;5;241m=\u001b[39m glob\u001b[38;5;241m.\u001b[39mglob(image_folder_path)\n\u001b[0;32m----> 6\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mlabel_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[1;32m 7\u001b[0m labels \u001b[38;5;241m=\u001b[39m [line\u001b[38;5;241m.\u001b[39mstrip() \u001b[38;5;28;01mfor\u001b[39;00m line \u001b[38;5;129;01min\u001b[39;00m f\u001b[38;5;241m.\u001b[39mreadlines()]\n", + "File \u001b[0;32m~/.conda/envs/dsai/lib/python3.9/site-packages/IPython/core/interactiveshell.py:286\u001b[0m, in \u001b[0;36m_modified_open\u001b[0;34m(file, *args, **kwargs)\u001b[0m\n\u001b[1;32m 279\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m file \u001b[38;5;129;01min\u001b[39;00m {\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m}:\n\u001b[1;32m 280\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 281\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIPython won\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt let you open fd=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfile\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m by default \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 282\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mas it is likely to crash IPython. If you know what you are doing, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 283\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124myou can use builtins\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m open.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 284\u001b[0m )\n\u001b[0;32m--> 286\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mio_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../private/katzen/labels.txt'" + ] + } + ], + "source": [ + "image_folder_path = os.path.join(\"..\", \"private\", \"katzen\", \"*.jpg\")\n", + "label_path = os.path.join(\"..\", \"private\", \"katzen\", \"labels.txt\")\n", + "\n", + "image_paths = glob.glob(image_folder_path)\n", + "\n", + "with open(label_path, 'r') as f:\n", + " labels = [line.strip() for line in f.readlines()]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a8957ed", + "metadata": {}, + "outputs": [], + "source": [ + "image_paths" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d44876c6", + "metadata": {}, + "outputs": [], + "source": [ + "len(image_paths)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3217581", + "metadata": {}, + "outputs": [], + "source": [ + "labels" + ] + }, + { + "cell_type": "markdown", + "id": "06669e7c", + "metadata": {}, + "source": [ + "Die Labels müssten noch umgewandelt werden in Zahlen (oder One-Hot-Vektoren), falls wir noch weiter arbeiten wollen damit." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "894ec1fa", + "metadata": {}, + "outputs": [], + "source": [ + "# Generate unique ints for each label\n", + "unique_labels = list(set(labels))\n", + "label_to_int = {label: idx for idx, label in enumerate(unique_labels)}\n", + "labels = [label_to_int[label] for label in labels]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e0acfbe", + "metadata": {}, + "outputs": [], + "source": [ + "labels" + ] + }, + { + "cell_type": "markdown", + "id": "ecedcd1a", + "metadata": {}, + "source": [ + "Wir erstellen uns nun ein PyTorch Dataset aus diesen Daten." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76830994", + "metadata": {}, + "outputs": [], + "source": [ + "my_cat_dataset = MyImageFolderDataset(image_paths, labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd0e74f3", + "metadata": {}, + "outputs": [], + "source": [ + "cat_loader = DataLoader(my_cat_dataset, batch_size=1, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3464704", + "metadata": {}, + "outputs": [], + "source": [ + "for imgs, lbls in cat_loader:\n", + " fig, axes = plt.subplots(figsize=(8, 4)) # only one plot\n", + " axes.imshow(np.transpose(imgs[0].numpy(), (1, 2, 0)))\n", + " axes.set_title(f\"Label: {lbls[0].item()}\")\n", + " axes.axis('off')\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "3bb87e57", + "metadata": {}, + "source": [ + "Warum können wir hier nicht 2 Bilder (*batch_size*) gleichzeitig verwenden? --> **Probieren wir es einfach aus im Code.**" + ] + }, + { + "cell_type": "markdown", + "id": "5503ccea", + "metadata": {}, + "source": [ + "## Transformationen in PyTorch Datasets" + ] + }, + { + "cell_type": "markdown", + "id": "e1f0879b", + "metadata": {}, + "source": [ + "Mit Hilfe von uns definierten, sogenannten **transformations** können wir unsere Daten sehr elegant transformieren. Eine beispielshafte Transformations-Pipeline sieht folgendermaßen aus." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17ebcd69", + "metadata": {}, + "outputs": [], + "source": [ + "transform = transforms.Compose([\n", + " transforms.ToTensor(),\n", + " transforms.Resize((100, 100)),\n", + " transforms.GaussianBlur(3),\n", + " transforms.RandomHorizontalFlip(),\n", + " transforms.RandomRotation(10),\n", + "])" + ] + }, + { + "cell_type": "markdown", + "id": "2578fdc1", + "metadata": {}, + "source": [ + "**Hinweis:** Die einzelnen Transformationen werden der Reihe nach abgearbeitet, also zuerst zu einem Tensor verwandelt, danach die Größe auf $100 \\times 100$ angepasst usw." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "af7af1e1", + "metadata": {}, + "outputs": [], + "source": [ + "my_transformed_cat_dataset = MyImageFolderDataset(image_paths, labels, transform=transform)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97904ffc", + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 2\n", + "transformed_cat_loader = DataLoader(my_transformed_cat_dataset, batch_size=batch_size, shuffle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa350871", + "metadata": {}, + "outputs": [], + "source": [ + "# plot all images in a batch for all batches next to each other. Each batch in a new row.\n", + "for imgs, lbls in transformed_cat_loader:\n", + " fig, axes = plt.subplots(1, batch_size, figsize=(8, 4)) # only one plot\n", + " for i in range(batch_size):\n", + " axes[i].imshow(np.transpose(imgs[i].numpy(), (1, 2, 0)))\n", + " axes[i].set_title(f\"Label: {lbls[i].item()}\")\n", + " axes[i].axis('off')\n", + " plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "c780c8dc", + "metadata": {}, + "source": [ + "**Hinweis:** Bei jeder Ausführung vom Dataloader (wenn er bereits alle Daten ausgegeben hat), liefert neue Daten. Somit können wir die Data Augmentation Techniken gut verwenden, insbesondere wenn wir mehrere Epochen (=Durchläufe des ganzen Datasets) haben. " + ] + }, + { + "cell_type": "markdown", + "id": "c98d593a", + "metadata": {}, + "source": [ + "Natürlich wird unser Modell die Bilder nie direkt zu sehen bekommen. Wir plotten sie hier, um den Effekt zu sehen.\n", + "\n", + "Das Modell wird die Daten in folgender Form bekommen." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b0709741", + "metadata": {}, + "outputs": [], + "source": [ + "for image, label in transformed_cat_loader:\n", + " print(image.shape)\n", + " print(label.shape)\n", + "\n", + " print(image)\n", + " print(label)" + ] + }, + { + "cell_type": "markdown", + "id": "4d4d885e", + "metadata": {}, + "source": [ + "**Hinweis:** Es werden die Transformationen mittels `transformation` library meistens nur für Bilder angewendet. Für tabellarische Daten können wir die Befehle mit `numpy` (und/oder `pandas`) selber implementieren." + ] + }, + { + "cell_type": "markdown", + "id": "bda9160b", + "metadata": {}, + "source": [ + "Wir betrachten jetzt noch ein weiteres Beispiel, welches uns zeigt, dass wir auch vorgefertigte Datasets verwenden können." + ] + }, + { + "cell_type": "markdown", + "id": "55721595", + "metadata": {}, + "source": [ + "### Vorgefertigte Datasets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4eb56a8", + "metadata": {}, + "outputs": [], + "source": [ + "from torchvision import datasets\n", + "import torchvision" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f33f9e6", + "metadata": {}, + "outputs": [], + "source": [ + "path = os.path.join(\"..\", \"..\", \"_data\", \"private\", \"cifar10\")\n", + "\n", + "cifar_10_dataset = datasets.CIFAR10(root=path, train=True, download=True, transform=transforms.ToTensor()) # other possibility would be train=False, i.e. the test dataset" + ] + }, + { + "cell_type": "markdown", + "id": "a7cf2073", + "metadata": {}, + "source": [ + "Wir laden uns also hier das **CIFAR10** Dataset herunter. Wir sehen auch, dass wir hier eine Flag `train=True` haben, somit gibt es auch ein eigenes Test-Dataset. Das ist auch später unsere Vorgehensweise." + ] + }, + { + "cell_type": "markdown", + "id": "b626066b", + "metadata": {}, + "source": [ + "Die Daten werden hier im Angegeben Pfad gespeichert (und müssen natürlich beim nächsten Mal ausführen nicht wieder heruntergeladen werden)." + ] + }, + { + "cell_type": "markdown", + "id": "bffc4038", + "metadata": {}, + "source": [ + "![Cifar10](../resources/Cifar10.jpg)\n", + "\n", + "(from https://www.cs.toronto.edu/~kriz/cifar.html)" + ] + }, + { + "cell_type": "markdown", + "id": "2d852832", + "metadata": {}, + "source": [ + "### Tensordatasets" + ] + }, + { + "cell_type": "markdown", + "id": "fe3b0e63", + "metadata": {}, + "source": [ + "Eine weitere Möglichkeit, ein PyTorch-Dataset zu ersetllen, ohne eine eigene Klasse machen zu müssen, ist das sogenannte `Tensordataset`.\n", + "\n", + "Das Tensordataset erlaubt es, wenn unsere Daten (Input und Label) schon in Matrix- bzw. Vektorform (quasi Tabellen) sind." + ] + }, + { + "cell_type": "markdown", + "id": "3a2bf681", + "metadata": {}, + "source": [ + "**Wie sieht das in PyTorch aus?**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae8c236e", + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import TensorDataset\n", + "import torch" + ] + }, + { + "cell_type": "markdown", + "id": "6fa5e7de", + "metadata": {}, + "source": [ + "Nehmen wir an, wir haben folgende Daten:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6242271b", + "metadata": {}, + "outputs": [], + "source": [ + "X = torch.randn(2000, 5)\n", + "y = torch.randint(0, 2, (2000,)) # 0 or 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34962aab", + "metadata": {}, + "outputs": [], + "source": [ + "X" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f3ec4ef", + "metadata": {}, + "outputs": [], + "source": [ + "y" + ] + }, + { + "cell_type": "markdown", + "id": "dddc6997", + "metadata": {}, + "source": [ + "Dann können wir nun folgendermaßen ein **Tensor**Dataset erstellen:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "122dbfd4", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = TensorDataset(X, y)" + ] + }, + { + "cell_type": "markdown", + "id": "cc4031e8", + "metadata": {}, + "source": [ + "Dieses können wir nun auch wieder in einen Dataloader geben usw., wird hier jedoch nicht mehr genauer demonstriert." + ] + }, + { + "cell_type": "markdown", + "id": "5668dd93", + "metadata": {}, + "source": [ + "### Vorteile und Nachteile:" + ] + }, + { + "cell_type": "markdown", + "id": "528700d3", + "metadata": {}, + "source": [ + "**Vorteile:**\n", + "* Bisherige Dataframes etc. können recht einfach in ein PyTorch Dataset gebracht werden (vorausgesetzt es sind nur numerische Datentypen)\n", + "* Erlaubt auch für mehrere Tensoren, zum Beispiel (Input, Label, Maske)." + ] + }, + { + "cell_type": "markdown", + "id": "722249a4", + "metadata": {}, + "source": [ + "**Nachteile:**\n", + "* Alles ist im RAM\n", + "* Daten müssen vorher schon verarbeitet werden und können nicht \"on the fly\" noch angepasst (normalisiert, augmented usw.) werden" + ] + }, + { + "cell_type": "markdown", + "id": "c2a1044c", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "a38a188b", + "metadata": {}, + "source": [ + "Nun kommen wir zu den sogenannten Dataloaders in Python (bzw. PyTorch)." + ] + }, + { + "cell_type": "markdown", + "id": "c5330a3b", + "metadata": {}, + "source": [ + "## DataLoader (in PyTorch)" + ] + }, + { + "cell_type": "markdown", + "id": "76495215", + "metadata": {}, + "source": [ + "Nachdem wir ein Dataset erstellt haben (oder heruntergeladen), können wir diesem einem Dataloader übergeben." + ] + }, + { + "cell_type": "markdown", + "id": "0bb27fda", + "metadata": {}, + "source": [ + "Der Dataloader in PyTorch hat folgende wichtige Argumente:\n", + "* `dataset`: Das Dataset, welches \"Batchweise\" zurück gegeben werden soll\n", + "* `batch_size`: Anzahl der Elemente aus dem Dataset, die pro Aufruf zurück gegeben werden sollen\n", + "* `shuffle`: Flag, welche angibt, ob die Daten durchgemischt (\"geshuffled\") werden sollen\n", + "* `num_workers`: (Optional) Wie viele Subprozesse im Hintergrund arbeiten sollen, um die Daten vorzubereiten (**Achtung:** Kann bei Windows durchaus zu großen Problemen (zum Beispiel Freezen der Trainingsmethode) führen)\n", + "* `collate_fn`: (Optional) Gibt an, wie unsere Daten zu einem Stack kombiniert werden sollen.\n", + "* $\\vdots$" + ] + }, + { + "cell_type": "markdown", + "id": "56ee34c8", + "metadata": {}, + "source": [ + "**Hinweis:** Es sind eigentlich noch mehr Argumente *optional*, jedoch ist es gut, wenn wir die nicht-optionalen Argumente explizit angeben." + ] + }, + { + "cell_type": "markdown", + "id": "b2a8b491", + "metadata": {}, + "source": [ + "**Hinweis:** Die `collate_fn` Funktion werden wir bei unserem Projekt bzgl. *Image Inpainting* benötigen." + ] + }, + { + "cell_type": "markdown", + "id": "5c33c1cb", + "metadata": {}, + "source": [ + "**Wichtig:** Es gibt für **jedes (Teil)dataset**, also für Train- und für Testset sowohl ein **eigenes Dataset**, als auch einen **eigenen Dataloader**!" + ] + }, + { + "cell_type": "markdown", + "id": "c2e5b3d5", + "metadata": {}, + "source": [ + "Nachdem der Dataloader bereits weiter oben verwendet worden ist, wollen wir hier nochmal kurz seine Verwendung in einem kleinem Code-Ausschnitt zeigen." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7568e6dc", + "metadata": {}, + "outputs": [], + "source": [ + "dataloader_cifar10 = DataLoader(cifar_10_dataset, batch_size=4, shuffle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "75c321b0", + "metadata": {}, + "outputs": [], + "source": [ + "for images, labels in dataloader_cifar10:\n", + " # Make a grid of the images\n", + " grid = torchvision.utils.make_grid(images)\n", + " \n", + " # Convert tensor to numpy and plot\n", + " plt.imshow(grid.permute(1, 2, 0)) # (C, H, W) -> (H, W, C)\n", + " plt.axis('off')\n", + " plt.show()\n", + " \n", + " break" + ] + }, + { + "cell_type": "markdown", + "id": "4cdb49cd", + "metadata": {}, + "source": [ + "### Vorteile und Nachteile von Datasets und DataLoader" + ] + }, + { + "cell_type": "markdown", + "id": "81d24aa5", + "metadata": {}, + "source": [ + "**Vorteile:**\n", + "* Sauberer, einfacher Weg, Daten gut zu organisieren" + ] + }, + { + "cell_type": "markdown", + "id": "61c77e12", + "metadata": {}, + "source": [ + "**Nachteile:**\n", + "* Um nun ein Modell und deren Trainingsspezifikationen (welches Dataset, welche Hyperparameter usw.) muss nun auch angegeben werden, wie genau die Dataloader ausgesehen haben." + ] + }, + { + "cell_type": "markdown", + "id": "d555b7b2", + "metadata": {}, + "source": [ + "Natürlich erspart uns die Dataset/Dataloader Kombination leider nicht, dass wir uns um die Qualität der Daten kümmern müssen.\n", + "\n", + "Daten können nach wie vor:\n", + "* Fehlen\n", + "* Falsch sein\n", + "* Zu wenig sein\n", + "* Unnormalisiert sein\n", + "* usw." + ] + }, + { + "cell_type": "markdown", + "id": "29d675b8", + "metadata": {}, + "source": [ + "![Reel_NaN_Values](../resources/Instagram_Reel_NaNs.mp4)\n", + "\n", + "(von https://www.instagram.com/reel/DMujUkYBXv3/?igsh=eG1uMDl0MHdvM3B2)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dsai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.23" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/06_NN/code/nn_5_training_tipps_tricks.ipynb b/06_NN/code/nn_5_training_tipps_tricks.ipynb new file mode 100644 index 0000000..d22983b --- /dev/null +++ b/06_NN/code/nn_5_training_tipps_tricks.ipynb @@ -0,0 +1,1585 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "01cb1e74", + "metadata": {}, + "source": [ + "
\n", + "\n", + "

Neural Networks: Die Train Methode + Tipps und Tricks

\n", + "

DSAI

\n", + "

Jakob Eggl

\n", + "\n", + "
\n", + " \"Logo\"\n", + "
\n", + " © 2025/26 Jakob Eggl. Nutzung oder Verbreitung nur mit ausdrücklicher Genehmigung des Autors.\n", + "
\n", + "
\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "1c2438fc", + "metadata": {}, + "source": [ + "In diesem Notebook werden wir uns mit der **Trainingsmethode** befassen. Das bedeutet, dass wir uns ansehen werden, wie normalerweise eine Trainingsmethode aussieht.\n", + "\n", + "Dabei werden wir sowohl:\n", + "* Notwendige Schritte besprechen, als auch\n", + "* *Tipps und Tricks* besprechen, welche für den reinen Lernprozess nicht notwendig sind, jedoch uns das Leben wesentlich erleichtern können." + ] + }, + { + "cell_type": "markdown", + "id": "d93c5568", + "metadata": {}, + "source": [ + "## Wiederholung: Welche Punkte müssen wir abarbeiten?\n", + "\n", + "1. Wir definieren unsere Modellklasse und erstellen eine Instanz $f$. Unser Modell hat dabei die Parameter $w$ (genannt *Weights*).\n", + "2. Wir haben unsere Daten in einem Dataset und einen zugehörigen Dataloader verfügbar. Dabei steht $X$ für die \"Input\"-Daten und $y$ das dazugehörige Label.\n", + "3. Wir haben eine Loss-Funktion $L(\\hat{y},y)$ definiert.\n", + "4. Wir haben einen Optimizer (zum Beispiel SGD) und eine Learning Rate $\\eta$ festgelegt.\n", + "5. Wir schicken die Daten ($X$) durch das Modell und erhalten die Prediction $\\hat{y} = f(X)$.\n", + "6. Wir vergleichen die echte Lösung $y$ mit unserer Prediction $\\hat{y}=f(X)$ indem wir den Loss $L(\\hat{y},y)$ berechnen.\n", + "7. Wir berechnen die Ableitung der Lossfunktion $L$ bezüglich der Parameter $w$ und updaten diese bezüglich der Regel $w_t = w_{t-1} - \\eta \\nabla L(w_{t-1})$\n", + "8. Wir wiederholen die Schritte 5, 6 und 7 mehrmals." + ] + }, + { + "cell_type": "markdown", + "id": "2e2c0294", + "metadata": {}, + "source": [ + "Versuchen wir nun, dies in Python (PyTorch) zu implementieren." + ] + }, + { + "cell_type": "markdown", + "id": "f9436ccf", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0a13287", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import torch\n", + "from torch import nn\n", + "from torch.utils.data import DataLoader, random_split, TensorDataset\n", + "from sklearn.preprocessing import OrdinalEncoder, StandardScaler\n", + "import torch.optim as optim" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c909676b", + "metadata": {}, + "outputs": [], + "source": [ + "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('mps') if torch.backends.mps.is_available() else torch.device('cpu')\n", + "print(device)" + ] + }, + { + "cell_type": "markdown", + "id": "fabf2ff4", + "metadata": {}, + "source": [ + "Nehmen wir nun das `california_housing` Dataset, gespeichert unter `housing.csv`. Dieses wollen wir für Regression verwenden." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6e34db7", + "metadata": {}, + "outputs": [], + "source": [ + "path = os.path.join(\"..\", \"..\", \"_data\", \"housing.csv\")\n", + "data = pd.read_csv(path, sep=',', header=0)" + ] + }, + { + "cell_type": "markdown", + "id": "f51ae055", + "metadata": {}, + "source": [ + "(Hier müssten wir jetzt eigentlich noch eine genau Datenanalyse machen. Wir gehen aber jetzt davon aus, dass dies schon gemacht wurde und wir führen somit einfach die folgenden Schritt durch.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de8acd07", + "metadata": {}, + "outputs": [], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3efae73", + "metadata": {}, + "outputs": [], + "source": [ + "data.info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5128e391", + "metadata": {}, + "outputs": [], + "source": [ + "str_colums = [\"mainroad\", \"guestroom\", \"basement\", \"hotwaterheating\", \"airconditioning\", \"prefarea\",\"furnishingstatus\"]\n", + "\n", + "for col in str_colums:\n", + " print(col,\":\",data[col].unique())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2167eb2", + "metadata": {}, + "outputs": [], + "source": [ + "bin_colums = [\"mainroad\", \"guestroom\", \"basement\", \"hotwaterheating\", \"airconditioning\", \"prefarea\"]\n", + "\n", + "for col in bin_colums:\n", + " data[col] = data[col].map({\"yes\":1,\"no\":0})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e37f682b", + "metadata": {}, + "outputs": [], + "source": [ + "furnish_encoder = OrdinalEncoder(categories=[[\"unfurnished\",\"semi-furnished\",\"furnished\"]])\n", + "\n", + "data[\"furnishingstatus\"] = furnish_encoder.fit_transform(data[[\"furnishingstatus\"]])\n", + "\n", + "data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa0d5e69", + "metadata": {}, + "outputs": [], + "source": [ + "scaler = StandardScaler()\n", + "\n", + "scale_cols = [\"area\"]\n", + "\n", + "data[scale_cols] = scaler.fit_transform(data[scale_cols])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5148147", + "metadata": {}, + "outputs": [], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a31fc825", + "metadata": {}, + "outputs": [], + "source": [ + "data['price'] /= 1e6" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96cf8a0f", + "metadata": {}, + "outputs": [], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cf149f47", + "metadata": {}, + "outputs": [], + "source": [ + "X = data.drop(\"price\", axis=1)\n", + "y = data[\"price\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "957ad9a9", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = TensorDataset(torch.tensor(X.values, dtype=torch.float32), torch.tensor(y.values, dtype=torch.float32))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6785b1a4", + "metadata": {}, + "outputs": [], + "source": [ + "train_set_ratio = 0.7\n", + "test_set_ratio = 0.3\n", + "\n", + "train_length = int(round(train_set_ratio * len(dataset)))\n", + "test_length = len(dataset) - train_length\n", + "train_dataset, test_dataset = random_split(dataset, [train_length, test_length])\n", + "\n", + "batch_size = 32\n", + "\n", + "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", + "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d92d4147", + "metadata": {}, + "outputs": [], + "source": [ + "class SimpleRegressor(nn.Module):\n", + " def __init__(self):\n", + " super(SimpleRegressor, self).__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(12, 24),\n", + " nn.ReLU(),\n", + " nn.Linear(24, 12),\n", + " nn.ReLU(),\n", + " nn.Linear(12, 6),\n", + " nn.ReLU(),\n", + " nn.Linear(6, 1)\n", + " )\n", + " def forward(self, x):\n", + " return self.net(x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72c5c856", + "metadata": {}, + "outputs": [], + "source": [ + "model = SimpleRegressor().to(device)" + ] + }, + { + "cell_type": "markdown", + "id": "1491a515", + "metadata": {}, + "source": [ + "Wir werden sehen, dass beim Optimizer die **Model-Parameter** übergeben werden. Dieser Schritt ist sehr wichtig.\n", + "\n", + "Wir sehen uns an dieser Stelle mal die Model-Parameter an." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37f97047", + "metadata": {}, + "outputs": [], + "source": [ + "# Show number of model parameters and where they come from\n", + "total_params = sum(p.numel() for p in model.parameters())\n", + "print(f\"Total model parameters: {total_params}\")\n", + "for name, param in model.named_parameters():\n", + " if param.requires_grad:\n", + " print(f\"Parameter: {name}, Shape: {param.shape}, Size: {param.numel()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "163cb950", + "metadata": {}, + "source": [ + "Nun zur eigentlichen **Trainingsmethode**!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2957083f", + "metadata": {}, + "outputs": [], + "source": [ + "# Hyperparameter\n", + "\n", + "criterion = nn.MSELoss()\n", + "optimizer = optim.Adam(model.parameters(), lr=1e-3) # or SGD for example\n", + "\n", + "n_epochs = 50" + ] + }, + { + "cell_type": "markdown", + "id": "163e1747", + "metadata": {}, + "source": [ + "**Wichtig:** Hier müssen die Parameter vom Modell beim Optimizer übergeben werden. Diese werden dann im Laufe geupdated." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60bcf8a7", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Starting training on device: {device}...\")\n", + "\n", + "for epoch in range(1, n_epochs + 1):\n", + " model.train()\n", + " train_losses_this_epoch = []\n", + "\n", + " for train_input, train_label in train_loader:\n", + " train_input, train_label = train_input.to(device), train_label.to(device)\n", + "\n", + " optimizer.zero_grad()\n", + " prediction = model(train_input)\n", + " loss = criterion(prediction, train_label)\n", + " loss.backward()\n", + " optimizer.step() \n", + " train_losses_this_epoch.append(loss.item()) # .item() to get scalar value from tensor.\n", + " avg_train_loss = np.mean(train_losses_this_epoch)\n", + " \n", + " \n", + " model.eval()\n", + " test_losses_this_epoch = []\n", + " with torch.no_grad():\n", + " for test_input, test_label in test_loader:\n", + " test_input, test_label = test_input.to(device), test_label.to(device)\n", + "\n", + " test_prediction = model(test_input)\n", + " test_loss = criterion(test_prediction, test_label)\n", + " test_losses_this_epoch.append(test_loss.item())\n", + " avg_test_loss = np.mean(test_losses_this_epoch)\n", + " print(f\"Epoch {epoch}/{n_epochs} - Train Loss: {avg_train_loss:.4f} - Test Loss: {avg_test_loss:.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bde6ac63", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Der Testloss beträgt {avg_test_loss:.4f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "ef7e3a3a", + "metadata": {}, + "source": [ + "Insbesondere sind folgende Punkte im Code **extrem** wichtig (sortiert der Reihe nach, wie sie in der Methode auftreten):" + ] + }, + { + "cell_type": "markdown", + "id": "10cf5077", + "metadata": {}, + "source": [ + "* `model.train()`: Setzt das Modell in den Trainingsmodus und aktiviert dabei Effekte, welche wir im Training haben wollen und in der späteren Verwendung (=Inferenz) nicht. (Siehe später zum Beispiel **Dropout-Layer**)\n", + "* `optimizer.zero_grad()`: Setzt alle Gradienten zurück. Grund: Wir wollen jede Iteration den Gradient neu berechnen und nicht die alten Gradients noch miteinbeziehen.\n", + "* `loss = criterion(prediction, train_label)`: Berechnet den Loss, indem es die Prediction und das Label vergleicht.\n", + "* `loss.backward()`: Berechnet die Gradienten bezüglich der Parameter (Weights) des Modells (werden vorher in der Initialisierung vom Optimizer übergeben).\n", + "* `optimizer.step()`: Aktualisiert die Parameter bzgl. der gewählten **Update-Rule** (zum Beispiel (Stochastic)Gradient-Descent).\n", + "* `model.eval()`: Gegenstück zu `model.train()`: Setzt also das Modell in den Evaluierungsmodus, dabei werden einige Effekte, welche wir im Training haben wollen und in der späteren Verwendung nicht, deaktiviert. (Auch hier: Siehe zum Beispiel **Dropout-Layer**)\n", + "* `with torch.no_grad()`: Gibt der Autograd-Engine bescheid, dass keine Gradienten gespeichert/berechnet werden sollen. Dies resultiert in einem Geschwindigkeitsboost und einem geringeren Speicherverbrauch. Kann jedoch nur im Evaluierungsmodus verwendet werden. Theoretisch könnten wir somit für das Evaluieren mehr Daten gleichzeitig durch das Modell schicken (größere Batch-Size), nachdem wir weniger Speicher benötigen." + ] + }, + { + "cell_type": "markdown", + "id": "11eafb3d", + "metadata": {}, + "source": [ + "**Wir wollen nochmal kurz die wichtigsten Parameter hier durchgehen:**\n", + "\n", + "* `n_epochs`: Anzahl der Durchläufe des *gesamten* Datasets. Normalerweise schon viel größer als 1. Zu große Anzahl kann zu Overfitting führen. **Lösung:** Early Stopping (siehe später)\n", + "* `criterion`: Loss Funktion, welche wir verwenden wollen. Normalerweise Mean Squared Error (MSE) oder Cross Entropy Loss (CE)\n", + "* `lr`: Learning Rate, welche vorgibt, wie groß die Schritte in die jeweilige Richtung des steilsten Abstiegs gemacht werden sollen." + ] + }, + { + "cell_type": "markdown", + "id": "6ebf47ea", + "metadata": {}, + "source": [ + "**Was ist jetzt noch suboptimal?**\n", + "\n", + "1) Wir verwenden das Testset zwischendurch zum Abfragen der Performance (das ist eigentlich genau genommen auch overfitting).\n", + "2) Wir verwenden eine fixe Anzahl an Epochen. Dies kann bei zu kleiner oder zu großer Wahl zu under- bzw. overfitting führen. Außerdem verwenden wir dadurch nicht das Modell, welches zwingend am besten (beim Testset) ist.\n", + "3) Wir lassen uns nicht recht viele Parameter zum Trainingsvorgang ausgeben." + ] + }, + { + "cell_type": "markdown", + "id": "93f1462e", + "metadata": {}, + "source": [ + "Diese Probleme werden wir jetzt Schritt für Schritt lösen." + ] + }, + { + "cell_type": "markdown", + "id": "319bdf2b", + "metadata": {}, + "source": [ + "### Lösung zu Problem 1 und 2 (Mehrfaches Verwenden vom Testset + kein Early Stopping)" + ] + }, + { + "cell_type": "markdown", + "id": "972fa0a1", + "metadata": {}, + "source": [ + "> **Übung:** Warum ist das mehrfache Verwenden vom Testset zum Kontrollieren auch overfitting?" + ] + }, + { + "cell_type": "markdown", + "id": "9359b007", + "metadata": {}, + "source": [ + "Eine mögliche Lösung ist es, ein drittes Dataset einzuführen, das sogenannte **Validierungsset** (**Validationset**).\n", + "\n", + "Es soll verwendet werden, um während des Trainings schon ein Dataset zum Überprüfen der Performance zu haben, mit welchem das Modell **nicht** trainiert worden ist.\n", + "\n", + "Wir verwenden also für neuronale Netze oft 3 verschiedene Datasets:\n", + "1) **Trainset:** Mit diesen Daten wird das Netzwerk trainiert.\n", + "2) **Validationset:** Mit diesen Daten wird das Netzwerk während dem Trainingsvorgang immer wieder überprüft. Sollte sich die Performance für dieses Dataset nicht mehr ändern, so sollte man den Trainingsvorgang stoppen.\n", + "3) **Testset:** Mit diesen Daten wird das Modell final getestet um nun zu sehen, wie gut es \"wirklich\" ist." + ] + }, + { + "cell_type": "markdown", + "id": "ad5d70c6", + "metadata": {}, + "source": [ + "Wir erstellen uns kurz eine Methode, die uns das Dataset generiert:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "087d9e7d", + "metadata": {}, + "outputs": [], + "source": [ + "def get_dataset(path = os.path.join(\"..\", \"..\", \"_data\", \"housing.csv\")):\n", + " data = pd.read_csv(path, sep=',', header=0)\n", + "\n", + " bin_colums = [\"mainroad\", \"guestroom\", \"basement\", \"hotwaterheating\", \"airconditioning\", \"prefarea\"]\n", + " for col in bin_colums:\n", + " data[col] = data[col].map({\"yes\":1,\"no\":0})\n", + "\n", + " furnish_encoder = OrdinalEncoder(categories=[[\"unfurnished\",\"semi-furnished\",\"furnished\"]])\n", + "\n", + " data[\"furnishingstatus\"] = furnish_encoder.fit_transform(data[[\"furnishingstatus\"]])\n", + "\n", + " scaler = StandardScaler()\n", + "\n", + " scale_cols = [\"area\"]\n", + "\n", + " data[scale_cols] = scaler.fit_transform(data[scale_cols])\n", + "\n", + " data['price'] /= 1e6\n", + "\n", + " X = data.drop(\"price\", axis=1)\n", + " y = data[\"price\"]\n", + "\n", + " dataset = TensorDataset(torch.tensor(X.values, dtype=torch.float32), torch.tensor(y.values, dtype=torch.float32))\n", + "\n", + " return dataset\n" + ] + }, + { + "cell_type": "markdown", + "id": "976ff3e0", + "metadata": {}, + "source": [ + "Erstellung eines Validation Sets in PyTorch:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dbdc45f1", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = get_dataset()\n", + "\n", + "train_set_ratio = 0.7\n", + "test_set_ratio = 0.2\n", + "validation_set_ratio = 0.1\n", + "\n", + "ratio_sum = train_set_ratio + test_set_ratio + validation_set_ratio\n", + "\n", + "assert np.isclose(ratio_sum, 1.0), f\"Ratios must sum to 1.0 but is {ratio_sum}\"\n", + "\n", + "train_length = int(round(train_set_ratio * len(dataset)))\n", + "test_length = int(round(test_set_ratio * len(dataset)))\n", + "validation_length = int(round(validation_set_ratio * len(dataset)))\n", + "train_dataset, test_dataset, validation_dataset = random_split(dataset, [train_length, test_length, validation_length])\n", + "\n", + "batch_size = 32\n", + "\n", + "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", + "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n", + "validation_loader = DataLoader(validation_dataset, batch_size=batch_size, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "id": "c16a2bc6", + "metadata": {}, + "source": [ + "**Hinweis:** Die Verhältnisse 70/20/10 können natürlich auch angepasst werden." + ] + }, + { + "cell_type": "markdown", + "id": "5148ab1d", + "metadata": {}, + "source": [ + "Diese können wir nun nutzen, um ein **Early Stopping** Verhalten zu implementieren." + ] + }, + { + "cell_type": "markdown", + "id": "ecd879b4", + "metadata": {}, + "source": [ + "Mit **Early Stopping** meinen wir folgendes Vorgehen:\n", + "* Wir trainieren unser Model auf den Trainings-Daten\n", + "* Wir prüfen regelmäßig die aktuelle Performance des Modells am Validation Set\n", + "* Sollte sich die Performance auf dem Validation Set über eine gewisse Zeit nicht verbessern, so stoppen wir mit dem Training\n", + "* Es wird immer, wenn das Modell besser geworden ist, das aktuelle Modell gespeichert\n", + "* Am Schluss wird das beste Modell geladen und wir Testen dieses beste Modell auf dem Testset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a73e0261", + "metadata": {}, + "outputs": [], + "source": [ + "model = SimpleRegressor().to(device)\n", + "criterion = nn.MSELoss()\n", + "optimizer = optim.Adam(model.parameters(), lr=1e-3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5bc4dbc6", + "metadata": {}, + "outputs": [], + "source": [ + "# Early stopping parameters\n", + "patience = 4\n", + "best_val_loss = float('inf')\n", + "epochs_no_improve = 0\n", + "n_epochs = 300" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "844fab3b", + "metadata": {}, + "outputs": [], + "source": [ + "train_loss_mean_epoch = []\n", + "validation_loss_mean_epoch = []\n", + "\n", + "model_export_path = os.path.join(\"..\", \"models\", \"best_model_nn_5.pth\")\n", + "\n", + "for epoch in range(1, n_epochs + 1):\n", + " model.train()\n", + " train_losses = []\n", + " for batch_idx, (X_batch, y_batch) in enumerate(train_loader):\n", + " X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n", + "\n", + " optimizer.zero_grad()\n", + " y_pred = model(X_batch)\n", + " loss = criterion(y_pred, y_batch)\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " train_losses.append(loss.item())\n", + " train_loss_epoch = np.mean(train_losses)\n", + " train_loss_mean_epoch.append(train_loss_epoch) # we save the mean train loss for each epoch to later have a list available if needed for e.g. plotting\n", + "\n", + " model.eval()\n", + " val_losses = []\n", + " with torch.no_grad():\n", + " for X_batch, y_batch in validation_loader:\n", + " X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n", + "\n", + " y_pred = model(X_batch)\n", + " loss = criterion(y_pred, y_batch)\n", + " val_losses.append(loss.item())\n", + " val_loss_epoch = np.mean(val_losses)\n", + " validation_loss_mean_epoch.append(val_loss_epoch) # we save the mean validation loss for each epoch to later have a list available if needed for e.g. plotting\n", + "\n", + " print(f\"Epoch {epoch}/{n_epochs} — train_loss: {train_loss_epoch:.4f}, val_loss: {val_loss_epoch:.4f}\")\n", + "\n", + " if val_loss_epoch < best_val_loss:\n", + " best_val_loss = val_loss_epoch\n", + " epochs_no_improve = 0\n", + " torch.save(model.state_dict(), model_export_path)\n", + " else:\n", + " epochs_no_improve += 1\n", + " print(f\"No improvement for {epochs_no_improve} epochs.\")\n", + " if epochs_no_improve >= patience:\n", + " print(f\"Early stopping triggered after {epoch} epochs. Best val loss: {best_val_loss:.4f}\")\n", + " break" + ] + }, + { + "cell_type": "markdown", + "id": "905e937a", + "metadata": {}, + "source": [ + "**Hinweis:** Wir haben hier schon einen Export vom Model mit `torch.save(model.state_dict(), model_export_path)` verwendet. Wir sehen uns das im nächsten Notebook genauer an!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3cd85c1", + "metadata": {}, + "outputs": [], + "source": [ + "# After training, we load the best model and evaluate on the test set\n", + "model_path = os.path.join(\"..\", \"models\", \"best_model_nn_5.pth\")\n", + "model.load_state_dict(torch.load(model_path))\n", + "\n", + "model.eval()\n", + "test_losses = []\n", + "with torch.no_grad():\n", + " for X_batch, y_batch in test_loader:\n", + " X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n", + "\n", + " y_pred = model(X_batch)\n", + " loss = criterion(y_pred, y_batch)\n", + " test_losses.append(loss.item())\n", + "avg_test_loss = np.mean(test_losses)\n", + "print(f\"Test Loss of the best model: {avg_test_loss:.4f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b4aff102", + "metadata": {}, + "source": [ + "Was genau passiert beim **Early Stopping**?" + ] + }, + { + "cell_type": "markdown", + "id": "f2d1086a", + "metadata": {}, + "source": [ + "![Overfitting_Underfitting_Loss_Curve](../resources/Overfitting_Underfitting_Loss_Curve.png)\n", + "\n", + "(von https://www.kaggle.com/code/ryanholbrook/overfitting-and-underfitting)" + ] + }, + { + "cell_type": "markdown", + "id": "e48c9e97", + "metadata": {}, + "source": [ + "Wie oben im Bild dargestellt, wollen wir den Punkt erreichen, wo unser Validation-Loss minimal ist. Wir sehen aber, dass nach einer Zeit, der Validation Loss wieder nach **oben geht**, während der Trainingsloss immer weiter nach unten geht.\n", + "\n", + "Der Punkt, an dem der Validation Loss wieder nach oben geht, ist genau der Punkt, bei dem wir zu **overfitten** beginnen. Wir wollen also genau an diesem Punkt mit dem Training aufhören.\n", + "\n", + "Wie können wir das Erreichen?" + ] + }, + { + "cell_type": "markdown", + "id": "90a6fc05", + "metadata": {}, + "source": [ + "* Wir berechnen den Validation Loss nach jeder Epoche\n", + "* Ist er niedriger, als der bisherige beste Validation Loss, so speichern wir diesen als unseren neuen besten Loss und speichern auch das Modell ab.\n", + "* Wird die Validation-Set Performance über eine gewisse Anzahl an Epochen (`patience`) nicht besser, so brechen wir mit dem Training ab." + ] + }, + { + "cell_type": "markdown", + "id": "0400b2e0", + "metadata": {}, + "source": [ + "**Hinweis:** Am Ende zählt trotzdem immer die Testset Performance. Im Normalfall ist aber der Validation Loss ein guter Indikator dafür." + ] + }, + { + "cell_type": "markdown", + "id": "503cdf09", + "metadata": {}, + "source": [ + "> **Übung:** Ändere die `patience` so, dass wir tatsächlich einen besseren (ähnlichen) Test-Loss erzielen. *Tipp:* Probiere kleine Werte." + ] + }, + { + "cell_type": "markdown", + "id": "d2991914", + "metadata": {}, + "source": [ + "**Hinweis:** Um die Problematik, die in der vorigen Übung gezeigt wurde (zu oft \"Kontrollieren\" mit dem Validationset), zu beheben, wird oft das Validation Set nur alle $n$-Iterationen verwendet zum \"Kontrollieren\"." + ] + }, + { + "cell_type": "markdown", + "id": "ee00aadb", + "metadata": {}, + "source": [ + "### Lösung zu Problem 3 (Zu wenig Parameter des Trainingsvorgangs verfügbar)" + ] + }, + { + "cell_type": "markdown", + "id": "5d33e757", + "metadata": {}, + "source": [ + "Um den Trainingsvorgang besser überwachen zu können, ist es hilfreich, ein Tool wie zum Beispiel [WandB](https://docs.wandb.ai) zu verwenden. Es erlaubt uns, viele weitere Parameter des Trainings zu tracken und im Anschluss in einem Dashboard visuell darzustellen.\n", + "\n", + "Wir sehen uns nun an, wie wir das in unsere Trainingsmethode implementieren können." + ] + }, + { + "cell_type": "markdown", + "id": "d144fe17", + "metadata": {}, + "source": [ + "Dazu installieren wir die `wandb` (ausgesprochen: Weights & Biases) mit `conda install wandb` bzw. `pip install wandb`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8dc7e003", + "metadata": {}, + "outputs": [], + "source": [ + "import wandb" + ] + }, + { + "cell_type": "markdown", + "id": "8b2447a8", + "metadata": {}, + "source": [ + "Zuerst müssen wir uns einloggen." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d75a7b1d", + "metadata": {}, + "outputs": [], + "source": [ + "wandb.login()" + ] + }, + { + "cell_type": "markdown", + "id": "301ec273", + "metadata": {}, + "source": [ + "Nun müssen wir ein neues `wandb` Projekt initialisieren \n", + "(siehe zur Anleitung ansonsten auch: [hier](https://docs.wandb.ai/models/tutorials/pytorch))." + ] + }, + { + "cell_type": "markdown", + "id": "b14a1847", + "metadata": {}, + "source": [ + "**Hinweis:** Pro Initialisierung wird ein Run gespeichert." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5abb7f5", + "metadata": {}, + "outputs": [], + "source": [ + "wandb.init(project=\"Regression_Network\", config={\n", + " \"learning_rate\": 1e-3,\n", + " \"batch_size\": 32,\n", + " \"patience\": 10,\n", + " \"n_epochs\": 300,\n", + " \"optimizer\": \"Adam\",\n", + " \"criterion\": \"MSELoss\"\n", + " })" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c00be10", + "metadata": {}, + "outputs": [], + "source": [ + "config = wandb.config" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76cd4e87", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = get_dataset()\n", + "\n", + "train_set_ratio = 0.7\n", + "test_set_ratio = 0.2\n", + "validation_set_ratio = 0.1\n", + "\n", + "ratio_sum = train_set_ratio + test_set_ratio + validation_set_ratio\n", + "\n", + "assert np.isclose(ratio_sum, 1.0), f\"Ratios must sum to 1.0 but is {ratio_sum}\"\n", + "\n", + "train_length = int(round(train_set_ratio * len(dataset)))\n", + "test_length = int(round(test_set_ratio * len(dataset)))\n", + "validation_length = int(round(validation_set_ratio * len(dataset)))\n", + "train_dataset, test_dataset, validation_dataset = random_split(dataset, [train_length, test_length, validation_length])\n", + "\n", + "batch_size = config.batch_size\n", + "\n", + "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", + "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n", + "validation_loader = DataLoader(validation_dataset, batch_size=batch_size, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4af93672", + "metadata": {}, + "outputs": [], + "source": [ + "model = SimpleRegressor().to(device)\n", + "criterion = nn.MSELoss()\n", + "optimizer = optim.Adam(model.parameters(), lr=config.learning_rate)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cbcf33a0", + "metadata": {}, + "outputs": [], + "source": [ + "wandb.watch(model, criterion, log='all', log_freq=10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b390f2a3", + "metadata": {}, + "outputs": [], + "source": [ + "best_val_loss = float('inf')\n", + "epochs_no_improve = 0\n", + "model_export_path = os.path.join(\"..\", \"models\", \"best_model_nn_5.pth\")\n", + "\n", + "for epoch in range(1, config.n_epochs + 1):\n", + " model.train()\n", + " train_losses = []\n", + " for batch_idx, (X_batch, y_batch) in enumerate(train_loader):\n", + " X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n", + "\n", + " optimizer.zero_grad()\n", + " y_pred = model(X_batch)\n", + " loss = criterion(y_pred, y_batch)\n", + " loss.backward()\n", + " optimizer.step()\n", + " \n", + " train_losses.append(loss.item())\n", + " train_loss_epoch = np.mean(train_losses)\n", + " train_loss_mean_epoch.append(train_loss_epoch)\n", + "\n", + " ### NEW ###\n", + " metrics = {\n", + " \"train/loss\": train_loss_epoch,\n", + " \"train/epoch\": epoch\n", + " }\n", + " wandb.log(metrics)\n", + " ### END NEW ###\n", + "\n", + " model.eval()\n", + " val_losses = []\n", + " with torch.no_grad():\n", + " for X_batch, y_batch in validation_loader:\n", + " X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n", + "\n", + " y_pred = model(X_batch)\n", + " loss = criterion(y_pred, y_batch)\n", + " val_losses.append(loss.item())\n", + " val_loss_epoch = np.mean(val_losses)\n", + "\n", + " ### NEW ###\n", + " val_metrics = {\n", + " \"val/loss\": val_loss_epoch,\n", + " \"val/epoch\": epoch\n", + " }\n", + " wandb.log(val_metrics)\n", + " ### END NEW ###\n", + "\n", + " print(f\"Epoch {epoch}/{n_epochs} — train_loss: {train_loss_epoch:.4f}, val_loss: {val_loss_epoch:.4f}\")\n", + "\n", + " if val_loss_epoch < best_val_loss:\n", + " best_val_loss = val_loss_epoch\n", + " epochs_no_improve = 0\n", + " torch.save(model.state_dict(), model_export_path)\n", + " else:\n", + " epochs_no_improve += 1\n", + " print(f\"No improvement for {epochs_no_improve} epochs.\")\n", + " if epochs_no_improve >= config.patience:\n", + " print(f\"Early stopping triggered after {epoch} epochs. Best val loss: {best_val_loss:.4f}\")\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6e686c91", + "metadata": {}, + "outputs": [], + "source": [ + "# After training, we load the best model and evaluate on the test set\n", + "model_path = os.path.join(\"..\", \"models\", \"best_model_nn_5.pth\")\n", + "model.load_state_dict(torch.load(model_path))\n", + "\n", + "model.eval()\n", + "test_losses = []\n", + "with torch.no_grad():\n", + " for X_batch, y_batch in test_loader:\n", + " X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n", + "\n", + " y_pred = model(X_batch)\n", + " loss = criterion(y_pred, y_batch)\n", + " test_losses.append(loss.item())\n", + "avg_test_loss = np.mean(test_losses)\n", + "wandb.summary[\"test/loss\"] = avg_test_loss\n", + "print(f\"Test Loss of the best model: {avg_test_loss:.4f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cb8bca65", + "metadata": {}, + "source": [ + "Nachdem wir mit `wandb` fertig sind, führen wir `wandb.finish()` aus." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d106b4ee", + "metadata": {}, + "outputs": [], + "source": [ + "wandb.finish()" + ] + }, + { + "cell_type": "markdown", + "id": "9b8816e1", + "metadata": {}, + "source": [ + "Im Anschluss können wir nun auf der Homepage nachsehen, wie sich unsere (Model)Daten während dem Training verhalten haben. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "4b5c1668", + "metadata": {}, + "source": [ + "## Tipps und Tricks: Dropout und Batch-Normalization" + ] + }, + { + "cell_type": "markdown", + "id": "d76bb7ed", + "metadata": {}, + "source": [ + "Um unser Modell noch etwas besser zu machen, lernen wir jetzt noch eine weitere praktische Möglichkeit kennen, welche die Performance unseres Modells verbessern kann. Die Rede ist vom **Dropout**-Layer." + ] + }, + { + "cell_type": "markdown", + "id": "afdb1f64", + "metadata": {}, + "source": [ + "### Dropout" + ] + }, + { + "cell_type": "markdown", + "id": "02383fff", + "metadata": {}, + "source": [ + "**Idee:**\n", + "* Ein neuronales Netzwerk hat **viele Neuronen**, welche gemeinsam ein Muster erlernen.\n", + "* Manche Neuronen sind dabei wichtiger als andere, sprich das Modell verlässt sich zu sehr auf einzelne Neuronen.\n", + "* Dieses Verhalten neigt zu einer schlechten Generalisierung (i.e. overfitting).\n", + "* Um dies zu verbessern, **schalten** wir **zufällig** einen Teil der **Neuronen** während des Trainings **aus**.\n", + "* Dadurch muss das Modell lernen, alle Neuronen zu benutzen und nicht nur einen Teil." + ] + }, + { + "cell_type": "markdown", + "id": "41152fb6", + "metadata": {}, + "source": [ + "![Dropout Visualization](../resources/Dropout_Visualized.png)\n", + "\n", + "(von https://medium.com/@amarbudhiraja/https-medium-com-amarbudhiraja-learning-less-to-learn-better-dropout-in-deep-machine-learning-74334da4bfc5)" + ] + }, + { + "cell_type": "markdown", + "id": "909116b9", + "metadata": {}, + "source": [ + "Bei der **Evaluierung** wird diese Funktion ausgeschaltet und das Modell hat **alle Neuronen zur Verfügung**." + ] + }, + { + "cell_type": "markdown", + "id": "4ad0b236", + "metadata": {}, + "source": [ + "**Wichtig:** Für andere Architekturen (CNN's, Recurrent Neural Networks (zBsp. LSTM)) muss beim Dropout aufgepasst werden, ob es genau in dieser Form angewendet werden darf." + ] + }, + { + "cell_type": "markdown", + "id": "d1b1e98e", + "metadata": {}, + "source": [ + "Wie können wir das in PyTorch verwenden?" + ] + }, + { + "cell_type": "markdown", + "id": "527957f1", + "metadata": {}, + "source": [ + "* Genauso wie `nn.Linear` gibt es auch `nn.Dropout(p)`\n", + "* Dabei muss $p\\in[0,1]$ (genannt **Dropout-Rate**) als Parameter übergeben werden.\n", + "* Es werden dann $(100\\cdot p)$\\% der Neuronen zufällig deaktiviert.\n", + "* Wir können Dropout jedes Layer verwenden, sollten dabei (normalerweise) aber:\n", + " * es pro Layer immer nach der Aktivierungsfunktion einsetzten.\n", + " * nicht im allerletzten Layer einsetzen.\n", + " * kein zu hohes $p$ verwenden (zu großes $p$ führt zu underfitting)" + ] + }, + { + "cell_type": "markdown", + "id": "409af618", + "metadata": {}, + "source": [ + "**Vorteile:**\n", + "\n", + "* Kann gesehen werden, wie wenn wir mehrere verschiedene Modelle trainieren und dann den Durchschnitt nehmen. Dies verringert das Overfitting (Random Forest Idee)\n", + "* Einfache Implementierung (wie ein weiteres Layer einfach in die Init-Funktion)" + ] + }, + { + "cell_type": "markdown", + "id": "e9515c46", + "metadata": {}, + "source": [ + "**Nachteile:**\n", + "\n", + "* Pro Trainingsschritt lernt nur ein Teil des Modells, somit sind normalerweise mehr Epochen also auch ein längeres Training notwendig.\n", + "* Dropout-Rate ist ein weiterer Hyperparameter, der richtig gewählt werden muss.\n", + "* Kann bei falscher Verwendung (insbesondere bei anderen Architekturen) schnell das Modell unbrauchbar machen." + ] + }, + { + "cell_type": "markdown", + "id": "41516046", + "metadata": {}, + "source": [ + "**Wichtig:** Nachdem Dropout nur beim Training angewendet werden soll, ist es extrem wichtig, dass wir beim Modell immer den Modus wechseln mit `model.train()` bzw. `model.eval()`" + ] + }, + { + "cell_type": "markdown", + "id": "08a1bb1b", + "metadata": {}, + "source": [ + "**Hinweis:** Es gibt auch noch weitere interessante Möglichkeiten, wie **BatchNormalization**, **LayerNormalization**, usw. Diese sehen wir uns nicht genauer an, können aber im Internet recherchiert werden." + ] + }, + { + "cell_type": "markdown", + "id": "c8534945", + "metadata": {}, + "source": [ + "Unser angepasstes Modell:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2623169a", + "metadata": {}, + "outputs": [], + "source": [ + "class SimpleRegressor(nn.Module):\n", + " def __init__(self):\n", + " super(SimpleRegressor, self).__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(12, 128),\n", + " nn.ReLU(),\n", + " nn.Dropout(0.3),\n", + "\n", + " nn.Linear(128, 64),\n", + " nn.ReLU(),\n", + " nn.Dropout(0.3),\n", + "\n", + " nn.Linear(64, 32),\n", + " nn.ReLU(),\n", + " nn.Dropout(0.3),\n", + "\n", + " nn.Linear(32, 1)\n", + " )\n", + "\n", + " def forward(self, x):\n", + " return self.net(x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5df36618", + "metadata": {}, + "outputs": [], + "source": [ + "n_epochs = 300\n", + "patience = 20\n", + "model = SimpleRegressor().to(device)\n", + "criterion = nn.MSELoss()\n", + "optimizer = optim.Adam(model.parameters(), lr=1e-3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5034b00c", + "metadata": {}, + "outputs": [], + "source": [ + "X = data.drop(\"price\", axis=1)\n", + "y = data[\"price\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b5bed05", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = TensorDataset(torch.tensor(X.values, dtype=torch.float32), torch.tensor(y.values, dtype=torch.float32))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c8597c1", + "metadata": {}, + "outputs": [], + "source": [ + "train_set_ratio = 0.7\n", + "test_set_ratio = 0.2\n", + "validation_set_ratio = 0.1\n", + "\n", + "ratio_sum = train_set_ratio + test_set_ratio + validation_set_ratio\n", + "\n", + "assert np.isclose(ratio_sum, 1.0), f\"Ratios must sum to 1.0 but is {ratio_sum}\"\n", + "\n", + "train_length = int(round(train_set_ratio * len(dataset)))\n", + "test_length = int(round(test_set_ratio * len(dataset)))\n", + "validation_length = int(round(validation_set_ratio * len(dataset)))\n", + "train_dataset, test_dataset, validation_dataset = random_split(dataset, [train_length, test_length, validation_length])\n", + "\n", + "batch_size = 32\n", + "\n", + "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", + "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n", + "validation_loader = DataLoader(validation_dataset, batch_size=batch_size, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70b9db20", + "metadata": {}, + "outputs": [], + "source": [ + "best_val_loss = float('inf')\n", + "epochs_no_improve = 0\n", + "\n", + "model_export_path = os.path.join(\"..\", \"models\", \"best_model_nn_5_advanced.pth\")\n", + "\n", + "train_loss_mean_epoch = []\n", + "validation_loss_mean_epoch = []\n", + "\n", + "for epoch in range(1, n_epochs + 1):\n", + " model.train()\n", + " train_losses = []\n", + " for batch_idx, (X_batch, y_batch) in enumerate(train_loader):\n", + " X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n", + "\n", + " optimizer.zero_grad()\n", + " y_pred = model(X_batch)\n", + " loss = criterion(y_pred, y_batch)\n", + " loss.backward()\n", + " optimizer.step()\n", + " train_losses.append(loss.item())\n", + " train_loss_epoch = np.mean(train_losses)\n", + " train_loss_mean_epoch.append(train_loss_epoch)\n", + "\n", + " model.eval()\n", + " val_losses = []\n", + " with torch.no_grad():\n", + " for X_batch, y_batch in validation_loader:\n", + " X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n", + "\n", + " y_pred = model(X_batch)\n", + " loss = criterion(y_pred, y_batch)\n", + " val_losses.append(loss.item())\n", + " val_loss_epoch = np.mean(val_losses)\n", + " validation_loss_mean_epoch.append(val_loss_epoch)\n", + "\n", + " print(f\"Epoch {epoch}/{n_epochs} — train_loss: {train_loss_epoch:.4f}, val_loss: {val_loss_epoch:.4f}\")\n", + "\n", + " if val_loss_epoch < best_val_loss:\n", + " best_val_loss = val_loss_epoch\n", + " epochs_no_improve = 0\n", + " torch.save(model.state_dict(), model_export_path)\n", + " else:\n", + " epochs_no_improve += 1\n", + " print(f\"No improvement for {epochs_no_improve} epochs.\")\n", + " if epochs_no_improve >= patience:\n", + " print(f\"Early stopping triggered after {epoch} epochs. Best val loss: {best_val_loss:.4f}\")\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99dd5c10", + "metadata": {}, + "outputs": [], + "source": [ + "# After training, we load the best model and evaluate on the test set\n", + "export_model_path = os.path.join(\"..\", \"models\", \"best_model_nn_5_advanced.pth\")\n", + "model.load_state_dict(torch.load(export_model_path))\n", + "\n", + "model.eval()\n", + "test_losses = []\n", + "with torch.no_grad():\n", + " for X_batch, y_batch in test_loader:\n", + " X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n", + "\n", + " y_pred = model(X_batch)\n", + " loss = criterion(y_pred, y_batch)\n", + " test_losses.append(loss.item())\n", + "avg_test_loss = np.mean(test_losses)\n", + "print(f\"Test Loss of the best model: {avg_test_loss:.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7cbd0fb7", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(10, 6))\n", + "plt.plot(train_loss_mean_epoch, label='Training Loss')\n", + "plt.plot(validation_loss_mean_epoch, label='Validation Loss')\n", + "plt.xlabel('Epoch')\n", + "plt.ylabel('Loss')\n", + "plt.title('Training and Validation Loss over Epochs')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "ff70037b", + "metadata": {}, + "source": [ + "### Skip-Connections" + ] + }, + { + "cell_type": "markdown", + "id": "08bf74bb", + "metadata": {}, + "source": [ + "In vielen Netzwerken kann es vom Vorteil sein, sogenannte Skip-Connections zu verwenden. Sie sind ein Shortcut für das Netzwerk, sprich die Daten laufen sowohl durch das Netzwerk, als auch am Netzwerk vorbei und werden später wieder kombiniert. Dies hat sich in vielen Fällen als Vorteilhaft erwiesen und wird oft verwendet. Folgendes Bild zeigt eine beispielhafte Verwendung einer Skip-Connection über 2 Layers." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "![Skip_Connections](../resources/Skip_Connections.png)\n", + "\n", + "(von https://theaisummer.com/skip-connections/)" + ] + }, + { + "cell_type": "markdown", + "id": "ec66da3f", + "metadata": {}, + "source": [ + "**Mathematische Formulierung einer Skip-Connection:**\n", + "\n", + "Als Formel haben wir\n", + "$$y = F(x)+x,$$\n", + "\n", + "dabei ist $x$ der Input des Layers $F$ repräsentiert die Funktion des Layers. $y$ ist der resultierende Output, welcher also eine Summe aus dem ursprünglichen Input und dem Output des Layers ist." + ] + }, + { + "cell_type": "markdown", + "id": "1c160267", + "metadata": {}, + "source": [ + "**Vorteile:**\n", + "\n", + "* Netzwerk muss nur den Unterschied lernen.\n", + "* Löst zu einem gewissen Grad das **Vanishing-Gradient** Problem, weil dann auch der Gradient diese Abkürzung hat." + ] + }, + { + "cell_type": "markdown", + "id": "2a8896f8", + "metadata": {}, + "source": [ + "**Nachteil(e):**\n", + "\n", + "* Die Dimensionen müssen zusammen passen, ansonsten können die beiden Outputs nicht zusammen addiert werden" + ] + }, + { + "cell_type": "markdown", + "id": "ef2cb597", + "metadata": {}, + "source": [ + "> **Übung:** Überlege dir Beispiele, wo Skip-Connections extrem Sinn machen können. Insbesondere, wenn du die eben genannten Nachteile betrachtest." + ] + }, + { + "cell_type": "markdown", + "id": "86552d7a", + "metadata": {}, + "source": [ + "**Hinweis:** Es gibt auch Ansätze, wo dann an den Verbindungsstellen (oben im Bild dargestellt als $\\oplus$) die Ergebnisse nicht addiert, sondern einfach Verkettet werden." + ] + }, + { + "cell_type": "markdown", + "id": "c6f9ac1c", + "metadata": {}, + "source": [ + "**Beispielhafte Verwendung einer Skip-Connection in PyTorch:**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "09f74a1a", + "metadata": {}, + "outputs": [], + "source": [ + "class SimpleRegressorwithSkipConnection(nn.Module):\n", + " def __init__(self):\n", + " super(SimpleRegressorwithSkipConnection, self).__init__()\n", + " self.layer1 = nn.Linear(12, 24)\n", + " self.layer2 = nn.Linear(24, 12)\n", + " self.layer3 = nn.Linear(12, 6)\n", + " self.layer4 = nn.Linear(6, 1)\n", + "\n", + " def forward(self, x):\n", + " layer1_out = F.relu(self.layer1(x))\n", + " layer2_out = F.relu(self.layer2(layer1_out) + x)\n", + " layer3_out = F.relu(self.layer3(layer2_out))\n", + " output = self.layer4(layer3_out)\n", + " return output" + ] + }, + { + "cell_type": "markdown", + "id": "ccf4ddeb", + "metadata": {}, + "source": [ + "**Hinweis:** Es können mehrere \"Skip-Verbindungen\" eingebaut werden, diese müssen dann einfach in der `forward()` Methode im Netzwerk richtig eingesetzt werden." + ] + }, + { + "cell_type": "markdown", + "id": "52c8cda4", + "metadata": {}, + "source": [ + "### Tracken von Metriken im Trainingsprozess" + ] + }, + { + "cell_type": "markdown", + "id": "eb089cc0", + "metadata": {}, + "source": [ + "Eine weitere praktische Sache ist, sich auch Metriken im Training anzeigen zu lassen. So können wir zum Beispiel nicht nur den Loss tracken, sondern auch zum Beispiel die **Accuracy**." + ] + }, + { + "cell_type": "markdown", + "id": "2b9615db", + "metadata": {}, + "source": [ + "Dabei müssen wir nur an jenen Stellen, wo der Loss berechnet wird, zusätzlich auch noch die Metrik berechnen und ausgeben. " + ] + }, + { + "cell_type": "markdown", + "id": "017b5b5e", + "metadata": {}, + "source": [ + "**Hinweis:** Es kann auch ein Early Stopping bzgl. einer Metrik implementiert werden." + ] + }, + { + "cell_type": "markdown", + "id": "8c7cec3d", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "dd90e899", + "metadata": {}, + "source": [ + "## Zusammenfassung Hyperparameter" + ] + }, + { + "cell_type": "markdown", + "id": "a5a96a87", + "metadata": {}, + "source": [ + "* Batch Size\n", + "* Modell Architektur (Layers, MLP, Aktivierungsfunktionen, Dropout usw.)\n", + "* Learning Rate (Learning Rate Scheduler)\n", + "* Optimizer (SGD, Adam, Adagrad)\n", + "* Transformation von Daten\n", + "* Early Stopping\n", + "* Train/Test Split\n", + "* Loss-Funktion" + ] + }, + { + "cell_type": "markdown", + "id": "d1f65cb8", + "metadata": {}, + "source": [ + "![Reel_First_Try_Suspicious](../resources/Instagram_Reel_First_Try_Suspicious.mp4)\n", + "\n", + "(von https://www.instagram.com/reel/DNiWMPWSjHc/?igsh=MW04OXNlczZjcHVydQ==)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dsai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/06_NN/code/nn_6_deployment.ipynb b/06_NN/code/nn_6_deployment.ipynb new file mode 100644 index 0000000..37f71c6 --- /dev/null +++ b/06_NN/code/nn_6_deployment.ipynb @@ -0,0 +1,1194 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "566e2678", + "metadata": {}, + "source": [ + "
\n", + "\n", + "

Neural Networks: Deployment, Import und Export von Modellen

\n", + "

DSAI

\n", + "

Jakob Eggl

\n", + "\n", + "
\n", + " \"Logo\"\n", + "
\n", + " © 2025/26 Jakob Eggl. Nutzung oder Verbreitung nur mit ausdrücklicher Genehmigung des Autors.\n", + "
\n", + "
\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "405e8e29", + "metadata": {}, + "source": [ + "Wir wollen uns in diesem Notebook mit 3 wichtigen Konzepten beschäftigen:\n", + "\n", + "1) Model Import/Export\n", + "2) Verwenden und/oder Finetuning von vortrainierten (pretrained) Modellen\n", + "3) Model Deployment mit Flask" + ] + }, + { + "cell_type": "markdown", + "id": "13624ed2", + "metadata": {}, + "source": [ + "Die Motivation für diese Themen ist einfach. In vielen Fällen wäre es praktisch, das fertig trainierte Modell anderen zur Verfügung zu stellen. Diese können dieses dann entweder nur verwenden (online darauf zugreifen), oder, falls das ganze Modell geteilt wird, mit diesem auch weitere Dinge, wie zum Beispiel Finetuning etc., erledigen." + ] + }, + { + "cell_type": "markdown", + "id": "9e220bc5", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "source": [ + "# Model Import/Export" + ] + }, + { + "cell_type": "markdown", + "id": "7af8efb9", + "metadata": {}, + "source": [ + "In diesem Teil wollen wir unser Modell als Datei exportieren, bzw. im Anschluss wieder importieren." + ] + }, + { + "cell_type": "markdown", + "id": "8800fc68", + "metadata": {}, + "source": [ + "### PyTorch Export" + ] + }, + { + "cell_type": "markdown", + "id": "e7e28854", + "metadata": {}, + "source": [ + "Wir starten mit einem normalen Modell, welches wir später exportieren wollen." + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "id": "18c53b1b", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import torch\n", + "from torch import nn\n", + "from torch.utils.data import DataLoader, random_split, TensorDataset\n", + "from sklearn.preprocessing import OrdinalEncoder, StandardScaler\n", + "import torch.optim as optim\n", + "import onnx\n", + "import onnxruntime as ort" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "id": "a63be8e9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cpu\n" + ] + } + ], + "source": [ + "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('mps') if torch.backends.mps.is_available() else torch.device('cpu')\n", + "print(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "id": "1aa73a7b", + "metadata": {}, + "outputs": [], + "source": [ + "class SimpleRegressor(nn.Module):\n", + " def __init__(self):\n", + " super(SimpleRegressor, self).__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(12, 24),\n", + " nn.ReLU(),\n", + " nn.Linear(24, 12),\n", + " nn.ReLU(),\n", + " nn.Linear(12, 6),\n", + " nn.ReLU(),\n", + " nn.Linear(6, 1)\n", + " )\n", + " def forward(self, x):\n", + " return self.net(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "id": "4d8d6d67", + "metadata": {}, + "outputs": [], + "source": [ + "model = SimpleRegressor().to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "id": "6a112ff7", + "metadata": {}, + "outputs": [], + "source": [ + "# Simulation of a train method\n", + "\n", + "def train_model(model, epochs):\n", + " print(\"Started Dummy Training the model...\")\n", + " model.train()\n", + " optimizer = optim.Adam(model.parameters(), lr=0.001)\n", + " criterion = nn.MSELoss()\n", + " for epoch in range(epochs):\n", + " # Dummy data\n", + " inputs = torch.randn(32, 12).to(device)\n", + " targets = torch.randn(32, 1).to(device)\n", + " \n", + " optimizer.zero_grad()\n", + " outputs = model(inputs)\n", + " loss = criterion(outputs, targets)\n", + " loss.backward()\n", + " optimizer.step()\n", + " \n", + " if (epoch+1) % 1 == 0:\n", + " print(f'Epoch [{epoch+1}/{epochs}], Loss: {loss.item():.4f}')\n", + "\n", + " print(\"Finished training the model.\")\n", + " return model" + ] + }, + { + "cell_type": "code", + "execution_count": 86, + "id": "63830aed", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Started Dummy Training the model...\n", + "Epoch [1/100], Loss: 1.2140\n", + "Epoch [2/100], Loss: 0.8031\n", + "Epoch [3/100], Loss: 1.2870\n", + "Epoch [4/100], Loss: 1.4929\n", + "Epoch [5/100], Loss: 1.7952\n", + "Epoch [6/100], Loss: 0.7782\n", + "Epoch [7/100], Loss: 0.6903\n", + "Epoch [8/100], Loss: 0.6253\n", + "Epoch [9/100], Loss: 0.8758\n", + "Epoch [10/100], Loss: 0.7644\n", + "Epoch [11/100], Loss: 0.7986\n", + "Epoch [12/100], Loss: 0.8123\n", + "Epoch [13/100], Loss: 0.9833\n", + "Epoch [14/100], Loss: 0.9311\n", + "Epoch [15/100], Loss: 1.1370\n", + "Epoch [16/100], Loss: 1.0218\n", + "Epoch [17/100], Loss: 0.7389\n", + "Epoch [18/100], Loss: 0.6206\n", + "Epoch [19/100], Loss: 1.3688\n", + "Epoch [20/100], Loss: 0.8149\n", + "Epoch [21/100], Loss: 1.1958\n", + "Epoch [22/100], Loss: 0.9766\n", + "Epoch [23/100], Loss: 0.9688\n", + "Epoch [24/100], Loss: 0.9180\n", + "Epoch [25/100], Loss: 0.9970\n", + "Epoch [26/100], Loss: 0.8746\n", + "Epoch [27/100], Loss: 1.3388\n", + "Epoch [28/100], Loss: 0.7889\n", + "Epoch [29/100], Loss: 1.2510\n", + "Epoch [30/100], Loss: 1.2772\n", + "Epoch [31/100], Loss: 1.0001\n", + "Epoch [32/100], Loss: 0.9372\n", + "Epoch [33/100], Loss: 1.3004\n", + "Epoch [34/100], Loss: 1.6018\n", + "Epoch [35/100], Loss: 0.6899\n", + "Epoch [36/100], Loss: 1.0878\n", + "Epoch [37/100], Loss: 1.0880\n", + "Epoch [38/100], Loss: 0.3583\n", + "Epoch [39/100], Loss: 0.8913\n", + "Epoch [40/100], Loss: 1.0012\n", + "Epoch [41/100], Loss: 1.6451\n", + "Epoch [42/100], Loss: 1.1628\n", + "Epoch [43/100], Loss: 1.2701\n", + "Epoch [44/100], Loss: 1.0167\n", + "Epoch [45/100], Loss: 0.7775\n", + "Epoch [46/100], Loss: 0.9025\n", + "Epoch [47/100], Loss: 0.8914\n", + "Epoch [48/100], Loss: 1.2683\n", + "Epoch [49/100], Loss: 0.9164\n", + "Epoch [50/100], Loss: 1.2637\n", + "Epoch [51/100], Loss: 1.0175\n", + "Epoch [52/100], Loss: 1.2392\n", + "Epoch [53/100], Loss: 0.8681\n", + "Epoch [54/100], Loss: 0.7182\n", + "Epoch [55/100], Loss: 1.0401\n", + "Epoch [56/100], Loss: 1.3447\n", + "Epoch [57/100], Loss: 1.2504\n", + "Epoch [58/100], Loss: 1.0714\n", + "Epoch [59/100], Loss: 1.2144\n", + "Epoch [60/100], Loss: 0.6698\n", + "Epoch [61/100], Loss: 1.0110\n", + "Epoch [62/100], Loss: 0.9046\n", + "Epoch [63/100], Loss: 1.0489\n", + "Epoch [64/100], Loss: 0.8557\n", + "Epoch [65/100], Loss: 0.9198\n", + "Epoch [66/100], Loss: 0.8579\n", + "Epoch [67/100], Loss: 0.9106\n", + "Epoch [68/100], Loss: 1.3388\n", + "Epoch [69/100], Loss: 0.7173\n", + "Epoch [70/100], Loss: 0.8606\n", + "Epoch [71/100], Loss: 1.2798\n", + "Epoch [72/100], Loss: 1.1241\n", + "Epoch [73/100], Loss: 1.0875\n", + "Epoch [74/100], Loss: 1.3110\n", + "Epoch [75/100], Loss: 0.6741\n", + "Epoch [76/100], Loss: 1.0264\n", + "Epoch [77/100], Loss: 0.6686\n", + "Epoch [78/100], Loss: 1.2535\n", + "Epoch [79/100], Loss: 1.0444\n", + "Epoch [80/100], Loss: 0.7522\n", + "Epoch [81/100], Loss: 0.9744\n", + "Epoch [82/100], Loss: 0.9571\n", + "Epoch [83/100], Loss: 0.9522\n", + "Epoch [84/100], Loss: 1.3742\n", + "Epoch [85/100], Loss: 0.8700\n", + "Epoch [86/100], Loss: 0.8842\n", + "Epoch [87/100], Loss: 0.8993\n", + "Epoch [88/100], Loss: 0.9452\n", + "Epoch [89/100], Loss: 0.7082\n", + "Epoch [90/100], Loss: 1.2160\n", + "Epoch [91/100], Loss: 0.9545\n", + "Epoch [92/100], Loss: 1.1087\n", + "Epoch [93/100], Loss: 0.6511\n", + "Epoch [94/100], Loss: 1.0392\n", + "Epoch [95/100], Loss: 0.8291\n", + "Epoch [96/100], Loss: 1.3929\n", + "Epoch [97/100], Loss: 1.1216\n", + "Epoch [98/100], Loss: 1.0337\n", + "Epoch [99/100], Loss: 0.9571\n", + "Epoch [100/100], Loss: 0.8764\n", + "Finished training the model.\n" + ] + } + ], + "source": [ + "model = train_model(model, epochs=100)" + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "id": "2860a48b", + "metadata": {}, + "outputs": [], + "source": [ + "aux_data = torch.tensor([1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0]).to(device).unsqueeze(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 88, + "id": "a42f76c2", + "metadata": {}, + "outputs": [], + "source": [ + "output = model(aux_data)" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "id": "553b8caa", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.2852]], grad_fn=)" + ] + }, + "execution_count": 89, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output" + ] + }, + { + "cell_type": "markdown", + "id": "48d2cfac", + "metadata": {}, + "source": [ + "Nun wollen wir dieses Model exportieren. Dafür haben wir mehrere Möglichkeiten." + ] + }, + { + "cell_type": "markdown", + "id": "cfb5079d", + "metadata": {}, + "source": [ + "**Exportieren via State Dictionary**" + ] + }, + { + "cell_type": "markdown", + "id": "f453351f", + "metadata": {}, + "source": [ + "Ist die am häufigsten verwendete und empfohlene Methode. Dabei wird das `state_dict` gespeichert, welches ein Python-Dictionary ist und alle trainierten Parameter beinhaltet." + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "id": "2861e89a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model state_dict saved to ../models/nn_6_simple_regressor_state_dict.pth\n" + ] + } + ], + "source": [ + "export_path = os.path.join(\"..\", \"models\", \"nn_6_simple_regressor_state_dict.pth\")\n", + "\n", + "torch.save(model.state_dict(), export_path)\n", + "print(f\"Model state_dict saved to {export_path}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "id": "82118080", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output before loading state_dict: tensor([[-0.1205]], grad_fn=)\n" + ] + } + ], + "source": [ + "model = SimpleRegressor().to(device)\n", + "model.eval()\n", + "output_before_import = model(aux_data)\n", + "print(\"Output before loading state_dict:\", output_before_import)" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "id": "1a036642", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output after loading state_dict: tensor([[0.2852]], grad_fn=)\n" + ] + } + ], + "source": [ + "# load model\n", + "model.load_state_dict(torch.load(export_path, map_location=device))\n", + "model.eval()\n", + "output_after_import = model(aux_data)\n", + "print(\"Output after loading state_dict:\", output_after_import)" + ] + }, + { + "cell_type": "markdown", + "id": "eb80d116", + "metadata": {}, + "source": [ + "**Wichtig:** Es ist in diesem Fall natürlich auch die Struktur vom Modell zu kennen, also es werden nur die Weights importiert/exportiert. Der dazugehörige Computational Graph (sprich die `forward()` Methode) ist hier nicht dabei." + ] + }, + { + "cell_type": "markdown", + "id": "825088f7", + "metadata": {}, + "source": [ + "Sollte es nicht möglich sein, die Modellklasse bereit zu halten, bzw. falls wir das Modell wirklich \"komplett\" exportieren wollen, so stehen uns folgende Möglichkeiten zur Verfügung." + ] + }, + { + "cell_type": "markdown", + "id": "2f8ec630", + "metadata": {}, + "source": [ + "### ONNX Export" + ] + }, + { + "cell_type": "markdown", + "id": "215da5af", + "metadata": {}, + "source": [ + "Falls wir uns für den Export via Open Neural Network Exchange entscheiden, so läuft dies folgendermaßen ab:" + ] + }, + { + "cell_type": "markdown", + "id": "75e2342c", + "metadata": {}, + "source": [ + "Zuerst müssen wir das Modul `onnx` installieren. Dafür verwenden wir `pip install onnx onnxscript onnxruntime`" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "id": "ec27f40b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_148960/3099032314.py:3: DeprecationWarning: You are using the legacy TorchScript-based ONNX export. Starting in PyTorch 2.9, the new torch.export-based ONNX exporter will be the default. To switch now, set dynamo=True in torch.onnx.export. This new exporter supports features like exporting LLMs with DynamicCache. We encourage you to try it and share feedback to help improve the experience. Learn more about the new export logic: https://pytorch.org/docs/stable/onnx_dynamo.html. For exporting control flow: https://pytorch.org/tutorials/beginner/onnx/export_control_flow_model_to_onnx_tutorial.html.\n", + " torch.onnx.export(model, aux_data, onnx_export_path, input_names=['input'], output_names=['output'])\n" + ] + } + ], + "source": [ + "onnx_export_path = os.path.join(\"..\", \"models\", \"nn_6_simple_regressor.onnx\")\n", + "\n", + "torch.onnx.export(model, aux_data, onnx_export_path, input_names=['input'], output_names=['output'])" + ] + }, + { + "cell_type": "markdown", + "id": "19a8703a", + "metadata": {}, + "source": [ + "**Hinweis:** Es gibt hier noch weitere Argumente in der onnx.export Methode, jedoch reichen für unsere Anwendung hier die genannten Parameter." + ] + }, + { + "cell_type": "code", + "execution_count": 94, + "id": "aabab8fa", + "metadata": {}, + "outputs": [], + "source": [ + "del model" + ] + }, + { + "cell_type": "markdown", + "id": "dc03976a", + "metadata": {}, + "source": [ + "Wir können nun das Modell wieder importieren. Unter anderem können wir uns auch [hier](https://github.com/lutzroeder/netron) (es gibt auf der Seite unten einen Link zur WebApp) das Modell anzeigen." + ] + }, + { + "cell_type": "markdown", + "id": "b9e8f741", + "metadata": {}, + "source": [ + "Wir können es aber auch in Python wieder importieren." + ] + }, + { + "cell_type": "code", + "execution_count": 95, + "id": "1fe0a837", + "metadata": {}, + "outputs": [], + "source": [ + "model = onnx.load(onnx_export_path)" + ] + }, + { + "cell_type": "markdown", + "id": "40d26d8a", + "metadata": {}, + "source": [ + "**Wichtig:** Das importierte Modell ist nun nicht in PyTorch verfügbar, sondern im ONNX-Format (ONNX-Runtime)." + ] + }, + { + "cell_type": "markdown", + "id": "6b581a80", + "metadata": {}, + "source": [ + "Zuerst überprüfen wir ob der Import funktioniert hat." + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "id": "4a742475", + "metadata": {}, + "outputs": [], + "source": [ + "res = onnx.checker.check_model(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "id": "b4fb9f95", + "metadata": {}, + "outputs": [], + "source": [ + "ort_session = ort.InferenceSession(onnx_export_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b85ba62e", + "metadata": {}, + "outputs": [], + "source": [ + "output = ort_session.run(None, {'input': aux_data.cpu().numpy()})" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "id": "71a1b4e1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[array([[0.28523487]], dtype=float32)]" + ] + }, + "execution_count": 99, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output" + ] + }, + { + "cell_type": "markdown", + "id": "60e01ff9", + "metadata": {}, + "source": [ + "Der Output ist wieder gleich!" + ] + }, + { + "cell_type": "markdown", + "id": "7f5af869", + "metadata": {}, + "source": [ + "### TorchScript Export" + ] + }, + { + "cell_type": "markdown", + "id": "5607d0f3", + "metadata": {}, + "source": [ + "Die letzte Möglichkeit, die wir uns ansehen werden ist der sogenannte **TorchScript Export**. \n", + "\n", + "Es erlaubt uns, das Modell zu importieren, ohne die Modell-Struktur kennen zu müssen." + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "id": "9aea4acc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Started Dummy Training the model...\n", + "Epoch [1/100], Loss: 0.8839\n", + "Epoch [2/100], Loss: 1.3471\n", + "Epoch [3/100], Loss: 1.5558\n", + "Epoch [4/100], Loss: 1.1475\n", + "Epoch [5/100], Loss: 0.9933\n", + "Epoch [6/100], Loss: 1.2930\n", + "Epoch [7/100], Loss: 1.3009\n", + "Epoch [8/100], Loss: 1.4032\n", + "Epoch [9/100], Loss: 1.2986\n", + "Epoch [10/100], Loss: 1.5628\n", + "Epoch [11/100], Loss: 1.6245\n", + "Epoch [12/100], Loss: 0.8749\n", + "Epoch [13/100], Loss: 1.0093\n", + "Epoch [14/100], Loss: 0.9486\n", + "Epoch [15/100], Loss: 1.3173\n", + "Epoch [16/100], Loss: 1.2826\n", + "Epoch [17/100], Loss: 1.0709\n", + "Epoch [18/100], Loss: 0.7309\n", + "Epoch [19/100], Loss: 0.7271\n", + "Epoch [20/100], Loss: 0.9565\n", + "Epoch [21/100], Loss: 0.9427\n", + "Epoch [22/100], Loss: 0.8496\n", + "Epoch [23/100], Loss: 1.3563\n", + "Epoch [24/100], Loss: 1.3468\n", + "Epoch [25/100], Loss: 0.5225\n", + "Epoch [26/100], Loss: 1.4669\n", + "Epoch [27/100], Loss: 1.0838\n", + "Epoch [28/100], Loss: 1.2212\n", + "Epoch [29/100], Loss: 1.4901\n", + "Epoch [30/100], Loss: 0.8856\n", + "Epoch [31/100], Loss: 0.9297\n", + "Epoch [32/100], Loss: 1.2879\n", + "Epoch [33/100], Loss: 0.7851\n", + "Epoch [34/100], Loss: 0.9544\n", + "Epoch [35/100], Loss: 0.7390\n", + "Epoch [36/100], Loss: 0.9176\n", + "Epoch [37/100], Loss: 0.7545\n", + "Epoch [38/100], Loss: 0.5950\n", + "Epoch [39/100], Loss: 1.0645\n", + "Epoch [40/100], Loss: 0.7515\n", + "Epoch [41/100], Loss: 1.0809\n", + "Epoch [42/100], Loss: 1.1745\n", + "Epoch [43/100], Loss: 1.3536\n", + "Epoch [44/100], Loss: 1.1044\n", + "Epoch [45/100], Loss: 1.9122\n", + "Epoch [46/100], Loss: 1.2601\n", + "Epoch [47/100], Loss: 0.6494\n", + "Epoch [48/100], Loss: 1.0749\n", + "Epoch [49/100], Loss: 0.8888\n", + "Epoch [50/100], Loss: 1.1439\n", + "Epoch [51/100], Loss: 1.0482\n", + "Epoch [52/100], Loss: 1.3640\n", + "Epoch [53/100], Loss: 0.7824\n", + "Epoch [54/100], Loss: 1.3926\n", + "Epoch [55/100], Loss: 0.8370\n", + "Epoch [56/100], Loss: 0.7971\n", + "Epoch [57/100], Loss: 0.9532\n", + "Epoch [58/100], Loss: 0.7365\n", + "Epoch [59/100], Loss: 0.9876\n", + "Epoch [60/100], Loss: 0.9812\n", + "Epoch [61/100], Loss: 1.2391\n", + "Epoch [62/100], Loss: 1.4859\n", + "Epoch [63/100], Loss: 1.1096\n", + "Epoch [64/100], Loss: 0.8135\n", + "Epoch [65/100], Loss: 1.0660\n", + "Epoch [66/100], Loss: 0.9529\n", + "Epoch [67/100], Loss: 0.7976\n", + "Epoch [68/100], Loss: 1.0494\n", + "Epoch [69/100], Loss: 0.8920\n", + "Epoch [70/100], Loss: 0.9967\n", + "Epoch [71/100], Loss: 0.8961\n", + "Epoch [72/100], Loss: 0.8740\n", + "Epoch [73/100], Loss: 0.9696\n", + "Epoch [74/100], Loss: 0.6680\n", + "Epoch [75/100], Loss: 0.7864\n", + "Epoch [76/100], Loss: 1.1901\n", + "Epoch [77/100], Loss: 0.8527\n", + "Epoch [78/100], Loss: 0.8770\n", + "Epoch [79/100], Loss: 1.1248\n", + "Epoch [80/100], Loss: 0.9718\n", + "Epoch [81/100], Loss: 1.4774\n", + "Epoch [82/100], Loss: 1.0423\n", + "Epoch [83/100], Loss: 0.8642\n", + "Epoch [84/100], Loss: 1.2729\n", + "Epoch [85/100], Loss: 1.3168\n", + "Epoch [86/100], Loss: 1.2866\n", + "Epoch [87/100], Loss: 0.7739\n", + "Epoch [88/100], Loss: 1.1269\n", + "Epoch [89/100], Loss: 0.7487\n", + "Epoch [90/100], Loss: 1.5029\n", + "Epoch [91/100], Loss: 1.3421\n", + "Epoch [92/100], Loss: 1.2918\n", + "Epoch [93/100], Loss: 0.9249\n", + "Epoch [94/100], Loss: 1.0778\n", + "Epoch [95/100], Loss: 1.2862\n", + "Epoch [96/100], Loss: 0.7971\n", + "Epoch [97/100], Loss: 0.6628\n", + "Epoch [98/100], Loss: 0.8082\n", + "Epoch [99/100], Loss: 0.6520\n", + "Epoch [100/100], Loss: 1.0457\n", + "Finished training the model.\n" + ] + } + ], + "source": [ + "model = SimpleRegressor().to(device)\n", + "model = train_model(model, epochs=100)\n", + "_ = model.eval()" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "id": "e8f24d61", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output from the original model: tensor([[1.1867]], grad_fn=)\n" + ] + } + ], + "source": [ + "output = model(aux_data)\n", + "print(\"Output from the original model:\", output)" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "id": "d8092502", + "metadata": {}, + "outputs": [], + "source": [ + "model_scripted = torch.jit.script(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "id": "1a0f45ad", + "metadata": {}, + "outputs": [], + "source": [ + "scripted_model_export_path = os.path.join(\"..\", \"models\", \"nn_6_simple_regressor_scripted.pt\")\n", + "\n", + "model_scripted.save(scripted_model_export_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "id": "51187aa4", + "metadata": {}, + "outputs": [], + "source": [ + "del model\n", + "del model_scripted" + ] + }, + { + "cell_type": "code", + "execution_count": 105, + "id": "5e19a16f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output from the scripted model: tensor([[1.1867]], grad_fn=)\n" + ] + } + ], + "source": [ + "model_scripted_loaded = torch.jit.load(scripted_model_export_path).to(device)\n", + "model_scripted_loaded.eval()\n", + "output_scripted = model_scripted_loaded(aux_data)\n", + "print(\"Output from the scripted model:\", output_scripted)" + ] + }, + { + "cell_type": "markdown", + "id": "ba55efe4", + "metadata": {}, + "source": [ + "Im Vergleich zu *ONNX* hat *TorchScript* den Nachteil, dass es nur für Torch Systeme funktioniert (Python bzw. C++). " + ] + }, + { + "cell_type": "markdown", + "id": "9e75eb7e", + "metadata": {}, + "source": [ + "# Finetuning" + ] + }, + { + "cell_type": "markdown", + "id": "c5626c94", + "metadata": {}, + "source": [ + "Passend zu unserer bisherigen Thematik mit Model Import und Model Export, wollen wir uns nun über FineTuning unterhalten. Dabei geht es darum, dass wir ein bestehendes Modell etwas anpassen. Dabei wollen wir entweder nochmal auf ein spezielles Dataset zusäztlich trainieren, oder wir wollen sogar einen Teil vom neuronalen Netzwerk mit einem neuen Teil ersetzen." + ] + }, + { + "cell_type": "markdown", + "id": "b2069416", + "metadata": {}, + "source": [ + "Für die zwei oben genannten Szenarien stelle man sich folgende Beispiele vor:\n", + "1) Man will einen \"Copilot\" zum Python-Programmieren bauen. Dabei nimmt man sich ein bereits funktionierendes Language Model, welches vielleicht generell auf Text, sprich alles im Internet auffindbare, trainiert wurde. Nun verwendet man nochmal ein explizites Dataset, welches nur aus Python-Code besteht und lässt das Modell nochmal einige Iterationen mit diesen Daten trainieren. Dieses Modell sollte dann das Programmieren schneller lernen, als ein ganz neues Modell, weil unser Modell ja schon vorher \"lesen\" gelernt hat und somit unsere Sprache bereits versteht.\n", + "2) Man hat ein sehr generelles Modell, welches gelernt hat Bilder in viele (sagen wir 1000) Klassen zu teilen. Nun möchte man selber einen Bildklassifizierer bauen, der aber nur zwischen Autos, Katzen, Obst und Menschen unterscheiden soll. Dann nimmt man das vorgefertigte Modell und ändert am Schluss nur das letzte Layer und ändert es zu einem (in unserem Fall) `nn.Linear(Z, 4)` Layer. Dabei steht $Z$ für die Output Dimension vom vorigen Layer. Danach trainieren wir nochmal mit unseren Daten. Auch hier nutzen wir dann den Vorteil aus, dass das Modell schon ein Grundverständnis von unserer Welt (wie kann ich Objekte erkennen und vom Hintergrund unterscheiden usw.) hat und auch dieses Modell sollte viel schneller trainieren bzw. eine bessere Performance erreichen." + ] + }, + { + "cell_type": "markdown", + "id": "d7c1307e", + "metadata": {}, + "source": [ + "**Hinweis:** Solche Modelle nennt man dann oft *Foundation Models*." + ] + }, + { + "cell_type": "markdown", + "id": "ba6eab3b", + "metadata": {}, + "source": [ + "**Hinweis:** Bei Methode 2 können wir auch mehrere Layers austauschen." + ] + }, + { + "cell_type": "markdown", + "id": "285f4bea", + "metadata": {}, + "source": [ + "Sehen wir uns nun an, wie wir ein bestehendes Modell finetunen können. " + ] + }, + { + "cell_type": "markdown", + "id": "7611d537", + "metadata": {}, + "source": [ + "Dazu möchten wir an dieser Stelle auch zeigen, wie man sich Modelle aus PyTorch herunterladen kann, welche für die Öffentlichkeit zur Verfügung gestellt werden. Als Beispiel nehmen wir das Bildklassifizierungsmodell `ResNet18` [Dokumentation](https://docs.pytorch.org/vision/main/models/generated/torchvision.models.resnet18.html)." + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "id": "f8968cad", + "metadata": {}, + "outputs": [], + "source": [ + "from torchvision import models" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "id": "1305fc80", + "metadata": {}, + "outputs": [], + "source": [ + "model = models.resnet18(weights = models.ResNet18_Weights.IMAGENET1K_V1)" + ] + }, + { + "cell_type": "markdown", + "id": "62fbbd25", + "metadata": {}, + "source": [ + "Dieses Modell können wir nun nutzen." + ] + }, + { + "cell_type": "markdown", + "id": "ecf7a7ac", + "metadata": {}, + "source": [ + "Für das FineTuning können wir nun auf die Architektur zugreifen. Wir erklären hier nur kurz den zweiten Ansatz. Der erste ist, sobald wir das Modell haben, selbsterklärend." + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "id": "09ec615b", + "metadata": {}, + "outputs": [], + "source": [ + "num_ftrs = model.fc.in_features\n", + "model.fc = nn.Linear(num_ftrs, 4) # New classes are: Auto, Katze, Obst and Mensch" + ] + }, + { + "cell_type": "markdown", + "id": "1844f9f9", + "metadata": {}, + "source": [ + "Nun können wir unterscheiden in 2 Teile:\n", + "* Wir trainieren alle Weights neu\n", + "* Wir trainieren und ändern nur die Weights für den geänderten Layer\n", + "\n", + "Für den ersten Punkt müssen wir nichts machen. Sollten wir aber nur das letzte Layer ändern wollen, so müssen wir bei allen anderen Parametern die \"Differenzierbarkeit\", wegnehmen. Das machen wir mit folgendem Code." + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "id": "45df698a", + "metadata": {}, + "outputs": [], + "source": [ + "for name, param in model.named_parameters():\n", + " if \"fc\" in name: # Only the last fully connected layer is changed\n", + " param.requires_grad = True \n", + " else:\n", + " param.requires_grad = False # Those weights should not be updated during training. This is called \"freezing\" the layers." + ] + }, + { + "cell_type": "markdown", + "id": "356be40c", + "metadata": {}, + "source": [ + "Das müssen wir dann auch noch dem Optimizer so mitteilen." + ] + }, + { + "cell_type": "code", + "execution_count": 110, + "id": "b7730b4b", + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = torch.optim.SGD(\n", + " filter(lambda p: p.requires_grad, model.parameters()), \n", + " lr=0.001, \n", + " momentum=0.9\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "ceaa5a17", + "metadata": {}, + "source": [ + "Nun können wir dieses Model **finetunen**." + ] + }, + { + "cell_type": "markdown", + "id": "e87de599", + "metadata": {}, + "source": [ + "# Model Deployment mit Flask" + ] + }, + { + "cell_type": "markdown", + "id": "4245a104", + "metadata": {}, + "source": [ + "Zuguterletzt wollen wir uns eine Möglichkeit ansehen, wie wir unser Modell selber hosten können. Wir verwenden dazu Flask." + ] + }, + { + "cell_type": "markdown", + "id": "d662da75", + "metadata": {}, + "source": [ + "**Hinweis:** Es gibt noch viele weitere andere Frameworks, wir werden uns aber nur mit Flask beschäftigen." + ] + }, + { + "cell_type": "markdown", + "id": "cb19e663", + "metadata": {}, + "source": [ + "**Hinweis:** Natürlich kann wie vorher erwähnt das Modell auch in einem entsprechenden Format (zum Beipsie Open Neural Network Exchange (**ONNX**)) exportiert werden und in einer anderen Programmiersprache eingebunden werden. Etwaiger Hardware-Support (GPU) geht dabei eventuell verloren bzw. muss selber implementiert werden." + ] + }, + { + "cell_type": "markdown", + "id": "dbba0be9", + "metadata": {}, + "source": [ + "Wir wechseln nun auf das Python File [nn_6_flask_app](nn_6_flask_app.py). Sobald wir dort den Service gestartet haben, gehen wir wieder hierher zurück." + ] + }, + { + "cell_type": "markdown", + "id": "6297559e", + "metadata": {}, + "source": [ + "Wir können nun unser Modell testen." + ] + }, + { + "cell_type": "code", + "execution_count": 111, + "id": "c8a7c549", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "import os\n", + "import matplotlib.pyplot as plt\n", + "from PIL import Image" + ] + }, + { + "cell_type": "code", + "execution_count": 112, + "id": "f90a2efc", + "metadata": {}, + "outputs": [], + "source": [ + "url = \"http://127.0.0.1:5665/predict\"" + ] + }, + { + "cell_type": "code", + "execution_count": 113, + "id": "143b4ff0", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOMAAAGFCAYAAAAPVES/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOz9SbNtSZbfh/2W+977tLd5fURkRGSfVagCUFUodERLo0yiSBPNNNVYn4BfQlOZRprKZKapJhpoIhphBoIQaQBFoIAqVJOVGRnd699tTrcb96XBcve9z3k3It8rKxhrEDvzxb33NLtxX81/9aKqynfHd8d3x//ih/tf+ga+O747vjvs+I4Zvzu+O/6KHN8x43fHd8dfkeM7Zvzu+O74K3J8x4zfHd8df0WO75jxu+O746/I8R0zfnd8d/wVOb5jxu+O746/Ikf1rh/8P/5f/p8oDsEDDkRQcSiOIULEQYw44Cc/+pjf+a2f4HGICCogCoKiKN0w8PrLn/P/+j/91/z+3/oZ/+l/+vdxGsA5nIDGAdFYri0i440IhOHAzYuXrNf3cF5wTspnRARVZfyGQ7zHMhsiTiOgKKAEYox2bwrO2f2O11OiBkjfnr6O2BlQUBVUQVUhKmgEjZR8inROFVs3ylm0XBsN9poqQix/2yMrqhAFFMFN0jQk3dv4xAriAEdEQQWVCpUaac64bj7gTX3J65vIVVzTri95td1S/+t/SffgQ25+9nd43L3i/Iv/mTPZ0rUd3XbLsr9iFg84EeLQIwREFBEhhAF0oHYVzjnQgKri0vM752wth4G2PYADV89xrgYV21ShrL1qtCV3zt5jfGARP+6Dc5N9kXG9caiq7W2iB1tXQZzkbUOMKAGHE7tODIKq3UcYAkMM5eoxRlQj3nm8E3wlOAcxarl2/pmvG6PR8X/9f/6/8uuOd2bGckeiQCwPnMlaVRFxKMLh0BGj4pympRSEaASb/lc3Na6u2O12dtqy2KQzAqojc6UN0PQhVWUIA06FqvJlAZxz5ft2b8ZsqgYDFB0vhiTiyS9NmN4ujyQRMhJKWmxNbJDOHWMioJjWpxCGojHaPXt3fIVMFOV6tjbCSHyqykQs2f3ktVAlgxtxHlyF+AZtzqnPHiFVQ1QhBLjeBP7NFwM/f/AD1lct89dfszm8RFeX3PzR/8SP/81/Q/Xhh3zcfUZz+yvm3RUuDixIglYU5zzialxZrUS4CE58ognB4REHIUaTPeIgDvTDgIrHNzNEnC2VusSs6enUnk9Eyj5P92VK7JnQMzOigjhHjLG8B4y/C2h0x0JVFXG2x6RrOSeEEIkaCx2pislZBCEJ7ygEjVP5enSP03t9l+OdmVExgorohCgVtVdAEhlFaPseSe/YZikxsWVMj+OrhsX6jO3NLRGlElcWZJT2bz9gfjjnHd45W8gjwldCHPj6s1+iUVle3md9eY+6moGDiOIyOtessSaaZarRRrEwCp4kYVXs+2oSxp4uSWAm2nRcP7Fz4ynElb8HgCeK3UVURRxGlDhjNHHEaBrHNKwD56FqaFb3qBaXqKtRHFEd0VXEELl9c8XL6w1/8HXPi70nvPpX7J//GcOLX/HwfIl6xyfhmrPfPie6PXz134F4XFWDmsDFBUQqEG8C1W7OND6CF0fUgajgxRUB4yYCNEZo+4HZYoWjTq9FY5AsdE1B4sWZYDe4kQSwCeqYNGmIhhZMSI7rLCeMWM6dGD/aiyCCwyGShUreRylyNF/XiSeEUcvGqDgvdq4oaQ/VFEnm6cy0zq77Lsd7aMZYiNCu5QxOkKBEUhOKMWMIAe/sM0IiepJKj4qvamarFTdXXyRIkh9fJwT6NhPaY9oihthTuQrVBIejsu8P/Mn/8N/x7Oc/ZzFbQFPTnF9w9uAxH/zgh1w+eoQ6LVpNROzSYiQU0zWzrE7UYn+pQSh19jvpbhUlJmKyw9t9Szlt0tEe8QuqqrbNdR7XLJFmgbqaylW2EOnz3tc4VyOuAhGGoUWjgnOIQt+19O2e25c37Ld/wuF2x2G7YTgc8KoEHegOLaHveKTCQwfEnioO+AcO51tDC3VlzBUCYIyQVt+e1XlwvmhksYUb1bqI3XcWZOk9IWtw6Loe52qc1EWw5B0v+5s+HwHJws/ZmrsEvUe6IMFQcC5TjDIhlyNElb+jGHR03id8J8QJkHFpu2OiZZGsSdX2Ppq2jA6cjghBmD6zFqZ24nDe8y7HOzOj0wzxYoFupKVzZRFNM3RdyzAMOO+LhJv+A3DeM1suuXrWEjVimD9OrjjRTpK2KUlOTRI5hBYnkTD0DH1gc33N6y8+5/bFc/ysIYpShcD+5TO2z7/mzddf8IPf+T0uHj+hns1xvppoHmM8UWPIIUS8IZ+kaRwSe3zsqQnGxBoZVOmkYUAQicCA14BLSKF1DTtZ0bslh2pFPV/xw3sLpN8Tuj2bmzdsvvwVN8+fczj0qK+YL5dUTkzzI/TtntAdUA1JAjtC39MfdmjX4oLmrTDt4hzqzD6tULwq4jwaQ8Lr3uw2J6DG2FEjiEeSnaYR1AnqXLLTjHg1CagjyJ60zBGklJERNUREKppFbecJgRAGW3dMQCdUmqDpaHNlM8LEo0MkM9yI0KKa1joBI+n6dg5BEizWI4iL90emQ4iREGNhZHGumGD52jEqTpQg433HbJOmz0G2lafI7duPd2ZGs3PNhZPtRLLRGxNUJSDOMQyBPkbqDGWPMLRJGPGO5cVDvt7v6frAzFXJaZM1jo4QBsq10noiGnn5+XMO7YFuuyX0gb7tkNgzX1/gqpZ42CcGt40Ytjs++7f/Dpn9KdV8xeL8knp9yXx1bs+hCs2CanWPTmdQNTxrG7Zn3wOdUbvAqu6pD6+pfMX8s/8J1/V88eN/Qr+4xMWBGA808ZYHzUDTt+wD/NmXr6kXDte/oml/zgu3odpdE27f0O820B0Apdu3xDCwWC9pZnO88zjnGIaO0JvjQ5zZWBkueV8nZJCYJjmritJKTgsRiJlyJL9m0k2L08KZMEp2qvgqwUA30YojjJva0faZtEOJOaJSnBg+CT4EhhiSTyFJu3TmKRNmxg5BOdaZ2RzImm5kwgKL30JTo4OniHvVwmjT40hpmO2RHDSYPew8GpUYBYmK985MMMlaO6180rDZ+fYux7vbjCeQi/SIyMTdIEmzqNC2PfNZY8ybHs7lDYpKxLG4vE976OnaFuZVscEkab6jhSrwx3Zh6Hqunj6njxEdIsSIqCJi0raqPB0YpItK6FoiHb7tkBDpDwPd1YagX5kW0GiSrK7BzxnEEdSx9Su65SM0DuAruPcxs+99yubVMx79z3+EPxy4flWzm6/wGjhfztmHwIaB6vZLmv0Vl7s3VAQEc4JEYFAIISBA5Rxde+Cw3bA6W9PUDaCEYHDKOU9IK24MZAzpvWm2mOw4ca4QXyFqEtGVLbLvi3hzVGhAnSCJFMwxFpCqMpsKc6gkWc9op1CEZdmzpJ2UkSHyPXjnk90nydGWUdP42ex4CUlrmZljGlySdjPhr2hMTOaEbCydMtbUMz4VIIVWZRQEU7U6FQaqkRBiuc/MpFEVp+kn2X4eeSV7b+9U199wvIcDJzNfLBg5JuO3EiFinlSDeErX9abSE/TMVogAQSMahWZ9TggDh8MBuVwlITlKJJeM9yw8JbmQNSj7w47rzTWX9z5Am0C32+JxOCdoFDT0+KoihgHvkpt/6OjaPY1bU1UVg5q9qM6knhIhDHhaGokMXc9F/zX6+j/gHFS+Qp9XhD+fc0+VWiKsKpYv/jWqgagGzb13hBDQGKnqCnyCWFGTQFP6vkue4BpxjjAEZvMZ8/kc5yq22yvmi7MkdSv8bGkwE8BVZndhgs3MqWTTpT2R5BAzJ0LSAF4SfDUnUkTSnkUQO3ckos5b2AFjhmBeD7wfXWsxOTzesssA8MmWjqZhnUO8ewvantKXRvOQG32YX8E5h1PBMzJPmHw/22oxO3oAJ/r2tRJ8F/L9Zo2VBZcJ5KlwyZ+JyTtuz+QIGhCBIQpeFLythc9Cqwii5EX/y4apZaHVJa0fEbxhZcmuD28LEyOHrk1SY2LnTWxGEJZnZ4ivOBwO6ZV0nfIw6bWo5aGi+ZdxzvMv/tUf8uSDLX/nd35GM4+02w0+Ko5IjENS5SBOqJuaru8ZQkedFtuLM0IT87DGZE+NmF/Ae5wH7+3+IVCH/XhvBEQdqgFHQAPEwWwsqQw2OvGm1NN5+6GnqmfFAaAI4jyL2RxX1fTdnhgGnKvMzhFAapyrRomNES8qqBOcjFpT8QUyTjVljnVmuy8m6AUuaT+l74Pdb45TkhnOH9mEFl8MR1rxyM7DzisC3o2f8d7TdZ2dS125N/MCR/oh0g8DAD4KlfdUlSRUkTWQA2Kx6VBzCsYEc6MK4iLejaGrAs+PwNb0+UanUH4mJy6FLhxKKBopmwmqEIPgclzSj1Y1ybx617AGvCcz2i4kT9FENWc8bbYIxKDs912SEMl9nLVccTsrs8U5UZXDZg+SbMnJvZuHNGYpYK+hBJR6tuD5NvIv//m/YjHz/M5v/pjebTlsb/BYcFYZ4bFLhBJCP2FUQZwa4SmI80mKm1Hvqwpig2hEJCIxlJghalA7e9skKmEY6MMATpgvFoh3yVaytYkY8TZ1Y97QfsBXFihvFitqXyFO2OzfEIaBoIoE03wiBsmyHRijTtbEzq/iMIeU2WKS7JiYYwaara9kF8bR+5khpog3D6ZA0GyvSVnLTLyQvZlyxJD5/TgNETljAo2BEFtCYp6swdKXDJ1YvgQ5mUCjojhUHF7UCF81QW17xhgD2as9+heydi6Wc6HXkljA6NPIMDTHqkfVkH81Op7G1fN7ChCz3ayQ4+sxIYh3PN6LGfOCx/SAox03vicJ9rRtbwubfAbT55JkeNeLFa6es91sx2c+OqaSa1T3GiOurji7mHP9i55//gd/ym//xo+TsTwQRBA1+yNqJMTkURNn0i4MhBBM6zmL2UmS5OocKlk7imk4IYFwRaKaWz4xeL6/SKQbero+sLo4x2WI5QwS2jNo9kMZcSfGr+raPLuqDO2O7rDDu4qoMdlCZot7kuCTZMepFqeOQdTkDS1wabrq4++KMMRgNuvE1kHAVVUisInHdKL1pnsxzXrK64CYp3IYhsKsObPJQh9JRsvx9zShHecsG8buPwmZjAQmj5STArJ2ttcyasufT1oya0Yo924e3GOhVu6taGtGbZplRowl48YE5MSMAojJTs9MegLjv+1499CGz86BZNSP+4dK0kDl00LfBWIE8Rbwz04SyU8clbpZUDcr9vu2PABM7NM7juxd9XXN/Qf3QH/JL5+/4evnz2m6G4auxVcVVc4Zc65oY5eCSEPf0R72uBqqqkZFEZ8C6wlYx2SsSiIOg7IpligpFQxjqhAG2r6nHwLNfEE9m2FB8uRcwaR9XjuEZHf45FxJmjMG2m5n7OLy2bN0J9l4yYWOQ/yYqZJUgkFUzYJyAisZkxuU5CzJzgeyMHUF2iNy5BQpgfPJHudnyUFtExTKECJdP5SddD7i8zrmFMnsSyiQz56zqmo7lwh9H9BE7ZJoKDrBFUaWAq8T5xhTy/jUpHvUiU15nNUzEfj5eWR8z+zv9LFoCQmRUASUE2dOpGjnd+lzmjzYIkyu9e3He2hGKQuA2oJEHTcoK3bxgovQ9b0Roer4uagFHqhGXDNneX7Bzc0m2S6Z0Cipn/k4djmbprt3eQ4idCGwP7Ro1xJDYBgiQ6V47/FjlgIuxTxjGBi6A4RAbJZAhWsUV9eQBUd6KpfSs4obSsBVKeEhDkRVC+V0AVxFM18hUh3bZskbZwxmzxdTelU9SZqQdI8hphhniESJxBSElwSlTJlryVSJeb1kIqZxBbFkpohElBS4jhEvprlVFVFBEjw8cphO4GfGuqqA5uQAJcQJdBMYQiAMWnKGDUikvwsdZQbOEJckkATnPUGT808VHUIK6YCXaAkWR5lSmUjsPwm4UOzKrIHj+PmCMk9hW1Y05XctNmdOEHFOC20fMfEE1ltIaZoj/euPd2ZGL9kOycRp+rH4VlUML0eT313fEqM5AzQmYiRnvthPX8+YL1ZsNluTaCl71CUI/I14O718dnaGE+GDizPOZg0xWmBbY6DvIoNE6uTc8N7jK8/QDxADfXcAN6TUphleGlQ8rppBSlnLwicHpXWiNUSEKMIwDITezjNbrqhmcwuqJzs0hJCgEBDiGMSeOBXyOUGpKk9VVcnNH1CNKQVs1F4hanImubIYlqIGUjwUI9LIBJZjfhkpHGen5O9lSKkFSmaGNlg3Oafa55HevLrZgxszEWbhbelw+GyPjdcrf2NJ+SH0DEMEZ17prhsI5mWjbiokYrG9EEYoOxXgGb0cPZMdGTa/5cnNNjCj9hwFWYbEmmVRcex470fzTAQNMSniUQC+I0IF3gemZh5P0MelRGmPEDgmKudgCD1xCJRgqCRJplmyGsav5iu2169SXqOkRGv9Vs2ePXbn5ysq7/j93/gBi9pzu8tQC0I/mOSqPbOmSnsmhmECoJGoPYrg1Vz+zrVUvkKSDTnC5pGowNzo9jxC30f6MOCahnq+QJ0n+5BjjAz9YPso5vXMxKxqEBnny0Zriqd5b4nKQQcTYEnixkTkZkqZ4yJOMq3EObwbmdHsMJeeI+fNkkyGHGQ/ZoyCeCfPPWoy024xuXhzEocTbymGGk0Yqp4kJ4Sk8Y0OvPeQva7JNgsh0g89/dBb0nlaD+89MdnpQw8pyDERCscCKGt5nCRfAUj89fTEBIYXD+2UlyWviBjCCRn6TrRfsYGxnNXMxO94vHsGzpEEtwV04kYCsXs1czBa4HgYepoqOReyrZg+aBk2wnx9wauvdoQQqGspxCFkO+WOh0mMdb5ecXE256cfPy4Ll+FfRAwuxUDjbdNzTMq0REy/D0R60yxOiK5DPBZWiGLQDwoUMeKya4UQ0/c8VTPDVU2mbmJUuq4jDAMOoaqqUYImQrTXRhvGWQwF8c7yROMJzNFkoybbzDuXUrNcscdjzGGOtB6piuTIATb5WZw4E6dMhpt5D8ZcVYOHIymcapgRumVhZHtikNu5CB5Esx0uiQkiITmUSMjLYxlDpjkjIYTkrPGEYKGjvCg++TOyjRs1JobEPpfuKWvhQsZM1iRqctwZArTnCEfMJPmLOtq6JtyyjZ95QN/+/R2O92DG7NrNCCaxYGJ+TZoiRogx0PcxLV4GtHnDNEFQe215ccEX+wNt1zGvZsnvoMmGnHxnei84RD1n6zN++5MPeHh5RntzXWzDI68YpgViiBPnRyIqbMHD0CLE9Azg6oD4JpUFpRCB96XmzdZZzf5yglQ1VdOkvE6DQ0MXiF1vpTa1BeRdtrkheVozzMz2sv3lXIU4JfQd0R+QelVs6kgumUw5OVFyuG7McILyn2Kn5bMXiZ+cHbmAJUO1CZTLz2Pw9G2Id5ewNMYaSG7e9Jq9F0Pa1xipqoS1kkDQqMm2jvY1yTmjWq5l9wek0FXW5KpasnU0Pbc5Ok2wTW23UbBkzWYrr1l7SpGn2VQcPysk82AS5sn/TV+PyUXmEsgTdaiMtanfdryXA4d868WDJSOkS7YCmmy2CMPQg87HU2RGjCM6X57fo+8jXTsgy3nJ7JDJA+fFOJbgjuW84fd++6dUVcU+xaRijMlOS/mRmEPB7jUv34jzjbgDBIi9CZYYI1IHEAvYV1WDqhB0jFOZBPXmuatkzHJRy6cchlCIJC9fDhc4yZrSkgUy0SW8QFXVRrgxQohIlUICYgXcGeKP1GLx2KghEZs9q8vlZUmouHKFcT1Jz5OjbmAmiS9xvMlnjrDhqF3KSwWmJca1F5MCNNgoMaIpCYAYEZf2LIYUL1QIlkWDvB1OsdioHAkIsD3z6X5yzHQ0AMf7G0MSifPKI1kMuaQOJseMy1qyhJeTcHCjaVaeO19Qsr8jeUHeUTm+h2a0e3dZyykJfpoNkwrFLHUpedksJe6YoUqwVS1jol6eMcTAoe2KJC3mmSQIcCKBs2Olmi346HufcHj5NYfDgTAMRpRhKPaQ846qqtNGxaO6NC0PkmrUokNDDyJEUZy3CnnVyjQKSXMlhs5hhxxHM0+t2TdW/Z5sJ+9LGVTOQnG+SsxCcnoZoxnz14Q+EIJafFPN5opiklZELCFcScH7MDJAgajJxnRYyhZjLaD937SmSiYhSU/l7HPBNL9z4/O9DXWnzEpZ82yek5jQOSnfsce1vFtEIFhM0mK/JmQ0M753VJVjCFogX8wXIVfy51iuWpKEWAUJSWGEEGwNvE9MpThvgszSCXOIxgTjEZ2RQ3IJY6VwTPEnZIGTwU1BH4XgDdK+Y3eb92JGKfaT3VDI3tXIJFOG5LCAtj2cwJgxrJE3pl6twMNutytEMT7ft4kUk/IhRl4+fYbEgRiHdG4jmq5vmc0aRByVT5JZTiRVsk+LcwkLWTCkx3HmDQ6T2FLWcHkTcw1choCK5S76qrJEb+8Rb0vtvTdmnFQMRMBpglauQl1lWUCxB7WsX03ZN1RWTa8ZFxXmgJGh0nNJismlZ/M5OTtaDFBlTCJXdcWmLrG/aJ8J4S6BWIJZ5LhhQSXJa+wmjJr3NSY4GIYRHYUQDX4WYZE8ugLOC7U44pBsQXcsEGw/Y3l+N4nuqBr/ZGFgQmwUDCbtE6NnwyNrWxnpfPx8YkTvkjLRkU5l9KGOa0Tx8L7L8R4lVCPsyAa+re6I6af1iKrKoe3usCmObZZmsUCB/X4//VDRflNocJqJM5s3vH7xnNubDavlgv3+gK8MWgYNbHd7vHcFkbjk8BjtEMv6kMKQ2e2foOMgRO9RWpy3ej/vUm1jcj6UxY4pfJA9qLhiFxYhVjSX5EU1+FOe214T76hqS3IPoUOGmqZeEHMSgWbHTLITkyAsXl4wgsZybkMhsJR8nzR2Vl+Sb2ZkZcb45GiH5TWzPTiGp/kzxhhjte6p8CuoJO1/TL1+ytowCXdMoTQp4K4x5etKyvk9dj6JjN+1vgqpO0FUQxYTJw4FpCVBLJLWyuxZUcGLxTvNbHLHym+6lulhTWG6IiByZ4x3Od6rnjHlvI8qOUEGpwbrplkaokI/TJosJX2eGdE+43DVHLwvvXDyu4XhT5RjOb9T6qbh+3/tt3j51XNeXr1EGDhvzlPgP+B8ReMrrKIi2WvOE+NQiMolNzakliAaEXWQGhG5RNQxBnA1iqWvxWLlK+I8UZLnMEZ0mMAY70emlelz2DWz9p9ul6R4p6sjoW2JQ0cMHa6eFxOhELkGSCl/GYaquBLgnhaZhzriveCdUGnSQIxeyRjjicDKaWM5+8o001iHOIWfiaBTtYoCotbkxATvKLSBFHvWgjCk7EAcnUU6kppZPgKB5OAjCRNXaCrbyVOPcJRkR4XkkCoJFIWjUq+bMT+VZDsKWhxikiqCwKqOcFIYMtuXWVllRZPzp9w7MuO7gVnyQzJSTXkYO2RCTvlzU5txdHmP3xegahrq+Yzb65v03eO0qymCz1cp4QGE+eMP+eBv/QN6FZbLVcZEqfDW4ysptoCkUpzRC0c5j8O6rolGiAGJ4GJEhx5igDggsSfSE4cBHUKqmhifbfqMUS2h3Xl3ZHNNbazpWtizZmFlObSVr0q3PKcRUXNS5H/OUc5NgpgZXoZUGzgM1ghqGAb6bqAfLAnBzLAp9DTCjjE7Sqx0aoiWYWNOKWXoIzFwtJf5uUPM8NxigRkmckQbb3tknYglD6lpn7dCJkQkVc5UVTXaamRo7wwBuePvZofPafe2jM7G9y3BIjuopuEZoJSA5Vj1cd+lLJCmUJbJPr8bI8J7wlRgEnbJlRATtznjDaooXd8TVfGavigJX6smyRipmppqteT2dotqQHJDkgzZpmdNNpJoNHmuSpCaD37zb+L6li//1T9LjDLgq4YYW2us5KyCXUhQLjeQkNw1zFpPjBLMoGqGYhqwGkJNoW9neagi3mocTwjz2ANJIjaXtMrYU4UE6bJ2zBtqDgJnTh+f2hnqgKVXCHh7HoNJFvervTAIhORAMgdFbvJljB404jEkUEWfNL9Q+apI0DDmAtriR6vnm8JFa6OhVhqVGDoMPWHoy/0UrZgQ0RGTTDKAcpKG6KT8KDOvTqBlRkpRijMmhxmsJryeMFpMexet8kOstUrRxBMYbEn/pj09ktpbJs2ZAECOiUrOc9bEBzpWibhxA4uwcD5nF73b8X5tN8g8krXUVFXar+bgsHdz5r53fiTYwog5EOuZrc+53dwcLXCW9OXUR1kWtiBelEoC9y4fMvzwr/P6yy+4/vyPmDUNIfQsZrOkXeyaIXkcteTIjucVyZo+E0Jq05ekqlMsnUstCA3gZKyOH936Y65l6f8zgU92LSnENfJxxt+CpHWpKoPiQ28JFLOqMQmdeneKWlvEkGJsPipdH4n9MEr+zIzJJiLAICmLxidEYJKNKMpRRUKBf6PWNwETECfEISRYG9FS7mR7LImJrJrEHa+Pc2hI66tainKniSWjacNonyNUtbPoTvayZ0End0DhJEyjKkMcyuvZlj9yymAaL/fy0QkdjmgnjspIEjSJRhvCxI5PWlREUinZtLfTNx/vX884PZKmEyE1rLKs/wLTw8Aw9FS1aYJsxxQJBiCe+fqcw7PXhJCk0Ft2Yk47mjAkEY+yqoTdbsezZ6/xD37IbLuDwxu8CHUt1L4G7e07ajcmSfJlYiB5SotHLDFihkIkr6KoA7zBWJefPdui4+aqamrGVZapOCZGDTERZHnzNRbNoJhzqXKVpZolyFz5KpWwmaPBO4/H7MW69lTeJL12wdpXKOTMb8VqKEMIOBECSlPVRHpqXxl8HWIW7km4UiRGzFpPQIdUjS+SQjSpSRcpBOM9PrUYKf1zMkRVtTYiWbOcmCWSGDYzYfluwWeaajyTTZ80pxexTEcFkdEmBfN8ZvST4amd921LbQx9jcya85KnccaMZDP6yEeM2Yn27loR3qfthh5DjYwfk+BDlKTmTcsEVWIIYwzMTnJ0PkUR8ZyfP+Dpz/+QGAJSFRY4OgRS2w2z6TQOaAxI2PPsxTP6Tlg//JTHn/41nv7Bf8v2V/+G5uzMFlvHzmCG7syVrqKpw7iUJPWpTWqpZvY5y9DoIbhUvCGpmhyLR043LzO2WG/XEuPSiTc4/51YK1EWEtPmZttZzHEQh0gYWupmlmod1TSWBLymcIQo6pXFzFNVjm6AvrcKipg8hcMwkGO9/dAzNA1VVdG7amrZTVLHdEK85sDxPle/hPKchXHUWbmdT53nNaMQKVo0d18D8LlZ1sQrn/NancsdCLJGNXbMgi8zmoXSjDujBohZ8I3wP5b1Hmm55OJOGH5ENlI+n7WLOCm5+Zq2rJD0HTa0nfyd2Av4C2rGrNWOnTZZchncye7tvu+ZV3WBHuPNRiz3zzFfnNH3gb4fmNf1uPgTO8VeDCTvAaIRx0C77dhtetbn59SzBcMQ6AbzmlVVVXrRaDACiKIJIjnGIG7K+CjPpKWhLpIaMKuCDilIbLcTU+gjeyCPwzhSCGuEfHn9kp2ouUlVdlrJyIgZcXiPSEUfcmPkaAxchNnELsO0Q3SSYC50leOwtw4EI+FZgDvGSN/3hBAZZCjtQYAJM46tKKqqsibFKfOqqh2V5CSD1E80NSYWTTZTSoccU+NSyVHuWpB42PuJhsoaVLKNP6GvHNfN95mD/nnN1K5pJtCYniiFMUYn4KkzKT/3VGjGlJAhfsJ5ImmnkjmSkQQjIxbN69345q853psZhQw7pNgiqC3KRDaQ4119H2AxkRbpfSWmTnJQL8/pukDbDqxXdb7KuEBiUDIU7G1ZKWgPg7Jan+Hma4Ye3lzdMr/3CXLzOaD0oaVvW3MQlBWDEHrLLvHVnV648fcEzfAp5WwwW80Wgjh0DEixETLsyYH9IlD8+Lsldg/FTs2eOxGxXqVqnl8RC0NEVbwfMG9KLMXcySJKkMghWLNfL9j64pDaU3tHP1QMg3I49OZJTVAqpB6mFqrpj7TG9PejZ0kMIU4TAycvcM61VbMr8dEYJ8HuUpAjCYvkGseJRiquBbJlriTL1iCpmj02dh0sG4UmU2iq3eBYsJDsu2PP9ghLpwgm04JoMlUSIiq2ftqDKfSdnsOJUFwR73C8XzocYywk5zKKlDLRFJPRBFatkrzvu/JQ+ShFo2lnZutL+gCHQwu6uPP6mggo21qCSaTZfM7F+h6HUPH66g34htkHP6LXju3X/wbXbVCFytelkLY/7Oj7ntVyaaGOJFykqOCJM2AkpbSohsldekaNA6EXfFWVQLqIlFq3cf0mWTsldc2l1mujHXkU/E7r6ZzH+woIR7bsFErltRSUypn+dGrlYdFbmEQbR13VDEFp246+7xnSc4YYiSEe3+vEHvaTgGUMEZwb826zWAjZO31sQ7tcg0nuoOaSZhwzYjNasgQdS65QZ1pc1GH/Mw04TS+baiE9WZcpEivQM0463Ovbn53Sat6/GHOPBZcS78fPJflyjIpk0pIF/ctnxpIlT85OMJvFScpdTvAju6hdKt1puy49TLrpvAEiCSYqs/mciHC73YLcB6YTmMDaSCTbBDUFkR62rjyzxZw3rw9sDweWizOkmSPf+5tsdlsWL/4ti3lN42xx2/2Bw75lsZzhqyoZ39n7CeTOYxky5pYPkkMfuQFhdj4oyJBCDcmGcdUovUj2CxNIJKMTKRYYB1kQSIJcGi2vMToQX6PR25L5afhkAv3IwiNl5ogQojGAVwHvcFQEhabxtG2Fb3v6rqMfbCSDnVNoki1ZVSNymA6aMUJ0qBcMjWrpZj56TzOMVlQHssAbGWHMVLFQTrbZx6wlhyCS+whx9P27jmKvTzRVoeGYhiBN2CMnOthWTrSaO2Z4ovlCTGxP6h1lkrQphghHRGy/HGnmbzneG6ZquqHkDc/6GsW8c9kektQ3o23bYsxq+ewUEkDdLEAcm51V/E8R9sjESWKr3YOtQIXGQOy2dIeO89WaejanDRCkJnz0Nzlsn3PBc0SVw2HP9nbLbLZgPptTskaUQjwWU0ptNaZlXFP7WNWq2V1yv0/sBUtKHuOJuYVisRslwySKHZusj7I+lFUi2a4OqTxuyM4fxvo5KLY6ZGiXGEFj6RnrnCMS8d4YpxKHzGoqX3HwrthV3nuapgE40u5TCJa1ZExJ9y45T5xUiSHzohwjg+kxhbxj46rqxEBJecCMmuctR+KUNjPxnGi2Aj3T2TMDTs95fE9y5+tH9y6jYAF7Zst7TTm3arsoJ6ji2473YMaMpRm1BrZcjhyDsVy+oBFhTIXL9uF4Kj1asGa+omoW3G72ZQvLgkweftqbxeBiZVAn9njnCMkJ0DQNdd/RnT+g/+k/5s2v/jnr/UvrYr5cWyduMS1uqVACqVhVTASmzH+SaWzCJMe3/CQVSo4WnqNMEBuh5sfuYWKbldcyt0vMBbT2+0llREoU8FGJqa2kymgukB0X6T6SKWsrlCS1tdLIyQQmSJ2oxRdVmdWe5t5FgnGkwvBhsl3H9r79S8+Zk7CRlEVjtmuSaSQZd2SjFWdVIgNb/9yRIH8gOwSnAfrIsSd1XNfRhpYi7Kfvn5Yy3XWOY7tYClLIn7VIgdmuaotYbMP8TKbRkxNL3s4K+rbjPZgx2TaquVqqME4BSRLHDJfsIGj3yekw4vQSb0yayVcz6tmS29v9hNum2oIkiRxKTBtuhOArRyWO87Mlr97sGAZrEbmshf1hYDd7BD/4XzP86r/nIv45uAHLvNHiFFBSD08nyX0+7pwFbg1q1HVNDBDEpUa5yc6UHGbgyNmR4dYIT0nPZ55knQxIzYKuPH1+Lbl1s2e22D95L7JjKsHsknMpkAu7c/6td9bUKWRbFME7T1MZdA3RSszIQ0/z80/qGHObfWu5P6LxDMNzNz0nbuwQP3mmfM5jE0smlfJTW1gM4kJJWzsdaHskKITimIKEGtM6ZaUZT50zE9sx30v+eWQnx1hMgul9lyfMt39iIb4rI8L7OHB0Eg/LryXsnKGqhRuM0C3pWtGhNazucxZOhqqUhF/xnmo+Z3O7MRKZYNMRthg4LoEFAe8jlQr97kDTrLi8XPH85TVhsHFr502FF2Gj5+gP/inu55Hz7a8Q7VJsNIJaSZFIZTGvEorI9pMrcay8aTYdyqcKe/Na5hqJUW+MDEO2LaKlkUnqFUNJlrZtvCveZcNfR6hb4L2MW5/TtXL80k2IfXRqmJAUMbWqKdMkarS4YNR0z6kCv6pKrxuXWjHGLP3BpvZm42jikPKTqvqpZ5MkQLLnOD9nzvPMzpDyeinRsofIpWfT4zT8ZY+fDUvK2mcG/SY781Tr5tcoTJ/NLIOgQ0hF3HJ6nm8enPoux7t3h1PFshpcgUFKOJL2uZJckqaIGtChsyqJIxtWx8x7LNNkuVqyuX2V9W/it6n1kCWQy25Gg8VecRK5vr5meXbB5cWa129uadsDrppxVs9wAvt+zu33fpfZL65ZDlcGo31qDBxi0opJq2UrbuqESHBJXErBEkceo64IuDRg1Ffm4p8+cLKtM9OpDlYAHceWJNkumjJitjNzzWEe223MOKaQGYMlnJKD5xlZZBgraa9IJWUk7JFza93I/ENi9pDHB2jWkOZU8j4lMoDtbXlIg+GZIF1+dGwUxMg8I4PCmGCBMNF86R7voOVT76fImBjgUxH2kYYylVbWLn/nVGvdpfXyfDXnkrBNtD7iwuNzOZdU0cn13uV4j9zUmFzK6UZkfFyrFldzrKQYWPTWpdPpYNky6t/SrHkBxAnnF+dsvvxyAmGL6XDHvZDsR2v5vl7VbELNm1dXrC8uuX/vjOcvXnPYXeObGfNqRjPzbNsZh/vfx7/es9DOoGU0d3qIEV/5snHTq+UFFiqs/b1Hxafwe/L8OcFX3nrlnFZpTGFqspVi0mK5c7ll+IxhjfydHCoSvMX1gCON4FI6WF6nvC+a4l+kKU05AJ4KdNEicuwZk6c6jwGPqlQiFkNN3vJcuOt8LkhOk5SzBz01GLbbH6EgZOaaQsExYJRh3mmMb4qL8vdGzVUo6C2oadOhxq8WhJE1sGpJFh+7l49MWn4mDVtihpAQkytmQVbExTbMza11Uqj8jsc7M+PcW1zNepoCkjPcNSkNTZjdoE+USHR5GEyP1HOSqf6WtPDOsb644PWf7en6jqrSkjuajf/pQmUNqcnuIwTOzhbc7HpevXzJ/QcPuX/vjKdfb9m9vqWaz5mvzzmbe7r7H7HpruD2KxYuQLCeNSFihb7FDpIjotFknEtVoZJqFHGprWBtUK/A3Nye0jSWFAaXhAjiWKWQ4Jh1wS4gsRTDFuomQkzvJk+1S6U9+SgQLRtIYhpRNDmncld0NQHqEpaJDmtslXvfirVPQWty6+Ogtka2f6elRloSwi1jyfCCkzEh2wSoQCoINt8BSQDoyF0Jatpz5lxYVzryHRO3jtCUY2bNAu2umGFp26kpRJZCb29py8KP9lmnoK4aGZE86CIZTyl538bVQW4i9q7HOzPjud9hRDONaTGRaBGcdfgKLlqnQUmu9tCVmzqGGDHFloTl+QUhBvquQ6v6LoV4fEiiSSIxtgTtefTkki++eMarVy+4uLzgww8e8/LZM25ur+nCgcV6zXy9pvvob9A+reHml9Q4ojqGIeLmrmiQktwQTVuUATdiU55cGvkdZdImIw/OkQRTpw+RNj+HcwSLA4oTqpR/egptnELOy9NiU05XX8e9lpEYxzhgspkYEG0me0XSwky+Pwazy8YK5IbSxqhjjucxnJQsi9OpRg15tGWSEc8k/qfmPMtrMi0UMHs5pQtma1wz2jiOIX5z6OQkMZw00CfEMUSjcZLRM4WxE3dwuiknwnT03kjXlGLo7KDy76EV4T2Y8X5llfipeSF59lxRyWohjEg0KeqEfoBBaxi6ZFGMkATV8eZVWazPQIWuH1B9mxl1QoIiWQuktehbXt++4uzhEx49vs/Tr1/w6sUzLs4vePjwAU3tefn8a/rthvN795ktZlSf/Abtr3r01Z/iE0w0j6BnOtPewjb21zihKjObTeO1EEad/k5OpilMnR45b5dUICxSMlnu9OyJJA/1uPFHIYLsaJgwlUYtlSYjk0ZrDZIYq3wWRaIwZhjla+fY3NhSMg4hpbeVr9uuZu2YFLNISg+cwMTpz2xN5tknUWOqos/PmDrA+6poMvVZ8EjeGnLMVY/OPWrKaVghdwwsn1UphQB2jlFZ5GEdJeZM6ryXQ1pk6J1UZClgSPRDZlYbJfeuZuM7M+NKWjIcGT1fORZkkCom+BOzye6ULioMO3J9YFHpOqp/Ucd8ucb5mvbQoueLE9uBvPMn9pwRwawW+v7Am6tr1qsLHj16xPPnT3nz8jlnZ2dcXF4gRF49f8H1q5ec3bukrjzzBx/TXT+FcIWWsEx6rizxnVqoDgtKKxkeK9aMyjQiroLU8GmUqtPbH5/bQhbJMzsNfZwc+TvjHEeOg/0iI6Pl13T8GTWmplNVEp4+P15ZvwzDslMoCxJIRdHIKIyyEBx1YLEbDfKR6lGl8NXUQ2n21AjNi6YvmT2U18ZO6FmEp3FrhelTVtCEMaZMeEw3x+EMS4CQQov59fxddSOj2XmzkM7OMMoaWXxVRnpm9Nxaep91hXiX4929qSW3Y8wMKXZiMu5cItqAMqhSoQQVwtCWDmoyZSYdYVY9W+B8xWazhceX6f3xh8gRRYxQTCO1h9ms5s12S9/DvctLHj9+wtXzF7x+85Lz9QXrszPqquL1ixfEriOoIw4drO7RtlskdmlCr2S7nUxskop3lWRXpa104hHfWKqar82eQI4dKuk8qCYPavKiqjJN+yqxt1zIm2xCTT1mSwMpZaJFsnZMlJ+Ix5ESBAo0S/B6AiamhJ/v0SUP8VQ4GNt4bEiXo+v6lNQBIabWI+naCUybxiER+CSFzhXbIodC7LpWYnZc/Z+dLt5bA2oT8KSePErxbieIm8M5pwkTFhdN16mqFEfNysDMg2GwsfRDCNbRLzMiU8+2L/dsTxxHGk40E4feku4LLLY2KY17NzZ7j8E3o6M6RQmLdIipoJiUoxixDA/r0yIMoU+5mBSpASnelqr2q2aGOM92synXzN6+QjIjFmNqGHsn9Pstlxcf8KuvX4IoZ6tzLh8+xFee189foCGwXC15+PgR++2Wvm/pDjvm9z4iOkf74s8tOJO0hzL2T7HIRpWYMWWuikOkQnwDvjb4mhkreyvVmvO2hz2xP1BJtOZSag1m4jAwmzXJEWNpaOLHKccxL1qMSQsnyZyIDBU0KEFySKnwq91DBNL9m8zORDjC16kzpTDzNC8zrbcCdRpiJIP12CEFwn0Oj2hK6EhOJSEcBdmdjBX5pStjqVxJn3GutOvPzYrFWX8iB8Tq2GHlvUuaO9OKJyeTxAwXRXCpRSMuoDqQxx6oCK7yNCJmXmnE20B6pND8uFbl2lENFUxK5+JgbVHqqkojFJXusON2f/XrGYz3qtoY+1zmBZyY/Am4FTZNN5/C4FkbTIxzph40FKkaqrphs9kYcWavY1nkiTY4ArzgUG5efs2T+x/x5OFDXr+5QqPjYr3k4sE9nHc8+/xzwrBnuV6zXK9ot8rgZ6wvHtMuVtzcvKQNHTPNleup41pUKpd6qOQMoGiVEFSV9UNN8BUmEEkj7WHLbn9L6A702xuG/oBTxdc1i8WCqpol8SaErrNBQS63arC+Li47Q0Kk8j5J+zSLoqRrhVLCNU23K461qCWOqJlhJceDE1QcNzqdM2f65M+aOVElzTGEYPA5CYncon9KsLlgOEPDkZimlJXjm7aTOTUQKX5lpgzhJ+mCxXNdHC+SfDypgbNqEaDHXlWHywOC8ve8x6fzxJSkkj8/ZcTjVp+OuvH0fU8MA+IiQxg4bFuG7sCh3bO5vSaGMbXw2473SofLZpTdiVKyVNQW0tCKue8lGduCQugR7QBfpleNY5xts6uqZrFcs9nuykUc08SAMUdxXERjaCfK4c2XfPmrh3z0o98EqXj6/AX7/Yb7D+6xWC356Puf8OLrp7Tta9brMxbrJU4eEjUyBGX25GfsN1/R7W64XDbU3jRhUMW7CvE1Ip4wGDypqpmNj3OeLCfyhg1x4HD7hv3tG9rtFS+efsHMWWezupnRtxXdYYP3deqWrVYiJTBfLKnrGhHY3nbmba0qZrM54KjqmqryqWt5pKpmxb6pfE7tk1K1EUOEOs8RUYOhSGq4lEWnrWUaOk6x+bKuSXbfoAHnTat6BKcTNKCpHkdjgY5po4q2HQXVsdNFyZ0A8u+TLJ6MokS4y0WbeT87ZQyZpYT/pOmijufK91IS3xm/b4kNsXSgC9EKrqP2qdueFbarKrv9nr7fE4ZA3+3YbW+K/ewglfsZrRyNRfiW491LqKYG3FvvjbAx2zsFV6sg/S0aBmumxGhjlgUF8DXL83N2t69TudLb15raA1oIx0LvFwv4oz/7E+5970csV2sePxG+/PwLumHg0f17zOcLHn/0IVcvzIlzcXnOrK4Jw0DlhPr8kvm9S77443/P5urA3A+sa6WuKqJUVFVjzyIBqWpcswCfNaJRRAiB/W5Dt9+wff0VTgKH3QbtW4bKpKhlZ9iYOFWhqSq8rwhqLTHa3TW9c9S1Tai6vb7hcGip64ZmseLi8gIdBrabrXVRWK5YzOc0swV1PYdcsIsw9AkSi8PXOS6KzexwWYhqmnWYAWm2xVOtMJPxazrCW1/SG92R5jP0aprAJceL9xWz2Sy9nz2aaX8zevS5fWa20exDGiaVNG95ZSfCmZGRU4JW7j2AaMrHnTBjtlFD6Nnvd+W9Q3vAidK2O4a+ZzlbEWLP4bAvLR1FYHF2zm57Q7vblLXRZAsPwVqbjOt6B9PccbxXCdVRBsPR61nbCUMePhNtnHQMwtBdMYtdajtoy3b0U6xdRLNY0F0dii1im6Zpk8ZAc27cO7kDFosFz59+zdVmz71qznK54uOPP+arr77gy6+/4smDeyzmcx59+ITrV46rZ0+Z1XNW6zXzuiaEltAHhuhYXTzm7GzF7YtfwfVrzi8r/NyS0wcN1PUM38xRtW7VIUTatuV2c8Xt9St2t6+5XDU0dcX11Rs0DlT1gvlijq9rhiHagFVVcIEQjVEql93oA0OAqqq4d/8eXdfy6uqa7fYNbb/hyb1LztYVr99seP3sCl9VrC/uM5svOez2iBjEbZomTcfqqdSS4vvcQqOuMY1TWTcTMVMvJwNGjcn5knrrJPiHTLSqG7uZjzZo8iokR4v3zjKT8mcTk8lEsDJhrqkdmwZpWsFxgq1Tsn5L40S7z6ABkoFjE9F62q5lGMxJE4K1b9nvN4ShpU8jz02ruzImIoSBm2QDZqibhysd2p11fB/6pHSSclBr/yhqDq73Od6ZGbuumzAjyUGYMLa64uwY1OBTCMqgMASlp2IRBlx1qupsyVQN/6vzXN1uUpv6U4ky2pen0kZRLi/WbK5egSq7rmfZCLPFnE8+/ZTnX3/O02df8eDikvP1mot7F8yccPXiGdcvv2K1WtDM50QPTz7+iOVyiaDUN2v2u1v2rbJ/dUsIkbquOWuEw/WGoQ+FcLr+wNPnXxH7jvVizmq94PXTZwzDQF1XzBbzlLEUmM0XuIWjPbSEYWDoA95ZFb5v6jIIpu12LBYzmqZhAHb7FkJHv1pSNQ1dN9AdWu49WFE3DXmcWoyRruvY7ncp7UvwdcN8NsNVFbNmTtd7hk1IuaYz1Dl8VTGfLQyOh0DUgHeVVdyLOUrEWVpjdsJosJBQiLFknrSHfbED66piGAZ2L3c0sxnL9arsm4vm/IsTzaqaNHUifI1mh4mrqKsmzfGs8VVl6YvISHt9T993HLpDqqONDENL2+7ouj1ddwCxEjGXnDMjzM0/x1K5DEtDn5Md7DMxRvzgUrM1c7DlDgCjCQYFXr/j8c7MeH27td6jIU8Qohi7eUDpEA049kOgD5peV1Qq1n1HQioTx43dvglLx2y1ZrPd0w+Bus6znk4PfUtCCsp81hB2N1T9Fi9n7HZb5vMZ3jmePHnC1avI109/yXa55PHDD5gtVzz64EP2r17y+tmXzOY189WK9aJB40DsOzZXV8xnay6ffAgot9fX9MPAbj8QQ0tIA16aWcNuvyfGQNNUrM/W7Ns9u3aPeEezsNHi2fYPMULliU4YvGPbtsyaBl9VDIM15lJVZnWNczXP32x4frVjXlecr8/AOV6+uub5qzecr1bUyxUKtP1AN1gH8Ny5DRXaYaDftqA39MOAr6wFSV3XrNZrZk20rnNDJOInTpFI5UmlYjm0YppMkVI1H1OzK1Kt32zepFS45KdLvXMr7xl6m/LlvcdVvthoYRjoh968kWlCF2qEX2mF85XZvwIQ2Gw23N7eMoSOQ7tnCIF2f7BE99SVMPfpAR1tvjIRKxQHGE6yoi/e39zbN6Zhsgoln9VSmDNkz0nzKfZsJ4SsZGQazfz2452Z8RdPbxMejuUhbLilGftdP1hreMx26odosRknSFXzuN3TrLVow8xG9sNw/mp9Sdv2dF3PspmdPMKYhpcXIdeYocJsPqPSjs2zP+f7H33MbX9ge3NF3dTUXri495DVvOHFF7/kq1/9nIvzB5ytz5ldnvGw+ojN62e8+upz5ssVq+UKEO4/epSaJjm221vadpc0YcehNem7Wq3oh2he09Cx6wPD647KC4t79xEd6A47rrf7kow+n/XMmjmo8OJ6w6EdePJwyWGI3Ox2dH3PYjYnAC9vb/jF05dI1bBYz9gNni++eMXNZk/lF3iZcfvihn1QukEYErTMMzKiRrxTKuetTMqB9B1hSHM0v/yKECL7biDgOFstuX95TtM0NFVD5X0arW7hlBgtjcwEZk3f97YlQnk/BoOGIhbbc86lQl0b0RaGgbppWJ+tQYShH9jttsYswGI+o20PiNg49aZuDH10bUFFWkrcSFrZGL+qUqxXrCWkS/SFJticS+Fyf51ESyTHi0vMmJPr3Ww2elgzfM4MmzOnkv0tEzMO5ChB412Od2bGr15uEx622BlqGlJVrMI72DINSXUbzvdJcyw4tB3rk3NqWqDMm/PlirYfbFbjOnkJJ/B01PjHZUaK0FSOSiNXn/8hH/3sr7OczyEEtjdvIAaW84VpyY8+4XD9ktsXn3H7dc9iec5ytWKxXlC5yH6z5fnnT9EQmC9mzGY1/eENm80t/WBS/dDuyrNe3yTDnkhVV4QgHMLA0Pb0nfWVcU6oXEXfdbg+Ug8DsttztTnw8vqGi7M1r756xe7Q0ncDzlkDqSFEuhBQEXx74OrQpfBErrIYeH7Ym587tWd0xVMacS6VFEWxORghYENBB3RISc4K81nFfDbjZn/g9ZuXNI3HH8z2tNFuPqGUFIeUnKJH8t6O8cMSfErOnjH4LqkLuhF1t9+z2V2BJNjnRm1yaG/JhWV9Z53A67pK8UwQl/q1gnmRqxTiiBbEz8nw+PS3y1B2hI+41NLyhJbycdoRLzOkRuuAJN4nb7+ZVD6bziUEMqnYmDiZvu14Z2a8utkWwoKxJUGMY0LyaEfaAztX4euGOCiHfWsOgBKayO7zpN4V6sWagLDfd1hfvDj5PHdh1mLY15Xj/GzG5tnXXP3Zv6a5eETdrJiFlsPNK66fbvDOUvTmleP+mRL2PTevP+PNc+Hm+g2P718yqxsWs4qXz15ye9WyXi9xztH1PV0YaLuObggMCl2MVHmGg6/oh8iQRmH3wZxY3TAQI7Rdz6EPNHWV7BYTWDEEXu2uS4WD94r3EYkJHokjhEiHFIiX52NYVf1gkE8cLnd3U2vrmBuGiQoMPUPfUXlvAXRVPFDVwjAYVHxwseTibGHCNYU9LOfYimlz5lFIRcdGxObFzNU8WXPlz2ebQmJOIxSiDmVfnU8pdVFw3k/ydI0Rqqqy8EpKBDC0PM43GcMjiR4AwWaR5AGyZWgTmS4thmj0PNqqpz1Tp0Nwy7clNQJJfTzyOIDcriWfK8dop/f26453ZsbN/pAeztuip/BClq6ZAbOTFBTvxTbSRba7Xfl8fl9SNVu2PeqFhQt22z05QWDKg1NvLnAkzbx3PDiruXnzhu6zP8CvL4izJX2I1E5ZNRX77YHt5g3bfm/d4ajZXu/54198yXo5ZyE9t6q8efOG3XbLbDbjerel7QObQ0fbD5bUXFdpCKdniAMhWvyueP3EM8Spq0kIQRikoutTRzLvzasazAvn64BEYWgNdjnnLQ1MNTVmDnYdtYyjpq6MsASaqmI+n9FUVWkrOKsctZgTwgFV8lgfuh5RpfaOeVWTZxwOw4B2gqsyoUsRrploj7XH1JM5MmauunApSyjEgHc1Qk5vc1iO7NgzNTeAzlUUmYmMqcemWM7ltL4RNk41V2Yg7ys0xAkXpDxgP2lbMvleqdSZxB9PW23m90OyIVOQZUwKSwZyyameMPBfene4Po9yTilOGYocGagJT2dIE4h4F5Gg7G83SXKb7FJJ+Zl5sRTqqqGZL7je3JbWiEeNrOBoEfPfNpnYcT6veLHdsL+5Ihy2OO/o2pblao2bz9nf7njz5jWr9ZL99Z4XL2948eaWASBGftX2yYFiGuj6sKULgSGSmMYq+6UXS4oOIcW/ihvKPqNj9oiKlexENftn3wf2nY1pC2GSnP4eiGaISjt0091BbvfJy2nr1njhbOZZVI6mstYhTgfqylOLEIRUqZgnVw2oYLal10lP1GyHjWsvkiMUlqfrxART0UKMifKeyjoyOIttOjHzRdWyXJwIzmmxRa28KjNHqhGaxBcLHE5hlMIvyXkkiUGipgqTzBwT+pmeb8qI+adOmSodMZqn2ovlC6eGpQhaxjuURIJJH573Od6ZGQfFoOW0Fo3Mf6Ndx+Q30YgPAy4EDrt9ej1l7meWLQwJUjX4ZsbNzfWRBB5DUXcbxE4c3lWs1yuGtuPLXz1luVowm81wXgi90vVvePPmmkPXc/X8iq4b2PVKPyhtUG6219Tzhrrxxb09BOti7sWxXC6Yzxf0vfWB3XddSgjOutv+RU31cgAitH3karOjHyJtCAZj73qIv8juTb+uU0WQUvjE4r50gboSayCc43xJU5g2sPIiye0anbe5kjLRhhrN0SapI5xmwToG/U+7q2WBnbWMZBglo3mRid6fpJ+dLsypljJGnFwrOQZFgJgq88f6qDKGDkaGu6vT3N33kK8VrUbRuRJvZPq5LCSShn3f490zcIb0UFOzL8t/PV2sJFkiVJbjxGG/Ty0fkr+7BIuTpgRcVbNYn3Nzc2uSszC6aaW37ikvpAgiNfPFkkVTJzi6TXaGeddsnLiQZ1Q4cVQu0MnEJa1pfn06dzcMnF9c8Ju/+RuslovikDEHlbJvD1y9ueb1mys2u0OaHgyHIbI5tETEuncPoXTU+499OBFmlVBXjj4ExHmayqofKufwzvakqlIbyZQHq4olTM8afFVb9zsoVQ9Gg5n4ldJbFiaMZA9pwiyU33N5qORW/if3PO36dlR1cXJk9DdNXZt+59T7TkJtWu4pF1xNFMCpnTeh4+k95NR3S2m1OZanA43yvUy/9649U+E9mDHkFuoTxtPJDZ8uXr7BoJaJ07YtQxioqYpd6XQkfHtJmC9WbLe3E+eAvLVgnFxXRVDnOT8749HDB1xfb9gfTBPnekOzZ6yLj3iXMkRSBr8OdLbGyU4CdYKvK370ox8SQ+Dq6oqu7Uo/0S4MFkbwng8++pib7Y4Xr17x+uqW231HlxgzH2PNy1/uMXUSOIHlrKZy5un1IqbhaleEUVN5ZrWnrsyeo1dULTm9aWbFGWSEFUdtk642Ojfk6OrHdtH4pDkGlxk5exyt99Dxfpo2eZuO7rLfTmnBvLYuIa302Zy4noS+974MX5W3BhUdmz7Hr2nqJEhBFW6i/U7Pc2rPvmvg/52ZsShdyQDzbuaw3638JRv2ILTdgaHvqbw/ak+QvpEEmWO9PuP2668KRnduSm7HNkO+rqb7Wl9ecPbgHvOzc27eXLE/HHBSUdUNs8WcejYjjykIITDEyM3NFr/d0W325nDJTC7CYrZg3szY77b4qqJpGrz3tJ3FQg99T98NIDsOXcf5+Tnnl/d48eaKF69vLGNGEkHqGFQOYVrfkgnAPtf3b6dQZRhnBBtyIXrSOuYAiVFZzmpW85q26wrR5NAPIjSziuV8zsV6SVN7G2bbd/T9wKHvzXOpqVlWscdyMa29njVD6SOKtarMxxT6nf6DDGwt0W10zBjzZ9SRjxBCKc+KUSeaOVdiZOY9bkB8Wj1yjKAm1CSj1kyPfMRgWXyUishfo0WnzzOFqXdp+buO98pNfZvDZaIpp0wzNbZN6oZ+ILQ9zCyYn5N57cgSWFidrXj2x1eWapRnVkxs0tEcm7qc7TPVbI6rG5azmsX5eYI/1m/TpYwP20SrZ4t9YLVesb69ITx9zet9N1lkc0bkbCKpjCGGfrA2IfMFs8WCtu05tD3bQ8uzp89wdcVqteb+j3/Is1evefH6TfK8mmCZVbXFaoM5wmKwPEZfG9EFF446m3lv1618dsePZkrWGiEEZpXn4vyMoWvxTqgwTWhdrwXFMWtmnK2WLObWHMw5z+Act7s35sGtfGlDCVYNc7TPTJkHVK1+T5J0mGaa3NnjFCZtJ9/WFtlWDyENdHWeup5Ue0zO71xum5jj3aM3lsn6FK2Z7GWXFjczJ4xlZaLHdKVY+Ec0lg4Xp/d7+owK7wS57zregxkdx+fMmm+Er9Of5YEx47nve7quY5k1WTnH9C9hfXZG17UMfYRKJp9IYZLk7h6vNxJM1lyV82hlZU4jxrL6SvuOw2tgkMisdsR5w2JW49uu9E7N0v9ms0HFMW+W1M0Cv1Tqrqfve/qhJXJAXWAtjqBK3wdubzdcbbbce/SYew8e8PnnX1inc1WaeoaKzSAhpraJUfFNbTYrCiGmnjxQ1ZVNuELQYMNjbNxcgkvec9jvqRubaR+y/ZVYI6b1rpuG5WpJ08wQcZYvutux2W7Z7Q7cu39OVVXE6XyRE9gmkqcrZaGYaCBXXbljrVHsqUIRb9uH+dzTITUxxpTTO1b85/vJBO8nQnMcMZ/uWUcHT4kXOps5cnRfksRHgrbTGOP43AnBTM2iOzTjNCwyXbP3adf4HlOoeGuR841qCvxPr1mgRYK1SuBw2OP0Prl1RzblbcPs99X6kjDAMITUSsEWu0CSskB50e1v7x2zRUM9m5njx2U7Ki1a3pyoo+NIFNRKqLyAF5/rhmyTROiGgdniDKnmVKtLmvmafuiIIbDfb2F7Q3f9moYVF67i+uaa4RAZup6rV6+59+Ah3//+p3z99VP6VKkxXy44DH0RHF3f4bwlSNTzJbvdhthbllPV1PhqBtEGnoJJ7LppmC8WBjVTls6h7ZK9l8qYFGKAKtmOi8WS1XKFiNLe3rLvOg7DAJVnNl8kQs4tJMd9RHNZ1jEcm47gtvREG2FeuXGS8zfNmnjL3hvfKPbjtOYw74d5SSf2qY5eUdJ+T488DyQ70LI3+UivpJhwvi9JaMxliCrHgulthh2vdfR6jLxrjBHegxmnJ33LwBUpzHTq3crSuhssodeybRJAL0syws/5aomK0HaDVaejqcGvMVF27Bgn5lPYdlVVZR67XKmjBocNRk3grlpaEzHa+6pUqQGvCfkUXAbarmN1OadanjG/fMRieYarbIx3P0ReP/2c29sNSiSoUDczbP7gga4buLq64tGTJzx+/Jhnz1+gYumB1b7l/PwSFK5ev8S7ivlsaTG7qk5lPweWF5d419C3e6ucKSsG5+fnpkGq2iBaiNaCpDtw2A+FqIaoBCyRf75a2bRh59kdWq43W8CEX7bDpmZHPkceo30XEWZnT4ayhTEmGu3Udvwm6GY0A/vdgYNrWSwWMEVTWZtlu5GpcEgwekKbmu49nyFD5HL3qm8JCxMy43i8bwp/nB6nsPSu733b8ReyGd+6mFKcxeXikv9jtXEBZb/dlZ7ktnMjAM2MWc/TeLjthihnuc2ufT4epxklkJ8cFCk4HYwosj/eHCdZulkLiuxJ8yqp8l1pqtT8ydlM+nx0bUfX97RD4OPHH9DMlgAMoUdvN6gKZ/ef0L98StgfCCpUzQwfIkPYoH3Hs2dPWSxWLJdLmsWKZjZnsTyj7Xr22x1RrdKlDhFXN1Szimou+KHn/P4j2r7n/MEjpBI2r1+DKLPZnMVize3mmqapmc/mrOYLqspxOOy4ub4i9APD0BNjYLvfs9nv6UOgmtWpvYflrnYhcmg7Vqv5kXc77+ddkCtrxbEHarbF7HUb7mp7e5ejbyq0XQqfmHPLahERYbezobZ5CnQ+smd/qsWKDZgZdyo0sgDP/gaMicdp28cZPbk38NQGPtWI+Th2XOqR48YeOZIGKP7a47014+lG5afLN24xuKxfIDqrFIgxctjvIA1+sc/ad13Op1Olahao92z3e5TkfEjaSxDyYFP0OCHLmLGm8o6+tdkeitXL5c3KnbwtSyOSFaYToc52mB9jYQp0bct+c8t2d2C7uWG+WFNVM/oYud1s8d5zcXHOeeho+5ZBA9vbDX0/WJ+bpoHUBe7e/fuoq+j7gcpX9DKkfju2PrPFmsVqzdX1axbLNTOE1cUDqkOLhMCHH/+I11XDzZsXNrYtWN3ifDajbmoQYbvbUteOBw/vs99sqVIDp6FreX79CqXn8b0HNHXN/fv3uN3vuX3xhvuXZ2UpTxnm1Is42ov5p+btS+aENXgWEST1G/0mWHrkcZ20bDw/P+PsbH2UEVPG5SXCkdxykIR+skLQLKQ1pUBLKlifMJMm2HSHplfV0tH+Lo1+GtCfrtHY6S4Wb+wJpX7j8d7DUqfHdLPe9jRNNCjmSTwcdunhJ5uRoUZiPOcrXNMk+CTlXIUJSRuabMZ8JkGom5qmqenbrmjRLBxM4qVWEGlDrQwnpvhbla9Gwb9RURe5uX7Den3Oy69+iaL0QamXl/zgp7/Fk48/5frVS/7kT/4dr14+p29bNCrr1ZonHzymmc/YH3qGIdL2XWrMZelazazhcDgwm81oFgs++Pj73Nzc4HxFiJFqfsYnv/23eP31V7z56nPWF+eIRtrDxqYN9z3OVyyXy1TmtaM7bJnNLOHaV8L5xRmV8+y2G3Yb4fOnr/nsy5c8eXiPh/fOkWZGUFJfnSygJkwooxbJa5+bGoMUYZLNh5Gox3q+vEun0NZ+jnMZnUiZlpyJPk+fUpicL19iIjTS4BwpvKbmST5lshPGmj5rhr+jauHou0CxZU9twZKzmsqpcjOuUTv++uPdmVGOswu+SdJp0WCjryV34LY+Ilo2SDRPsRptRlfVBr+uN6eXT8X/1nBodNGlc4lVqldeIOWWRrVkdHF5hkW6qRCSZzLiArg4MKssv7WPgNfSYRxAh4799oZqsUK85/6Tx/z4r/997j98zGZ3y357ixGfweDLy0vrnRMClfd0nTWfijHiq5rVxQW7tk1eyFvmyzWf/vBnrNb32G+2Bdo571ktLvCPG5aLMxbzOeuze1y9esrN1RU4TzNb4FxNni2Jc2y3e7wTC5sgrJZLGxLraq6uNmy2B/btM243Ox4/ekxdV6l1f57nOPFgJmvraI8LbsjwrrIE8NKZzuZPSim94oRuRoYFikNGE/O9Jdynpklem5SWl+3GRHzH1zkSKm87do7pa3RQ3eW0iVAaT08Z2Hs/akqxShlUiyNpgo5/7fF+NqPd2ZFkeMtVPFnIqeTRGNnv9oQQqCo/asjigLGf4jyr80tub25GNa/TBxp14fikCTqJw0uku3mdmiKZJ9a7sYGveRjNKxnU6jEPfWAfHJX3mBNTs+Avl+jalmZ1xtn9Rzz48IcsVudsNre8fvM1GoeyHovVMlXPz9gf9nRvrsyLGqGpG0JUFvMl4mtmzYLPfvFLg8rVjOXqgh/+9BL5RcWr16+4OH/A5fkjqvsrXF2xuX1Df/OUX/zRv8Z7a6mxXJ5ZLJVgWqSqmTUNQqTvDmw2Gza3t6zXa+7fu8+jR4/Yt18hXqibOavVmtm8toC6HHdxy+56m5kycUZoZIrUjBFNIIa0tjEqp5lgo11mwrl4tXN+0h3OjmONhRF6use05ZRyvGIvjt/P9ijl62+zxuiHMPQVYxgT5dN7TnP3u2NtOj2naLZGJ8KB/wjMeBeH32U/3uVBMmZUur4lhh6rwnq73ETFcgcXyzWbzYvi4ymlVwlC5to5Qa0dYdGWDpXIs5dvsCE8kxImtSC3Yt5FVeEw9IQo1sMHoRNnU4mSTZKbBseo9P3Aq5cvOX/wAajy6vkX9F1HCC1tu6euan7rt/863aGlrmv2hwPSd1R1Td1Ym0VcTdd1XF9ds16v6do9dVXR7neEGFme32e2WvOJVCzOn/O9H/6Uxeqc83sPQYT5YsbL/Qvi0LFcrvDVnIvLBxzaPeGwS7ctVHWDhp7ZfMGQUsS6fuD65pqqqVguLeF9tlyy7ztUxAYVpREAIql3K5CnMysjzcdEcBWSuQHIYShPbiJWoKobp2DBJIMrC9oTiJtbXoy0ZZcoXnGxFpRFex8r2tJao0DrGI+UxV10Ot5ZLM6tI8UipIZXx+GaY+bWpBmT91aE/yhxxlPocApZ34oXTY5c42btODqaOrc3HBkxPQviPLPliuvnvyAE8/aZw2WiEce7SmGIJN2cw81XfL3tUIR2CKktvqMbIuJyVULKDKIuTiRBGJKNNH3G3JWbCNv9luvra84ftrz8/GtEHE+fPWU+bzi/uGR9fk7fHLi5uWGz2SICTTOjaeZoFLqU49rMFixWK262ez754Y+5eX3DvXtP6IbIk4cfcXbvQz78Uc/5g0vOzy9Zn80ZwkDlFvxyc4uK59GHjzkcWpyruX9/xc2rF/SHPT4E2kOLEKlT/DD2PTFafnDwgcvLS169esWhPXC72dAeWgaN2NxpLTMTQ2o9OGqsjIZyE8Qc15sSuBvzUPWYbr7NIzllyOnnRsehpd1NR6nnsFqhilPmKqb/mIkzzQx627M70hJ30PNdzpw8UAesu559Z2w7+U08cdfxXsyYYcvRMJOTG50+JJgeyw6Xoets5Nt8Bgn2mF0HIxwVVhf3ebo/mBFsdaiTY9STJ6YHTiqWqzW7Llqxr1QQbTZWHraiCEHG/q3iju2RzOBTWDRCakuve/70GV9+8Ss2uw2Hds/3vvcJn3z6Y/aHHV9+8SWKxQBjVNbrNSHArt3TxwDO4euKV6/fcP/+Y1ZnD/nBTx8yWy7Z7loWqzOW52vatqVpai7O1jSzGSF4yxoaen7/H/6vWMzn/Ov//p/jnOfVy9e4qMxmCwtr3NzgBS4uzmnqiqD9xAGhVJWjbhqePnuFamTWpGLoJHuGGCcMZc8vWSjmAt/iqaQQntHBOPgnT7LKx6mWeNvUSYovv15g5/H3RGSc9TLduSRURzuVdO+TTJw7aHR0XKVnQFL6Wyzfz8jQev+mqyhWbiZ5KrM9QG7inJn0XcP+78WM04WbvnZqM54+rI2nitYw9rDj4vzs5DNaNh1VFqs1u8OeoR9sLPcUJE8lqyGCgtFFhPV6jfPe8kl97jA3pm+pc4wj0dL5OIUtbz+HE+ufeXt7zccPH1PVFcPQM18sOTu3Jslvrl7TdT2PnjxmNmt48/qNaZ62o21bqvmST3/wI5yv2O1akIrZ4oLv/eBnrM/PefP6DU1l1SfDYo6vPMvlHOccvQzstefTn/yMs8WM2zevWCS4+cEHH7K/vaHdO148/5qhH/BNzeFwYLmYoQmVVJUvhcLr1Yrr21tQ6IeB69sN54vLvFo2EWCiMTQ1s9KyDQ4RK4kLqaJhDG+lzWEyFzGtZ86qqarqBL6d2F9ZI2cmdSOTAcV5c0SfShrhl4yThJbuCs+M92TAe5pSR/bwKpAh81QhJKY/VQYQUzbapHbyDufRNx3vHdq4yx6862f6sOmx1DEsO3HKA+Rc0YkDBxGa+YxhGOjaDmbV0ULcdf7prszmc9xkqhFlwUcDPOeenlrBU+EwJR5IvWCicthtWZ+tWJ6tqG+uWK7OaBZLPvvVZ7x+/YLL+/fZH/Y4b63431xdsbm9pe977j16whCh8o4Hj57w4sVraAbqZsaDxx9xef8xirKY14Tg8LVnNqsS9AuwWvKTH/2Uoe949fwpTz58wqsXL5jNGq5e73j+4mu6vrVxAM2Mpq6xTnFjNYT3Vq3Rtnsq54q22rad9ehpMsqZjGRLcwezQMv2IIwNg4W3BXWWr1OSyQjrLvo5pa0cWin3MnGcnCoEmfzLCY2nPo27rzXSx/R85f7QErdk8o2iD4p0ykopkBFWTgWNekxn33S8d0fx07+nWnH6mdFDNUI81cB+ty2PyeR9IWm5qMyXcxTYtwdggWlOKV8pWliny6ggynw+p2lqutCbhE8LMmrUSfKyHBPDVNTluBGaRpGln9vthpvr1zx48ID9ds9ifYHznpvba5vx4D2Enu12m/5t6DrrDvD6xQuW6wu+/4OfEoJydrZmdnZGNV8yXy6paoeo9V4Fa3PpfbJjouBnDdGDzhwffPCQL/+84vL+BZf3ztH+AV9//gvarmU5myNeqGrrFpdt9XlqO5jXYd40DCGABvo+8urqhlldMW+a4rCxtfapBcbYwY0iBLODxwjRzAE/7lGCq6fJ3qd+hmk8Mu+BTuiCiR2Z/709LevIiCmHJEfK3cdxds5IC5rs1FHgi4ihvBOFXu5TJ2gxaslz5i+bGTMEPa2uzu/ln3fFIpmwZde16fGOVb8mzKlEZvMFeM92u4VH9yghBp3Ypsngd8lYiIlI5vMZs3nD5tCRs3eU48VQDbZ80aTieL/HknHa2s+eXTl0Hf/+3/5b7j/5gBAjq9UZu+2B5XKN957t9pbFYsV2c0vXtoVmZ7M5rq7Z3FwxryvuPX5EH4X68gn3nzy2qpFK8FIld3wEiXgXUmOnWHrWaFQe3j/nRz/6Ie1+S+gOvHzxNWEYWC6WzJuG+awmhoG+b4tQb9P9OCfUdU0cjIhCiGV989j0qEAwrXRsoqc8VKKZ85kpUpgxK5Gp2TJ1Zpwy5ZSGMpOX0EH+jJvcwzc4gE49o6rHfXxOQ3Djz/xsWePLhJ5zF+YprH0bnVHoJ6OopEhzIsJ/DJh6l004fbDp+6cwQKIiIbLb7o7g4KjuALWms943qFRsd3tys1qSNhPVNPMhr6JtmsMMaNc4lssZr6732JCZE0mGlMXVdN7p/b7lJSY5NTQS1Dq0bW5u6IfAYn2BNDWVc9S1o+sOeF8jIjSzGdvd1uZdVA11M6OqG1xdsd9v+PTjH7K8fER98Ziz9dpaY1RQIYhLUlmsptBJxHuLi8YYiASWTc1PfvgDrq+veP7Vlwh5JoT1qqkqT9VUVE4QXZrzrO1YzOY0zQw0F/PaoA0n1rsnRCWohS1ygXjukwNW7GujsgHChHFye467E6vvoqVjmpKiSIqjx40Mir7NUHddK+m0YmOOdu94HNmNBZ5mZ1ImiUwjauMNgDQ7IQmu9L0kfYzOLM95mo73Psd7lFC9zXxvLcQUopb3Jw+mVsc3etUyRB0hqGC9WKqm5s31VWGmI2SgKX4ox4yEKk1TsVouUH1jL002cpSEGUoct0eAt4epZGa0/inWyUxJSQBLq/Rdr1fEFPjP6VJNU7NerZON1DAMkdlsxsX9exBtRvxivWCxXrKoPJUTKgdeFHG50ZXiU26v5dQGAtYqpHLKfF7TdQv2h21pQy8aGdqObeg5X6+JUdhsbzkc2vJsT558wPpsjTjhtustKQKTUREb2FJNWl1kp8td5VDj34kYT+hhTJ3jrbWefi5rz1yneSoUvwmNHW3UXa9zTLOnHt2pcZI+zUhO2ZMq5PaSGT4jOTeWosGzzzUz9J2a9FuO93bgmFEu5GGWxmvxiIjLIpLHbytOLQvnsNuPTDq9SVUbygnUVc3Z+Rmb65skqjLUzNoxay2Z2IKQm+bOF3OG1Cp+wnuMxvrxYn3zs2qyhYTSAsU5s598xXw+57Ddsl6d8frla1bzBbv9jqqqWMwaRKFp5qxWZ9ze7qiqmvP1OWfrNX3fMXQHm/yUu3+j+Nx9WwQ04h1ImmI8uIhqj4sdTgOVeF4++5J//s/+W371i18gDj58fMlHH6z4+c+fErqey4szExLAoR1o2w3iXvLw4X3rDB5ycN+Yrg+BRhsTBjJqK5efmykELcVKjFkwFGg2FhF7rMtfPKqOOA1XjJ32xrzYu447nTGjPE70NokZTr7zTde+mxayRswimRKDzd5eEdCQvbHJhiZnJB3HSn/d8Rdw4Ej6/+jnvsuTqoVhJtIMoW0PqaVCVZ5z8q3sh2I+X7HZ7oo3y9zLMc33zJthjaaY1J95J5ydrUbtPbFNcxPmAkCO7vfuDtJRLWNnCGMWR1VXzJZrvPOEMHB+ecnrF1+zud0QQo8X4eLxQ+5dnuN9xdOnz9EY+eqrz/mzP9vz6NET7j94QjVfcPnkE9vUVLMpAl4ygViKnmADXUPsIXQw9BB7Xr74kv/H//3/xrOnT7n36D77q2tuN7f89j/6ERfLms+/3vKTn/yAz375eeqTE9nvW16+esOsaVivFlSVp+sCKPRd4NC2LOY258MxwvaxB08maA8qxGRG+LTpUx0w1SyFgO847jZtjm3Pb/v8uNfpq3f02i0hqpPY5l3nHV+fnj9pPzcmpZ9+bioQ3obhv/54/3pGs45Nx6SR129BhOS0iBrLeGsRqzaPMdq47Gp66WPbUcQxX56x3b1K6t/+5Y9lmDCCjJy9YSc4O1sng1yOA64JP5yChlOv8DRNK7c8yt/x3lNVNpKsrmqaZoaizGczDkMg9AHRyLypiRrZ3d7w8sWXbHYHDoee58+e8fnnv6Sqa/7oj/8t/8X//v/A3//H/xU4iA409a6pU6sI7wQNphEldOjQ2ZSsoeNf/ov/llcvniFO+P5v/ZRf/bv/wOGwo+17vv/pRzx9+XMePrzk+vqGqq45O1vz9NlLNps9VzfXzGY1F5eXvH71qvSc6brAMEQcAz5NgyqlS2SzI6/bGLN1lSvMy4mQg28myFMGKEJcFZlU6kBGYW/nhR75KhBrkJ3Mo6miyAI3w+6pM/DU6Wifh7FVy0hrLvVNnYLckrSumux2U1bvyojwF6jaIM2Ch7ftq3yYFHRvGbC5iLbve+azJqPvdP7MjOaRWq7PePXiM2unnz739jTjPFsiMZIqROFsuUjTh6RUekzM0jsx/KkDZ+zFJGP8KmmrZr5gPp/jqorz83M8ViHh0i11hz2vXjxnuVry9Msv2e82/O2/9w/Y3/sN/sO/+f/RfvEHLBZrbm5e8wf/0z/j3uV9fvv3/iGiznoFV1Zb6VNeqBVjBxgG0AEhcnP1inktPHnygK+fPucP/sX/gHeO1bKhqmd0Q2CxmrO6uChDO9frNZ/UM54+fcH19TVPn73g0aMHVE3NsO0JCsNghcbezQhO08yKTKjZVHBl/4+aRSUBmR05Ix2OhDxd71NCPbURY1TyJDrUetXKHTHK6Tmn2tidvHc6S6N81o0NlY+9vFmQZCHhKKPCkwaOKVtpSls6WYd3tRfh3TN1xsU7WcS7jOpsa2UnjaYEW42RIQYOXUs2hKcMWabZinB+ec52vze3+6TmLdsBmUEkaqneNwZ0XKxnVhkyub+YpdYd9zp9liwRo+rJHVoQ13uPTx3n2nbPbrfh5uaK/W7Dm6tX1krROc7O1vzwpz/ip3/tp3z40SN+8MMP+eoP/0de/uIPrMnUbAYCv/U3/wZfP/0lm80NQpVIyGJ1koqSiyBMfUydKC+ff8n5asXPfvJ9FvPabDuJrNcNF5eXbDZ7Hj3+iGb5gCef/oTLDz7hwYff58n3vs9Pf/Yz5vM5m13Lq1evqZu6jG6zNnZiydJ5byZ2VkY40zXLndqOCJ0JYU+Q0zcx4Tc5iDSO/ohSW3GHvXf0vdRC5YhuuZsxSp8dfVtQTD29pJDOeP0EvSeZQGMN40TyC2890zcd79431aVuyrkS4kQFH3u/YOwmY5BSY0RjQIfA/rBH5XKEjGI/c/WboizWF7TtQB8i8yon9049tExRgv2RPJHLVTMSxYisj+Z2nErhI2lJXkspHQaskFyomtoC+1gI5WK9Zn9zxWe/+Dmh6/n44495/OgjHjx4wNnqDP+9j/nqq8+YNY721Wd0N895dutw7iVnlxesLx6w2bT8+Z//KX/7736I94JLCdGGQmxcmmiw7uxVTYw9N1dXbK5f89Mf/wDnIj//81+xaCr+3t/+LcIAb24O/O5/8k9olvf59MdnPDn0bG9vePn1r/jgyYLDbsPXX31VtFs1qy2PVxOBxgguDwGV4sjKe3AKD5XUC1U1NUyeEL7kyU9aCLq8dcJYx4cWJ8k3pbVNOwHkcziR0l3vLk04pYEpVL3rvkbbcmqsuKwJCnFpFASfXsshNVNe4RsQ5OnxfkF/KJXTU0iQ/74TtmZjVkyCxBhpD4cTA/jkK6o08wXdEOi6DmbNiIK+ScikNwVYLBa2SXkybbkVLfdzFxG8ZdRnLaxCdq6oWl7lfGHxusN+yx//8R+x3W5YLRb84Aef8uDBA5xzDEPPvQcPUGBzc8Pf+v2/zquXb9gdDjhX8cMf/YR7Dx7y2Wf/hptb4ff+zj/CV3ODw8k+K7MQo+KddbALKux3LX/yx3/C93Yf8oNPP+ZnP/yUEAfevH7JV1/d8nv/+L/gx7/x+7SHjv3+wH6zoXaeRiL9/oYf/+QnrFdLfvXZrzgcDkZmyS5yzo+2GTLa/Sdhh2MklEqWBNQd22r+jrV+y9a7Q3vZNXmLgbJDafq5acgiRi0o63Sf73IIZQF8+nk7V0r2Lp5kyEkPmUicSyMSy3cnnlylNEH+dce798BJgiHbA6cPM/1dmbStE4gSQSqzfwQ2202yDe3pRo2VoJhANVsQouNwGOAiIjETBW9dOzMJCRo1ixl17dl03eS8xxkep/AlG/bHGw8k77HzjkFNu7ftgScffY/lasmLF8+5ubnGecfPfuMnPHnymK7raA97Nje3PHj4kJ/+7Lf40z/5Q377d/4G//l/+Z/xh3/4Z9x7cJ+//4//KdvNlqdfP2O5gqHrqdcXOBlMwgKqglOl8p4Q0gQkET755GP+v/9SmC3ucbMZaA+BIPDRp7/D3/vf/C5PPvkpotC3B+qbG7RvYTGjCmtC5agdtOdn3N675PnL5/RtT87YHYbBcmOdIwr4tO+W7K+lQdSpMC0VPcRUfuXKvJTpWLkxbc6OzEzT89lnXUJPWoh7qum+ydQ4Zay7hP7R0JvJ+6ctNe6CmKMNmes+payPNU9LQiGl08X4l8yM3uVR0G9LqumNZxdzHpNmCW6aaZoQh9ILJ+eWujyUQpTsv6ybhqpuOOzbb7yncWNgCiPqumK5mHGz22O5OSOiPbJ9Jm7u0+oCO2O6QR01lc2ksJ6lzjt22y3r9ZInjx7w/e9/Qt/3nJ2dUTcVu8OOV2/e8Ju//bsMoeeP/8Mv+I3f/Bv89Dd/myHC519+yb//D3+KqxorfA7RIB5pVBohBftJrfcrQm8w6Hvf+4T/7D//L/nd3/u7zBYrfD3DNTPEzyGNUtMQiF6ovbBezNHDLcFBNauJYc6Tx48Y4sCL1y85tB2zyjqXhwghGjTM3cUtrBHTWo1jF+7SkgLEbKak8QPT9omne3h6HNuOktFqQUVT7TylvXHfTHgyseFOhe2pVs7PN6WrKV3c1ZCtaP/xzrF46iTJQI9sqW893t1mnIw+mzo5mHigys1l71JiSPNOjv1R2q5LzgirQo/5kfKio6XU52ZzCzzi2ECcWMeZYRh/rSobi+3EEd4qA09/ZSKaMKRzjhgC2XVrtqKj8Y6AIDHSRwtv1FXFi+fP2W22fP/jT/nd3/s9Li8u6A4HVJX7Dx5yzzlUHHWz4h/+p/87rt685uunT3n5YkNVL/jpj/8W/+Af/lfgGv7wTz5jiCCi5bqOlK6GaUaJFYQeVceTDz/gH92/h/MNrpohVQ1+ljyDNrg1iFI7gaZGZhVuPWdZK0Pf0TTCdhN5+PCSTz/5mD/e/alds6psWrIKQx9xOlDmKh6FKKy2M6MSLXsX02RjBSIxOd80E6aOWTbT4y7HzvhersIJRSN9q2NGLDhfOVdGwX2rEJCxsv+b4PLkoxgPnJ4jpcJhaMIKjDNa+0tmRkkjwKx6Io/XutvotZouRkkGOI3WgArP4dCjQXEu14pJgWU5XFFVNbP5nNvb2zJwRd4ypGEaahGEgHk85/PZqDFPv3LkrNHsjE9XFzQ5lKxqwlPVNThPF4Cup6k8L55+zavXr5gv5vz0Z7/Jw4cfEjWyvlhZyZKvWazOmC/XXFze5+zyHh//6K/xe3Vt15OKGIUYlGGIXFw8YrlcWNt67JlEDTh6Z/hevAPviAEWsxnzeUMIYgRf1ZDyRmOEIVi7QEXxoswqoVrO6bzSdYIy0PU1Z6sl3//kIza3W25ubhmGntvNlpubwMVqyXo1YzGvEYVKnDWMUusJYCMfphoGxGnZw8ycKramUfMQmRHiTrXQaPMda7Cp46TsdUZhd8DV7HWNGktu6l2MPr2HiTS5kxGn9zl+PP1dFBWYFzyMJg53d1S/63j33NTpzeWF08kyTVubJyKPIkdlTpkB9vt9qq7OPdvyAhsBgc1TPDs74+rV61LgKeWB7eHt8/n3nOVhUny9nBUpJpMFPt7kpMETrlYgZEdQqtrOQf6qaYiHHpGB2+s3DF3H+fqc3/3bf4/vffpD1qs1qsoQekSEZr5gtlgyX65YnF2yWJ1R1x7vKpAKxKFeUKdUlX0+IwNJXktxghOPF49IzaC9ab4QcEhJCsB5xOd5EsJATCPXIoQB0R4ngShKXVdYNsnAdutZzud0y475bE7btIShp+96lMib2w37bs+9ixWzqka9t+Ex3qddshVUJSUHULzn07U2P4J1i5uGJ97yyOoxo+XX7O/sg7D6yQnvHIOmkyPqpFPbiXl1lKZ2B1w+va9TmzbD9/L9RIAiOf0vv/WX7E1VlCig6SZcMkxzsvXpDQuC19y8LzGkmr0zxIEYBsQ36akVFas2MGwb0QiL9ZLb222y9cCroqVE7KSrdLpLxGyd9XpW7NRsQ+jENpVEzE580YSoeStjyswXbD58U9dcXJxz70HNy6tbbjc7nBN+62/8Ho8++ATnZ+AttunVrlXXc1bn56zWK+bzZYFmIoAn2YQGaYJCDJoLpMjWtmRoiG2wc5YapwwG8fGJGTOxWRwyakTiAKGH4YD2O+LQIcRiew19z9ANgPUoXcwqhtRVQKNj37W0vcWEN/uOR5fnXJ4tqZwvcyskOZOyYCti9S1NpIhU6Z4zO7qkQSKl4fG3OAbHF0cvagwUIVyGYU6dPJJDa3a/p97/IwE9ceTc5ROZvjb1L5iCGTN+wPKYi/NJvkVSnBzvlYHD5CFzUP8UKowB2lMrzQ4n0HV7a1nR1EfWXz40SbPVasn2xStgzIQf8a+5Ce46HHCeWtVP8zCk2LIpkC5iHcezhkzfdSkP0/k8ndcCz/cuzrl4cJ/N9sD9Dz/l+z/+ayzXFyaaqhnVfIar5ilRfEYzm+GrBhBisHQ3IZXnOcsRcZLbIAI5kTqJfSepWkIp6CCEYM2p8FRV6jOTGNS0oQlIQk/sd4Rux9Ae0Dg633IigSV/C6vViscP77NYLNjtO2bzBc9fvqDfHugHOPQDXf+GLgQe3fdUrmcxm1P5EQ0ZtDR71bTXlJmk2OEhhR38NzDfXeGOt4+Tz6jCNK7JxCuuOTf2be02vW66S9Ns6bW7nHrTe7CvJeUkRlcx3YukxPncQPtdjvcuLtaQHS1jatDp50YHkiYbIiu9tCgxcOg7VswZ34TsYdX02cXqnK8/2xBCPM76AKaMWAL1qsmGcZytFzg8OG9V+ql9ht2DjPFSJRG0jg6pVMlgnb8CdeXpuh0Sey7WFzx4/D3OP/gJq7P7VM0CX1V4X1M3DeKb1Ig3pYxpZAiCMJhOEI/L15FIOLI/kjALgQBWFhU7ogac9oSup2sHwtDj5ykHk2hFvfjUTArCEAldR2x3DO0O1YA482hqJIVolKGzc3WHjvmsMRTQ9PQxME8NskK0RPm+i8SrDeI8F6s5ta+pxKWGv6lCI/XJyXaeMVeVkM1xvWMORZlSOoak+fcjujuFsIkJI8mzW9ojJmdSIRQ9QqB3obj8OnKsXL7ZoWTX1mzbT4RSNig1U6p+g4a/43jv7nBg/UhDHsM8kSTTGJCZYUopDsaa1sYQiCHQtR1ZJ8nkhpUxL3S+WtF2PX0faPxbbeLSs082SZOFKI75cmkay7k0mDV1hc6bTYY4GQYazItpNkQMqWQmWC6t3XvHvKmpl2ubs5gcMeLMyVNVVZmdCEIYrGu5aI1IhQspbS8VDpowwwQIo+u871oOhy19u6MiUkmkcUDsGNo2MXZtcJ6cKmfCJPQ9h92ew25H6Hu0twTwPCpOsa7di8WSi4sLdptbNEYuLy7ZHVoO3TV5nmGOreY0r76PPH99RdsuqHyF9xWVS1C6CDxH5cyJAYyMmOEkWuy/3Ef1m8IN33Tk+LWSHUJSaluRMcieaWmK4o49oydQlVEjnl79LbhqRD8KEJHU0jPzwJiM/q7He3eHi8lAFyFhco6wdgn0E4ttlh4HMGLv+579YZ+fJ5tJR1jVagEbtrs9XT+wWswhTziGkQiZnD4tkgosVxYH1NR8SpLr2T4rozTNRjcWTsBhME9iec8YpAcNNLWnnjV4J0mTpCr+qsJ7651pSxIZhmAe0xhRiVYFESJ91xN8qusz4GoPYE2AaA97nn7+S778sz/Gac8wdFTi+PyzP+OT733A7/3+32R1Nk6MMo0e0TDQtwdu3rxkv9ngXcWsWSZIaPdlLnyoc6c4EeqmQYNjUGjmcx48fMjN9kB/ruzbgf1wKEH5GB1R93hv06/8bE5OZ9SEmLLgzd47UySZMKW0xzy14Y5oiLcZUkTGUJS9AAnRJBRcXtdoAfecufZN3nT4hhhioutTBp7C8lNbkQkDvg8T5uO9mxhrEgCGxSlacGoAZ8dJ+WNy5MTr/f5QJPoxrEjPpdZcSoNyaLsjXH+qH0cb2e4pKqyXc2aLOX0OYJPGYiebM8cYheONBU/UoXT0MnPNOtsd9nu6w56LhZ3bNVUZzGKBai3Ma8nxFnPTGHBO6OsexOGiOaO896nrderRrQYlh3bHH/6P/5x//f/5f/Pw3op7F2fsdgde3V7zyz+EDz+6z4MPHtr9ZelMso/7nu2bV+w2G84v72N10NNJTs4KvZP0c+KofE0QYfvmmlevX6POc3l5j7YLXF7eB3fFfre3lh8RDl3k+etbBOHTDx9TV5Yc4PxYqSNSY82Ow5GcjYbbUpuUb3fa5Pudej7FyfTDE5IZf8seXUna6i4XypTpT4+7hEC+n1FoZCGf7+Xt7576U37d8f5TqCZY/+1nSeo5e7C4+8E0Rnb7fSHcwoiSH8Ag4nxugzIPuz3CfTN40oeNkdMtkX/PoBMW8zn1rKE7DGMW/8maWLWWSTQRsVKjBGnzmWNUhkE5HPa0+5r9dsNwaGnOrOGRMaJBXBiDzEkH2HpEJQ6RMAScWNmuqBuzUlBrZamGHmqvDIcdMQ70hx3z++f45YyXbwbW9y549PgxIglKq6ZWjIHDdsP162dU0nO+nlM5IQw9MFbdZ3OjrmvWZ2e4ykYOHNodq/Wae5c9+64lbG1/1pfnuKbi5YsX7Hc7s99VGFBeXt9SeXhwecb5ckWVOtpN6YFJHFIZhdXbZPW2NrxLW95BdOO18nkk7St5f3X8fXKcDnOd3kf+zmlqXNGm4kr3PDSZNydq4l1txXy8BzPaovoiydMFJ5LBYITY+zokqS+T76efQNsdyA6bkwR/+4rz1E0DVUPbZk9gnDBKvq6OX5ycp5nXVkblbP67rZlObkcg9bPJE8qZesjMwAK1cWnzZcN6uUIEhm4PobNJVtnpNLmXNPozdXMzh75GJfQDFhT21Gnzw5RZQ0+3u6Xbb/j9v/t3iLsb/sU/+294fbM1ezAq//R/+5+zvrygPVjHhGEwaPr62ee0t6+5vb7COcflo+9RVfVR/V/p5ZP+Oe9YrpfMwgLZeNrDwIOHwvZwIIqHV1cM3YBAqsQY4V1Uoe16Xl1vWSzmrJfmGHLRnHZeKPs97pX9R7Ec5erEjvsmDfJNsHV6TG257Bwa3zSTyk2012nSwen1ZPL73RfE7EOsY31GiOLMfp5e/i/dgePFHA3qHHhvHjlGRrLHjxYHzI4UJg+biNwnZ8JhfwDNhJuhhHGmpHBCVc9o5gtut/sRauSNUVLHsnF1slUrAlVjDLRpe4PBDqLLTqfjYK+Wn2LOlJAcKqmEyWJGjrZtzS4ZdmhMNZllXt9khgOa6ixjgdX9EGwmozrqqiF0A1GCzbYIxojXz7/i5vkXNAS8RP7m7/4Ov/jsF/zBv//3eIQf/ehTfvCzn9AeWlppCSGiQyB2e7wG5pWnOlvSzJc06zV+trCsnLQypZQnaUfvPTFEuq4lhsD64ozF+ZrzPtAsr6kXS66vd7x69TI9q9LuD0xjvPtu4MXrW1bzGcvZjMpZXq0lhqfV0LFtR2ECESKS6Oc4+fs428W0aUFNZdtO7c3RH1AcEfnX7ONIdHNXhs3094zeTo+3PLGF8cdSLkhVI5PP/6UzoxmnOasgpg3RI8eLMrEhUzMqEAgp3OvApQhD2/ZlxsHIijKWviBQ1Thfc3Nzc2Rb5hkIJEjsklZ2qbuAqHVVu7xY8fp2Q4wZelr92ei5Bas/s52yBXXEYg+oeVYlBdJzP5rDhm57jZ9fMlQLW0i1YL9ILnkKI2TNjoYhQIiEvkfUtI19NJgH9faam1fPWM8qai/MKuEf/L2/zWeffcb+cOA/+Sf/kPVyzdB21qZjOLB584Lrr3/J5vlT2u2e1cU5j374E6oYIUa8F0scV4uflvtyNRGH0Fsyh3epbtKBVy7vCVJVLNcdl/cv+PKLhu6wIw49aeob2Ut6tdnyy6+UTz98YM6ipiaqo9K056dOmcxoooT0vrXb/IawQ96sOxTjsQ2XbZ1EezIyhZ1CCYlvM83cdT4BmAiPbwtxqOYUvbHqxLmsfHjrWb7teO9WjVmK5XqyEIcivaydefbw5KWRxLMpEJogQD8MFryuJiELjaVDHGpDRZaLFbe3mwIZc1WHJIGQtsKg10ThORGW8yVUHcTsVEk2pzJhFMXVNWiwuY1CKlNyafqvp2k8q+UK7z1917HbXKHV5+AafL0wTSCWJZNr+USmzgYpntOhHxDfp/o3kmZViL3ZcB99THvzimG/Ybe5pd9t+Mn3v8d2v+ejJ4/Z3l4jAn27Z/fyC25ffo6GjsV6zUc/+Al+dY6bLZitzlE3A/ElNJH3DleZ6Ev2sqtqaipm8xkxKl3fc73ZcLu5RZxHdWCxqLm4PGO73RnkDmFS2e55c70lhsj3PrzP44tzGqmI0TOly6kzxrScIZucpI3qGJa4A5pqPI7pndp6ow2Ys5eyo4pRSFJIYGKxfHsVyLcdd/lOpiZLfu53Of7CY8SLBOFY8jB5eDTFktINFzc80PedwSxv5T6F2/JZktZbrs/Y7tpiB0iejiuKx2rncovGDJ8M8zvOlguk2uHV4VPT3pg0vIlGxvtVb5oviLWgkEglKdNMlH3bUlXCkPJCtdsTD1eE9pbK10QnDLHGp4ZSKpKqGk6ToAPd/lBa91sSeCT0LYpjtrrEqdLGSLffslit+Pjj7/H106e8ef4MFwecBNrb1+xefkXoN6gTZkE5tIHzB+ec3XtIp2rNoNUETkze3bzKERN8iGe+XJOzkqIqbt7wpPbMlgtiVK6v39AeNjx8+IDDrmVzuxvzi51pkADcbA8srzbcP1/h1VqHEKwP7jidKgftMS2sI8NowpKn0NFMT9uMU6fOFNLm6U8FQiUFkemrfJc7lWxhoIzwpm0l70puuYsnjPbMGfi+4Y13Ly5OUq30DDlJFbKwx3jDpzdfWC318OyT84GmQieV9DAumIqwWK24ffUF6DiiOScnG+xMyeVJK9vmWXrSelUjVYPk2Q8OJLoyDnp0/CmoNQd2jU8esgGGnj50iCpNXeFSXFFSAkLoW7TdI/MOpwni6ViSNSZKHK9JjL3ZlH2kH4YEuwfi0KOhQzTgKg9OOGy20HVczGd89fM/4c2XjdVT1p5aHL4+Y7Fecf+Dj6mX91Ct2B865usz6/LmquTtVUI0727UMZSCuBRfdalsDKun9BVVNaNtbTzAs6dfs98fGIYBERt113Vd6kquEBxDGHhzveH19ZpH9y5RZ60xNQmnuzSaZIdgtuVlXKdCW5LtcBD/7c4e+6TFGEsBcxLkbpps8Gtg5Knd+s1OpDGcNhUOVVW9VTD96473b9U43sN4w4hV8yebr3I+aSCDDKimPFDKaLGuH2z2w3LO0crYSpSLLNYr3ny2I4YBX5mdaLa5JgdOaSvLcfU4rFcLXDVD8WaTuMqyi9Xifon7rbwLD3g0dIRhsCbIGglqcUL6tMh1Yy0a6xmVKP3tC1zlWMiDpAl98p+W05fDe0cMkRgHdOjo2tbsUiD0B7r9Le3tC7rXz9m+ecXN9Uti33M2r7m3OjMBIUpdReqm5uzsgqaZM1suqJolvpnj6hmI0Hcd9WxBGHIiNskJ4izP1XykqQ7Z4HWZ+EtmEmtc3PUtq/U5i8WK1WrFbrunbfdH9OBFGMSx7Qa+fP6GxXzB2WpBFDXonp0vBdqONGXtOnJK47HWKg6cyd/FDJhARJH83khCbzl5ktA/vUY+pv10fp1WO/pc4gFVUiWSHgmVdz3eI+ifCowTbBPnLO9UU/paSo7VZCOOEISSB5ofwhjHs+u6cZZe+q/52LLkhtnqjN2+pR+sIZNELWl2tnlSvp9r3ZwIAWdd03xF1CQRwe7NVYxdojXdnyMOZgdFHaxNfhysDAkQ7/BVQ9UsaOYzfO3RwRIXhhvoU4e0euFQX5nXLvmGvHfmWYzBYHbsaQ97UKXyjhh6htbCJT50uLBnOXd4zm1mRu1TXqnZWFXtmc8XloTuHOJ8qmmscHWNOo+q0LUds/mMMFiQxTlnmtgWydbK+wJRy6AvNYHpvVVwLJcrLs/P+SrNVAzBsovMTstcZQwSVbnabPmzX33Jjz79HherBS4EI1axSpXxyOjKiodVcr4nJee1eEHluNUiBR1l+D9qLithmvBwsRcznaX3dLQdM9Pna+VzTU0rJu8de37tPix0opAKElRGh+S7HO9RtaHpXywL5hFC0JSILeSxbTGG7LhMC5o9XfnRTYrsd50p+QJvj7ADIjBfrmj7QD8E5rVpxIQJiq0xfmWUaE6U1WKWhlWmpO30exEeCdKgpi3z5otYGp14b4NnMI04my+oZzOkquw9JxZPG/bo4Q3SzNHOo/UMmhlZqGcoRrZvsW4BGiL94UB7uIGuo/GR2XyGvzgn9DNWel4Ir8qVFt6lpsIw9AHE42cLqvkKEfM+R5dnKULbdtR1QwxDcXKEVGFR14LirAvchBmzU8k56+rmnWd9dsb6bGU1mV7oOjUzIyeGi5kGTo0xrrcHPv/6BasffELlYxph4MkhitRX820NIrnqPtPDaGdq2nPTQm/D1VMHzMgoUxszfVayjaplXEQ+4oTRpvBU83tH5z82q0jXUMY6ync93tub6pIKjgim/BQGu51ItNkNZU3yzRhUSjtBTtzdH/YTRqQEUTPGExUWiyVDjLRdz9k8N/kZHUWZ4vMmlcVxjuViRtPM6GN6P0YkeVQ1mL3pMDNPo+LwKNY4K/Emqg4vwnK1Zr5cJoZoUpNhsyPr2iOxo7t9QbUM1O4ebvD4WpLbfkgbm68p1HVFkAGNVvkwhJY+DNRO8PMzZqszcvEPoU/PJAWaqSr10iPVHD+b4aoKvCNoitOKNURWVdq+o6qqYt9VVcWQcjedq2yac0JxqlpqVlFwTqmbmvlixqypmc/ngFjTqhiTFk1FSjnJQyFGuN5s2ex2NPUFMUJwCs5Zy0nhqHWm8aUrSSOgozZTx2hV5IB8WZ1MoUnwTRlySsHZ2z/acTJhzML730T/Ez6YMugoJJIgE1KYZlRG/xFCG8nosLJinJgHLW1DelwSfuZI4k1hf/5sCIG2bYsHVsq702Q0qGczhqDs9gc4r8bPSIbHjMXByMjbAnVtZU06WOZJ0AFxJizGkiy16ns8jgodnDFPcMRocct51TBrGvbtgTaVfok4i6FVQggdQxhwdcQ3C4bWEsnVKSoVQ0pIl0TggsFey8aZUckZcVET+wMy9AgznJrNCiC+SjpCS/W/S2Va4rzBbu9TPDElLohlQxnD2PN771PSulB7P7aLSF7KmJ1wRVOk/UvJBcNgAcaq8jaQtu8TjB8dMyW8ECPtADfbLRdnZ1SS+9KkZG+xdL6KEwZC7HlVIeTMltQEy5P6p45ifuLTZ4S9b5tFOaZbphAfMUjWaMfe2fw8OnlvPOeoHQu1JtrXiePm1In3bce724xJ4mVwp6Jlw623pxlI4xZOviuMC4UvG70/2KxGN30gxkoPUaiaGa6uuN3skA/Oku2XE6MTY2Y4yMRAj1BXwvnlGZvWPIgxOW/EOWrvsba7IX3XoK3B4IF2d0W726DJ3kn/J6qkkqmGuqpA1QjepYE6cbBGUmFAvRBSCZN18BuTi7O3T0SQqsZngVI3aOgR7dGY0u1St7gMeXNJFGIZUU4c0TuU3M1Nyvm1GuGZiuCqyuwr53DeiLzYSk7M1pcU/pGYOhRA5T33793nsG15+fwVu+3OGi5n30AiYp8aTochoFH56sUbzlZnPJmbVxsMDjvAupanQT+ZUJL9LpIJRwu0zbuejDyytjOtGMlJM9mWFDmOQ+bSqGwXaqbrRHoikuKYJIHvkrl0dxwyw+CMer4p/PGXrhkzw8nE5iPBzUAwW2FyA3eOiJPybUQjh8N+dKykFTG4ML5Wec9yvWSz2xWmsP1JtmZ6zvIeSXeq4quaywcXzKIniOWAahxMgkVFQ48xpPVDJQ4MXY8O1jk8Dg2x6wqxmTwSI2RXIVKBExvr5iuqZkFdVcTQgw6EAcR7QkhDRgHUMX1ixDQBCW1oyvRBK1xMoRsZv5FrDU2b5WZHyYQQZzgpeS/NnnOpxs66oIvzuDKM1eErMWhLMiYkdyFKKzmpW/XiLW1uueKqviJ0sTBRjsm5FD8Fg/6HduBXXz9nfbZkvXB4tdCS8w7JwjsJ6yxccysXcrtFSTdW0FmGgLYfLrVHnCxqorvM30IudyqCSVN/VBlT35yIeZUzYsi0Jcex4vzaSOcntYvZRcC7MyL8BUIb5vswqJo9Xa6yqn+n1l6irEcC0tngtQdOoQTN9uUEcwDjvI18GsF5bylxxe4cmfDoSLZUEm3MvLJohHYQlKQJXWWJ2YQ0kTZpYclQ3DLwi9SUpGW8OXSaWYOFAjxUVh1fVZVpRjFNWdcO6C3zp5zcTZ7Tj+4sl4jSzRBN9xjNqSTJUUVRGJZQYEM643g+VXxet7z2iWEdxpiSqgzyPmaPo4qMHdSKg0PK3qkqYRjoDi2bzS273T4NU61wbijXnyyiDV/1Qp+6K1xvd3z94iU//uTjNN/SkWOKlndg0FtDTENfJ+fLz5ttHyAH1vNnVEeTIyNQmXx+eoikKcwJulvT5WTqqEJMAq44i95mqOmgHHOmTZyUhX6zZvj1YZJ8vHdxcXZ6ZONbxXLxQiUw5GJLyF2LhBRHSjfmAaeWTxqGnjD0VLXBFyEmgpl4zrxnsVrb+HG0MGLMNuPkKIwICA7vlFmlSJ82IWVjqgTEm4NIYw7B5PF1oxB2DkJOb/NGzDZ5ObmLxJuGdHUiioir06bEiLgBjeCrhrHFoCspYmWwTQL/th2ZAELSEJS2+oVIVclODlRxpZN1ZnFGyJeECWntTPhV3OVYKKJJx+vEGOnalkNCJt475rM5vvKE3VAYI4eiUGFWe5raE4J1II8h8Ob6huETLBXP+5Krb3LDlTvwKeZYQgeS4CLZWzChxcSouX1o1gDJ75k+l7Vi1mTpqvm7jEom5u+qGpqwN9+yJSVp03H9chpeTHM2R4VxV/bONx3vpxkRbJxyAjUp1ke6QefHuRAuZcvoZG+djB2ao0YOXUc39Mzqu1tqGGc4ZssFt5sdUaPNbUgM6yalXDDB6GKCt45QOywGN3k/B2hlci7y4Ln8mcQEzpG8ktE6qg1DsScUTY4U+4KvKsSDhgFziICvK0pgODFGbvLsSg+ZbCtP4mBUFNdYqujKzGZCJ69/ipNmEyDl69pruSrepXWK6USZCIvitfuFJMltPVUtTBWHAQ2R66sr9vs9Q+jp+pYQBsjtPMomWzWIrz3dEBmGHnHC1c2Gn//iM/7aT39CVa/BZeIe9y53/caZli4hh8wI6RHKkknWQGrPqpJGCCZ4PjZUYkRA+br+6L4znAXK0JxvOjLNH404yNeRJIjzufiPYDNKMqyLRGasD4siqQlTyv9M/zOJk7SdJnyOVUAQLXDcdwEWp1ez1DenVve2PD/jzVdvjLgLfKUQ4BiAzdkmHicW6F56xYtjEJKtmiVtTB0AHMpQnglnzBKTxq+8p489XdfR1BVd2yGAr2rL7nEVMW2Mr+qy8FZsatre7KGUJJHiay73hZECuJg4yFMrx/Raym4qki1Rpc1wyI4zX5i6wLRyblub0mR/om1Mu45mwWh3mxNnGHpCjIQYmM0auq5js9labWZqY3+cBmklZV4dzunINcDzV2949PgNs8UCX80skSJ14MthG0dMvWTi5P4mPlMvZe+nzpUkOWx8eKGJlCASs9FhSMNs2hHyFjoiohpGJv//M/dnO5ZkWZom9u1BRM6gg6mZm5l7eHgMORa6hq5mNwtEgU/Ap+QNH6BvCBAgmiyA1QTIZnZVZWRmZERmDB4+mNuk0xlk2AMv1tpb5KhZeJglI8GSCHVVUz1HjsiWveZ//SvXK1AFvjQwDwTMGK1/o+HPvLb/DEBx2RQ5RdWIzBAnXRRrLd77atZPU8GJqD49xpFMYqqzGreL18lnzZYAzs/PeaPAZOPUXizW4sQVqJtJ/tPZApZbLmKe31dXXArcJb4yxhAq74vOo48SYzTtRunqlUrRCtExRfyEOHMRK+R6XdRXieVKGK3TzWkTsWxKAg3qgkKdG69umXHI+zTGLS5uPYtZxFfGajyurpxZtEAvLUQRXO3MGIeBw25HfziQUuLi4gIw3N5e45wjpJkh0OgGFrZ5TTYZue7VqmMaJ/bHHts0tOsz5cLJ9buoHuVSTWL9c9kHuoQ2L9ZouTdzxmanzH7FZVcEjMCX9BwGiPW5VwQRopjlmBVBWvx1GUMukTklo1tAE0tMbBk//iHHR7mpKUViCkQUd1puaSF4xizaqxYj2WKaM7HGSPYrxaBzN+TG58PIgwBMtnSrNeNwJMQIvsQPJ/t79ukzVAC5ga5RX57qXEtSQunvywbOplgPOXtZv6T3YK2lbaXYH0MkpoTzswIqgXyFzxnpyHgnPizxj5HfOSX2tTqsBhOrsplpHUy1dABmER9ZrWGKVzCbtwoZW8RZpbFW4lZTpF7eu7h2gBAjMUZMjozjnpvrN7x68YLbnYxbWK06Dr2QdJX1zzmTrKFpmsJFBRoSpJSJMdMfB3zb4jqZoWmsILmMKmxRcKEqQYOhzGmp1gnU2mkJREdNZJMlyZWSloU0w4u4/UkBFylbTOGRlQWt97DYUXVvo2tlSo9uTLOPDwsq0HpGQTDVuJ0POj5KGGekQa7xRUqzlUopSeE4v/u+otqKdUpJ4s3j4cCceCj+lbod6gJ1qw37Q88wBjZdJ4s83/eDzzq1MKvGkFOQGt+JpZ55Uu2cfps3VXnA5Eo6lXJiv7/n5uYNz9crSppcLLM5ifoEMFzEv/ReKhuAsdU9tSX20Pgpq7IqLk5S1zuDVOmqHFqMEbpGo8ky0NhLhTGzdPeMuqdGw4VqcFgCN6rHFxJTf+T++i1vv/uWaTgyjgPjOIgicpamcUxhqu8DBLeqJMsmF8qTRJoCJsOrl2+4v9vRrbd0XYuxRqB+iCW0ZEhWu0Ms1jliCPVRW73H4oEo9RA5R1KS7G5WT4Asmepi/GxBxibpUqnd/EbflWQEgZxQxyOkAjJZuLVlYkiZy1j7RZe+fq7QuRoY/4Hjo+Bw5UEnjA4iFa2dlS9yOXpb3zULmlELVXGqEFNkUlRHleiFG1kEv1uvlXMlAJ3ersYMJxspnwT82cC68xiCWuZTLSGDWGarmHKuPSASoDtp+YqJ4/HINAzcZxlWc/XJU7rVFkmViztSnkVJTMzWbL6nImRm8dzE6ywZWrVazKWLlDUsKJ6FXp/8LpOV6t+aZXySi/ldfPb8c9Z/yra2QohsiqLU+vEkxMnrtqV3hs1mxe54UGLmCe9FwEuVpQwKqpYYBImULKuuY5oih0PP7776mifPnoKRmNzYkgRSjyYhkDm9TeuchsO5emHFMxJ2TSlvmJIZT1HuPUEhNy4JTmO0fJmioJYWiv/ErQdsjmTlf7UKJintZ8UGGSfTik2px8LJ92pdP+D4JzcXY9A6odX4qOqDhdtqIAsrd4FAqX6pLko/jHWDmLxI32P0JqFtO4x19L3ElzWzVh5KzguRLJsVjLW0TgTOFjWR58UyxYfUB10TY/rebAWDOQ4j0zhp7kUgZuKCmxqv1o6RRZwEM9a23LdcpfT5SZeSJIvmdD0zxpMkw22yPqbqTi3cq4wU8rGzAsglFT+vtbxvAdNaPMpyTSlnRSrNNWBDYuiFlfz8fMP17b24e9Zycbbl2I/0g7Cll6J/hcV5j0/SSJ5y4vzijHR7z89//kvaruNf/7f/imZ1pjNPikcUSXFGG+WUpNslU93suYTlZtcyZdljZf4Hrs6FLHEeaYa3mTK4pijp0lZWpDwp1tk69S4kseOyKK6kMahACNHcZtLPtyezHv95ShvFR6qkP8U6afCqDzsvNkO2Bp9tFZQSHxZHsj8eyfXfpz57+advhFRpvztgzBNKMA6LB1N2aM7VBTRIAqexmZSdBtwzOXGxrlWTcbpRpylorWyOfY0xOpm4P7nXnHNlwytp8nJtBsXSlrPr35ezIZfHnIyydVOKUJVrV6HOVKbwWYnNfonRc8/a2c4CvNRDCNtbStKFE6Mk6ZyFceg57vfkGHHW4r1jt5+wxhCmSdnaly6uwAPnkYGyNtM0MQxHrM3EKfL3f/8Lzi/P+PO//Au61UowtsYgPZYFTAA4K4mZtHzGSNwG80iGLIIlrqmEFC7HmaktZfFc5QlhEaGMlGdbst5iUCTGLjVGcXuLzOWKGxYhTnpNxhpxgcsYc2PIKVZL+4eOj7aMs9BojGOkAVf+OC9Myax5Y0mlu6OY9ly+MsMo7G1OF6GePVcbh7OWzabjcDjqNRikVcTUh031z7WMgJQNhOXbYNKc8i53Ui1M+b26qRK+iIVPSp+gRoacZdjocb+Th1uFEXWPNelQV0y0vcSMabaUZrmWVK/BLGtX+nvZ1BHD3CNaBVrX3SjCqNhjU3C2cnML9veymzWTmWFizhrHJJ7MOA5M48h61bHdbDkeD4BA3ay1pDERQtCijeI51VUriRbnHLRlLHlkmkTIjDUc9j3/6a/+M+eXZ3z+xecYZ/HWY42v29KUoasmKTfsXHvMPs9Dgso9xpJFle4ci5Munaw+SioN7xmTipIURZdSAJ2HafRZFeIz4+xC+JE7VoJmh0ANa67BaJnGiiBa4xfq8fuPj2YUryl7M1uVJTqhHHkRvJ50UC/OmVkQUzlf1nR2ZbPiJZ1nuzljP4yLd9rqtpnil1K29uyOOiMJgqovjLrCpbC23Pi5eChZXTWZPZ+zFPpJmZCS0k0IMqV2fNj5ekTQS+HXUvCvaJ2rrt982eXiTtd6/gPglLcGJIso1tbW5mrxBaSXJlcnpKznUjnkhXeSF38r1iWEQAgT/fHI/d0dx+OBYRiYQiDGqVJkijsKxmZpCdXPS2o0rBNCr2mcGEfZ2M45XMpMMXB7s+Nnf/P3XH3ymIt2Vd3HEvfKN+WePSlBSLbSLba5uLOaeEuRigAuFjWjwitjzXO5yAwpBXVttSyj2WpbFRegHS/krAkz6ZAR13VO+tXyjkkaL+d/DmHUB2tM1cji7Zk63msplDV4RetIxmIKE5r4TxgMISja3qnFyCIoVSARTha/6ri+vdcFLA4Wc/pYLWSFNaq2txgFRtckf3mc4oYsYHVZi+tJ3W6crXMhSJmo8VTXrokh0h/2WGB7djYLYjH7IJ9BImeH0XHgxqAwLNH4xni1ZKYmnarpLAtPcZWqo0sFcJmitAp8S6kDl7HKwkouhbwWd/Kc2hLgRoAYCONY3c7buxumEBYut2BQu8YxTkFH68mjiTEyBKMFelOVnPce33hCLlyxmbube3b3B7bbC3xbYmNTky1C/SG74KRG/KCOnlXATBUmEdCsWIgcM9ZliEn3Q+m9TJhILaOQgmb67ayw5WSy7jkyjxJxuoCpxqNJY1CTjCit34ejfs/xUcJ4UiQ38wY6fd0slDXrZUqcJOYddEPlTN/3jOPIpllXDfbQw7bWsNmec3NzS4pJYNZWrdsijKjXoOeWzFmic4GUkGBcr6e2DuXi38t7U0qa2JE44aQDPoM1nq5bc31zw8X5htYbwtRKF4IV508KzeVNi1qZAuQNBfywXNxZSOduDDkqR22NOktSZp5tUTsHKGWN5SQlo65yXpyzgCBKBw5Vq+ccOR4OjH1PGMc6E8QaaQy2RrpprHE47xmmoJSdcpkhJIwVEuMy3lVmtMhnNk3DNImLd/36mp//zd/TdR1Pnz4T6EMGo61Yskzq9tWwofxl4UVYsEbiTjGGCQrcUB2pFLUpsripGtMlq2Uj1aM2Gilp5EV5IitAoLB0l7DEqvLSnMBSeS/Jzz7k+GBhLPMNk5YApN3k3cbL5QLV+lsJoovgZI1ZsoABxmnCsKp/r8JlipExdKsNd9ffElPE2VyhRzAL4skGRky5M7D1DzZhvVbd4Po5BunHCzqfokpLqQtai7Mt3reMQ8+rl99wfrZlGgdxJNcdTtnQKh40I0X6LO5TSgGXm9k1NKUsIlOMRZhO15HF7+YbLn6HuMezR1C+m3Jr1RUvYUNZB9QtTSEKUZaWqxongPVpHDSWyoQowPWkiZzsgQBt47g4P2cYbwgpz1w0IdE2VssWUn9NFPd/bkgG+Oarb/nsB59xtj1ju9nIGiRRhuWGigu91F/zd1MViVWPK9ckHzWeXbKam7KG2eCyNl0n5W31CaLEqdYUAI/E68U7cpRM71xJSKlCxsUT0e6Qd6zL7zk+MoGjruRC2Ky1Ndv4Pmq6ggR53+9iStiUmMKk7pai/5kXS6y8oV2vudsfxd83Sdt9FHBtZ0tyImg5Y4ismyUWkxPLXbtPslhuk0Ugs/eE0VBqf1LQEhDCMPSMfaRtshZ9pWm5cP9YHYhqjCYPkiXbKF3tqNWun6kQitoGZE6u83TdlqGC3GMuXjtmRtWgPl6NJ3NxbapggsZZUUoHSePBEEbG4SiGPQfGoadtW5x1DL0oncZ7Qhw0xmsEaWQthFBBfVljPeekpDVNRVAzIQZSFO/AO8/d3R0/+y8/4/zijNUXn+Mb7YKp8pirMVquRVkvua3i3RhkSK4689WlmddzuU9N4bnNWSxc+ZQorXSSDAJyofCM2JhU4BFvJzUioIW8u3hBlWnhj2wZ6+Zl3vTLoPXENX2QfCgbR0JFWxc1WbG0++ORxFXtLSia3y4ewGa7oR8CU0y0hpr1fDfZsYyNMhBZNam6qCX5kbPUoyRm0r9YKzFcCPXhCiJjTliU6x37I6uVZ7/fs9mckaMlTlqqyMjoeY2ZSifFbGxz/VrGdWQzYzFZbji9s+pNlziyLrEg6tUbyFVpUnHai9NKV4JeQtKaaUqBGAPDYU+/3xHHgbZtaduO/X5H13XSsWIMx34kjhNhgqZtaVoZaT6Ocm5p/hf2dLvqNOs8CozQF/7dct+S+n/z5i2/+Pkv2G43PHni6LpOqfiLUtVrNyUMywulNO+Bh2u29JZKM4F1c1baLFa0Ka6wurjCeiBeWMmu2+zBlgytCJ6NWkIpJTDdvyFOeHtqCL7v+GhCqoeZoVP6vKK1TdU+xXrCjNWbU+Aywed4GBRtv1zQXN1WyKy25/RTpB9Gtr5c9hLRs7QoeqXq4jU2U0aFF/cNtHnXGjI63ThLgilWfKkn08seNwVjGcnZcL8/8ne//A2XFxecbTc07RUpRG08VaY06yTTSMLkqF8ZSnfA7EdSui/Kv4oRq+5kViiYXVrQ2SqUPZc0HS9LWTC6trrtZe3rd7JQRcaROA7Efs/h9pr+cEfoJV4UoWy5v7/XGlpSIdYxd87iFP6XiseSs7IKWLS6wDAFrLeKw5WhPzYLhjfGwJe//h1ffP4DHp1fEr3HYeVVlQ5yxhjPca5olYdCWRzGCv9T8HqNgWpoUndN8SEENGfzjHJzZa8ZSUKamb5DxkpY8S4K86DiuFxRhg88nN93/JMQOO9DFDy0UKdBq1IMVku3hAllDn2PQgzfca/LvTSNtCcNfU/ebKk4y++5JpChnGsHxkSE8iKjdHYqC6a6RAWib3XSVrSKYUQWNcTEOEwc9keur+/ouobd/T27/R2r9QZag6lUiqoMSkIoRcHIpgmiJD+Ed1awZHNUZ6sQZo3NxQ3OZOPUE5G2o1LdcVncwpQjOYoLL83xiZiT/mxEAShKRAbARnKa5GsaGQ/37O9eM+xvCePINE0cDgdevnxJjJGmaUS48pZ+ioRJJhoXhFV9AqoEUc8pRioznTxLj3PSI1namQyWoR/4+uuv+fyLLzCNpzNOlaibN395bsXKnySl5n1Xjcc7ntrC3YdiZudYUl+wbEgGMF5Z6SkxsCg04x0mW1I0+Bq7p0rvv1Qcf+j4SHa4BwmBhQC+TxDLd2utCppuMMRNxAi/5xQl0K1wODPXwsr8jG61whrHcRiBs7nxdE73LK9WrYMkJDo/l1kwQpUgXkeptZkqlNYKZ6rwphqlxaDGFGEaiUnc5XAY+MfffMnz50+5uHhC06zkYWVkiA4Cf5RzJ3IIGCbVwAa8+AY5qztrTd3IogCK+l/c4qJ0YiUzQc4ybi7FiM2ReQ9pplhburK2ghWIWYoTOUzEaSBNR8JwYDzeE0PPYb/jsD+w3+/Z7+7Z7e5ZrzesNltu7m6ZoljEEAPZoNOLxYMIKeOSrdYzRWGCSzmTYmYaguJfOVFcxli++eYFr1694ofrFckLyECypQq8NzPoAeaSTim2fejGL++toPrq+em+e+DvFoLiXCoIWSad6W8lS6sIKYP03c7e4Yddz0fFjHPg+yAI+UNHnl3UeeSbprqzkOHmlMjaPb+MEUTDCpu3857dbqyZMz3L/Lr6eeXa5DytNXjQBmOQ2pVcR4yqwfJMsOuUods5R9M2pOMARl1AjSOnMeIaSzaecYpM40TaJBUOAS8LBUPWjGnCGHFVSZHEqJ/rFbkcawKijAcoWWhT19os/l3gWCXLGTBK/28LU12Jf1CGBc3o5jSXWVQr0R/uub95wf7+lsNuL2Pqpp7D8R5jxDMJceLm5oaUDNMkbt+SucFYnaYMxJSZYmaYogAj1JqEmEhxtlbei7WfJom3xnHisN8T40RM0ijuSvLFGHJSAWaO95bK6v0e0sPtOBuT972nuO8lcDKg5S0dtJTzzLVamcyLuQliC0xR5B+uHT5+JJwxtXZTu/++120tX7J0Qr2PClzCGk+K4p9bs+isqHEPQJaxZW3Hbj9o1rPk7d5nGTn5XevtTJ/x4LWlHadC2pjjWTIzgkKFIKXENI5VOO/u97x5c8Oji8e0q05cOdcg3QKJcpnJWIyN0kdnEhixTJaShCigAFu1v6nXs7heVKGlDGi5JAYBU5uEM1bxk+IiZ528BYgg5qh6qtxjIk4j4/Ge4/01x/293J9eV0oCsdtst0xhYnezY787SmXJKs2KQSd86Zohwhhjph8mDJO+ThSy9548BcoU5eWI8xgSX3/1FZ//6DlNK2P5UkzaaEDxflGt+I7Sel9N75366smCmvpd3puYC8vU+8n1PBU+Iaq+KMry/lQqAnrulD9IQcA/BSheH+T7j4c3boxA0kr6OBUQcUlMxMSxH4gx4p0suElU17CAoq21nJ2dsT/0oBpq1lvvu1arGkqA4o2BwQCIIM/2Odf/lYJxLN3rSciU5o8xNE0L9FIzC5nDceKbFy/Zbs/YbLds1htoGi0WC6DYZflciRmdgKKzFUEyk3aVqFvqcgFAAurq6PoZxJXPivrIORJ1ipXLEetkSB5ZeFtTmRBV63NiFeckjljUcTjSH3YMOmVKvEIRgmkKWCt9h/e7Pbv9gVhgZ1paSlkA5s4YAlnXMCtbQJIhrKrIYgiYVhj1QphkT8RY48s8Zd6+uWE49mzPtlQqFdlRNZ62i6x8ru7CvE+rkDIbhveV3ZbJRxah2EmM+v4dpo5KAQUsBHDxpsqG8QHHPwkoLmDmUmt5/w3Kl2YlCze7vrYIpFXKu74/EkKga1tMklioevC5xBaWrluxP+xPPm/++IXrodaupLIbE1jbzCFm0hy9Vm2c86lIV/dEm1uNky5957wkP3Lx7gyH48h+nDj2Pa9evqRtG5rG09jmpLfQaLaNFHEpgJHzCpSqDKPR1yrmVAyjwPaMtdp4q0osRWKctCtA9bXGhFEhXUVpzgXuNCdwtDNjmkbxUKxhtT5jGCShE4PMxui6FV23IQNNsyNnAX63bQvO0g8TY5AYUGgPS+uQADq8klYLm7nsi6gDhmJSAuhF3Gat4+bmji+//JrHnzypwjc/GaoQLDz32XuhNB0s98hSKB8AKJavQ3opK7vJg/c/NDTlPksKKpPfec0i0/IHjw8rgCxvqJz6e6LSUtS3Rro2ypx3wT3G016vlAkh0oeBxbKeCnmWc24vzrm92xGZheh9+mup37IG6Y0zStWA9Asu2pMks6kZNGvwjcf6FudajGk0OBcIWAzT7LJmcI3jeBj57sVL7u9v6Ycju/0941gGvOpjqoIgSZMcJ2xO2Cy9cjkHUpIMIwp2zjFo94F8mRwxMUg9SzsWTE6YHDAkUpqYppEwTeq6itXMCKwrJh0Wq4sawsRhd0u/uyVOEYyVmSKrNTkbGt9xcX6JNZZpHIlhxFpDt2rpNit801RhKow6NYxT0yKfm+ozlD1QmnMzYcrkVG9J94XhN7/+iqMOyS0xXNkMc7knVwGvG+Xh3qn7cbGZip19j/soDDpL7pvytlwJ2OZf5aU1OHFpy3tKdvtDjo+n3ajB64xNXQbCSy1kjMXVPrWH/rr637phwxSLISz/OflsY2Cz6fjumx1lHHipLcEMOFicuWomYzNtwQyWXVIWb6F1ZUKT0XmCotkzYtmbpmFKsolK4imnxNBn7u72/PDZY2JM3N/faz+gWKxu1ZGNsHiXhElilBCxkBKXMEWCasHRlkx0uSG1yFkTRMbOWTrJzovbmhVqVmbXSBJH3NW5L1PdyxQJw4Gb61cc7++ZwqRtUbDabLi7uQWgaRtiTmzWa/b7kfV6RbaWMQTNAcz1ZIH1pvlZIj2XqXLlUIEixZqU/VKEyxhDfxz59sUL1pu1nDu7+dGdujEqBMWUPcwKPDx05sYJo92pEEssmN/xmBab68GxsIon/K3mPa/9/cc/oc5otSnoFPp2mqEqP9sTbsklDCnrxpJWscQwhTkQP7lNU7XNZrNiCgMhSxwoRE3z1KvZ9y99iTM1yMpEknK+LksGzjnJRCIujMUSM7V04pwjGqtxkiBVxmkkJlnuBGQL4xT59sVL+nHkkyeP2azXpBRI0VaWN4jCJ5sK+ZQjY+tTsHVdrLix1c0tBGBlnbVTIyl4oPBe5IQhzsmnGi7kSrpb6mMhBI6HO+5v33Lc3ROi4HFTNrx580aG2sTENI3VtXTW8+jykmwNxyFyfXNHWEwdi7lQr6j+QFqdsnKkWmOIOWl1RhV0spWdrjy/GBP3d3t+/nf/yNXjK548flJDmqVHNqPMSlvT74/z6vlPjejvP1SRLV9e0FGnrvP8TFJ9SLq5mNM9H3J8fMxoZi3wTgDMLBBiFe3MD/oggF6+L6XI/nAA87gu0tykNbvHm81akwBZoGbm9DNPjrLxcoYUWTfqRtXsqLpvBWaRTz5O3lswl0b66cI0iXVZxL85S4P01998x/Onj9lsJpxrmMaJME1473AmagJGzYUGJTkFiEORy/k60Gss4wHqA50z1Glxb9RYR5qYy6YoSYycSsE91SL9fnfP3ZsX3Lx+wRQiwzhxOAwcDj3Ho3TS9P2A95aYE9MwSqLFOmKE+7s9x8MgisYoyibN7YPGMAPDsyagrA671edjlTkAdXHn55gJU+TVi9e8ffWW87MzHcBjkFHncw6ivKN020NW2pLfY5RMEZBaGHnXXV0o67z89+JJlPctk0Xzjl1spgfy8X3HRxNSlSsqPXRVW+viWHW9XJklqPyb7wa2BlPYyXOmHwbqQ1m4AnNcCK5tGceJYRo46zZAWDglpma1So7UMK/j2gmMDZJiiM0siPr6So2IbByczGWw1jItaCSMOcW0TlNk9JGbux3jNAl8rHF0rcM5j9Xm4wodzMjOzUGZDvVmS5OC0Sta6omswHOKW1ouQJuWc8mOlnpiqm5gjEG5bSRzGaaR+5vX9Ps7cpy4vbvn1fWBX//2a8KUGMeBJ08es+5ajsPIMPQMw6Bs6pAwHI4TmYw1nqhN12kBiJ69f61xFiWtf0xqeUo4IyPYi2cgb57Gia++/B2fffqUrm2r51SOUhaqJQZ1DZXNBlNeo9dxajhm4zCDDh7EksuQZvH7E8Oy8GXnBM735fl///FPihlBY8ZKgy4NQ1k5RL3VtL3WkZbu0vI8jtLSAsOgA0HrhzGntfTzfNcyhsThOPDkfFOeHLMzS3XZ5MHOZduVCdjsSQXNv4xVQAvWUOpFxjpSHqjIGL22kkmu7ksWV2wKkRAiIWZev71ms26I054YBi4fPcaYMyk9ZHWfCwLIIkpJ3USDEaY4VRhF8RVTJzXeVDdfEcRi8QTULJnWsrmkcz8wjQOH/Z6xP3J395a3r19DGNnvj7x6fcPNzY7DccQaw6F/wXa90sRX1ElaTka+ZRmgGlKiHyJTijWWXm5etWE1v1CUMEAZGY6Sbi05fACl94AX377k5uaG1XaD9a72LJ58kqmhI0UK3ueNvttUMIMW3m+9Tl/7vvfLHlALWPeVqWFC7e74gOP/B2yqtB9Jh4zFWI8rAO4s2LycZjKnZQZ1vvFIzuLGjmN4cNHLQFyE13spF+z3PTALbp1cpQ9XDLVR/15c0pUVprAySrxYmqokcj75LINkT4NzFbojszcMrlEtvlCcIWemlAg5c78/8JuvvuFut2GYRj7PGe8anHP4pqEOVapadKKka4Sb11alIMz0tl7TbDU1gZMzKSgkTvGmoZI/CYooTpO4pTfX3N/eklLkzfU1L777jsPhwO39Aaxkj1MaCUjp5Xg4klpP13o2mw1d2wBwHAflMOi4O+wISuobT2VxfkKGmuhLc8pUYlHFcIrHIfeGSTgva7zbHfntl19z9fQTmq5Ds1uKwikbX59/qc8WA2pmnvZ5vU/31MOjAsvzbOVmaT99fd3L5vRPJ3mRhTv8h46Pb6GqgiSjsK0tbsbcQRCTMLGkhVU8KfbrIdOchKjo2PeYqFOs6s0tBTjTNiuMbbi/P1Q3J+dFwVytx0OQuiHj3QzylfvJLGewlwWXoSeWrK6T8548yIcI/UOu3dxJN5AxhkhinAKrGInZcb8/ykTjMJGzIcfE5eUjLh89pu06BUirRs3iQktZNgtEzpi5WyFLddQqdeHM/J0VY6qQOCWTmsaefn/AOsfQHwnjwN31a16+eMFut+P+sOerF2849oFhHLi6esT9/S1f/PA5z8Yr7u7uZLaId5ydrWgaz6pbEaeJECOd6Qh5wrpM03imNFI87+UhbpuEA2VQ9CmsUt4TNNvrnKv7XmWCaZz45d//mqefPuWnf/4neLxMPy40JHlWnuSlyzpfg+wh+feylvcwy18tn9GgJ88Ct1S8Sy9PnoWEMOaBbBd2A/eB4vhRwli+Zm5QcbFyLhZpFrwZpT8LYnrgspb40hgYx5EQY518W1YwL3423pGstC/lpfnPcx3q9+ENG5cxjOTcnZy/vKdoQWEwEBfX6ueZ8hp5k5BcWUuISblPFSY3TfR9TwgjjbOkAN4Zfvvll+xvr/nJT38kUDAu6LpOiHEpjTu16YdCAJMrF5BYyoIzNSlSMZIKV0spMk0TMYwcdnfc3lzz+sVLDvd3pDBx3N/x8s01N3f3WOfJIeFMYtXA43PLZ598RoiJH376DMwzwHB3v2McJx2FQOUQdd5jbSRWPC0qCKdlhZQyUWEWU8h4HftQEjkAQUEL1mq3vZ5HYmxD23mGfuA//9Vfc37xiCdPr2pZ5KSpvC5drp+xfMzfJw4PE4zvdSsz799jizxK+d0ysVNf9wHHh08urjMl7EK7aemAksFTzs0QTwRwWeg/ub8s/YyZSMyJIQaaxungz4XXr3flrWe93bLbHSnc3xUUZeaetnIY5a80GVoDnsCUgmbkOAnqi9uXrZHR4QYBrtfYTMDuRlmonTeUWaFJLdqQE7Yf2aw9jfc8urjAWUcYBg4Hw931Wz558hlhPWFGI9lm52SuqSsujbhnSV0E4eBRZ6eAkgGBwyWNWRMpSvdFf9hx/fYV3/76V/z9L3/Ds8cXnG86Nq3hx5895snFisPxyMXlOcdhJJNZrVq69Zag/Z3DOBJikvmZccIYGWvXp5FRQd3HfmQYJhmIakoXzMMHXJ6xUVSM3Ic1Buyc1EIRQDHoRrZzBj4pmfDN9R1/87O/43/77/4trW+FdUGhglBieRGXyntzklgRQY2mxK+zP7ssP5wkZt753btykZdJwEXCZ07kGFLlZv3+44OF0XtfB9oA843mrOKU5/89SNjMJQzJlColbyUVLjT/IQTIbbk1DchzfYjZWNr1hv1u5GRs9Ek7++KizVxQ99bSWsuYC7qiuKvzd/k8KcdgIJmksZvDuUZiY0PV2saWbggqtth6x3azpnOZTx5tcc7iDTy62HKxPWfoj9i9J63W+MbTmJaScarwPKPcMTJadlYY87Jr7kpbpFIgK1g8TCNxGnAu8fyTDdt1y2bl2d3esV13tOcdZyvDau05X/vKgBCdgYD0PwKH3YEQIsZ4iR/7nn4YOQ4DwxDYHwYOw0gMGm68q2sXetFUC1JDCP2VtI56DQECaUw462QaNJlxjMJNEwy/+vtfYYB/9+/+ey4vz6VGrLH88jOlabnEbEXgqIKbNbxYelbUH8ue5f3C92Bvs3xZLqRg5bczfO9Djg8WxpnMR2/HlER/sVFi5QokanZnC+ZQqXoKFhGdFqi7WJi7Swf1rJ1EdNSNs5ZuteLmbqeERRK5weniLa8M5dYxTmpaeYyQvcalmSXlfYkXspGJImQ1Rnov1AnCen+mDBiV55lM5nDsaUziX/3p53z6+JK7uxu882xXK1IYefviK/x1x/b8gvXZGZePn4ETDVrB4Er5YAo5sdX+xxJQk2FJGJ0Tloh30LWey8sLOJ7jwxFrDN4bzNlGEkc20XVrUoZxGiEnnG84HvccjxPZefbHgazgh7fXt+yPvcTFOdMPI/04EZJhijCFtNzL7xzVXcu1a1QOrfcKJWam8YL7zS4Kg2BOeO8r1xF657/7zZc8++QT/uzPf8r2bKtK0ZOMzk2hLE2a48c5zVGPspdlNYt5KMqiCNup213u5x0vrzI0FCCdxpvoMOH3Kar3HB9VZ1xe4EmZolxoLv1zWRZSL3x2ObTUEYtbK1hQpxOBhnFU2sQSnaX6eUm7NDbbDdfffUdKQZMg6OItYz+9ToMOFJK6YbtAAM9XPbvE1f03UkQucYj1jmDUO/BeRp6ZYz1XEUhillYpZ3BEXr14QbfqaFcN+/2B67fX7Pd7Ys6sN2dcXF3x3/yr/w7XtGw256w3WxnCahW1Y5XrU2/EKLVFyRBTKTbU4zCGtvGY7RaurvAETIqMU8CkiX7oiTFyHEesbxiGkURivT2XThKXcc2KsBv55tUbhjFyt+vZ78eKrMEYQillpKzWhxrqUldTf04CnpCNL886Ljaz0ylUgsnN8u/G4V2hhLREA9ZbjBE44Tdff8OPfvJDNlmc0spcqJ7FnACcfYniZcktLJsDSsKnunr1Dsp9PbScD/MSNeYsJrjgZc3DTP33H/8kYaQ6pNrTuPiwgo5YWsOTmzAywTeEoEXiefrR8dhLk+rJB8v9ScYUNqsV3xx3hBRpfZGYXDVdXX6zWFArjGGbJsNYYoj5QVUcqDY356yZ1mLd9cs5J4mGoljkzaDrUDRzf4z84rcv+fPPH5GyAOFBsszWeYZh4O3NLfeHIznC9vIRV0+e8fz5Z5ydXdC0HbloeiM6u+AzMyXZcCqMuvjScW8b3NVT1qst03jk2y9/RUxBGQ6kn7BdrcF52lVHiAnbNExx4JdfveLLr15yPI5Ejd9wXpE+EsMlWHSuzGs+b2MhZzNmFgWBFj6gXsyCoZV4UeCQbdPQrSXJljR2F1JmnaeY4btvX/DtN9+w3WygAwExWvEw9NkVT2m5l8rPJVzKD/622BH1TooSeShQxbjM91IWI1fsclmcPzpQfE5Ha2c3C0iWZpXMwp9eao1lqSEbnWFgMzkoeFgpIPqhl98vKOtnD0NiptXZlj6MTGPANK66siU2gHmRxUUqGbzMxk2yf5Xda85Fq+tpgewoTbol2HdWAO/ROtqmY9W1nJ+vCfEo5L16fRGDiYnjmGkay34QlMr9/Y71qsO6hraV77e7PSSk02M88vK7bxgPN3z+459yfv4Y3zY6VUnGm5XsYIITjbvE2crlG+mSYS3v846nP/gh4/6G2+s3lcozZUuLoR8CY4i8ubvjy2+v+frlLeMUCSmJBYvi/q67RkoNMTJOkWgkT5BiJp54GvVyFvMkc423JdkneynGyDBkUnS1pe4kFxGVx9TMMXNMEWLk269e8MUPv8B5J/Md5aaowxiLNJZ9WNRCjelmy1Zen1l2ZijPkO4n6QFVT+3hPq/hgrymVBKM0W6W9EcWxmma6j0ao42iisxYsAOdaAxj5CZSLkiErDHjHAfK68QCjeOoC7dwIUstCTBZehpTtgxTgPQuTcc7FJVG3SMyLaNmv9qFS7tIKjy4poKFTGSMk1HhmR0W6JqGxg1MxlRXNcUsAL2cub0f+MrDjz+7pGs7phA53h3ExXWyMcdpJMU1TJEYRl5+/RWb1pKnnqunn5Od0MfP4Pu0uCeNi6osinqsGt6WpNsWLiAOe8SyRYx1hJz4h6/e8uXLt2xWG/bHI9++umWMIvzWWJIKwfm24dnzx9zeHjgqd6oxMqcypMw4BcYws6mdWCO1Us5ZvG8wOQk/68nazwTDKSXiJEONoirr1XpNzpn+2AOimF589S1f/uZ3/Nlf/Klu+lKHLR6YEcGqCnoOq5bPvIZbyx3wMKFTt3dZ78Wk4rKn1U1ND94bU/7nsYwiHKlejHyJNpgR96J6pc1UZ2vkTDZOb2ahXTRJIpbV0g/aZb7ceDDHIybRtSumlDkcB9KFl37AkqZegFozRQuq1TaWzaoszAzqLZulvEdcKhYj7sBaTzSWKcqsibJxCiInK7t50rabnA3Oe45T5ru3B3749JyzzRnWeI79Xq2TSFTXNay7ljO7YtV5puM902FDjqOyAgA4cqo+HwU6tshZAQVKZ2bX3iEgbW/JObLqWgZGhilxGCf+7stXfPPqjq65AYxmTy2NM1xdnmO9eAl/8dMfAIZpCHRtIx0dKTMMI1NMNN7SxaSNxgsbqYrbOfEsnLXSy5iyAmmkZtt4L088RplaprDClASC13jPFCeZxZHAWMuxnxjGSdzvLFlnU5R+SspWN4O437+fl8epW3Xq4an3lUtyR+V1EV8mDXuSgpYNoqzeV9L7fcdHxIxBTHmK71zosnQB1AWQ+KuMJwIBBswuSvluMNgQ6PvjycIZs9hsUgSkbVfgHLv9AcxZ3XizTC8wkGoVS6fJunEshdzaOb4tyqEonRLzNo1nGqWMYd1pB8rM3TK7Y7JZJJF1GDPD9ZHb3cAXn2y52q7YrDe6iQOkzOHuFlYtj87PuDw7Y7vuRJHFoLFgWsQyyxkcCxeboo9mm1TqvmRh7V6tz2U0eLjD5cDtmwN390dtZLasVw3RGYyFTdfy/JNznl5dsD3raJzjZndktW7J2dDFlnEQ4qU0Tnr/iXGKQNSShdEJVfMeijFKrkBBBAWuV56f1bprjJExjFgMq9UK6yyNbaSlKybiKPw53379NT/9kx/iu1azlrm2E9a68UK4Tvfzg26KrK4x1DedIG2SKP26V2KufbWwRJcVhUAN5f/oRX/MXAZ4Xw1xvpDiQ5cuh7I4cxtPjPN5pLdN3KuYUUCyZblOaDxoMNqB77nbHatrgFniNjW7Vd+vT4fEqmlxjIQsxWKT6ziheq1GywfWiNgaI0icOaiUB+LUKoqga0NvptYbQ8w60k++39z1PNquWHeeR2drLi7OtZVpx+31DY/P16QwYljjLeQwYvKGYsXVX1Vro+ijUjagPPlT9z8XxWEczdljOlr6CfrdLV99e02Kgc3KsVk3PD7f0DjHat1hiXz29ILt9hzfiFuY7w+03nG/O4rJtWKh4hQJMcqw1SJk4t0rbrnsj6RWUYS1ax3rdStNBTlDirVzI8ZEnMB4EdOksx3748g4xTqM5u7mnkn7YCs6qcSPSH+lVZLrh7XBec+e/ly8qdITilrapfP90BjV8ydR6BInUp/FH10Y3yeEJ+WNPAOXa0aS/J7XUd9f2l2ybrBxCoSUaDxztquugQiyb4Q+8W4ng1Ozjicr58ugg2dqlEDZpK1NNBZisbjlAdaPschgznKN8mW9x3qvswPF3RLXWZWJftacGBDCY1PO0RiePDnj6mKtLT2JdduQk6O7vOR8u6JrPf1xTwgDzjf4fodtV3izwtosXDw4CmayCiQlHhKldIIIybnOc/TdBjtO2Kbj5ZsbhnHg2dWGtjFcXV5wcb6m817cz3Fgs1mzWrU4ZxjGCe894zgyhUkQJfoxzguXrXpptQZbvCOrSYzWCyGxtINKCaZtXM0lOCdonGkStgHvLc4V7tVCL6nWK8s4OiGTvubs0QVWca02P7z//E42dN6v7+nCUAGrnGA5V4TVacKnPPDleVO9voex6occHyGMcuoaX9WbUXA3BunAAJk7KC9cvr4cdsHSjVO7kktPXInr5G5LMkd1pLgsTccwCuXgSbxnZjtShKxYvQw0JuGJ6s8uYsTi6pV3Fm2usWbOMsyG4tpoYirreaqrarO2RumXKRlFx2E0fPndHdYmVm3L3eEF3mY265ZnTy/xmgVMOTL0e9qhZxoHrJfJtzPB8Wn7Vo1b6oIu0uzldSCJqGzohwlrPT/45IKmabi4POfi7JwQo7C1p8yUtjLsxstQo/v9nnGSeH4YJ6YgG65rOpquwY+B43HgRCOhzcPO0TbyFXXKlbU6Adk5fMWkap9oVCtU0E+lqfh9exKJW2dW88K4PqOZ6t576JY+2JOz2Mw10dnwFS9P9jsI5E6H+1FCtSIHJST72OPDgeJF7XGaLTX1d7oACbLLJ+OwlzFWXXT1t222Jz14MQRou/Kh8i2VfxiscVxeXrLb3avmRd3HXAVS1/b0yBnvbKVgz9pxftJsak5eXjwQmSplZOZEUIp7p95QTMXaz3a4PI8iBNk4fvPtW6zJtN5ytml4dnXGxbqFMdL3R862W5xzdH5F1wnZk7UzB06tZZUEVVro3JN1VjKrOWCpiibo75q25dGjC9brNdutQPbykEXRrVpciGAtISQO48g3r2/4+rsb9v3EcRg59DLvcrtONI3V9bFMWjcxFqwTN957S9s4nJMapzUygNYrWDykoKGJJ4bIMMgkKOcsbdvQtm21jsZIYoykeQZrlTTMi7+RZg7WZUyYq8icaIt3drgoMv1Z3V2Zr6joscJ2mBbCnWcfDCtjyzOxxq0f6qLCxwijLVqGhdUrLScarBuNL7RxtrirZZHEjC/A2Sf+IJBhnCag1eUpbiCUJTXGcnFxQX//HSlnfNER5VVKIVmPYjSQ17bOwlRfLtdmZNCpy5AKWsRIPtgaofp3zmP0yznBTnrvyVnKMQJ+RtAm+rCcxk2T8oMSE/2UmBKEcM9uu6JrPM+fXgKW/f7IxaMLfLvCNQ2uaYTWvm6G2UtYbinJoC7j+ahrvQBdQN2wt7e3XF5e6tDSgSmIwvFNQ4gR3zS8vr7mbn/gzc2OYcoc+4n7fc8YUi1jhN2Rxhka74g5E2oIItbPN5bVqpUYWHlenZO6LTkTplCV9vF4EKa4LLM4msbTtl11j2MSMHnnPFOo7ha3NzfqRpYG5fdbv6JoTfWTTv+u+T5dK90DRSMbfaYsET4lFFuEX9U1lWtbusMfcnxUC1W54WUrFFC1HainhIDAy0Qka4yO6C7aZXalXJZ0fEHwD+NUzzPDqzLFAbVY1tsNr18cCch4uFLwz+W1xmAoJMbLRY9sXIIhyjyNcl8a3JVZgCKAGbIVRJBRXKpz+Lah6zqatsdwQD1TjDF4a3A5E20mJJm3YBFaDkn6QGMs3ll2x4m73cjZpuE3X3f84Nkj7u5uWW3OmWJm23QY50kIZEzWoUyuerdDLuVEzLFmu8ni8holvUpkcC3GN5xfXHD55AkxjIz9UTLFjeBAb2/e0nQrhmmi7Tq61UiyiXbVkvZHQpRFLWPZpgjGxLqZjUFazJyjsRZvDV3rIUvXzf4ojeGjMopbqwLrPWEacQ7aVsaNQ2YYBqYpVBdY6qfabOacuN7jgPOCPZ6nkj10Q3UfLOMR2aryn4Kd1LwHKrQnnAI1hGIhhFneV0MIFfbFZ39oeeMjm4uXPxc/2ZxYSpNzbdo1mFpqAGFYKzGvoQi4FmyzZFLHcZwXq7h/i65Ng8G7hv1xIkYDjaOywC0eQo0cc4kZBDTlzOJBLa97/sATLTk/uOIezmAHQ0bZ+GuanpTIIYs7DMQoWtNaQ9tYPrncsl41vHl7CzHRGPjyxRtxDlKgHwaM9kq6GPHmNA4SNyzW2LhsG2mjUqr/nJVlIeEcYA0xipVvui2f/fAnRC1VNSkT0sQ0BaZpwjct3WrNJ5stKRm6zRlffvOCcQykKILQNJ44RcYxCiNcAu8NtpV1bLzUKr32q1pjadoG6yIJ6PueKQZyEm8lg3ob0mhVSkgxlh7NWGN3SebImk5h5NWr1/xpUNLmd+iLy3ObETzVBVv07tTttkjopKTteeZdoT4tv50mgZYW8qG7/IeOD+9nNNoDlnUuRnFXF57BstO5tgPNEkzpdMBmpanP4tZR3IBMr0if6h5w6pZla+hWK8aQGKbERedqAqi+D8imcIeXXKqo1cYEUpa2oOUQy9I3V+bWldNlbRyO2rMpyRTp7Wxbh1PN6q1QkCSNZdBGWWtmcMCqazhbN1xdbnm09Uz9xDj0xBTZ3d+zbj13tzuePJvoQsSOkyTFbNIMrlBHOhynvJ9yDRoMy1BS9V7kWjImSdHPWU8MEylE4hQoJM4xZ5zztOsNTbfiOPTs9j37w0CYIuM0CnDbWlrviMIVSR8ySa/FaxKjcYZ110qM7g1C0CwKq3Ge4DyTjcQssV9OEOJEStKp4XWMuYxX0BEBTor4tUkdsK6haTotifg5cVaSfzX8OeWhKUZgtmqlfW5+kdXhLCdZ1+qunh41Nk2q3LXx4WH57w8dH93pX26iXLxYw2LUF0Jzoi1mdWGs1vdSksnZ1pQKA85aJmUgW/r+RRjLGduuxRjLMM6d4fII9CGUa6O4zqa6Vqv2VHvWWLb4tA+SU6VTwmrs5pRd3KhbLbyr8vnOWbyTs4dQYozZffNOyHxNDnzx7BHrpuP169cMfS/QNZuJceLmzRtSs6bZ9mzW51hj6dZrUow03tN2XW3AnQ9NjAEhjAzHo8SeXuLviGSrY06EJCRS2cA0jDhvGKeBKRp+8auveHR1xWEY+fblG/p+JKVyP+IZhCnQtA2sDOEoMbMz4pI6Z1k1nq5tscYKm0EWwi6yzO4IIYhb7z3Wzq15xfLHOjlalE9xPec1tcp2lzjsD0TlvdXtqSD/hZAsrWXVsuXF1D39MPP/Do1GEbAHQraMH9Up1M/9cEGEj40Zzfsxr8X1LPFaeX3N7i0WoyyaNJOq7XLolKjINAb1x9O8wPXWJKZruxVtt6I/juSrlqoL33EJCnTJIlwsCW+NdAOo9pXrK8XyWbEUFSD4VCcd/hqfCqBZXmOthRh0w060vqHzDl/un6ydKEZY2qL0Cw5Dw1nrePbkjDBql0IKXJxtySkw7O7pDwem1ZGM5ezRJSFMnG23Epc1vnK61vqqLvMUJva7G1bbRxjXSIkiWWJCSaeOYFp+9buvICfu7u+42x0ZkuPXX35H275St1cEuMRNOUshPaRATInttqPxlr6X4T1Os6TOykgHDMQUGMfAOEw470natwqIO4p2UVhDo89E4Hbz75w1hJg1ds6EKFlskCGtKaPKSfedar9Un+Iy8jHznl38vuzvh90csDAstdQ1j5VfCqQ1pe90lgN4V6h/3/HhzcXGCB9MnmOXmqkqX+/LHtWAWP9pRaKNcdpiZeoiStG3jCyTuyhWTiyxIE2apiGlzOFwoA5OXVxJcdvmT9X2GwJnViFMWcdBLwahyPAWmQuStH4qw28k9m3aluhdpSS0aglclPdC4YtV0mMdCONUMGNMHPoJkzPX90fOOs/FdgO+IaUkTAdJXM3xcA/GEY9HvO+47Q8kk0njkabzZJRBTYmiwZBSJIZJhPlwS7aeptuQTWIaBcVyOB55+foNL1695h9+8zV9fxT6y0PANJ4xJEIc1OOQr6aQUVtxHb13WBOxBkHRGKm/WSeC2LZeGPTI5GAIUbCmU5xUiFDWiLnf1XmHc1Zc7BDr50rHv6FMm7YGhmmUtjRjGIZhVvzVK4JSm303ibPoB5UznmzTE4+OWWjrnjZWKT/iiSCWEt8S2MLJmf7w8XFuap4FRz4iz1e7uOAlflNQOOpEmhlCVohrpZ9NtLwUgHVEmGrB992Hd54YI3f3O+CZuDja1T8vxJKiW35pMaw6izFBs7lGA/RlEqfcy+wsWzIOiW9jSsofqlQROdWBnzU6NSKkOcs1LOOOfkrEOOKM5dmjLVOIdF2LDdogmzJjP+BDBmOx1uM3hnHq6YcD07ijXbVszx9B7nBeRi0ATGPPcDwSpwlLZjzsCWcyE3KaJsbhyLcvv+V/+c9/x+vra+abd/hGlFPblDHmYmHWXceq9cQUubg4Z5omNps1OUb6wxFrEs22lTXIZfyb1mEN+NyQ00AIkSmmmjkv8DhDki7/PAtpAVR0XctqtRI31ALGcOxHsaxGwopxHGUEAXNnEPosc+FA0tzDTL+foXS45LJfdZtURr682A9L26pbfhGyPewAKUJZzvOepO57j48Qxlxq62rdUo0HMjpCul52EdJcN7XFYJ2XDm5bir8ijN63ErTnhPHSSd65hlNJLOe0Ug9LmZu7HXP6eRYuuYZY35aVXS0DbeOwBG3hmZWH/oB5sPA1K4CM6s7GSLOygpqlfScrq52rL/feqaUvI9Dk01JIJO849IEvXx15+fbIn31+JYktRE+HKRHiVImDs3VShwsjQxxJ1vKpsaw35/jG18RY6Ef6/Q7SxBAyYTqwOhxpOss4TLx+/YL//LO/4+Xrt2RkkJBBmAy8M4zjBLatnfOucTx/9oTNZs1ut2O9XoviyIZpHFi1jjCNWi8UZe29FyufpA2KKGsRkb7HutfJSouptV48NsnP3hmMcXhvcL6gpxwhJqnZUsplmbOLLY3zKljisBsVzLIn00Iw5n1kVOAKptWcuJYZjT0rEbacQ5rI7eJVsxDWBNJi+5QmhQ85Pi5mrC5dUjqL8tcFREv/KxpqkcSwFm8tjaawrXVgrGAQvbJEJ2FjCzFC07z3OnKW1Le1jt3uQCWmWmAQH4gwlAW34J3A4oI5PW9JTpnaBLfsdyyxrqkseW3bViYDcVdlE8YQaxazcQ4ZbqO1VT1nCJl9Svzj12856yyffrKldfLIPZYYZbxbzBJHXh9eMY5CkZHuDZs+YJo1T55kmqatGcihH+gPe0zOhCFxd3tPt72jy45+GHnx7UuOxwPbbct6tSGEyDiMlSbCIPVK79XdbDwmJvb3O4a+pz/2hCBJpPWqY7VyWON100cMjpzFPa9dOWNiHCeWrWr6IKsSsSX7qO5qMgoOUNpO56yyVVpWbUMMiUmdp7Ozrba0FZDD+zf++2qOBXA+5+x9fXtV0A8SkTXZxELIy+t0n/DgvX/0bKqcFE5MMCV79D7YT178TTKl1lqatqVtGxFAqFkvYwzJastQFl5QkxTou/DHAYx1NN2K3UG4WIRxbsax1mvT/0oIKX9rraHxcAzFhX5gCR/crym4SWe0QdYxUO4rybRluV3h9tH+TmtHjHG4xuEpVB2lFad0GQjQ+uX9gctNS2MMUxYLJdObPLvjwBiltrkfJoxxtHcjybRY17Jer+ialm7VEbV2F2OQIT0hcNjfk21D34/0Q4+zjrNVi3OJ6djz5NE5XdfQeEfXdaxWHU3r2W62oGTMQ0i8+O4lr9+8ZbfriSnQNo5Pnz1ms+5IPkmPZmFoMKbGwMMYGaZMzoLSQtcuIuCK1jucNnCHFFl5UW4pJuI4YWLGtw22dTBFnLa1WRNZrTp++PkP2KzXutdKPfIUBL4UhlPbp/u5ZlWX+7q4vIpD1T/JNCy1qmnB8SQ3JrtQU6rl93/05mKrliMZQeEsb/jhz/V23/Ozc47NesV2s6oaJNWbnnlkZiSDImDKIiFp9G61Ybe/I6ZcQdYaLsyfWVzl8m8i3hg6E5EhMrPDAbNikZ91IU1GQMvl0+dFbhovSYcYazKngJZTkruxHlZNQ4ECWmM4HicZu+08EcfXL/ccLhJXFxumceT6Zsf9YSBmwzglQiw0JzCOE5tVR0iWpl3x+NElXdPw+PEjbJbCdyaRrSBvpnEk728ZxsDQH+kaS9s6Hp13/MV//5c8eXyBMZbDccAYS7degWuZYmYYI4/CI2LKXF2e8YtfZrp2x6vXt4xj5O31HevVJzSNp49j5cetuQM8KQVxhXWDxgcxVEqIidM9sFqtRIinQTt6jDCsx8wUxHW3BrrNirOzLZvNSu4XgzGSjZVJ2QkWcMD5mkrvoWZaxW3TZ3ua8HloHU8ALtlUy1ggn8ZKg3OKgZgCMo9T6DM/5Pi4WRu5XrK2pojBkY6CGQwk/zcnAlq64cHQtp4nj87ZrFc1eK6A85gEVIMIk80aiNeEikjcervi+vprYgwijJlFvFcEZna/Ssrb2UQn0b1yjiYUB3f6AEqcUL0OvTeNdZ1Vej+bRVYtOAwpWwpqLWtcaq3Ek0IIBcZEnMtYZ7g/DAzTxJvdyPrNgWGYuN+PTEFS+c4apiDKwCsgYQoj0/SanDI/+uGnUk4whs1ayKVSMmAbhphww0gyhjAl0tRzvu344gef8N/8xY9orGOKo7h/Qdqn4jCATyQcx/2OmA339wesMfzLP/sx+77nF7/6iq++eck4jIwhsG5XGDMxTVNN2uWcmSYZRW7SYrLYIq5KKTJmiRed5hFWXUtwTtxyVWoxBHIIUpJCKTycZbPuaFuHsfIAMkVxz+FFjQdVaDLaCVIeb4kVS4BZPamS9pAkHepmy3ChRJ6CkjwHwtgThhHSRDjsmY57coxM06DC+UcmMTbK2G2VL7VQHMDM+1EGuszh7mz9S2CdU8RmQXIYpa+vQquUYxHA2dntrKkurRuSOduu+aY/EsLEqm0ofX7lU5eg6nodGKxJtN5ihvk1teeNxftL9KtuZ0nEOecRciXhdHFOEwdGziO1toaYQ40pS5HbGEdOqAUV1rhjPxFyIsSR3X4ghFxZ12rsyqzIjDO18P/q1S0mQds6Ggc/+MFzUpbexykbhpiJh54z19L3A5cXZ1w9uuBPf/Scx5fn3N3uGY+BbGSUXEkBjMcj/ZS5u9sxpUx/HLk8P6NtLMNoONu0XF6uMYh34K1kQFNKhGmSpuOHG1Czn8unIUkwqU+uGlnLw36vBM4Qo2FMkZhGrLXSxJyEtmNMI6tVx2azEkoPV0bVqwVjKWBLK6e7oshg+aXu55QTcZqUFDqQp4k4TaTQE/p74jgwDQPTMJJjIgwD/X4nnaaFHd5KTsQZgQPWQSN/4PgIen8qjaLJarEWo9GqZdJNVBtuF754zmUOfRY4lg0zOTCLGE8bSeU9+UE0mIHEZrtmGCPHMXC2bcqnMZ/p3RgW5AF4C3MmzWKKsOUFhEpnWaDJJ+tKbYyayGnbhm7VkY696oosyippih/DVJIXWqrJSUiTkrp01lpcQjS7diY4K3fTeseqtXSNJLjONy2rVYfyRDOFwPF4wJg13373imws3bqj7VrCFOjHiE+GZowY53h0eckPP/2E548f0Q8HhjEQI4QYMF7Z6AwM/YGIpVlvMCERI1zf3HA87ME4UgjYDGfbLTlEbg83Mokq6/OKSdve5vJWiZkLB44BjLM0+uW9wRkkkUOibR0xSsuUs06Yz5XzJsWgbVUTOQrsESWisk69JINYMH30hYYkTyMpBeLYwzSJsI0DxIjJiak/gk7zIkdiCGLdKbA+2cM+FZc8YdqmTkhLKeo8lSWW+I8sjLLRxRWY4ydOMXjFtlezKP2GKoqVX8aWYmyJN0snVf2vflUtVqxiSVsjCQY849Bj2KibEjEF1fMgIJ9vI7N2WSY/KUrfFNS9nUW6vly1PmaSyonC0KzTvjzvsBimKL5prhvSKj401dpZykkygUGHIaRM4wxdK6MDYoxsNi0X25Y4RVaNZeUN23VL2zQ0TYPzDdMk4IHDMLLbR+7u7tnvLfeHiU8/e0bbtaxXa2KG/njANS1d03BxueXR+ZZ+GNkfBnb7A8NxJDvHdy9ekjHCMtCsGCZBznjbcDwcePnyJbvdnhgS6/WaVevoWomZh17KGznEedKCs6CZ5RwzMSVCFPSMGAtDY0UxNq3TkofsiRLJr7ab6lGEHMmTeBQZuNiuuNiumPa35M6Dk9mbqcRsMZDGkTQOpLEnjb2wxU1RZqlY8e60fKk71AjBmcadOYF3zezpmQZjEyYlvJ8ZBn0VtkyYxgUfDifcwX/o+Hhsao1pMw8TNhhOgLTvy2iFGOjHkcM4yIsKCAARghACzsBZK9bOqGadLac4kW3XYi30hz3weL6A9wng4rDGsmkiNjsCi6bTGl/M7y/FaWMcxgioWbJ5DUFdqa5pOZojBdhQgv1kkmTgCluYFbp8QaPMLAVlSb2TmO/TJ+c8vuzwqtnD0NM2ns57gYlNCWsc665hfzwSoli34zRx17/lOI403vPZZ58yBnHrjkchnhonS0yBmCxf/vYrrm/v2fUTr+6O7IfE3b6naRquHl9xfXOLyZnnT5/w5OKMtm1ZrxP73YFpmnj6yTNW6xX7/sg0jOK2MY91SznTOHHLJ5MXm1PQXI2D1gt4ftV6HKWLvmA8xVJ61zBFwcMejz0hSEPz0yeP2HhLuL7m5vYWSVfJLqnUGxZRumESS+fFdcxWs+/Wab6j0K846tSYLFQ/xaqlqp9VgHU3Jk1eOo2BpccVtcZJQ5plCPT7j48XxjwL1rvljIJC0cte+LBJrUSYHHslk3qTUwX8xhAZdaTa1cUFn/zLf7HIgBUHtKaI6FrZsMdhoDSMzvUiZuGoN6Cimg2bRiy8TLtKZAqlBmL5ccyiX8DibhHsi8hbJ1SKvrFMcQk2zjrbPpOTIdtU2QViSrUqmxTAPoaAmQIpOl6/yaybS55cbLENjCRSNILcUa0/xZF9H+mnyOEgQ4Cyg6ZxjOPAMA68fPOazWolfLRTwJiMNRsaDyH0PP/sE5pVi787cPH4CRiPdQ3dak2Mmfv9nl/8+rd89fV37O9ueXxxztm64+mTK8Zx4tmzJ/RD4PrujmEItfkXqMV8jHDqCjmVKDZrwDnp7Ggbz6p1NM5qq5vQdMSUaLpWlZjB4Uh9xBmYjLSCheGAyWtsjviSeygN7tpdZKwRKKIBYxXDHDO4Mtyo9C+WJ2KweBXGAhoQrSlKWwR92ZEkTAxRKwIyNNgipRlnXE0ifcjx0ZOLy4UsL6iUNlKaf5YjLyyoARL9cGQKE9e34lLEEGpbTIgB6yxt08kDVepAEWoFc6sWa1qPbxrud4NkD0vQ/r5bX/7CGFoLJgUyjVybWYqt0c867bDPVc7l4SzjR4kXFj2OxkhWMAbGXtnw4qLBNNcQU2OfJBtmgBCObDpLYzLb9UqB3YFhHGXozJTY94H7XS+MbIqhvdxsWK8bfOMYpoglV6qKECYab/FNSzaG9dmGbrPm8uoRn0+ijA77nsOxFya5tuOz51f8i7/4CV9++x1/81/+hs2m49mTR6y6FmM9uI5f//YrpjEoPyu1QTzGqEV/IaNqrMFbaZvKOdN4y6pxdI2l836BoJG94pwgSmLOOGC73WKcYQp3mKbh0bbjyeNHbFdric2tBCeSlBEQf9kzfZgwOGzSECsnTIjkHAm5jC6UfSWWUjLTBisgeVdwuY5V05ILdaZWB7I6sTbnCiL4/Zvv+48PT+A4Q8YSowwgOSmkVjfkXZze0p0tCZE4Tdp0G2uNZsayGvbHI1OMdM4pAFsVWM2PJ5nv7jw3d4NoJEoCKdfumBo3Fi2n7kPbGKzJAonLcv1GX1cc14eehfCmOqHDcIMCm50kF0qIG8U1bbtGUEKaEMoJocqv1I62KqCk1ysWw7LuHBc6XWkMI9OYOPQ945S42fUch8AYUlU765Xj+ZMzPv/0McMYCRkOw8QU5d5jjCQybUocjgP9EHDeala3g9Bzc3PLbn8Ua45hbS3rpqVx8MNnV6z+9V8y9oMU53Niu10zjLDbHRiHQep+bUtOmTBJo3fWtbUGukb2T0qeGKQ3sWuk5SqFSdbFChJLmOrnWuDh2BMSrDYrHl1dsTsc8M7S+KZu9JQjMWVCSEzxyDglbo8T3769535/JFTaFxlbnmJmmgIxIXjZrDM/FDDg3YyhttbStS2b1vLsasujs43Ey24ub3mb6OxMEemsU+X9bg7i+46PcFOpLmrGnHzAaex4CnsqaIzyOvLcIDr/nmpVYwr0wyhcOE390HJmdTcV2tY03O3L2IGiqZjjvmLkivuqSaPWWxyZMUUwMjSltns9YCOrXkCJbZ3Ft54UWpz3GG9rK1BOQrAvMyqCZG4bJ7oz6EyKElOqy5p1bZ2RYvzl+ZaMZUwQpsDLlztu7o/SqW8t1sJm3WFz4OnVlsuzhiePNpX46e4w4r2VORiprGviOAwc+4GU4Pb2yIvvXvLkyTO++uYlb65vmYKAya21nJ9f8Cc//Qm3NweB27UdY8jc7g48ffYJTdPy2y+/5s3ba8I0QYamEfyrRTb2pu0kYRWLkgbjDMnJM2ycuDw2QwoB1zYYpMMFjeVihkTibrfjOI6EBP0wMvSREBPn2w37w8hhGBRwALvDyO1xZJyKN5MqobJkO8vEZ4hz0l6PqEpR9l0pMRlzJCfDL7++wVlL5y3eGRW6zKqBp+drPrlYc95Zrs42gjBT9jv/gUmcD7eMiP8dy0Y1pzc3Y/KKSZqt5JzImf/+0GKW7m3jxCWYpkCmRWDocyxI2bzOs91uud/3Mi2pqAhTAvLlQufFPwytc1LeqHFkyZeZ+j9goWD0nQoQN85inKNpW/rjUa2lwzqrcLSosxBlE8h4s4aQk3C/qFAmdX9LrNW1LQm42w8Mbwf2x5Fjn/DWsl55NivP5dbx+HKDt5l15zg/PxPGNGMwQbS6C5L3jjkzBUHARGvY7+8J05Fx2PP580cMY2boA4f9yMs3t6IEQ6Jb3/Obb66FRvL5U7qu4fZmzziMNKs9Z2cX0nFhlAYjJ2IIeGfACVBh2wq51aj0HgYpOxhntPTj1XXPtI3HOyf43pxpnCdb4dEdY2K13jCME/00sTscGafAi+s9KSlMzcy7LiXhZi1Jv6W0nSQTlXkvptNnrC/UZzQ/mzK6IcWomXNO9uS3NyPW3NI6uDpfsekcTy42fP70kquz7ftE6p3jIxjFjfbaFXdPfjCAXXCP1LnA72RS5wsvQjy/JiNsbGCMkAwdp8CJiVUXVzxPSXc36477tzcie04+fbaGxZ+vJ6iW1VtLY5kJuDVutNbU6bbz22a3Nes6OGM1Ja7dG0bB7s6QYyJOwirdeI/JUZpiZadg1HLGtBxaI+1C1jqGMXE8Hjkcxf272HQ8frTiyfkKZzKfXG0xCTarVugQvcS9wUgm2LuG1ltijvTjQN8LHtVkSBE6Z/j0s2dY4zkM8Jd/8pzzswbn4a6fuL09chxGfvvVHcZa7g4jn336hN39kaHvef7sMa/eXPPixTdM/SDFcbKCyw1t0+AMUouzGdOUNYTNdi0ZYWT2ousaUo60nRcS4izPPyro2zYemxIxwTDK2HLjLC439EFGnYNiXouNKBOrlvpXDUExDqkKYK5CXI6UKdPn9K25xvfvHkt1L57PMcDx+gDAr77b8bdfvuXHzx+/783vHB/BmyqHNWIhhbJGhCypB5iYOUNO3msK5O1UO81JoNMm42zQITOni1D8cL0Szs4vuX7xFSFFGm8WL363aaWoihK3No2DYX61cF3KoJdy/hkALPdvrQOXyFYbbSmFbnlP6WvMocwQkYuWwm9RPgVMPl9b4amJYeJ8u+bR5RX9sadzlrON5dH5hsdnK8I0EPojzlr290ecb1ifn+O9J04T4xirS+29xzAQYiROUYmhHP1hR9w6Qp6wyZCnnkfbNVcXW5I5sm5afNPRrjb8zd//I7dvb3A28/STT3j+/AlXV1d89buv2N/fk+MkSCqjtJSlHFFicAzJOQ0rLC5nGpPJLgnfTtfg3EoIlpGRCNYaMk4tYSQhVCyDTrpqOqHJHCYt+i/WcYkdJc6s4FVp2hKOxPoM3idjc8BVWVj+SUfKmft+4m+/fPlBr/8IN1VrRLnavuoOVgv3Hpeg/H2u373LmFXiUN3ygOE4jPPnVC6EEhRmsJZus6EPkSnCCkMdiWbMe1VZuURjDK1TEIMxlba/CI8xRtwSkgLk5cxZXVR8g7OOtmnpuo7hcFR6QktqLNZ48pQFHaJsx1Y1eOOsxHIssm8pK19skt5It+HqfMvZxvHkcsW2lXRl5zsmZwSnGYRCJMdAdoZhGnn99p4xRlyzBtMQldBpCoEQE7vjgXFaY80FMQag4dGjSzYjWL8i/vYb7vcDP/riCzbbDU+uLvj5L3/FMA60Dv7iT37Kervh1//wK0xOkiV1jpwj3mZsjtiEtF7lhWekQmBzFOCCE+r+bLLUY2PGWF9jcpssKRsOY5AOliyg+m4FlL1mWOyhuaxkbWlkMBDz7M1o72nJ8sb0bhjyzn753r9++JH+wOeU48NpN6wj2VRvuhwCVdPay4NYa5m8Eas0J2rq+xfJmWWT5jgskO4PbiZr1qNddQxTYhgC52tbhYnfo/HkBQZnIq2ZSDhsdiKUyShrnbKwFfVQOD0SknzCSDOAurXigZvq7jp98LFcRxbkjXOONlNJfHPO2iyrI7kzyhUTuNv3PLlYkR9vcExsPrmSovfhQONEsa22ZxhkeOmbtwdu7va8vT0Sc6ZbR7bbc2IIhFEz12QOuz2Pr37CerUihEymYbfrsbbh0eUZf/rTL/iH33zLX/3sr/nv/uVf8vhiw7//H/41Q4g43/L5Fz9mv9/hyKy8JWYhuccIObQ3Ai1sLTr+LegQIyl3ZBJt04glTzJRSiIHS4qBrLA36VcU8qopBJqmxbmG1nop/Ouwm3IsQ57ab2hEiZsHr0kpz6MZ/is7PjxmtIJCkZFfAgcDCWhNnlETcrzbWiVH6XHUV9UEjz0lLc6JfhxIOS1anEpQMDcRr1crUkwcjkfy5ZmApPVzvv/IdMWV0ulamCSlCFs+TQu9KZG1LUrcxIHhuOf++i3Dvmd/f+CwOzAN0mWQMzislA+ygsSVMzWEhPOZVt2mlD19HximWEeRJSSpcLcTjOQ4rHj9Zsf9/YEn5yu++PSS7dkW6xtCSLx6/Zbr3YF+GCE7tqsVF5cXZJ1zmJPEQInE2WbD2XatHUsSmwo1vmO1OuPZ6pKLq+f86Cc/lsE0bYuzDt+t+OTpD7DO88u//k/s3rxk5Q1J549Yg7C1N14sppdElvcOr4mtmCUnICwPUkMOUWIXSYo5nG9JxmNsIFuL9R5hSRZv6XCUWmjKSWcfnuYEYK51P5S1kmiULPYf2B7/fzo+gt5fFtWmpe8tDZaFGyctWk3g3fiwLIa1phak50PgTEWXWVdQMA9WNc8RYds2JCzHw0DO53J+TvM+J2+dtQDeJ6bDDmMbjJMRcNM4SVtMmkhRKAVtjIRxFIHN0pUQo1BphH4kTwMmTTijbmcCa7OyWxu8y3XDWi8uqjRaS1TlsDiblYhL6lvdqsWayHrdqgWJrDdrPnlyztXVBa7x7HY9t3d7+mkgjCNPLtc03rNatXRrz36IjONATDI9t/GORxdndM0GZ1vIvbiKydK1K9rtFuMbLlzD+aNzwLLeXqhVb2iaFbdv3/D2u69pgK5tJGuZkjCnG0PjDd5I90RWcIv3kh2P2RBSqusSc6ZbCQjBtiva7RlDSPRDZMoTx3Fid+gF3XMYwBhN5Mg8yIexXM4aNZlTYZzjyPcL6X9Nx4e7qc6Rk8MuhMgws8RJxGfn4FddQuq/5RXGpFozK202dbKPkWxmTpn94aiCqyeoyZviymZ8J9QXu8NAjTdNer9dzIvrMZYzEre//jnWtRJP6FMSxjUwRBpkBsVKuTtjNmQ/x7SOLHGTMRwOAykmjMK/TNYkRdORYpABP0qCVWF1ZFwrtI6DMRobGdYr5UZFujusNTw6X3FxdsbhOPL6xTXXN0fubw+0K8fFtuVffHFF21qy9RxHydZ675mmkZjEIvnW49aPME2Dd2tyzKxXF3Rnl6zWlzTrNcY35JiUdU1czTiJJTrubujvblm3DeTElIX7x2RBoLTOSMFc3R9jrSRunCfmzBgEcOC8Z4wRbx3tZottV5i2g2FiP+x5e7vj7d1e2ssmYS3PC2+LXLr5F63oueTLDdJYrLVNS/XG8rx9/qs8PkIYLTG6St5brR0qaLWOOCdqjJFJweLDW7puRYyRvu9r+9A80aq4rLKAh1Eygb4iG05X0gBt09C2Dbt+LoO8L5s7v6d0aWTWPuNyonFSlrAFVmUtvm1w2r/prGTvQghgDZNOQOpWHYdGYphKspQy0zTXTk0Gb52cP4kglkRXVtfLxBGMITkpQjuFY3Xe07aWbtVyebbl8eWW/tjzu6/e8O2rO0CSUFcXa/7FTx/z+LzFmMwYEjRKluzE5Y0BoklE4zCXP6Q5u6Qxhqbbsj5/gnWOxnl1nSM5TnRjTxgGpqlnMhP9ceLu5prxeMDkhLWwarzw/OSonQylYI7gQ42MFQhxIFuDV7iZ9x7TSKdF03W4rmPKkngLGkuHSYaqeu/wRqBpBToIcyx4kr/QjLVhBnabk31TWvHeLWn813B8FFDc2jlFbMycvEhmAQ8/STWX+DDX+KnrOvp+qAJzAhxYrKy4Z4FV01KE/OHiOefw3Zrr6x11ZDDfL5ClTLFeNVxcbMl+hW9bmqbBanwjBE1Bkg0VNylYyRZhI/BNIBOr5Q5TIHQNOUdhRdcpveCVhEu0eIxRsqEK4TJGBuJ0jWMYpQ1pPPasVx5nG7q2oek8X7+65rtX17x5vWcKiR89O+OLZ2t+8HTDp0+2WCQ4nKaB3e2B3W1kHASoXXbs7jCSt89pHz2lWXU0bUtjGy0pqfJIiSmDcaOijubcwO2bN8Q40XpH1zi8lQyy9AeYilyp7XG6B1LWPk4nnRyuaWjaFREjySE7kaxntz9yHCac87JeEUHtGIvNbmYSqOFPec7y7F195mXkwFzq0Lzc7OIu6hfFda09PPm9AdI/+/HBwtgInohsIKpg5mwkZjTmvVee0tKtFCLbzq7Ynp+zu7sTAqVlVixLQsUhs/yO48RZ11UgAcX46s/eedZty+1+wJyw2czHqWDOmd2uW8uQUNeACqHVuFjSvoJFRQHhznrN+CqweJpwbqRpEl3rmNYt4xgZ7QQI1I2QGYZMjpbtuhP3Td33FGW2vUXS/BbZsFOQzXB3fyAay36M/ObrN9zteg5HobFfdZ5PP33EF88bHl90Mpat8Qx9oHXCxrBuMod9T4pyC15nVbimpWk7unaNtYXhTiwdKZLQbvqYa4ZYstsTxlga72m8oesaKeoX8mLj6qi3aQokkigbCy5bEai2wTcNxrcEDGM2vLrvOY572tWKQz/x+vqOMUbpiNEETRlAU7oyahlDa96K5RC7p5SRuSABcsY4KK8gF2dWE0oaLkgFTc5deHpkkppsh5AKnLFsxD/+8RGlDUMwSh7rpJPbKDXbw5rhe0dgZcloDeNI2yzIqJa4VVNaYUSFTTFWFuv5mHWWtZau69jtdghz9/fcwCKpJMBgh2+ccnkaua0CdzMGCgGxFtG1gElB5DgnbGrWGB17PWKdFNvHSZRQUHbs1AiWsfWy3MUdLjGxEfNBtILdTCkzjYm3b+6xzjHEyHEQCNnzRx3dusPZxO3dkf39Pc+vtnStZd21NFiuzjpCGvnsccdvX/aCj/SWi0eP8W2nMyZlXklZl6RZ45wEyD1mKoMdCnBwisc0VjavtVZ6CHNSrLCvcwxDiBVyljBY1+DajoTjMEWCgde7nptDYH/s6V/eYpzj5uZOyxAeslIwol2GKoROFaY3VtFNpTtdXHxnhXvHO2E19066R2TL6Wjzuh8WiUaUdiOJHya1SHFtp5QZp0gfEkN8sCX/SMdHsMNROUZEYoq2kjTWabdGcSNQk28qtYaoljK/gZO4sQSMop1EG50ecu4ijjnDetNx890bYpJWnu+TR5hF2XuLbyxjUFY65xdad4bFlf+W2HgJl3O+wYBSIwbGfiInQwhCVhxjySAbpjHIlCardcwimFUpCctdyoYpS/1tmhLTKJujHxOfXLb8xU8/w9nMzds33KQJGwaYRp4+6uj8JV3XkK1hmLIwBbSSYHEm8fjpU3wRQoziLtU1S4IzTToeIOuEqJIbcE6QMKYR9JExFt94GmsI0yAQuNKVkhVPnMBYz5Qy+AbjWkJ2vL4/0ofMbR8ZQuRwnDgcR4ZxrAkba6MIHRBjYgpC7++MgOUNpTVLgP7OSoN26wQFZIzYPq/cOAW+OA/TXcDmVCCLHZ60rarUzWPOhGiYLLQO9mNkP/3x7eNHlDbQhIbFGUs0lmQkyMbm6lJU/7uMlkK0THEjc0onQXehJSgs1kBNDA3je+54GZMi4OphGAgx0Pr570WollZ7qSicSbTW0iMwt2U9VEacm3ptBeRe2mpK8CHll4z3DW3TsuoagWyNnmmYCCaSEKt7nBJTHGm8pWsbMTa13JPq+b1qZoskQ8YxcxwCGFi3ltZl7vcH3r49YGzip5+u+ORqK0IHElfFyNna8cPnl7zZXzOOUlLoug7nGgzzkBi539nqmGVSrTwjxOKvNxuunj7BxIBHBNxaaG0jHDSIF5GdzF2cdMKxa1rwLdl6pgD9lDhMCesc47EnxMSxHximQAzifTlXkCva7hTFojkvtBytM6yc1G+dpY7kcxaa7CioL6lr1nRPjYGLolmWO5KOgOu0TzRnUY4xZ4LL2CBtCzEljvGPX6/8CMsoiQbvrY6dZpGiXwTGlIf8UAiEgiDngvQ/BQRYq6ypi3rkoR+0bFIkcCHwGmRv1htBmoSIWUltctlL+b48jiHhDXgT5wxnSqjK1dcs74V6TWYppAhix7cNbdfK6Osm0raZqfVMMYpWVxfHGMOm8+Q00TaNeBpQXVaDlEYckg30xgiXS4ZNZ/nhp+f0fc/Ll7c4D+cbx2fPzhlDwJsM9KQYpdeSSGMSrbYtOWc4v7zCadZ0uRqn6YpcQ4Vc3HOkTLE93zIet+RxoLWGnCdymLCmFRS6olys9vNZbxWg6EjGcZwSh5DBt3hrMCmzWq0ZRuEcnYL0GhoD1qZqtYTECm00l9JR5w3b1tM1wp8juFiJc4sb7Ywylus+LQJaS3Gm3q1kYLW9T1gHdO5LljqtVe+OxglX7xh1gMQfso9iwT/k+ChhtCXrVut9NaKtpn42/actVOVB55ToDweEhWohVSytEJAy/SiF8LIhqN9miVlv1kxTZOgDnLn3ZMHeZ1rF9KwaJwVlK30nJs/w/9oSeSLNmcJ/Wq4164yPUqyfAowBxtHjp8AUstL1y7r1Y8RZR4NkoeXWlKjLSUdISllT82otXeYHT88EgpYirUv8yY+ecL5xOBuwSE9l0zSYVoiSmrYjpWuMUH3T+DVf/OTPhI1b4+K5bqchh+EEeFG62DMCYp/ihPVSl22tJacBYis12pxIQUmpnCG7iEuS5JuS4RAyb/cDY3LYpqNFZmesjOMwBFbryOE4kEI8zXDm0gw1M0YYdDpyY2kbq65qQV+x2JfKPVvcOlUqS6/gJDurJNMp5blzIwte2BtJCsUcaSy0Bsb37K6yMaWDxrBpHBfdh4nZR9L7z0KwlA9pFKbeHBQyp6VPWYRWN548eUliOCl4ywLMixRTrK9fBoPF3bTGsFqtyFiOxwFyWy6P0mN9ev1zDGuApvBFIL8XQ6CRgjHvLHRBclBiKS1ql83tvaf1mdYnulZKC4OJJPUUYspMMXEYJePaOItvnNA6aLeIt4bkBLjgjeHpRUsi8+zRVgmdJ/7sp8+43HrOWmibFpIQOY+TkFFZ3XDrtpESS4SziydcPX4uAIdFz1FNhOei5/S+MDXplnNmCgPGQrfe4I1k13PsyGGSDKUK4zQIs7ixksyxSeLsfpyYNMvprWezPWOMibQ/SIZ41fH06VNevnzNPP9QHJYlyVnW5+6tdohgaRdJHKkzqitqFCtsTOEF071r5uafPAtULpOKS8yr5ysTq72FNkknyraDIUWmRYNyyau03nHWydemsZy1ZSLa9x8fwZuqMZNerLGKNkEzg1YC9mUT8WxUUrWES/SONF9YWWCrjctaLDbWCrtZsZ4LwRKhikCi7RzZRHbHHjj7/VaxLrrVzRPpGn1oWqAHvTdjFw+o8PssqUaWsYYRKocs8YRrPM7JEE8h17WV5Tsl6KPUGmPItI1laxBSZe0akREAE84kEpHWWlbrFpsDOVucgcP+wPnqjClEvPPSBRIDQ39gs1rRNFKyWHWOprGYEKVlrNTeThC/WgpAk0tYTPZE50lmrEpp6A+SsNps8UZiW3JDjsItalIEOwl5lPKqNhnGkMghYRw4L/jc4zTSOU/jLWscm7OBw+vXnJ1tOR6P7Ha76n0UtTo/ey3FkBWGpy1qqGI1As8rXCYiTNIlo+xRwturG6nO2sxI3JvR9ZCoJWf5NIvmErwksKwzrBrLELQUYg0rb1l5J/Sb3tFZR+vVff6A4yP6GWUEnHeSwZqyBMvJGmwWfGXMszZZHrKci8my6vej/Kkmy8/WSOYRhF+mzG7IVgmiFmcsyJ6mbbDWcewHcWFMPtF0Ji+EGO0wyVLL3LQNMYy4xgNW0WpW3S5AF7lkUJddAdVijKPMCBwFujUMk85CDAzDSAwCJYshEbLQGaVU6lZZEkHbDkfGISS4xZNOhbsnJaZpojGO1dpyvmmxJrFareSZeCdxZgig54gpsG4NXetwI4wFX0uJf1PNVubyv+KyMjdiGyCmyDiO0s8Jul4ZkxLEUfsxhdApJQtWxsbJmQIhjEQyUw7c7g+MOePWWzZn54zTxHq9Zr1ec39/Lwzl3p2s9awCqX2TEv9ZzZTKk01GkoMJwGZsNpyAI+fHOot4oWvJnFD+L/ewtQpKt5aGTMyGTc5crnQtodBUSfnHME9wdvb7S26L46Po/W2K2JxwRtLw2UK0CZ8sFDrCZR+bMWoRlwkDxRlaiaGkVCmWyAmugBIrZWMJIdK2Zu4MQRQDmtbxTcN61XF3d0+JK8qjM4WZqi6GnAcjD2m7aiCLpcIYnPUKRHCUxyUalEon2R+PQil5nBjHkb4fOO57+qEnxqTJBsM0JqaQCDo2O2oBOSWhcAhZhrVkgmQ6vXS4k5S+EvWgkyB+Ygi4dUvbOFZdKxQXIMRcADlibUPGEJT3xmTLdr3m7W6HdS3et2RMbe8yijPOWeqLgvyWGFfoFpXFwEiLk5RkGi0DiUeRrMUaKYkY09DYFUxBFFtOGBJhSPSh5zAEjuPEGANvrq8JEcZpkrKP9xUmWfoOZQ8t0kv6OCVrahR1VBJrurdK502xhpnT/UgNNBBjaWpLVQVl1CTPzBBobfHqtFfSSP+sEIqVfIckvqwKs1e6f2c+TBo/WBg9iWwSnkRjZKhzKvQVVi/elsDbVM/S6C+zXm1W1xRrqnWJmTkrq/FK1nkWU4kX9DVFo5V+Y+cc6/WK3a5XrVa0XFZLl6vbQU0FqKiZTN/35HHEuoachHclBumlG8fAMEyM48Q0CpX8NMm4tiSlQGWwViykldayGEXQpDxiBByNjDSwVnsiM4QgG2B/mJhcYN01SuZb5g1G1eZZWdMCZC9DZ9uG4yDewNj3TGHEpMjZdi0UG5m5NuccZ+cXON/IxrNzl3tOkoTKKZGjUNpP06QAAKWhyIZutWa/v8M6D9ZjrBIBuwZ8ZhoGohb6YxbkT9TEjm06hrTnMAammDn0I9i99JAyWyFjDNM0LRoINBQwYvlEqSZ1S+f9tCyjWWvE5WSO9cohDc7F8tVfyr4wc+a9jja3BeFTwicULGCrlqidQrVmSwWtlOv84xNSmYRVX9lMicY7UhA3znhDzlHGkKkQLONDEdBS2pjrf9k4smGmJlb2ZYGlebpWWog4yZPJAy6zHKwxrNcbhiHW4LtmJPRbccNEyJFgP0un/K//8SX7flQmNTRT/DD1T1U2pmQbFfNpSuklGm1QtjrSINUuFGcl+4dq/OqJ5kyImWNO5MbStEIXiH6OxE1zDRAERH17t+fVq54nj6/YHwdurq/xxnB1uRFn12Sckbapw/6OHBKPrh7V8kxJJ6QsUptSIEUZ9pJjEKHKUaYsR7kW37T4di17wTc0bYvBkJK83hoPY9BN3DEeB8YwyFAf47jbH7i+uSVjGceJnPdMo8yTbJqWcRxYrzqGvhdlknJJH0hDNWCdMCW0jaPQepCLVaN2i1QPdP5BlfQMMK/ZHBYoKMncCXLHmDq4xhihLDHKIoCOWEfDnrBodpC5LJog02TRH90ySgZL3NHGWwxR3LlsCES6pqluGO40oylCOSdwbE01WxVCuYlUokurE56sr/hA2YtJFZIKry7uo0cX9Pu3ylhWbKBuuEXiJSmjNcic+5WLjENimpZujK0/56o1T2OIh6lxoYuQRx9j+bz5+tQpWghngVwhbmwSwdsdJxqnWValfzSq5VsdFz6GgMmW1XotA1Tv71mtVjw637DpnMa50PgWGyJd4+l8JoWB2+u3fPJsA4vmXLGAAZLMoBCBFObynHQKM0U5eIw1rFZrmtUK7zwZ6I89Qz/gbRAeUjxhiIQ8sjtO/ONvvuSrr7/h7u4gVt1bjiEyHgcMgqstHpUzyz4Lg9OBOFaFctNato2nsxI/WrVeNhfW8tk9qogqeVCagNRnaO0MBijW0mkDBOJqugKNXByFrj8zW+7Gu9nzUo+udI8Ys8jc/oHjw7GpZROKmseYTCRgLDRZ5iHEpD67ETo+SqBdtIS6bWXwajUwChEzTmpPxgjsyTlflJ9YtvL6OSzEWMuzJ2dcD2+IKeLdvLh1THStIy2sZg6sfGLdWQ6j+vm2iI6mMGqw8q7wARpXSfxpKsWDrUI8h/XzeZ0tWztrHCkJrHGMTCHhraKDWsVVWiMTsLxnHCP3u4HHFxu6Ts7/+OqRTGwaD8Qp41pP1iEzxsrYbecTL373a778za94+vyHOqJOuV41mZNDqtSJ5fEUan7jJKHmvQDp266l69Y0TUPO0DYdfTewPwykfsKkAWMbjv3E3/zNz/n1b39HiJkYM1MYpRXPGIW9GXKKwqWDAB+8t0LDT6ZzMoWrsbDylrYxrKxM6vJqICR+XGx+WzLGqsV1jQvvrBzyZKyx2u4lDe0FEFEU8DLOzKrYjS3Z1Yz6vQDqrvr5OopAfiCQ9SOA4ojfjAVESzvfVB6TFGGMC/rBGgPOmS8Zry3zFGYfv2gas9BWM/rB6uYtWdCa+ipJCJPZrFtuUpRsovUSL6ognmhKtbxZM6zOWs5XHdf3E5rUrw+pAMLyIpaQ+6prL7jxkhpXDSiTBjI5GRJzZrbUvDK6hLpoxWcIOct0XkWg7KaAyUK43HUdbdsAjl0fud0fcS7g3ZpV2zINA2RL8rre+OolrLsG3wYshovLR+rSzW1hOUWhFSFr4mOOqsWtF0XVNC2gvZxNS9c1GOcKNxjWdhgs46gDTpNhf3/g7evXatWscrkmReroWmrnj8RXWUMhi0nC1N56K6TTeo7GyCh4a00dQe4MVRgrusaqB0bRv1lpNdXrKbbXLJM0s9UzZk7E1ASQc+IdqJYuXSRlv5f3slDYZaLxhxwf1elf6m3Wyii0hLI+a1vRipq3qTeRy4WbJQigxEOLS9BuellAvfgss/rq/piXtpxGv0lyJIyR3EizKwvWAVB5VH/XKO+NtZnVuhHgM4aCLDIGTc3rgGpTSLfKfcEc9JexYvqAlzGulUxoQXHkWnbJmgOwtUtASKkkGSDj4+QUISUa59l0LZmJzz5ZYW2DNxnfNGw2K1q/ZtW1pDAyakZSrK+hWzW4XaRdb/jk2bM6loAckSSy1WdhSxlOFZ1YKptsjeXF5Y80jadxRsKRZLBZylupa2m6jrTvwRp29zc0JM5ah46zQAluMTrARmZZiB0T3HMrZZ7iRUmErm5jeQ0VAufVSs34U6oQFe8LZmGpxfzaVSCvKzBHKaUUpM5sbauXp8Jb5ODhYU2NVE+8qA85PrLoL+Owk8IZcoKE0961efKtxHOaBazaZZ6ngRHwksEIOiUXrKtM8WmaRpixMbgcII4ookDOXqxvFo3onKXrGsYpFtunV1IcxDmQL+6jPILEprVYdZfEYGmcoLXNJT1zSZkXTTeVeeFQsj/1ASaDWnZJmEjJhzlG1gxzTDJzIigTQNQaZKkJip8ulr31DRFLP47c3B/Z3+8YDwfO1p71qtUWrYzznhRCnfuR45Gzyyu2m0tVqnLe2rVhPNZql0aWLhSr/Y+JrP6XJHqaplGTVixKUkUlz7gMvemPO8Jxz9m6YZqoKBpTY3BRvLPglFY2qV+KZSqiouuPEF9ZLW14I16idQtFUsIgTbihQnZaby4WUf5ergxj50ytMVWWl2CPSjPznrCl/Lt8P0GgfcDxEZaxoQwjqZ/tdANnUy2gXICtvjo5aewmFIYZKdR753He0TUtq5V02rdNGQrqcerWhSnw8puvmfaHd26YXDYCNN4zDCM5d2Vpme00zBnVhaBaw8VWgMaWrBONyquXCKLyQBZaACNwPnVti1tbG2G1sda4TDQgTOkiZEmLl6WWmDL4rKMBErgENgp8zprM7jgwxcjZekV20hb12ZNLjoe9Iku8kD4lWHUrYgqEIDH80AemYeDJJ0/xvqlrYU1hQzdYm8E4MV0xYC1El0mT1CpDDgzDwNAfWXWduKVqeUssJQnqSBh7+tvX3H75K+jvuOgaRpNIJO26Kworz5Zn+aQUeCGuv1xSeQ4SwGSK72SycAaJ3CzDHMTbsUXYNU4vCZ1lPGhKd0jJEzxwOzXr+jCJ9/uO00Sf+Sjr+MHCmCmaMtULLFFY0ll3XuND560IlPe0XcuqbSUIbzzWOhqvSHsnU39lg0rhXVy0oBT4Unh99vxT7q+v2d/coDSdlDohcabsEDqPy8qrUCxArSXNy0QJ3i7WDkeZPDTHAQXNg5kTUPO5ZEW8sSSTyMh3MSIS6zrJtePt7AVkpa0v8XDCEJJw30RN5UdTGl+NWgBJOuz2R/rDQVuhGg79xO7+iHWOm9sjnz7e8ujijBhhGBO3dz2H48jtPrM/DHSrlaxpSBgv5ZNCoSJ0GYmUgsbIsq4mO1KQZ3zc7fj2t1+x/ou1jkwTYUgpEqcBsIx9z/XvfsXLX/6c/u6GVc40HjpUkev/pENflrJOq1StZOypwGR7+uy8U7faZEG3OO2iMZqMYbZ6OWeFJMoskhlhNFONAkpdmSl549oql4ua1T1QHSy1mgvBfV8Xkrz/w4QYPkYYrcc1ltZavHe0jfKzqCVrmobGy0ItbzQjiYmTsW9alwkhvldzVIhc8eed4+LxFY1z3L99S8yj6GQN1p1raJyl3x/KUqErQSmplFnleSGYFsf5ZsXKg/f6EDTwF1e7LDp10eskZWtJUbLHEkJaYshEq21DGntFOwMOcrYKiJfOdWnNgYAUo6Na6xKHxuTwjeEnP3jM5dYzHEdCDNzd7ei852ztuTxf0Slr+TiOOGvYHyd+880NxnmyadkdA69fvWEce8GXGrGQjZfZksZmdeMyKVvITmJAVLnEyOuvv2K8P8j4dgMZQTpMYZI2r5i4/fZ3fPs3/yuxP9A6Re44SG6OoyuBcIm/k5VJzhIY6g6QRbcU974kW5DOFmc1sSTJQKdEVxIBmUV7H1IfLOUINb/VOhcXVYUxlbkqczICa0pVtgAlTr28cix7X6sysUKO/UcXxr/8i59U5jJbs1GLhERZ6CTdFuXCKK5M1Uos/jZrl5KFenjMsRMyV6JruHn1ktgfa+bM+4bt2jMej3O4mBfXZhbQ+iKkRsTx6nzLo5Vk7MroMOmgKORb1OK7xUi7ktVAPWl7VBYU0RSFniEkGIM83BBTUQeSqEmQrSS+YoRkxS0NGIImcWyWh9sm+Ozplh99dknrxTN5/OSKtmsZ+iNJOV1vXr0hjD0g7VPDlHhzF2ka6FrBU375y5+zv7ulbdc0TafxudHEVpkVqWP6NPYLQRi9h/HIYX8vRFoxMo2jbLoY2d28ZTzsOdzt+N3f/IxV3vHkaoVBuuKtdUyj9LB6RQCFEAlZpvvmrK2QWuZRCdAG7+JCznAzNEeQ1QV1jYyPc95Xxr3SfJCzgCjKaXOyGiOWvXea5azd/kuBqrFhETSh/bBWJiyfbCceurj6+w/M4XxUaSMnoUQID/72+yS/Xkyt7xUH81To6gKU35dvJ0FwIpmMbVseP3/O3ZvXDHd38vnW4HMg9ZL9E2uV5oTNO4IvVgoi5yvPJ1tX+W9Kz2ZJc0uiUc5hkdadAr7ShLhahOarmgAAigpJREFUEOVPyVZ7GhMpG8YpMqXMFCIhQYjCpJ2w5EbiwjEkokuEhMTfuSFmccOeP1nzyZMtP/7Jj/jshz+kW63w3kMZgw30fc/rb7/j1bffYMjcvbnhZgdffvUdIY882VjO05H7V19z/viZro2Qfs2Ma5EUJ1IKkIIC3wNDf+Tu7Suuv/mW119/w6/+6v/FT//8z9g8eswwDHz9D//Ayy+/Ih3v2TaRP/9si3eajU6BEAPJR93M0p86ac0RI5Azo83IQsUYNcPrBZ2l8XU28mWck+drJEI3Whv0TUNIYJDMrbEGrMO6UiYzxBCZ1Bsr+1PCCvXANOOfsghZcVWLMMmMjqy9vVkaHTRDr47QA7mAylj/AceH9zPmGU3CQ2H6nqzRstUIig+uWJtcJg5nylz1moldZL9KBraMF8AZLh4/Zm8N929fkUkcrt9wcX4mC6yJI1mNVF2TaitrAsiw7iznncGrBbSG6iqamk2TB1NoHTT/LTW7QhtS7jPrePRsyAmmKFYwBMk+hmgYs6UfAyHL3/sxE7MjJOG9KfdwdtHwp3/yKT/66Rf86Mc/0ZkTDueEVlIvlqbt2G7P+fynPyGnxHgc+fN/fcNf//Uv+Ief/RfcAOt24Nu/+r/zyed/znrVEa1eP7kKYwwTKY7kMJLixDSMHO73/C//4T/y5T/8lr7v+S//n5/x5utvuHr8iKHv6Xc3PDvzfPZFx7rTWNJlwhgIY8SZSPY6QEhLYKkpO7wgriTDK4N6xBtyTtw837QCvhczJ4km1DJahzEyTi7XXekquB/jVOmUwMQToghl2c0xieMZg5TQiqdzUidEnn/KhqjPM9uSI5APrlYd2TC1tPcRGdWPEMbyrXzAMmv0rkCWV4hxKtpDhMTkUpCP9cXVoVWhlOkyQEkYaXq/dK5DYn2+xvCI/XBN1zR03hN1PkcuqmoxwariD/VvGYNznvONDO60VhIExmYFJ1itimk90ahbp5rZnty73q0G7SGZmS+0xCPaNxhSZgxiIVJyDMEyxERImSkYMsJa9/j5FZ9cXfL5Z18IJX+BJuh1lI+0WDAZZxzZetpNy6MfrPg3my3x7gX7b7/FAoff/S3f/eKv8Kt/z2q9xhhhB5AEhs4UCdIhkiaZHv3muxf8+ud/xzT1tE3m8cZgxlv84ciPrhouP2tZNV7hjLrpUya1nglJwGV8zcTnnCTzXmNGXURrgEaVsIqVDs3JOnConF/6NZSDqPIXQRXw6ooqNNEVg2DJHuikB1RmZEpTQk7SvBCj0bmPei/ZkWLUMQeekJxw9ahvlDT5FjQWTqlElZCNloY+UCA/qtP/90n5O7+vVhRZkCIYpWv/QfyWFm6sCHtSIdKSgWbaYppgmkhhIAwH4v6GeNzD7RsabSpNqeBPCyaRhZuRZ0FUsHjrDZvWkKLRh28kSWDBFwhtUSK6gYqHIAzcuvQLTZpzklJFEU5NQqQsvYFdhpUt6ieRsrivMRtSlnFoE4bnP/kBl08e4crn5NNEGNV7UG2N4HsTijaxjsY3tN7jrYAzvvtf/ye2z3/M+dMf4JzMk3QaJ5eWs5QS4zRy3N3y3a9/weXa4boVzy4iP37iuNx41uuGxjoVEBn3hrbEZSDZjO3kvmMsYxqMbjlTrYexVoWzlB/gJN7S+8NoI7BmulMCoyPJQa2QxonGlsdeAopchVNKYY6kLH5FhsV6yn5rTK7IMXImRUNGEGQpCywvIaUkGUcuCjbExFgy4xRcb1okpr7/+PBsavF78+xwzg1JWn/M6hbqTAl5VQTdIDWpkgWNYlLJrmZyDAJOHnvSNJKHPemwZzrcMO4OhHFiOu7Jxz15OJLHAcYjOU40XcPjRxuSzeQQMI12ViyVRKkT5sUXMnG3bS1x0pS7NTWxUSj+MaWqhuJo0fS3FJNLtzioNpy55UUQk7rgWefEG6R6XRSQCqy0HFkp/BvHp5//kO2Tx3KuJKPjjBdrClogN7qFdG2lFSoxjYG72ztylqzpyjn2Y+T62y/55r/8R37yv/s/0K7XkLOgWJxEvzlKt8bhcOS73/6ar/7+5zTWse0cn14mrs4Mq1bm1FtF0QjoqOwHQCd8SS1xDj2slcYCYwRmiFFomZ2FsS4cc5xfcDMlC5uRWM1Yje9q8mQeOJ/LZqVkQcshNcxkwDdG40ZDtppEMmJZJcElUp0KciqZiqJK6u1EHcqacmaKMGiuICqK6sOiRTk+wk2NJxnRanVArEZZLN2XWQu9hCiNqXkijT156kn7PaHvif2BcNiR+gPj/gbT77DjnhAiNkn/oMkK4DWWrqhdJ3Wy5qzDuw2u87QrL+SyIYL3s2XOCyUAFfpW9KA1hrZxTAr6luwpoDSEFQlSM63LOFLu2RVBUO3s9PcCB9QwR2MOsFLakaq1rqFdWM8kmthZQr+DdCXKLUn6P0eHNstodz2grGZJ64ghJIb9nv72htevrzmPE76D8Thxvxu4+X/8X2kvP+FH/+Z/TzJilU0U6x9CTxx7vv7l3/G3//P/jf6wp7GyRtvW0FkFr6sFswrSjrodClY1VRkpbHFiqSvpdbYkKXaqwNagpjypOdFSBXMRDuhzqG2E1Zsq75i9FfWcWXpltbvHGBwlwZTBWam3amiScyZbyZpLy/6iBpmz1pn1eXuIrZUcQcyErOTLHyiRH24ZY9B7ES0cU6ptN8Qghd9+T+gPhH7PdBA41LS7JR12YvGGA8QRM0md0Hlp+ZHkiBXWsa4htR3OrCWUUHhSirpALADABUuqyZcyATyXhS/PqbjN2nVrsIvXRJneNM0xpfCqFKo/qW0VDid15Ci+TdlwORUwRNaxH3PwXhAndRyIyYJIMbO7VsibhQkAphzYv/6GdnuBubjCNwaMuMDGuZqkct5rli8RwqRNvpmx79m9fU1/d8tPPl3Rect3tyMhGXb9Hf/P//H/xMuvfsuf/Jv/DavNOevtmRAV9wd+94u/5T/9h/8L/fGexjiEwgI6VzyCQleplkmbbQUFA2RtPyqW31pyVvhcgRsCURkNitfxfZHVEmZGSa4Z1PqewnSyagV9GuRcG+MWxrfkJKo017/FOGeqi/xmZ0UDqpdU8NiGuUkejIYimeiQJF5lof/DxwcL4/7Fr8n9xHS4JxzuCP2OuL/BHA+E4xHCRB57ISdCOFatuiDe6cTe1uJ8izGtWLtGP95qYVaDc7DoIGF15gM2LFAO+tjqclqpEXpb6BocJUadYXD1qZKJ8wg4hHZ/NMcK8xI3S5wjq4mbpRNVA1EjbqEQchkq+3BJUGlTbE1WFM/BGqz2MFbAqzo0MRuikWlW99+9xHpD6/4FdnNOJhOVHcA2LWM/slqtFLCfGPYH9vs7Ugq8efmKX/7sZ3x2aXl87hiLq4zDOuiHI3/9P/9P/Pz//R95/Onn/Kv/4d/RtC2vX3zDr/7zf2LoD5I8MdI58fjM0HVUqsO8YJE3Vut5NTg3lGQT5f7z6QYXd9Oeei2L1V3mIZY/n3C+FkuYC3yxALk1ftezznK6DF9KPDl7OBVJs2jqts6QlcKzVAZMCUUUDyBeQYlrJV+QKv+OCuQHHB8sjN/9n/+PmCmIxtMFazph5Nr4DreymHWL9Wtc01BYunGSpCjjpmtsk8GUVh59AjUyy/qQjMZo2YMtSEh1MXLtM6BMS3LOigWnqded64OnLjx5vhZjDI13FeJV+WGdEGTNbHjLAnHp2CgIDsG1Sh63tJHND8Cc/gdvCvxcHmOBToGgcJI3TGHgV796S/7qnj+7S/zpX/4E4zxhCOT+QHd5icEyHgPGOMZxYH9zTX9/y1e/+hVvv/2Of/u54/mjLTHA213EeUPTWIIRCxazJebEm2+/5D/8j78FEMii0Xu3lovW8OPHDT9+Ztl0TiGDBe8pXoMxUvsz2ixe3fE8uyrV+usziCnOCkqW8MQ6nvz8oI5d/62KuYQNqKUsS1/4UsvzOEkIgVot+XeR95pxl2CRIpQ1QUcGk3Qeknyo90qeTQlV5gZpedcpuOD3HR8sjJdXG2kwbVrx8+VuBONopGhbhMW6ctokAUTWGCFH1ZJU7WOZERKmtk8ZAVonKfSbjDQ1n+rORXSsLUdWsJKVfkP/NtNW8GCR5L9tI9wzsoUKHG0WuEJTOcfLdVhBaTeRh+hLm1j1heZISB/w7JBl/TTUxZPfeA2Cuqbl3//bln/8buRvf/Z3vH31HVePNhjnGSfDZz/4ASkn7u97vLMc93dMxwOX24YfX2b+2x88pXWQwkQICds2tN2ar94Gvnk7cn+QOmfhoSmufNMJiH/bWZ5deT5/ZLncGLrGLyyiPGNrSvP3wvfIyoguUKPFKpv6mlLwqC12NYzQ7a0x2cNQaxZemY2SUUUMlWayRJ/lgRelsYSpyb9nRVC2chHYwhyv/CngTy1yyZ4vk5pylIneZU3UQVso5u87PlgYzy4fqdDow0syvEYW1qrwpxo75MWWry65KT53cV9ESEtcLjcpwmty1NeqJaygMlO/Ct37vBqKe81VbvSaJc4VxnCNMzVVLg9IuE1qB4CRzyyub91Cprgw1P7gYjGNBhdiJeY8czkettzUJlhjTjZJ6aPDObrO8ORixb/6seX2CPf7nrvdyN19ZHh7w9lGpks1m5YfPG7ZrM9w1lVWNB1ZjPORrYNtm3hy7vnzzzoOQ6KfoJ+CcLoinR/eO7brlsutZFBrOccYSmmqQOlmicmL9U4sKSGXPawGRdJkDQW0fHWS9Ya5TWlhkR4e1lkZP4C2SJnZlZwnF88wuuJemvcIadmK5ShzJmc39jRIWbzy5HmWZZeYEsVgL7Cuf+D48Gxq6ffTvZ9LP5gKqLEGKgubvHCJ60P9eavxoahUM2uRnKk9cgYRWJOxEemIYCaeN8V/N4rl1D5Io1yrp45CnkmjFmmtWs/M0HadgNzVfSuJoxJDSouRLOrJRoSTha4uky1AuXIFuTjgulwq8eqfG91MMyuCntPKzMOLzvDosqFA4JIWyq0rZEMGJUaRKygdEiaDd9hceEgTq9awWhuuxLypVTLgGozCx6SOp5sslRqdABjK9apJFMrNDEbBGSW6tkanIJfIaYEDNSYrvlK8n4WPOfsURvlwqqJ6d0M7Y8E6uU+lvlyGpRVxhcR+LARyRtiU86b6ueQqVbPQplKWqxdYPaK6TkpWRblaNyuDDzk+vLlY595XWjw162XD1fhp4cqV9PGsWQqId+nALTQLBUlfNOIDl6M4h7pgWTvaxfUX3OJ8CQ807vLHakjFMW2aVlqVdH/WjKFRi2fyOw/xIcgYTq2f/kLuMs8/l9cVJyOjzbCmaPjiTdj5gSMQMKMlENmbBTWiTOR1/ZH4Ohcvw4ufom6TUTiZcVbY+crnq+tVn4nVe9AJYwLcLkkP3cg1oVbWt4itqkKzDEFOlXV5n3XuZF2BOgULipU8fYwFeFE8MLOU4sX566ss9Tr0sVRLKN/nD6hXWK9pFkwWCZzieta9a4qhsNWVXzgFH3R8uGU05VLN/MD1Z/1TtVhqdygyJTdVYFwqMGWEmFlYi4WQ67ObN3TdzIUdYI4z6pQhiutZ1k4WqWq6E8ex3FRRFHo9uXYfaqNwcU2Xgni6ug+F8Pe9Tq51MTFXvDVq0sCg/XzU+ynXWOMtM0/rMpU+UNetJL108UyWjZxqUlMsa8aAFSSMsYoPpZ5iftzq3yWSCHaKUs5a3mdW/GZO9XlQLEUGoyMOKonYQrKW6/SwH3D5N2NL6LN4bDUpVPaPq3s+I1yYJcG3FMhT0PZif8wbUfYs8zM1y3yBnT8FihVVHFLWDh9hcH53u/2B48OFMZUo0MyLoWgUY4z+fdaUMt5sFsSqjoymP+zioSyXRzeALLTTXrtZIwG4LC05EmDL5opAU+NRJZTKxU1VDffwc/S7856sdb/CmlZjQjPXNZduyfIc7+taWf6ufPIyCVA5W0oavSo7dGMIGLpYSFOUkLEYUwRpVvF1CrH271UMTJ7juTmkKMkOWS/56DLAdhloLDyhco2qQJaHuPe+bsxilawSjxljKTg5W9i6q0XRzZ9nK+f9vC2z7HD0ccy/M2U9iuDMYRHZPrBGsiOtqb5Wfd5Vc5PnZJAKWd0jBT/LfH9yHam+tsiEIN1TXfcPNIrAR2JTKUtikE2kKflKTqwdGdUHI1elI3GYzLSQ0yzt+HwskxkAxivDGsVUyiI4kf+aEreuIDxm61zWJ5Pn8QDvsWpCY+/mfxtgkUmtAvjAnVqe451z1gfGor48k3KV52v09wsnVt25UpeT7SF346vQlTizuoKl6GUU61meRrFYUJLaugklf2yKxTcOKbXMvahFEWVbMLflRPM2qM/SGDWvaoLUGhr93NrBX73AeU3LbcPDhA/1WljuiYUFXR7L39X1p2xXU2kxjSl0KYtnp/vxpCZdvLFaCzaL+r2GSFkEW6Kiovwkb5ETsyL/gOPDhdHNLxVPxIrLWXa8k01tUlS0TFkcxVjkprpX5chFyy4szHKU3DLDKG/QTZIN2bh5UElZuEXAPNvH4r5qadb4+aEpd4+wExjVaqV2qZdaN1pxGyXz+lBp1LWpm0zOb046/aNqS1OVhqSm3IJWUGtj/9/23m1JkhzXFlugR1Z3z77Y1jl2JDsm05v+/3/0pAf9gWzP9HRVOKEHrAWAdI+qjFFvs3kon+nKzAh3OgnisgCCoASEdDY7uHn2aN/FufeyClGdXYicMEmBBxiXoQYc5afpf9Pi3SETld4opBEMHUwpz1DaJBh05NS6z/AzuXw6mttwu9nANI8DfUEjUcLNlW6LpA1InjOLosZ9rX1K/EYsH7rLTdoQjCPyi5k7HY3PBQ0oOptbAauzUAkZ0TXXtD9xvXHwzUH6U3uM0ubqcoFBA3LCgwKLteu/J1R53eOMWCWDSZF5fhZLlB4BB0NCZ0D3Vbg9fzqg9CxN7mCSuLWgzTLI1iflokohLPDVHVIJSslKKychl/WzONkXAKPUBhsHhCLicJkjkIgd7bsqNxjbddS3ok1fm4tA0FFIwLAAU0tlyRxSbga2eeI4BqY/wm7wuLpEGlSoGdRA+Gix3uipRPe5XwRNfoEfDTGtwrsITh/bZjkVadd2LKfB0Dlmc1MKaa1TSZd1TD6BaDWzf97GFX/HeaKjtcnB4zPXP7CfUWJXAlKMCaRPQszsYPm7zMDGhcGLkbVGtWotS0lgxbARGjTuOTJHMNggNFPcIgvlyTCK8Kp/A4YnIpl5nmfzmHSEHbCqNiXYNaIkcWgpmiCCIwieDQtn45G0UL4j6K8iI7kUHKv1NFmpyHahdaWFcsTuAklXVzqaIy3E22El9GBfM8gjhaKdOGUxAK85bIqwZCFZuATTgH6QadJXU9No292INXnCl2e74qt24/4OCiP3S21F4rZ0x6lNw02RKh7iVKD9hQ5P5S4FG/2M/gWimxyvtQz2T5pFvGUZReIGGdE1r27kepnP2FhroG+jCs3KbQxNErhc4GrCGEgYjTixBoeoPg1vAhsaftJaznHkskeaUAe0oTnWFZW7ihTQoSJUgm0GCkgf/8poCjakwWu/xB690T4DjuMBH7GWZ/aoDbAOJHIw1aYVczwKJgPIDdfkU5+CSeTsAk0UFFuEUvOYZ52kdWaigHU2M5ggMJgGps3gZlmHNMP/UjiCa9lKW29uv+sqJFXUVcAyI/PuBV7NWKWd9PdSjUoOT8XD+xNWo4zUEN/Sv5ZCazOe3dI8aXmZowVGHYTq7rUMM3obnxfIN4RR57euC/vg4HfIYBZlITQB4zgy6AAEI0WhXMdDeaw8Bk12QNDyeOi50E4mZ3OSCPSXbACnqV+cMFemUKyTZXIyNX9U5zAcjwN+0sdrW5u6Y2IYucFVhkCsOOGsRMbzGgYjiRRMwfWsMWsAwNIZcm5MvzPYYEwp0+QukMibEvBkOpfqX5hewtKFkM1xc2/usHejJZMFls5qrLoH2frzpF3DBMUnsrrd6jQ+AipQFJXV4vPZoegU7PP6vFth7nOrN8rHP1O5r1copAy4m8ZXVs+y8Ya0FuSmqg5gYnlrmsbkM9fnF/0TKoiDGcBBVHcGhcepIXPzraGY07TAHMnEB2jxUvNPYCq/z6FDdHRGvRa8jTDMTfvMI/VpGIUiYWm0SQMIRbjidbEn3tq+O8FFiiy0JmfcXSGn34eCGpbBJYdjDO2jVFsV/Iib5Ks0gXE9j1h7NUQAYAScNN2b1p70suKdbB53f3TOaP1A1HCNSIdoIkvTAjZ+5vkYaFboGojpCfLRV82T3mntOc2v+uge27E8xxptFZSuscRrPHRb2z0FRZjDHiO3bDUE12Yj1w/zc+tvsaRJCqB0Zv5Lvtf4hhDNP3a9Ud7/gM6icGd6EQeoE6pAgYEhS1ik1aSFEQQ5EOffuQo3JGIYNGqxVjM8lksMg1khHpuOpSRZy+Tbt6/4/feJr274b//rB74cH4SShHKGTO0qvxFg1nqk042Dm4gfEZnl5GrJRJaOu4XRZ0cL7exhtc+Jkiw5UNkt6BpYDOqEn3G6Uhm+VbteZMGDtjMDVW2I7Tbrn6Y18PZNjSAzxULLrXK9vd9I3/iDGTt50K1zK6nKMa79L+XbSopIEHfLEloiM44s+RAwY8Q+j10Y7SHx71gi9jH/q/APKV4JI1GWJ2JwOSQaQVnJxW27IdR3rk8LY5yChPQdD6W1ZTJgn3YSMQMPMjpF1IKiIZAauNFvnOfEOb9hno7nCXx7Or7+/Ym//fUP/P77H/jrf37Df/4eh7d8sdjx/uW3f8F//I9/j3fDyJgKNASstSEm8WREd8fj4xec8xvseHARW5WrCTMX/F/R4YyoJTT0bL6eoTKgELQTD1tbRBZ+snjV5JkTayQ63tn8sIWrwVIcIWjThAjYUXPASqrK5vS5kzDqHu4tGdx1Q2xvXhtq1Y+e3ZI0dxCpEHK2VDc0/unLHpUc0fskOlawxlCoSn5w/HW0c0TWluqzwqW5hixrKKjPflfOSAlctnDRiis9Xy7n3FyfFsaPXz+q8xK2hCzswOzqs8YEoHmbCqSEFf32jHMcvv5x4u9/+wN//c+v+Otfn/j2+x+xa/r8BrcDv/7yK379ZeBf//UL/uM//g3/839+wS9ffsXjA7CHwXi89fQRQXePI9VUf8aonZsZivVKBkG+fv3KuT4Y+Y2lBNNpzDmr4Q9n9sjRhUIrkRVIKcSAhfWLbgp8VEK0mGP6jGRorAL4uUvM6rTY1BCMWOeCu8n36fCTwmNRz2j6k7toqvOzzXVBz/b6RmcnIb7HlNEHHcpztTAdRsP6M9kAFIGXa4ONXl0wlvac/Ll9VpFkR3voAkt6X6vPvnz/mevzRYwfdS6digrF5EzGUjzLKGow57cn/vj2xB9fT3z944k//j7x1//8HX//2x/4448Tz3PA7BvGOPAvvz3w2y8Dv/3lV/z3//Zv+O23gS+/DFZvDohq8sVAOKwye2lxzwApDsBP/sfFXleyc5zvoYK57gN//PEVv//+Df/2b/+G42AiwHEwiVk7HkqhZrl4pQhaJRcokufJtJ4QtkdkNXE6WVnfTQyWE5Hm5Y6VT07ovoZrAIs/IZWg2bP4aTomNwfzA0BR5wn+3q0pmdqM6G5VEhdLMKyq/9nNKU6osQ80lIHIqoIUFbSM0ywT0x+7YhneUjQ3BWBdmDo9qTAiocSbhe2ErXt79tCcte5YfrPcDwbBPqlDPy+MmZ8ZBIzS9hSy37/i9//8HX/761f87a+/4++/f4UN4OOL4e9/feKXL4YvXz7wb//+K/7H//IFv/7P3+LcwA/ulxsjN/XmkoFjZV5pWP0El1mYlCvEND2qo/3x9e8Y8FIiZjwEJYJO4xj4+BK1SH/9y1/wL//+bzifccb8BJoWjEGnxUDrFyuNK+pY/l8TCKv1p5m+5JVBkolNzBD/mzOOlbuDqne/y40I7gqFqZ095gY/HT6fYTGHM4HAodL46n1YhYKz/QU1RIOhGP67TNflYG3x4g+nS09BTAuOQBeefaMlHAqmGU/MPq6+8m6Z2w0ZhHNfnmt2eXksW9nmcUULYbg+G0kF3hDG//v/+n/w1799xd9/P/H89mQFsInHMHx8IKzaLwP//X9+wa+//oqPL3HkdJRw4B5GY6FbEUYWZc7aBoVIXeprZLLIQahyonmUKTzTYSPH4gFg/PIR26IOHXwiqNfyQylIWt6IAzQjcqhkbE1AJlE7UNGMYtbIsmGu4rJU4WTgmf5bLTSvUErBjXXpGjyYsxhG1/J75gTv3zOqyGBQlKM3Rmq5dNK3RoUpTSE1TCAP+BFcG2kt74DnqzTB3ue0jhkoQdK7uYd1n2CJNiZrjNYUlTsV5OS5GmgwvChVVhkNlZC7tHZLHziOlesJHKU4Ew6r3XFckI5l7vCPr89bxj/+X/xv//ELfvnff+PJUwMPs0zAjTd3huVHhJbaQaAJACq0rSWHGBDXF9uisgYIyHfbiRtWsnk+cdxcMnDBxnUXun4aH9TJtagd7CiLlm+WIlBb0tZea0zBO4ZT98lnFaSCfsY1ERq6R/lCm/P7NslnVlVvGl+Lz4LF8s3BQBEFNVguz++FI0qfqMKEIJ55lcaIwQyYdnAY10r3ubZab+5Q9HsQe12v5FipEOBekU/Cv6QttF7beAjK3FKQaA8EzUQegEfebuDTaMOseNgAHfFQfavx7eus0YMwQLUhmxHfP7uI8f/xf/6PxOfOLBY35h5Wb9O6FaTsGrHWc9yjkGwynIRldjbBAidu8X/+HWIjSxQyP+thab6QGn5k8MmUgBlHtMFV85PWkxW/u/YOaCbR9PazJiaUq/a11SCUpeGbMOq3Ds3NjCUg4tLJTBtYCiFrzFpf1bNHYYrWVyya/xQEhQSwlIap71OCpnXXYtTep/TjXpiFlwJq1U5myQCQr4bWZFnENteMxOSSFvuVPmerCgcmAwg9VNVzKTNt6m5C2kB2KR11Ubs1nP0NNOh2T4P9+rQwTj8avQ99qDGhSlrUBt/4SwRAooycM2/wSkTQd8MYpkdOUAlBCaYikEl04zv9ybcbau1SFoz9cWP9F5Zkn2UtJ9azIzOM34QyaoHGOmhowIKxdVQIE4cjkx0wFrdqY4CGKG3quh/tPm/tc0wiYt64QrIU8HxffGo8ccsEYTEwcwllFrt5JJe7McGCazYxHc7q2lKeltYXjtytkoGeTSj3oE/7Q/Uc+KdSGnWPkW88avyUNkg5NshK0w0gPQ8zVFZVvHeMQlDSHcEnKstc+znVLfVkNkUhhSAjpJszKPSJ642Ti7ES1J1bTOhBeSGw6cUIvvxbOZ266qz1hr1zXQ9sFBVmbu3VtUKjgnsMTLjTCs6Ey0szW6Ww6Bf9QvkFbrBJNjWVVYgJj3S30QyWKmZa7HAyAD7ykNjLJQUiJYdVCRR62Bf+HUWm1dImLfh56nQyaU/IAIDBLU+yvIoCR9sK3jkwJnAa3I6IfqZrEpM0MZGoOIMubagbzNPvff1xh4ARuZbggSMhfVMY1/EkXckP8sPn7L61BF4/ceEDzYfuq/lovi5RiaLBnmPrT//4enPXRoMvXgIY/k4JWzdwOVjiceqw9p/uKSuXi+DAElmDjUqxatLfz2N0OMY507En4KCcbGF3o+VEnVY89YyPPM4bhHO1f1dCQ0YRq1O1TneuuTZ5dyl4Wb/IlQWUzVEsdnd9b+G/0AEg50+L/wqM3bS4KljViGV7FYMSYgkBjGDQxGBJfEtGjiWkw4+0ltlXTAV2l21t/ecupPsaIrTFDDJ2zEfOukBMgYPBtEH6jIJYNrQrpuaqG5aLAGqXCV+YwK33swaXPBTVEvLjkI1XCvjmemOnPxm3O3jWOtrwfU0gxcBKeAHJUDFQwBmlqlkSOm6WmgW1r5iuCXl2gpZ6cJJMlowqf1hYc7Ms2RF3HGDZpmTc1IDs4aQVyngG2iQyYX0ydU8nUdWgWUwqaelQOUE0oFcDvdfQ3Zr06Oqt/2Xi/qLxQiWv37vS0XtSKEhbMXyljxEVZGn76+Wcf/NVHexW8dXV1yIXATfxlqxvU+Ld75WQeHta65BFmuwLf+OTlcKX7+z9Cs2qhymgVzqMcfno5fWWMCLLFrHTRsZIf6NrOU5cs2KD3Y39edrgWq61GNi7oKqaACiLUHDGi/BWhJdfVFQ4mHPKFDRCvujp4CGeXMCfUfncN98r/Yi+n40T4DxzIWGUjhTwsw7aTEaWb6Id9UnJVXHhFVzap6TC8neQaGe4lLkcm3AKoX01i67chFqSyS99o3VXHiC43isL07fcm1FJ9jbjJUvAprfeAnHZ995k3ztIa6mgVgHahh4cbaw1TilHzU3V1Q3hG6bzFnX7gv9SOah75Tp8Dqi+5zOiWSYKWsiNYAAKM3Ntr8MYjj4tldsBLEyv9TsKCBQFa7SX/2TFJFa5TJDAJUOZHHm+M/2syqntWtdBRlqqBGzQqv0+m2UQM4pWKttvkEWtJG6nFV1TtMJip7LwNrm7ZkbeIJe4ZIRWMaGh6MdNyZbzMxba6GSpnoaWKRYJCTSgEDafsbsdUliQQFZesOrtiTknUH5oy+7Zx9k/G6MKBuccYLOsqZBEjI13mkwkwGuJ4s7yGupnh87h13qVw9kU51pxvFn1XSN+53qvVKMmGUDbQYlYIAe03gbgZpH6ZDMStGLbakhCWrDDgCpr2PsiC4iY2BJc/i+ZSpHBWLusw0ZLYENPdGunI8LidVoINr1rE8710mxZ/lQEeE5ZQGbAoKK3MY7wJVPxbdZgFRSkhncv6iqTSfPVNXVm5AwtVusdJXzdxymlIUKw/6ldPBXNfiWvW9mmvPpDLqVupYcbrC3r0hfnG+80v6GEtSx0CWT2KOmK9glca5TrGIS+HFKUE5YFrUsY16h7Wf3voPDL9XnLaI90z+IFqi46MzdU1aNj6MGUgiSVgxj/pShbW+TlICwd9oIzy9VglLtzrVPEyBv4c1a1BHRm9iZwtfQxpyv20oSTgtMqRufm5N1qdh9jyipUe3mOYPZVtxfiiGMIWo4l7n96Y5RhoWyMwlYTVxocxuQKi6UbQTK3mRu/+9pdXFefcMCYz1pWY40X5F2xGVwW1cX8nLt2CvPo0F19NtHTSPtmQTU/aamUOdMQRCt9khZKVq0IdE8rvWXjPRUOG1Iahgt91GbJy59uGY/SN7LVBsAfEERFi2rJZxupF/sifglLMOgJJUPXV5Y/Yzp6SpLgD6FiTnSbrJcC2LNDjmQQLIKHRQjV7Zp0MejNUkP7PW2jBIh5sSbfZJskM+MeQsuZvIOpy9T3CDV13UVTNyZzB7MamJmTVmVtuaVubBZ5Io7InhhzvlyuWSr3pV+BBnyr3Zz3NHakPQVS5T6K99rl8bKuOAvKnol+EhksfKTn17ZlxU3v7+OyIpiCR1I2gbbKV9ag/nRhzHKJ0p4WllBlImIwwaARbu7RyA3KeIlRAwqrz3OBgjP9U3fWAyUD5Rp+A0Px1Wx/FSSU9UwmTqHrgshsCjJzVWGhUnLG+HxlqkYxgGlm8alU+APw58b8ukkCQpr2PNNGIGn5ZOtkZO/kJCm7pQjaxnAnMJ4cv5DNyMa80QzQwvWZi/9ahxUtQtK83lPk5MX8YwPXjFc3JgpENYWzWRsp+0WO0jJdx1pUS5WIcDX0mZQ9x7cpu7u1zuUdbf406hxXojw0Hvzx9fmd/o8PSOhWpmZHDACF0E07rUkEWpS4lUENQ1Kxsk28EXsnrMntC5IqnzTRi5ioC8qE0aKreRC6Slkvb2kLwdl9tyR2jbVH5/aE4XqXW2nNIBqZzNhT9woEQ3NYmUzFQAb3J7S0Y646pyjruTHp3eXtHz/JMKPWQ420aYMverC/aX1gmD6QRY+bIpOoXWICpMG8CNrq0Xn7rHzAgowLVK+RLe3d/R7EmonTBEIufWkIKT6rSvV6RGg4Ekkmag1zG+6Scvb96/M+43iUBlQnRfKFs5tApeMNCNL1NUioaK3Sa1JhbeodnGQpAZ4Hyb1Oy+DBx2OZQJFCfpnK0CTVnGNa5rlGEq2PSZ/k4nOl+MkmV7RVyxuVIO4qhoVi6IF6dwXvd0YI/9uzIHPzg7B0D6kcmuYvFYVSBvzDfMskQTBPPtkUScJdXrldSGMUjdwZYS2Gzbqk6guVWR/nUrxanyXiGK0//JmQ0nP8BXk3XoO372x5xyootvQl+br/59LRvhU2XuesXy904+X6tDCeuQMAhPJlKTq1RbgrZNj/ZkdRE1kdXwkYn5cOlFVc1wIrTQk5IZZ9XqHHwN69hK45WSzy6zyAVW3kcFY/RZ2L32qLlRCESUkpQNOVjjuPHmHJDuqqgmBSXIB8nmQPMVhCozWndfFjL9RvcC1p6KUrfWL6yW2bs/6brAAwJzAZx83AG7I6wsrwkTjhWZm98xCyDf28WHe1DWbepKDdpdHZIni+8UpGrbv2BkrRbvGCcFOY96y+JTNYI2znVd/a+vH1Rjqcli9OgFtUyjm+5h58X1MUERSIKU2ZsbblGdum1ngU9xK9TKsQT6w/Wh9kFPnuPZJ79VdqeWJVChI2ranphTOfy3vNYHjkyBSJBuL1Q0LbtXr6J5aMIEbz0T/jSy6CVczYfdRi+jYHbdeL86GqyQPsa6LZTqtXk9uqmB2VqYtGtyAr6QGD/l9S0zbR3IKCSOHDQgvv0dPSlMtcrfPo1++8rUf2SFJhlUQHNQeo75LmEGMttPrThbE2zDIXcNtgm/dh1eMv/ZcUIj3nC4GvhlVWzrJ+azVAAprgFNC9Cefjy04W91AsIqfWPndrlwpnpOLQF/KdexdVX6Z8K1nImuSC6sGQsXMn7rNMFkfda+13lNKYG3193k9++ffr5+c5YfbkceCWSKANHEeOgMpOAj4sLPyWZF+C7E2Ajky50506jSoEUvRo7YymMKwEvlCT3qZkk+rzjpgu9EQoPTPNYT5ZihmW3fEmYBEbCD7oqYCytnq2rnnl5RfXGxk4Z+Ps0uDOrUpmXTQrcrmvQXVGFxENmjjk5zs8zdalzedR2hOr89yVG5EhBamuTp+CFetk6hmHM3cWGcGNGwwVJWXS9wZjXDmpVpNc/eyWuNn0O5i29XfOWtVNOn83cHcV1C0/RS+vntiBNahTdMklLFkJzTdb7ko2/uYiVxc4b4vniwLsKKbRLS04e998RDPc0qxo2QOO6/edBNGm4OoZsB19PmaNbR9L4+/kqRsl+Or6/H5GFKGN1isKHQl6acDKyrhOvrI7FAwwlLY5jrC4aAx6tz7j3AAc9XKkaXWEtd/unUvhv/AeD3ghjDJYlhMBkIKokocFq6idj4hsWmMinTMoRU1dUXPSlc4LWu+T3F4JADg3EF9QGbTg9d3uhyXM4vkncVZm20Tbngu9ZwU8tL7MLWUL3D8jEHc+nzz6j5Y0OhhLIWresSiSfb5sE46yqM162mz37NawR6S7spX7oAr1im/4hd/CIMgA9MBQV/p2P1f7d5+TxffOZyyyFLSLNw+oNH/rWXZwsYjiagQDKS3LcwmhnrvTdCVYs94BJMTKZPT9Wb/9lRpZSqL5hojMoKpLYx3FFIRJ6Bba9GRfKiPNqlI4Lb6nxkRC5z1YtfydxKdvfWsFXm8PuEsuMB3Eqs9aH3o/KlppaSXK+ktDEdlM41kiW1TaSOfZ1p4NFOq0naksXfOSvYsi1uxkToBp3Mv7yir1hP8sij2OcgUEyC5KvwI2m1pf2qv829V9EO3ECJ/c6P8GTJ1nBTwQ2sV9hMVkbmh06oTBcaiAkVllaXhNsDB++l3iXr8yWgpCCoYT/jmRgCKWwCkm6b6PRRuFGLx0grSnbCwnTef4JnzMCZOWrd90rDlQbYrhJsDzQytI5ezz8IJtXUCd7+tLVI5QDBLw3PmRiq3Ry1bGuF4NoKovpnlcr458JXNBHwoDKySg8cbFZ1U3TTWGGkRRcSvvNzfUIEFmBYJSVBwoHCs+79ZJKZuOUJYOuHbwOMrNCSGb3BWvyHevSdRdLdFCn13WmPP9CO39yW1Ub0RT46i1gB61aDoAzDFzv1+KioryJA+XRpOvEduPUIQZu2aR5gI5mt/ZWLJQdGLsTJzulX3f+gu+CrLW0nw2uBdxxLFqyuHsAp2aIsnBb5rE5FAp2GLUFPR1fL1Cj4R9aYmvLaUvGN+T9KtvnWG6P7pbKsg6LT6bvmpwT6/0AZWhUNAFM5Y2dCLU9LOswTIALO/V/FAjoS8xZA+N40SRfI9OuqNV0OgCuLsBxjGKlp7V9ozCKKRh49pOp2EI7HUp5RrfaPOiSPInrjeEsQSoT+x5hJaRhlF2TFZPc63piSHo02T0cvJsgyJfXIOQhXUrWvs8H67ZKOTAg6miHeZEr1dCRWQ0kvqNzJc4KeFO5r4uDLEQB441etId+FxXa4xSu1oaWlD/knHZZZMlvVq6nUl3qKsxr+Rd2/G0CHv/m7KRj5iCFG5CVe6rYJoEaFn367Tic6ACjdL81f8+DkU8O7xN5XGha0cv9e8a3dyCY1aKbNh6z6t1T1nmvoNn7Uc9o75/5nojmmoAt0qB5lf+kt7l2lTnUWlsMNqYTngZLl4Dqidqjwfrlqqt/u4cIedgx2Q0Il67L0rbeqtJshYX2pcC5py5yTh6vEZ/X/mxK522Ky3ZqwkJouSkiZGXNhDLOUmba8AhXlWCmN9ZvmEnGWz/tzMWCIuJJCJZe1U4xsYkoBnEm1puKDQSqKUJcIN5tWvkCq3T2gojt3GVAtAHqzAmKnJ9ZjzAiYYiy7gw/pDCdY90nH3QuCnGN2jObtv50fXGrg114qgBZg1KEV85ekAI2mhlG8qv6es7gOE0xzyfeBy1bpVaNoaJJHZDhQVrspMXAnhjsu7zxN+rcKXSd5H5Koh3DN/h2QqSWr9vru9p1W0QUEDh7vlXyQre+Hb/PpVoYsHEg/0FrJhXG4Ul2uAG7NwfOs9lxL7QRZNd8yf/t8YbPutdFDITzEenURXvqv/Ehzu9u2CICR3uZxwFwUsQ9JWS4x9LBfKRQrnO8tVa//h640g41cnUyboNOoA6SNkjImKWv3cqN3bbunAyCJSQp+DbWhLOG39bcZru1e9TkV590rWUrQwCHaRJpm7vWRXH963QepVVEIx5FQT43hJOja1rZfWzBPwOGq1+k7XiyTVGJy1GbnKeMBw5bqclC2EzxFJC+I2T6XDuHuUSwakiosAiNJoDNL5cBfHS54ti4fd0KeLefVFV8xH3a/NezGdv6ySvWsJl9fEyDzeWWv27XNL0/L3GdDUQr663ljbWTrQXU7vKP0+8DNQZf+QEzcnszdB/jNN6u2QBCloMK7gRzPhqx72CG7NNnCGivPV9RjcB5BkfI5jkPGuJ5v4dV0u5dtouArfT73sCqADD7qdoUl8dE7e0K8vXldkynpnwEhYR7zm/5VEH8mnr/AhfFFcbSHNF63dfbrGclgiUjJSPos9VuakypQI6P/AQLpfmttMgWKgrZilOB1r2Uxf+/FutbB0pvxlF0+W+P1sY9bLsUfwzE+LoKOY6iQcm49bwNSUyadTi9wF3bqJVLl26rHIGxoey+uty91iMjz/Y9bZgu+wG7/3DImB3kbMf+Ywd/nXrehlnZzr+r8+ZTm6KNcE+FtKht81WLgwTA8ra2glDDVklfc4uPqUw4/PWdvpupBXjAyNnFnHArGi40WXsywS01O5YLOkeXCl+a4EYj+itgcfPC2qjfuaxd4tfuX4PqeOWZXOJPBMhYByLjbO9r94FvStr3f3j6/O7Ni47ujNGmILpS08lPIR9mFkeI+BtRGbHsIBH+WC7BK9QEbVYkgjL2AHbIiKEdGktvCasgB7vSchEwvOUXlnVXfheRdiW7wUni3da1+4tbao6GQkyi7vDn57kCHnxsuzyWfy+/R5RXTNsOpHr9/q4rFUKW7vdOAeZFNAs3x2tdK3bk7D4gXsAZIW2EmJQSUk1d/ru7+4M2S+qdC8/Ve3rCL5+H+Dws/gv+NcLXcCWPZrfW/L43vXG0kZpTk2+JwuJATkrFKJYdHUOMs48jALAFCJpGwdqi47aIYTMWqmW1jFrkMqCFb/l586ivIuWA5cQUludOWmpoXkqU5SUsdpueQNT41V9Iu9hSecVv3xPCwivAAL4bpOV552GyHKh4pj80KT8TLD7qom1fNH7EZBRqmhFC8Xqr9fJbAlelD/X0cWd5d4DT3pmjJGQT8GR9rLSBkbL3Oq33lAbtSyi/ywFUJXgynJajlX3q6/yK0NXWyKyEmq9Nx7aXYw/fWnDxkCBeCSh1CV1PKUV0WnNseUEDIlEMc+w2Ng7TZzf2tGfQaEevAihGotSUKBjCF6YyMQcUhUPNjSCqsnStBLYTEcUmsEKX/do7EqRHcqULzlzU25cQh5+VkRaR+rVSqjGj/RpkHCWHUhodrU4d9a43VXKIoM9QiCvNXsin6awXmWkZFmWqOaEVdZWRpagxzv6WNb7FzZPa6hPDcajGPrWtKDdWB42A47jcXEhipbcsTFjTbkvFenAnErtVF+/H5zbr89bxvGA2VG1QSkYBsAaYd2tzmmwwbmsBW2HVVn2lFTElhlHHq6yMFSHKgv5DVBSQLsD7pg2MVShLO9YYVD060Wk1Ms6L/aMA1mTH/Rqgl5ad63T9dbzRCs+O8V40tpjpBaPJptgJwUqQBb91QCNAnVvmRbtJmXTgMO9PY/DfS4R4Rzyq0BWb4lz6ojdLwadM5uHAK2NalD6rLlA21iC4kzOa8qu/MUjixxb9sVWq9uuDqMVyFssnXhH3WvthJKOd70riMBbAZyPsEwHgJ0JRW4LQZusCTI6vqb6LrK04kcOLklMglHCF0HV5U0q32/s/tVBz+VnIRn10FDnggLsw1mEnuUH9AAIboYcKXisf4L2nqoLmYp6hY59gb/S8tIOvBDEiEiKHs1qWLGmxjxKOuHE2UoJNCKOLOalPvDD5QCfjZcm1xsXI9z7l0pMNCLtsnoA0QokRI4TOUFsR5lX/JBKI48KXGSXY1Gern7fFNFgtXQHA05mObddmUjIUg0RNmdbXnzUL/PahVKEu4+if+/6/BYqCpbqkohZJWRJKVd0UyLBBN0FBmm7TnUyUVdGDvn53hEzwI+EUvlx/21z/rs2tz7RGhsrzWXC8/RMGM/LnRlgzr4Wk68jWa/d8i7w0YwFrT2F8C5gdOevmtrq1gQl1LlQr/d3gW/NdcEv+Hi3rLIxFelYoQ4F1mZzScp68ZFlP2hGkNs4MqiSzws10Erl9idLBLEgCyqtpb/UilLyAC2yrxXhYagqiB41isYceD6f1d6mlBNON6Fejvybfnnm1fVWRXGHZ7DEk8T1rpiEkZo5I359+4GRiMka7LgZ3A/M+YzfB7Afpx101GSwSdtFi++g5i0fIOBMFm/0YNip7VMURGnMqpkrZWMwVdBGMUppmGuAAsDCKJglGNX1Wu/UlT5TF0Qygnlf+K6B789rjLhhIMBypWWBsV7zUe3ELpyylFZtSMisFETZ6LoGzXfoHB4FIASwHfEAlM/Zhf/xeJDx16SQPOeF02RjJILoisowuGOstLQAQC6tNAOQ4GAYjo8Hzm9NIPul99oAzhmWtyGf7k786HormioCdGe9hCpeOigw4EAK8RQM6wKoidR37tpXHhNVY5eW37olJ2w1lHH/UmQ3BC8deL7PUjJqmtJHmFPu3+LY3wVtDOs9/ZL2rbpdVjLim2LzqyW9RCGX1q/f78++DK0LKgDVtwbxcryLgHkpBgm1Pu9oqY/FSVcz6MTrkMOqEftqzH1cx3FA6ZJjCHbzmVEGYHjvV/Wv0uSavz1s479VkUrJPMYRx7fbta/ugPWlv05vM1y68uJ6owbOo0FTvtBaKL5tNN7hScmhtGlMaK4xscPDD0x7QoSLjI8OQZq1WvpWlqCRaYOITW2k0CQ2gtSFfMY6k4Ntz9ne6qmcZHV1tkcJUxfYHarQH1Ypj/ZNHmUtoSB94kfz+1BMvL9vp8t+7smu3ZelCL6vf5YK1cDKcEjlF6yqE5xzFNmn4YA2JY84WSfiChG9KWFqY7xXLmR61Hj7qnHR2VPxVRW5NoYLlfobSkECyCIH8k3BqncEgIXthkUt20uf37vezsABLKESXOt98f1Y0rRkUqwp4KZpHVAAQy7DMQxf/eTZ8VuSgBc5j+PabTFRF0ZtXA3t/GKSawCLRt+t4MIW6TN7acEXVmhhHOP6J5DJz+6AjkyLfkZJ+loLBZbdBNLEy04QisUmZNeIKi5/X/xR03zVIUE2BsZR8NDg+XedCNy6I6FPWIFcfk5ibhbjVfRRlqekqZDQEALztpyAUBjp30lv6dubd4DNYrQ+uRNVGQVb6CsmTYchZZCnKeKuzN+53s5NDeshK2dNyBqjG6DqcXnUdu7GjQ5OB3AOTvqTkUyHivQWI/X27dafEFuUoYuZswbD1EpBCEcvHVw31FgX69AhYpnolYHM2jpZHYNgzrP+2MBsVlkyVwhqQOkNDsuA1mL7FisSDfQ1/X4Zx1xopvunNYb0U5vgDTG1tKVZ5nbM+SRdihaJEogcBpWpDq0x1dlpgrhYRlvHoH6NfL8GWUo6xqYI9XWN02UQzDB8y4nu726/BwJA1p5yFF/jGLFxngKXx5J7KbC93c+K5FvCeN/w1SJEyL+WDOb0jbEFGWcsn3pBrzE+1OxyBcSZCVN2IcgfaXGJU+DKzAuCeYfKQKldQAmzYwyc54mLIOqdDrgNvkOaOjXBQo9YuuD3Df7ELgmTy5RWxEbBHTOebHiRWORz3pRL1Cxv+aGtX7LGaHAxCW3Rz7R2gtxzpGCJrjAtVXH3/0Qq2rRiUERd5VY0XxIq6LRzKu3Jwl4Gs0f2XTWDaswmjwJaa1UOcpTYXysgaD27gm18n6CmdTo1ZvMmfG3ODch0L/Ur9mbWHPUsolRin7SQb/iMvUHPCRAvLlGjduyZOl6W7iDvesBcd2BMRIXO+5B6Dt76Clv7PFO6+JwpLYy/O7MwqABC5bFUiLcgQDS4wI3r2OOeYYLrkwyqigerRZ0iECSz+4JwMMqiW1DR1B5sSmF2rQc2RUK0IsFKAQYwjkdZtsX86N1snDBQMN3hqfk3Xk0GV+nErlANkeY4DssAmi3vjOvI6gGgf88Nxih/X33JMhk6Kk91bOb6fdGHiAwS1kYrjcKvwr7P9xoHWBWzLOH5/MZo77G0s7f7o+vz64yEJcLMoTk8Fb4N44IqkEet8R72KGkQP2npUsXvGvDK2EsKE32ZtA7siOn5aZhMhcr+wjP1I539xY9g0Ii5wuZYEoSbexiTZMBabcgy8aGCHGsgaY9wLhY0f19L9HeB7JYwBcKr3XQllP0Ey4pt+6W1UkHYCg4RbDaLOMnwoTxbMazcYzkaSOB62wRwHLRaUtwTC4THo5DLGIkyrDpZGl/vMynVPpbVYoUQndCp2shgmeWOmKx9sz+HxiPbPPR5MzNGeBmp91atgOPrsPVH1+dzU1NrkTaN8NKu7CaIB7n7gZyZe75KR0lnoX3SCbsTYvmOs9uZcLHORgHPqnMVuYy5KQuxMGrKZkVw60YvRCtBkN+1vLd/gFRMd1HMlcb3Art8jrsgUfSgGAEF05p1qbY8hS2VF+RSVPCjW9ExxrIMsFsIgICoDT/9N1nedn+Oz61q4IDKS7zU5rfTKCD0QvWLVdxpj9FyZDriWu7TJ9elpnK59orxwDEeyz3ZIBQv+ZNh6hh94yU4WU4tVlkLtXsj/ktDho35svMGhdrurPkddFh8N3RBQAkLlOJVglN3W6xxTYLLrgHb024WC8WzLUGkdSrtrRIS3bIs/Xxxdf83FLilVbxoYjBRfMGzDDZYpUf49l20JR9G3zkZXd97oz+lQmPpwoM0MtyHO7jOS0tJbvLzzCTcCL6sOz+KfpWmZjrL0lo2cQpiWUIVsW4Rr6Vd7wJMq1hW3jf3rWg528FOSuK/u/YAorunkTYpv9a+tXn40fWGZby2uPtv0bsVn1duKGrsmVA56m8Bn4uGuX52Cai0zzwdnKoOYGPAppaIQ2HIM5H2BXoNFLCoEgle6Tj3tHApJUvNuWv0/lwyNrlbE/bKX07YJvTT6TUslyOC3oos1sK3CoBrbMjRl6WXEawup6TmWEZjQGPyhx19DfhKo741rsN9Up9zFjQ2K+u7rMA2y+6zuRztXYpu9r/jHh16W7VZX1kq+a1JDyqCNOw5X9WOwSrNOvuLHKMQ3Geut4UxIU/TMmswQlWiz/wkblN9FafNijS5eXHwm2DdwdP+vheWNH2d9YtGFFtodzdGETJ0xoCf58K8KVhj4MFE5C7YHYrdCWVpfA06vwXgOI6j7HhbtF99aNCirbg4E8Vp7RKRp3GsAE/C7BtlO6dzF5y1CRKNWteT8YPG8ptNvtn0rIkrqzgXgRLPTNi0DKxJwQHOqn16xZUX+rpelk2xiIjnflRqmztlDqKSVRmtSzCp6LsRQJ/fWg+WlYTZcv/3rn8wmoqF+XpQgR8Tlp44c9G/nQFBJu/HAVwnFwj7/0CppdW6AMF42gapZ4Xl0xDa5mvlhuXzMq5s2wDkOY7t9CF36vS1+rY087DVIl78RNJC72jAOe6nwko772W9LwKzyHNqiejdaP49/EI39fmurwkjjfDQ1JfWp3X0DRG1uUFY0KltYSznP6dHLG7bYeFACqLmcLqKYq2v3H3q9e9ceVbXEoGIXjvaClr1IB2FnA0YBfHKL4Y+EQlTvQJjP3JXdP3/KEiFpXOr5YyBQeebW3/+Krz6rluSgjXcHdGVGCuvCOpkxHJv1vPVrd12w27l2gRNlpO35kMV1Im3lsCEdTqO42aCiz79my6Mvb+RocM1uk1BmVVSAQ3HQrt6a6OVtfc1Wu8+aX+Hfs+2ZoOJG+RYgyXr2qbzHIxpa98D2cl6z+QjR9WuldAXEgNKOeLCJ4uvmLVxvPFe9TznlVk0WpIpycViQQGv1LoZvJ3DzkdqnkebaxsDf7owdj9IVkBQxHHVGHGfTP+NVocGRbFK2OdRjBgAcIZIEGIGk44cevh8QcQqtqsIodZnDeZ7OT62McDDWmIqelW2JXjSIaQJxkoJGCpoSDCTwquzXVxPN4YnpFPKFZ8Ndr/CsB7cqQ25u1Xzy+d31x4R3T+vNj0tUiCLVj+1zakBrSC0EjRiJCc8rd08zz6ymCMVLm01izJZa1MwNYfxhwEsfL0JQEng+jNpH4p1umeCOQIo5f0rLVqQMpHBFWmsv4fiGGPf/P76ei83tQ+YmscpKPePlKDda4ddazgAaSvnHX7JWgOoSc2WLUXhH+jGSavmiMrRRgtL+CFfVYvHFNy+DBDvWUtspC/A/oodV8YROlitTlnJHeopiWKNvNVCNtPC9lSvi+9ztfD73xe/M9vRe3r7TCxo9Y1U1cExM3m+LJfGTmWGifM8g/Gfz6TVOAYwDp5FsqKqfRyAtc3SpbzT02g0u+OwpLBcF6Dmhs8rnKeINvBifdBXFLLTuS4ZoDVB/0fXe0fCcUDJWFZRwK5xinBAZ9S7YIa3bB1zh066ckPuMQyBqX2S5XPZclhOtCHmoM9EJJSB6wUXss9A80k0MZ6+SjTfYBucPoHzQUPV+1FQSlpemUotytkEYw/krJ9ZWpr9bMCMODpin2RLZes/ky7W1mFlKNKKU9eiVekOiYtI8qz7BMn6rhZ35yZpcI0waGJD455JGyW4p8VtJBVczCJZY6D7lUsfNP+LLi/AmDghH/LN4lF5sy8L3B1B8/55p+NO1+sVfDP9icZAP7zeCuAsu/w5aoMqZnUHWl87LKuL1eCqzTq/wayEVwOwQ1Z1ELoU5FVIH52ZTaIFzgihoNqFbjNUBJKCI19N47AZPiCXNSRgCuDIsrlLYEuIGJ6CIKcEUX3o1qmEoU/YiibSijZGqhO3apzXdtb5yyWnxtw96JL9TDRAOHeeMBimT5zPJysT9KAJo52O2Ng7DLklBYRsh6y6AhxFj+4C9fH7TgPp0rS+talAiiaFR3PfZn6HniLfReCApRTKbZBrVjplR4tX+menf3i9nSiuaxnYxkgxqSccPd3tWG+Dc5nxCmMN6+BNFcXEzPy6YFud0RCO+mQ62xEWkelChtDeQ/4a80oljHAuuWjKxWi9ZybGMHHzQo9obrIERR+H51hFiBzzQsqe6dSgbb9PVka/d8g5Z/5dz4txmyXj94Jj8zkzTUxwEj5xnrXDXSu1WnBYxg9UrI7SJiRwCRRlwTKmk+l9Oz1cCl3KlvM2y8qVEimlckVfDYnkWHC5v34vv2iH0Avcn+nc9on5NCzdr89bRru+JH2NhAeyMI3xmDTsJqbomqYYtgcgepmO+HYts+dkQPVnjLVNE2yEY5wG5xkREdEzQqrwFUK5Kdm7FpttAn7OZf3KLjTgeyScmuRex9NKvO+sVkbyyMge+Carxl1gjqy/R0U0FZBKy5GnAzvmPEtxMDqRm6Qbukn4rE2JTXBrnlH+NcpX9Db2GP+GQHI+aEUcKYDpE/OQ1qQtb6+9BqU8duWj/t27QE3xiP76bPO/u9B5mW6+Xeviq1D3Z4vngKym11HkJ663hHE6kw12s90Ly6AgzNJjZ4Iwts/7oFSh2jwSnQEM+wCKnJx0vU5BjW0SoPmKI8CCv8/sojV4Vk/ICgYsG16TWQGJvcPWmL1bx01DyoSVjqk3e9RxcXhapqXtPjpBItfv0d/76GhFZReD2j7YeK610bR/+314COG0AZv6XVMi/66TR4qWSzVGRdhgqRRHs8cBeaWnF93XFDfHqgX+O1/c9IzZWiTrghxWnhZfZXuk5iVYuNE83rsG/P5LhHHYF+Bgzt6cC/8PZtv7LB8i6KWzMGqwBS0rKLBepV1lLYPQmjjdBQp9QdQaeNPqxn+cG5xtLhtxZ9Zp5QT6g9+dSfhiypmMY6iap/neKXiaIDJ/TwGFR2S3Cx3AKmKr+CR5zusOhZ0J9PNifYXwGmdfhdBQ59L3Lwu2BxPHcYBwuhjuGfA3O8LSqZFB35nQf/oqLPmTVjEzIzU2adQOP7Pvnj/RBOY6dltgqbtq2ZaSTTVwUwJkI2EKddcTQhRdabkD7wihrjeE8QPBoJOFWibgJy1dMaB7K4fAGjZhOpUjiGYdbzpsnY2xCH1GA4FcUqmB31uH9DFEdXdVhW+CCKjqnBjvEtl2QMeV1+YNwUsyzTi4ztwhd0HOyxKDlJrbBoctn+vFdneL25VQH/ddRFXBogv0TIUXNIC8Qlt3KAhplEVTRtU6h06yiI4xzds9HXEE1Kpk62X69rYbchDUbFD+Mm7nbnz+3hHOHeroZv2VMMnPTmPQSCBX7s7afuZ6I4DjZF4AFLo8c0DrTdp+w23ctU3J4E7NQ18t+qlI6sx2NeAajG1zIoEGiVdLKfVMKc20s2ZQBodxUiuiOmjwJ3Q8tqzxIuSu/t9fc2rFKpr1GZ/loShW65J+Tpj2HIklGHzRFqI9te5KG9Hjeh6GDvHsTOq+PieSmyyeBI4BsXEAOtW3py4W05e8OPsRUxJtSaH0zpVSWsejDi7y3RRAh/pDaXX7s03gak6usLHHJ277kvO18qOUu95NNRbcNw5WE+yFyz6ffQO8tbTRsi/YCfVtMBS+VCGruaY4jK1BoLbBhHXtFi4/b/Dm0kBq7XWXN7Di+g674upbZKw+o5/Vybca2onuyNfHte5mZulCl7Vjn9wj9O8AzpnMtPTVLMnyY6hDZWJtYdzYXg9QjEIquooyDrMogRhGqooEq+8hIJtiyp/xn+rB2CL86/iKpkJMBTf7UHO9sfmVNRQWwKJQ7BD9FUq4UK6dILVv5u7X9fkSSElCaaXGOynw/wXCGB3FNtAh9yChzOpjoYTSViZ24RkAkemRU5NowSys7R1PBh27lmsBgZzcJBcnbqYfg2S0giuyuAbUOh6kcWsrkGr6OCHwJaCVfax30MZDp1oRS6BPlpmxcJVln/t3aZFEHAbEhkuoKJhjMCmbn+f6aSUjSE328ikS5Jy4fqDqAr0LUs/z2e4h1mnLUEtzoEJCp1FZkjknjmPAuFm3Zxyl0EnoW3+uDK9v75WZ8wh0T2sC9DXsO388vgO0RNSVb95nIHS1G6v64+uNshvXfWnZYQzAFK3sDjgu5NhhxNrZgajF6vnOdELuNExqQUdrsoTAeU+zUBLGIB4/SwgazHnOCZ/PHHNVASstnOM0aXEsE7NMrAR7WKuChzw8CBoi0JhotQxrKpxHwMGOFNruL8mngxLOWQYlccpieclsXKE551lKYBuv3i266B4Bd9X7UaJ3QkCjSvJi6myTgRMbA4/HsdDxuj8RrSg1a+HQjVkRUCnTyghbGKcUcslNXp3WdSrxrAOdmoHpljVIbLk75Tag9p3rzUX/lenSKnFOZ05QWZpX10uNcfvxvZaxE3BM2DFWAelwmlXFp8/Y1Z/rXZooj/XE8+SkyOKtfuzyXo5/0Prs91wDJ8GRE8xWGlFkh4q0aVG9H1FT6M4aArDjQJTFHzBflzaW4M5OU1bG07jz3V6BsL4VbbXM4t89DW7L3U0vCkXnbOQAmDitphVXMBypDD23bd0hDVIj+ZDxXu/+4YnznJlQP0bbYMCdNz267Q4cxxqbqMDVOreaSxGluweJBuKlWUXis9bxzS1U8arC52KmEsLqtR76XNs18NEelZUsAuw+gs2BiedyYImYBgDmOXOSQyA5GQPIoMRiWeJVNg7Mk6di3fhBRZRtvNuYAKxrVobw0co8lUDw3YfF+ujopQf1inz+4GsHH5/ZZPlAq5+vfkpcumJCg6umXy4+4sxEAglrWKaAmovFY68DPcSc9hXhXi4yFGbfW+qVtG0Fr9IhoUVMcbJIcDjPb6tCwci85ttlj7YsVmOt8da8l1UvOEq/Wop5g7doSmhPMHh1vbGFqhSlolsKCNQZf6VtXrdTVvWVRhe2WcucxD0BkbTQezC53IDzhKBdMmBCT8K+7MNI5gME9fsie8+L7HsIu6WwhXn6tp8dMurq8HUX7lI3g5HQvigui1Q0rrGUjyyGgVseN2BL64RlFSFa6d8hNWoppAfvLprevYvFMvaJgwvlYUF6gO88zwrGbKVKIjNHGUiAyqQoCi8e0dqozlB5PD4WYYx1pu0dotyo980FBa1jUCwEWYqzuGaZxzYfCyp54/qHauCU/1W5oSq7uDDAKyTqnvBAoXNpn25qghDa4tSCKHrriOWHWtcEIUi005lDGhrGbAwbuUMkNa/MYkYbOlK7t44p9FgFbxHuBms79Nm1dfkgxkR4Qn/6KoOlOAolSEi69QJgHvhCpna33q4c3h4dp1Ja/HPWmN0mMpVTvrD6P7mm65yQCUTSwhh4stznGAPj+OAexKiOAI+sl2OM3O2R7SaU5di9XKJOwaxc32oWyVrLBakgS5u/Vq/33lgso7+Zs13RjuTJy9GC37negKmd2cIpntPrQFRt8pVWcAA+l7Wg1TH3jIx1odF1Pp+Y54nH42OBC70/CZPRywLynxu8H5c3zPf6KuTSYMjFN1R1tBrfXYCr/658zLvJj/euDODuuUNlnVNffr8EcOKPdVC0NBKUqhJHtWoRnKhtXzvsrD4lSuECuMg1jgPP8wTmxPH4IOQeGMcji02NccDGBwyxzSgUrGMX7P2dNRuA6kJ6o3spOgsXZ1z5ZqGze2/x8t59TvoSyN6v9adj0LJX7OHH1xulGlsNGxjcz8UCdodfv/vWySIatQWP2KpBWMFURi8XEUiNds/odCZygtQ93ReCL8lppu8inLHwHXcwmKDz3NNax132UDGlaGAmI5ewvdKMd7A3P1M387Ti5UnId9Hm3B7ZG9rCxJS1tT+ocZPeroNbsvW1//m8RY3TSYQSFjx8p6ioPfD4+AAwlqoJNaZubb7JS9O0paXez+W8C6T0Il17P/N907lBQUkmd3OgPlT0+z5odBW8/f763SMX2u/7/up6wzIqUOAAj86WdTvdMc+5MZKgZQlp33QLePpliy9HeDkeD8xvf6T2K1+pYNT3BpkkvGjbgYlIBBfzUm1sJUTiXul8s7BOw1Q5W31AWZxtYu6E8EeTs07sQJ7ynMIzMKxq2M5R9AxaNgGgyxBWrNYUg4kCgob/HD50RCW1VGCwEX/b4FopEwIGAJ+s+TMjcDTGY4HLti/FcC4U+RTtOgNTHOFuubn3FY160OX2O7sT1m49K0f6e3PThXS/59U8dsPzzvV2dbjn84k5n5jzxAAneG4M7yIKkEIHRd5w08lVc0WA6MAxPsrJVz/6PWgD57fWbUiYyPYWjaUKD6dLn/80xqG1yT63ZGqTOtd4pdu9oo0dOu2TE3DtReaHqcr2IM1EH4PZgaodFKorLP5EpBWK7lRaoV2WvvTFf3dZVNGyaJsVSWRdWu0dV3XxEe3G+uRKa69cpEURmIHKgVC/CVZ83/nDlmh0ooIm0Pu1lycpGpfyXII5y/c9aIP2+9UK9jmbs44w0L0az2evTwvj3/72N0K9s8GKEBqMiT2zWhG+AiAzrUdBA91NpueviaTGOiiJD9gipT60ttrp1XpTtrpFAOBHf2PeF5spOLIM+8d7p+Cq/KrWc9BihJWYCYlfXd2nvASFTDSt7UhrXVTlAVtyYtDiTLrmIUIwqLP3PNE/HCmIi8BaKBq32eCyI5dNbs4htFJx8HyqDlBNaLr5c2Li1f/rMLOOFjQqhEbUREF3QRWwNzZqSUP3FJKwLJKcCsKU6FJ7ajW3FR2O/bJL1Tj0937ueiuaOkataa0YYs0LvDPTAZcqH1BEjnuzQs0FeZTlK0tVtyn7gZOdk468D4wa1pMU8lmAN7VZ184KKuRpT7Q+OWL9Y0wHDC1v1OLdX+pRVP13l8AMUcFWrVtvbKNgSf1JuFk0l/JjEoNvU9Vayn9XWJKbt5U0P0kPOCkpxv8e87elnhXi1eevgjT7xcWR9cOjbddyT7ovfen3b++7KBCjIu8PlXxn27uSjTYPaA7uoO5nD795e2mjFkrroEhN/wUmog+6Q1nNZQltBm54p3aICMCoTGOHpOHrWOZCpmAaCKmcp9PRQmQ3aH3QUr+A1nfPzdSpANrjgqQRLJEy4PmI3q3LCkv7hN4GA0Sc9qb92dVCvsqjbEtHLbfXVDawM+IAevV3LWYnTANalk0TYfFDhwiE7CqLoY8dO3SktQ6QvfRdtDAIqVQ/47GR7eYpZBO3tE6BlMA2ZdivVJDmjA67htLo3f3UNT2xfNAzdZQVcMFdFPbuersGDpOPYrA5+jZ5N9ot9u357T096pb3Nx/CsW0lSoNHobO+ZhRPWLIAlmcAWT0n0BoYtq5XyXLHclm103exa89i7XAIq2/KYUVZBC02SwDvjpdefEeTtW5WetPoC+yyVWsnKvA7Rcg5MFlg0WQ9ULQH3dSHH11yAaxjYgroYZVVlVCaHBs1cn1vDHIzhFdsGJWvIsnsK3fM3C0Z9b2gYRk1xErS2H07U+DInTm2Pcr6YuxjZAL8nPLb25j8x/QD3lxnLILeZWKUlwB4n8tFS/Rrh7PdjyrYG+/uwRSYwv0rzJJ/ksV3gWLOeOF1TDag02x7WwNHZHbQb0C+31tbCthwUpXYbjeTzJ/7JuLO6Mvv8rvYXloRV0ZLUwSyyou2Rs5GBitMnwj6yVKs7w5FWxbAyZhuG/RD9Q1SitJfVvOENpem+zN9cSKP7oMvSjb6SfXvzrlA9tdgKYSiX8/q6b5f0VIThOxTuhMqjrUpIhmE+OoGcub9jM5TMOvwpD9ZGD1fqgFYdUzSJli3DKYE4hanY2WEeE3X/oSfMhl6z3I/YEyZ4m4hPZp97mPoMOsyzvbucJtqgXv6s2q3OiKfcqGRv6R7Fzy9o2vz+yBIpWA5PNbNCPEygKVoKAzya9UbBaEMg5CvBLCgr+azV10vuieSMK93dsJnfwMyxs6Nme10hVqQfuZ3k8LQDn9q9F+DPvp+zlWRd/7JXfjsX7IAVIgr/0mh0fxc0EAT7nSh/LqWXPTQm0eOo6O8H13vZeBsk/FKmJaOUhtbe2Yn5B2Gr58r5KmP1tC0YMSQar4RCh3BVooFqInpv8eirc9J+BEwe+San4QBiyDdQcpLH5rfcmsVm6WJz2XtoiJ3/65MEJluRo2fqD9jwCkrUCUzy4+5Lid132aZEiGV2Ys/XUaWVkzBpfRFs11DlQIvug36fvDahhdXQf587ywB66mRyfjurSRL7+T6zmp3zT0uuDqyVk6adPcsgDUUp+AWsHM6/WfSSu/qyOwH1+cDOP1f0+ADEuYAgHQH8pLy/qQQ98+luc2MQZr6vgIrwURjdFxPK9HgdO/CKkAEMZzh7meNhFwHM8cm8lhq1vUpqlyt34+uHkF1ONO3FKgaQlOQcqEu4TMt/A5aPYu+xvOep64LqtKpI1Tkm+2AhHT3G++CQzUHjKxPdQyZEmr9e3PAD8R+V8I4GICqxxrtC2VhCQqNMbirQ+pvUBCfEPwVLyrgJuTTrZ0sscayBNik6DVOPld6v8yQDW31Io3OM8aaa9D7KoLBPxe/eTeAs0MDTXW9upYWvA2+dknou6XV1t4anFgjWHdCbOQs3XfOb4ulWhhq1mRUO93KdIFyVnvTexRl5DPmcI/z3ANrFXTZI6U/pmrQTtY5GCCitdkWQD9rH3tFF3VacNLpoC+0KYiVftrhrvdJMO7GsCox9assNpbvKhkf2Q9v7UhYkmba9qZ3OMfvyPVM1UtKP68HbspMVo93wcOGYDpPdYFkP7Ku00aL9fcJw2NDHVcI/aPrvbIbqGhWhXNLIMMPX+FQWKW4N9fx2FoSQZdhKekOwQ5HpK+l2ZfCm5iQxr0y22IBF1CLS2g6M0DFUy5NGJp/jFEFuQwIdRfV8Zx/x/vW9aYLbCW5kjkp3M4+uaDPAmVjQbkYR322tA7R10ZK60jhwL4Nqs8qwMrisVs7y4p0RSWlVwp3wOfgssdK90k4ux8lZyPundxA7GT26Ne69zIEFQgDaLdzdlV2M9CaaLXBVLNCTA3PLH3vbfdzUu7cqQxKYTTaa76khGalj/3geusUKtmR72avY1FM2/Pldy5+QBJtGyyAilYeF8tYMMMSxu73JBxburISXoV18+6Qs9TobQj5Xgkeluja1X/cgzY2LC20aAIe8DKYNHDHGIs1QVm0fTzFXOqnskpkzVft3tPUpCRKYJ3vOdiymGw93KeEfH3euY47SaeZhbFsyV2Va7HTbhCq9890xelWdwvxEeFeXRJW7iNN5KN2GvbAjwONBjt9V7iLfq85DEfNl4Jon7SO//BZG9XJNdcxmFu1V+6FzmQ5tXbXmIKjgwTI3fH161ccRxUpCoGviJUGG8x88D72yzM4vjBbJ+JIU8gZVEHhhTGvV3xMJXNEE51prN0YRaJkxdh+DCDGMqr+i5hTfVzf2RnQbmgc1kFBJmWuaC8oW4H7xFWILedN1mNRQDgRiQbfOMexlzRqBJVwVBAERDUDYZ0jK2iMkVC0niEtU4l1SL5auIwBoOBrogMzltxY106NczzGkcGXnb4l1KVcXl3F87SKiuZrHlDr3Z9Fqm8LY0Gf228TPikBeRFSM1qS2gxcA+ZQ3BdNdz7jWK2YRGsM26BcvJkNxmuVUwp478YyjvqdFi59hQmbqyDu+N/McByPhQHdHbWfwgGeYQirNVrQOgJCfWF5VJW917V397ZZ13MQqQToP83m1wRNR7aTyy0WonO13LUups+BEVXalu90onCrvI2+LOPJ8NkPtSeBPnPUGGPgPM8UmO5jdXr2uXLvR9FxvjT/jkQVj8fK1quSaFZMMC55rhDPfsJVv+6UR5/z+LX48ntC3a+3ihhnMCVhXe+cZRlCCVkJW0U0HR4hcnfg2fIzUVouowIhVdycOhLaGLoG5cOybCPSvzwLynJJQvctsLMsamCZJnSGBNXYBK6e7VHI6EPcAxgTqzs+zjM+ep1OFONCiILWLbJL2C5CvuacGPbAcUR1BcDW05FlVbz6G19NQjMjs7GGTnayXIWeERVWGDifBbsaGEl69lOu1GNNoSy5qqZpx0weB4BQiF3HC4YCFcgqoeA67XEkNJbFFf3v0ExVSAfhIxIBhTD2fOGC+ndXRfCLpcpaankjrle7SPbrLWEE1nLzuiZmCCEXpbu2dHBL0devmPNbCtRovkr4BwY0H6EIFPrdpkvPM8J5ougkSBLZHLEhN4g90NKiCAslXPX+mVWwdZ9z/9DQga0vNKR+pi+iV+D63BQ8VdCJlurEE2M80LcO6bzDeMUo2HX0ya7Idf7nBe9WqGkMmtQuft2zK9YumAAwhsM+HgAOwlEKD4VS5U+KT5gp1La32AC32q38k+UwzzPSHgcWJt7dhLJEff5EqxLEEmQihM3KJp1v3JCe5tjv6++33jcta+Cg/aiaPoVnfnx9Xhhl2drkZQfdgemY89msp0LRfGY4HvbB8hHngq+LKJVaZrYWih1e6z494JNOMqydYly+DoCWvMMWtgwKB+JZGEPxypcFYLNOQ9ZbW+M7lOWnoUx8TX3Tu/p9+m4PUKQVA8U2tWujG2nWoMTdxEV/Wi2bHQ7qM2XICM3ocUHkgrVtd+kYoVTOUj5Kml4OBlKUWbVHYxAZpT4eXFZBn6uV3jvd+/jis+uG4U7ju5/XXOFV0Pv8aa6fzyc+vnyp8ZoCcepfbcRf+fX719ubixeoIDAyK6QPlyNfWlYTNM0wfHKTxVj2wlWwhAJ3npxry81RC/ZPOMYCR4MbcbsRXDBiW9xo/skycQZYY1p27AKRejLwyiCki2mz78rseolBkeErk2VXlLA8kWhjfdfMzzs0WtoQM/WxA0u/1Lfdyusdq8JAWu8MfpHuY1hbCrira0TQb62/6W705RSgF33qc76UvFyMQvW7+851iR9LSOq/yirSUp34QvNc9ZVCAR70SdNtEi+ipYi2/n/2eksYb5lGfhJ9CzHkol1jaLG9CGUt987WhBvsqBN8gfhb9Td1ryahM5fqVy7+5c14VkH0EH72Mz/z8m8jWifYNVobvU2NP4TtWozIWZ+m6gldhbXuLX/EcmdCjh3FVGElV3osLVGq777ru0zqXqENzVG9K6urqx8+6xDVEUI2UOVTVrSw038XDGVKEZ8bljH3n99bc7y3oNWHOSfO54nn8xvOeeLj4wuO47hAU73nYFW+yWDczk/FS1e09871D5VqBF8r91tgz6ADSdA0YPwI2q6R2NVvW7UcpG2Ta0dBs9RE9exlMtS2rOqcSzi9rnLcYx2th/wdCuRgUSJiHPkjSkJuG5u99n12SxBuo5YlrkKzvFuEbj7gAt9goDMGGAMefgC51lWMEnS9wln1UZHNstgN7WBFEI/HIwUz7pN1Ub+L9tHvluq2jaULudVNWGZPkH1j7t1Sdjo2IgXPzFhsCC9l4tvzG+COx+MDx/GgTz4y11R84KikiqLXymtV9zVf2Fwsx2dF8v39jL0zXsKmjAqzpCQZljApd6D7Wil70da2DQppbhKuDAlA9anDSMEZ9DQmMLXpVlPR1/RI3VqJ3deKJOASLjIB92qCQqrF5RIgKSJFG1/TdaVv83u9vn+tbbXMM6CCxxI0p2DNKUi4lSFZ1hbvIpKvUIz6Opc+JKrogtIYVVfGA0asQ8oV0S25j9UWx+ECWfvVx2bu4cua+hZ8MMaBf/nXf0+1tPiFHG8oiIjkvjpfc3+23xNsW/PwmesfEEaNGgvTGCLClDv0BZlmt4aWirNjfGljo70lVgntkoqaGobnI5pdocpyCS5vhKqARIeVu9W39PmwKB8yey+PZTUZwZQ9iCBaSNBEPy3Et+6mtUR+V0Ny7q1cLXaP24jeswfBaJ2n15Yl9bUYa4fgIVy1UbnR5fIBal4azfNfazyiv/lt+e/xbSnom4DY1aCn5Tn7lilDthOuyoFxUMAml3a81a5JRV91bmzETTp+PvpdyxQi+15WJd5ZkWp3RH1YFeH+xPUPwFTntqJVa6qr0Ukx62C5PzFSW39Z+jeacAYFMmWM0TcfkZU/xrg9RHSxsLLOFHIHoANckUzXDDsFNBIpEixhWWgHYDdniiz+SYaaOjztdKplh9WCCCJWykC3opFCd+bJSoaxNBttnZhTkCkinF1LR9t9Ha0Hf2jdElZ32l4FVyTWwrKxj1Kojon5jCDWcay7QtKP9JqAmKpg6HOeSdOurG0Ynu0kMC31RCkRh/dJmZVArgme1qGx3BLBZNGFylSnaNENUOxCPmPGJfrce8H1Cpp9XhCBfyCA4xR7+4Fzmr6HLGT3FxDkk75J4huzX7xBU+3Na/D3M2Zf79KON63xXmmjqKxC7m1iW9J7t0paO+1jrYSQZr0ufa3vCkH48l34IGfzxwpuypT3nEq1kZCPW6KAEqDwFefy3r1vxkT+qCYgf7wvp+zjUT8oHBa+qnAFjieAx6UYl+NJpl6RgeIBg7VY2WgwNbOTBqGnWUXNUe7oBcoCazGojpAC9dBYoARIfBcZUdddMvt79I7uZgzWGtpT8n50vXlYatf0fdD83AzY0uBChR4oEUQ5xCTI3Np0CYTV31y+X3C73t99Qzn/BbvyxmCew5Y8yvqeVsm0Jaq6X6+LTceCSB2qyN+qgMd91kVZmF4WwnAc8d15PtPXGMxXjUAWd16Ynq8+hcJ4QIrDAVYJrzvGmMvyQYxdQveADpItcLFa6srYqf4L9ofsH01hFUMWf5Qx7C5Kn0fAYDP2jFbgzrhO7AV1E+TQDcmKCEQNo/j0TnHrJDDT72OkYYAlhtPAlp39HUJLgZSAH0zeGICCjxu/fu96H6a61os6I4Mce4WOiaa2A0BMlG3XEqpvn2nQjpOavbTQLow7xLnc5/V9jqubTEJZb/fsfQRA+NX6DToKWVKi0cBj/OcZ4XT5GNKcYwS0y74mI9Tz6lz5KZ1GFMLcgNsUkcnCVSpbMSrae0rYhGTS72/CE4ENbaoutAJtvIbGfo1yh35VokZrU4p3QRYNQVn3w4u4C62IanqVNt2zzN/ymdoqeinY5HIHcg66cSkrufydKwiTWY1zlZMfXG9YxkmlLKZrE6GhbcZIVw+PG15oq6Z19s9DiFSK3i8DXLSo4Np2FgOATDK4FIWCulefLT6frdHBvqFV627hduik4xCA6IIYOwTo4+Oj3psy1xBHrkHyfSz1MWe3Wiu8rVL4MQE9eqkDWNSPO18baBVhdaBqrxHbaZgwuejbfc9eRa/a17Pn4tt58tKWKWfilVKUeqdO5RISqyrnslgzoeb+fugV0yOHWfNMnim4WQJ+pVXRI7vcILHZE2ZVYfzT2/zxjs949qjVZl0EISAm60GcfCjI2xjiDnL2n/vvIlrXbiLgqFr0wSBLifiStT39KazaXMagz8A6mj1T6K4UozI1zIBjxHHY2jJUTLJa/Qrg7HSayYwV5r9fFrlCbS7Ch8aLUVulC94hl1B0fZ9hedt3VeK7RdPvuxLdmda4FjpTaOM6ApsDtJZxcwn80kf1v9ExlE6lnF38+CY0SASElKKkk+bkYABOtY6WMVi6EeLhRCI5X4BZJImc5xmJBOPz9u69Gjje/0B1lprYSlfcTlpE+XwRpj7gru07MZfJTavgkADEd5EkPpifuvihrmURtm4y8NTm6Z+GBT7Pb3lUGaYvpRd6f/rSSlU+A5l7H9ONsnHPnRxiGzcDT9xD98nEkIqOik49Q6VSsjwZpEexdWVQIyFBlfbXF649nUXFBdZ2gn7Pj28vykqWXTV4a09+fs77WAUiBTGDemitxXsK5dT45Iro44LuYPUBFAga2n4W92t9On6nYiZda828rYNzZIOpmv8lR8L1ecjPBPFukOtVwEQFWc221pQT6/X39q5syxWCD4yeuoAbdB2+nAmZWUKL5owAyczNrgacwOOo8hRAJTAYc1/vYHRnvIAnQPohzcfr90ZRponsPP8zsy3uE4LSjywoOV6TC7RjBU3gw6hmZm/RN+mJEP62MVYT4O5ZKzXbZDuOiKYb+tjXn91q58bmZFakYMn6a1uV3g9UCmCiCFwX37uy0VLFohiXNoWQHPA6lNV9wifTCelOVN+klAIZ5nF7cEZyS1HMllklqzvGqBqxP7j+f+301+iKSQWlbqJO+yVN2P5OAd1uT4h4xpmQxrXGEtpiNG/37xo74SeDLCMTDCRIE+YnFr190fJN0cxtgtUDzlUGdm6IZnzpXKKqqA2vmtCGFvLpCz1TradWD5fKaGS9xFEL02DbPqGtPwudyHMZB9jnhIrqbs23hL4yWaafhUDc4efEJMNao1dAZQa4LlC1rlqyGFQKoJIGeiGvheam8ceaoJ9UEg64V7AxyjF6DqNXSTC/yefVRvTmGi0Q/s8+a+P+kkYPqZCWic6IkHdZ7AK0TqYBwgQdhRZ3i8L7zScGU6fK8iAMyIxo69D6kOQ1J3LSmmrSCvJp7grBPaAzDLtyWcfBPrR3ZOTXDgrBCrXrKiiUuJkkzS/KKL1oo7e1WQP+Ojy2amqulMgQ5y4CxYLre8zREiXWdydiMIsgCICDlimXARBHxEWpjll0QrdqDVFup0MBa+7pnVtzR4NQquWr7wRxFnp2eG7nymrm2hfrE8/ns6EFblT3mpCwBasfbzqqbzsu4dWs7df7wihIKeIQ4rz2ja5WRZDCoOOjaYVurGK2k4vkE57WazJvMMLyWpdzf824peW45WobWmB9Hn9mgExEaPUNegs6buOOrkWof+ARvkU7/HORPSqe2JUSWRtuiGO38Xosl2WD9cv2XQ9NAZFuWBDXsC7RBLyN6unG4MbYBLHHA+rZoGBB07Wv/XeVxcij1V4gj/26d33qUrKE5fF1CiYKOVnNvyP24DJ90IbhYJL4nApeMSBGRcbgeGkQrOA/YXsiG1vG+aPrfWHUJLZ9aUi4Zm2i1VWtcfETWbxCUzDqKg1ai+d9sdUAjMfB3EFan1CpkYOYSxk3WhyCJwVJIpveMoG99ECa2RBy4jT3CW3gCmUyawwbY0bEthSSlNLiOlgESWzUYnbQ9cGqB7bovB6NfSWIBuQG11UY9G3jIs1QQyaCeobrycFdwVb0ur+7IuXhUw7Ct4E5nwlF5SuqrakUN9HJ4xRsAAyWsO0RxZnLnVnhu+Y9ih6fDWHV4bVGXs1AkaCoO+Zz5hawUirBu9PP+N3EJMF7Ct7oyIdgopGT9lJZvrg+LYzHcSzVs7p2L25yHrPdNQEBZvtM0DK33sSHHKwybRxHj1aaFseB08+QERuIjboH3+EXJqk+67zF2vMYTlHL2WyYP+vmsL+RUSYIjoXgfU2zh/PXcvl9gtZlIgJH0lTs9wPrByxMLYWy4NrLe9UvWYh1icB94hhH9cGqjbv/yk1gRg4oJsn8dFtcTL7SqJ5F9t/PKMFhNnA8opJ7RE+1RzLWKvelBymZ8NMR7lP/vkH0xZ/3umcuUXMplwH3BxR4m6rVQ2XZFYlJEKVU3oikAm9aRk0i1oWDDEIIVkXWvyZfGv1qQRT1GvQJFRSAE/dbMUScKqtMFUW3rL7D6m9cfbwEwunzDJbCF6MYHG4nIXTTvmFy8pgCnakAgKFwfr6E4gcTAVYr7T4JjRRNpe8BWQAFJOq/VwKpiOl9kKju6fQQZMv/5W6GSaaPcRzHBxUmBa0LuXv6ZVOhfm++M+f6PGsvJ5jil/eTsBnplYCPcCDcnRu+1f8n+6JkbD5vbTnDkOvCiUYy51aoS6hMf1+RVBfwKDR95HeV+6vxcua6f4uOiD5/vZUo7mQirX9p3QWKUInwAIDamV5pX2suZ4c7Sxi5FfTJ94vtvN5X22XiDrVz72NFpDFh1nJPg2BgVEzwNTfGGiyTmCOZ2oMICe2WQkR+MlHEAora4K6KJ2oLVgmSHP+FhBv96z6kFlYeazCZrNGq7a9toSFW0c1w2FHCpAghsLQn98T8WY3pPnVDvlIK8bpMUIC5IRFuDjh7Ti0bnM9n3teP8BYfykqVFeTvNhuN1ghon3q7Izi8LJ07FbG1HXLdgiaEyI7nyBoS+NH1aWF8Pr+h9rkVTDGr2pcOVYGuNZ/oY2lnCaDShfbAT6xHNZL4MjP8ISwf1m2tTm3Lz/39+bfua2MU1HAeyoI505reElSa1p1LCFzs58SpFk4yswO5vgjyHGknZb3zxe5fTfo+dihaHe9qfLgIWvcJa4xxw7QSEmu+Zp1J0hAFFdj0J1TnqCvA62VVEeA8qcSofD2Wqbpifro32khc1/nbf99ptMBfzlyfu8XvhcUxd17CuPOH4hPJ0zbSwrqvlS6EDAGgimDLcHzu+rQwKs/yOD6Wz80eoZE3OGUIxSTo0n2FeA7ts20yvRopn4bLIaP8w2Ssiy+zEnYR0M2vsv4+vlzvjLPsW1qVDvKkNjY7UgsaNWyNcT0w5ZL2RngajNELMDFYgLb1iMLgxyO+N8CcG1ftzPfFQOiHDzQaHegFjBVAMxX7kjJ1p0VUyYlV0DyXqQbH3aB8ClFZLz8DqmoHibsOiau578pWaaZxHqTVzoftuszppT1rULSeGayh1EGBIOWCqIz5uaZtUNqBEgfrumrQxs2hpJSjTPSoDKbexx9dbwjjTRaBiTD10pzwPEdvFcIVRq6CWM83X7Glau3QdY/w7YIYj5cV3C2P5y0NDm+W2sYA6Ksa+4OxQeFWWzUVTyfTBsezPkqasf3axkPLdphh+AcE8EK5PBC7FDrNWv9NrFd+e1pDULt7ZDQlvHKPurGN7ur/6U+4fwvrlksABYnDV9QWtaTwCyjYBIETMsxY+nXkzpTLnLxgbt2XuzBp9Th6IjY0mlzbHGNEWsjkzpiNx2LITo+Cy3pT1p/WXWc1ekOzn7j+gf2M+ZeGsgiYocGcdncXlDlPjnFCex2NflCD3ctzWY5xI476IthrNnCqsoCUpLQ8LRF2batcR/eUjZpEfT8w3DEn4U1Cm5lnNyiCKJ/2fuxaQEZarEIOpTd2wVa3g9YHDsaddO58CfVVG8e8VTAka9JynCaxpKWcpEOUpQhqnGdAyPN8Yp5P2Dwx/RnZKl7vCQXG97v6cb9Msl8OwDnPsIoxmBmDMpUxtfvEHSmF7qqaRkUHYA9G7X3RPAS8dthhpPOZyCcO0Q2BPqmIJxWc5+6mfMPtWO+uNwM4bVEXMsPFNOlQk6lSXG8sWnw/ap0PIUg7gXbNvMBTft/9z8XntNbWJoBpWYCsCBbLkFFo0LNf2tALRCI681/Td65E4WCQXs90RQu1YySeG3aglhrWwFa2Z6tNCYW0ERiCyJUt1OlTfdHf27ym7bBUXMGRlgwcW8NObsw+YRlBJ7N2xs9MlA4BF/Iv/co5kDBtShfG/E7x3qyg1e6OLBXj6mULSksLeqnDOpLmiujO5wkjKpqoZZjTtzFQHDrPXwH296+3Nxd3gguMrb6iJrQGfNcOBFzqnwVu3vl88i/ddV6DitIqoBAavgH6zU8i1HBnKfnmq7Eb8e5Rzj9QRZ5iRrlssyqiosFGp3YNruGF4Rn5/E6fRRnlv+zNnJgUxlgdqb4ATLhO4+8Fh41QTQzsTaNDijDg6sK0TIbOZRD45UzGXfjlZvgCw3cr9TnoudDBFd3tSxU3vNkemo4IxG3zE2iquRb5LdEBJpwVIXAjWBLuutSWKqMHrd9Aqf+4MForKNw7mBoRK5NeiV3QtJvye9igiezHmg04zrSIsjZ8qPBeNLpA3MfjccsYtVGV/UunvazaMMNhDxQzCl6SKZlqNX3tu09vB12VL7InHe+/CxbFHwMK16s2qoQpto4deKTAK2jSGHEYD2cliSbymDoxZ3w327uJDlowZLe+urd4I8m+QPi7YIzGu1qoTTh9LnC0L87XVK/v2ZFGNwI7dMxxw0MI29zdXTrfcTGFKEU0b5TVZ663hPEqUKsF006Kri13DXidxFVu9km5fE6mcHLTUH0dLb7Kr2gc0QUxJwdFxwxoDIt4zxhldQwZ2saMo+KiH1y6kJXRYCDYt8EiB2or1MDKvCuzhk+9IgqHA/YMCOgAmIjdEYAgqI6RM0Q0OItyMVgTObORowqP5anYwcD3kCFNWiUebu9hj7w+2+dJY6uTn1aa6J47HlnG3WhYiER8U5FgdVKCVJ9XHm69x8vKwqNqPMQ2A76lAiaCMTQIzRuoZKt27ob+WiDqR9fbljEH1ARvfXk9c4Uva0fj+xaOTr8HyVia+FuCwzHxLfqg/T5W60P9WixQNkyo4sGAsWCu9kdGxzKLKEaFSiWLdaS+njQ3pstDTXVeYoOPNU4tsCsi5xg6szG1/ZljqWVzLr1cSmDovoDDNmKXizW/GyTXdMd89ro2vghexoVdFtM3Ru2K4HML3HfQvAvrnNw1sQjrLnhon+1jlxB2azmX2AIlKOGk9k4+jgOKaSz9S5ITEVK/rtULTLOyxCQ+e30+gHPzyd3LdouG1qnSKLW+KG0XlmDdh1ZyS2095Y+IkR8hEJDjfyQUE2TuyiLhTVqrs9WArbMQs4xGDIjpfgzYOJCZSJgMZIDQ5AXtXInjJwxXiOxUBlFwS8WTmzB4X8JG0i8s9rXWj2g9YMBBxUQFlEoAtqCMjjwGduZGe8e9sC2+efbhuJnzq2KvcWps6/flh97UNfoOkupwe86TzZdQT9IV7i1av1rsogP5RmiINF0QFhpYaWT683dt8K1JSH4YHyuk75dJufqMcXTcnCfmOfE8g4iPxwPHsS/QUyNmUKaOPzMDdzwc7RlAuTDudwwfvw9qSce1anYUl3oyJ9FYK0ICQYYWLJ6eZRkuVmnWM4WujIvx63F5spZmA8eh2eywqgsikj7h41KgtsDRkDAa/dcZcCwZJxXS3GZypn+sIlil0FYhMEF3o8AbfWxV0baCzndzsfAJBTZIXEtcq6+3fnZviTUXayK8OCnHYiM2acCo1FcrO70fjU6EYQC4e6RiGHynlph0jEUqh0IdP7reEEZDp2pCNv6c7nmO4eoTUCt5pNQ9v37DfH6DY+Lj8QE7PmJrVFqxGuj0iemWmzYV9apF+mJWHT7ji0/XIqZ90qXhNUlNIg3cQDWfvKfeZaajz0BrWMGVWG9rUOvJmwYhjg7CgZhCvWnZHItmx6L8RM/dWqRf0qihPni248DJfaCl5IO+55M+qqGvj4rJyp1YYaJR0Cb7fbVeJF8/g2Ubg7svywuagHM+uWvmDprW+F591oNW+jms4KcZI+qq3qf0yqkUPbkfHD+M+cWRxTMOKZxY057nuVh2a9vu3rk+D1PnZFJ0p01ZwyhZUcIBAE9aGTPDOB54PD7w8fiCKDlK4WllE/LAUhPEULoViSe/Uq/waKd1smAXibiHQeJfEi59A6F8pBMvpRAHwCq4MSO9iwvdAX3qNCY6EQygdAhmmQi0ls8PCohBR1uWodcDoBe7ko8dPXUvITbzTGXTHEy9v1nBNQHccc4n3J+wGYEw7aUcEjQ4T4JG0qSYzkqRbtavLOK9JeyWqNqMsagkbS+7ofGngCwKAilEBfkXW9ra8fTzYsNKHWo7WYrDxEsAsqq4CkzZxPRvccaHGUt0cIMyHqWYjFzlbZ36B9cbwljbfqwRJs6dpzl3x3g8ImBghi8fX+DzQW20nyOhDae1IKzKW7AqcxAMHObfx8HIVMECs9kYPAQhJkQlGGrngKJa5gYM+VwrZDNfj46LhGZpTI6TzFAlJhRSISMfIYwqshQlJQCVwnaAZ5CIZdox6u5p3cMSM5Djo7ZeJVPO9Hnqsxh70Iy5ksopZSqXEwoCwfBuB3VaLK6rrs0QxBrqD1WbIc8kyWtFsTnPtxkytkbV6/dVQEuRzHxHIJFraqYSE8Kf03w6YvN50CiqtcfnQky9eJnei0wqiDF0PprP+i76OCHYGvM1MuI655lpoZ+5Pi2M5/ktutaCIIKVj8eD2jQ6kn4JkJkTQYTeYmzqzfzFNCa1XCCk72SU+LgPrWAykrHFNFcfK31NGBxPnDNSxNwnzudXuJ84zNJ6AM4qaC2PtKncZKr83PIce5e1MEVnkbBT7pdib+n1e+fogqHhcnDRoVm/Er6zEVw/pTCjXVlD9Tvhr43YlT+6f1gbo2MR+6x54nszWWCxP6sgRXsDCj+9uvaAid6xnhnZ6AUk4+9+o+VUtEwlXvOsJQyfk2uumj+2o7zhdhpap10diSB3KjNh2W8liSN5/nN28Q1hfDw+Fv/FvUWSRAylLHl3YNv3QCNc7+JMRgxmO/H1b3/DOAYev/zGSZX2b7vtZe38gHaARztRNsN9wmZoe/cCxvAToI/n6Ss8MczhxzoJBtRkJJMzoRmMovJUHaMgiyPGod0SFblMv9N3WtDSeaGIaE/nKhoi2tqWODoTChICyLB9QsIqD1EJzgfvZUI46WLpfPZASNszeXMtcy2B5o4d977vdOWDu+jqEmHdoOjKN6twd3pmX60pLCrGsK5d2SOFMH16zRHQMA+VZhJ79X1BxyhfTaXrG0z/3vVW2Y16cWk/ck+Z/w3Hl0B2wolAE2jV40zbjvzAr7/9BsEmvW9VBqCfFI2fsyCkrGv0qbSrFIeZw6bqXDqFNsLWJ5QUEC2dqPQyByFcamf5cx5QzntAYkBR4H3Rt0cmO1Qtumo3O4VbRaNaQaz000WXBBjemKH5nLR4Va/Vwsfv8wgwOBUKxrRjb4OV6ziw+IwtjZeKDBvT4raNGhfQAzB9vPVMfZ4J79HRBXa6r3TuwSMgpywEdIzk37vx1eFdUpqr4gi/2pL22rkCDPqVP77eShT/3t9dAJfvfdci8vEoFOynqlcYAByG6Y8SYGoxlctP32VOEmnmWuBAQa2h14tBE3JoaSPEDR4FseAeumEgM/4Hy1Dl6caI9ai5jI9pXyaNW0zoZ1h9RZZFm/KFKkdSFFCfS/GFlp9o1jetbG493wzXpOtwYIwjyjV6MGhEvts85VwGtDMW6o0p5UnIaKF6ryUhA7LaQWOOBcp3v1Gf7bA03q00w/W+mOpeRQHkA1vppvebxTYm/p4Ct6M5WcociIXSyhaN7damZ8UH0hp24zODD3///e+QW/U4vuDv3/7sdcZ2LRE6rBqkfy+kv5hqj0LBYioVyfXciIwUnLif7WVUc7Iko5KaY7uUp88V75xGsJpMI9UdjOY2AK67lXXi8sNUDmf5mUPVySys4+AzsTwgeGkyLQF7gGRkX/zBuDIKq17bgyZlPbIuKtk1jX4cYokMJqWigiXM7oEmLqJAmkJHY8MsixVHOqAljSznbbMYaMgofdNGX81bjuuan3n9u4Jwa/BmtWzZeNZZKqGSguqKPxEHLejOs/o9kFOjOVR/bCQdIk/5xHlWcsbXr3/gg1k7x/HAMQb+9V//NZXkcRz49y218dX1tmU0INcTQVjR96+BbBKToQkCTUVkrYRwMXMFcV/XRG0aAdAv47wbPJYWzIrB8xGDqwCRG87WhqPWJ1M/HqzTy4V9wdExBsZxBEMOE/exNUsFMf2sg3HgYWTnk3QhI1HjRtuV4resdwK09jrHUrtR4ntP4emT6vkejXKPDDpRQ8CPI7U+X1gbr9PKsh8NYwZzjxT2nFP39FOLPziLGXW1Nq9XoQ5UEIvraOOAGc5U5quFXOFmQ1uQIDZjQEaUjyjlA/eWMF907FHeCCBFYO/rt7/j8fjA8Xgw4cnx8csXfHx84Ne//AZtvZJOCKvYSkT6n73oDw0SRSRUdDIVFDum9DBnrVLLhVRtTSnNlXPPiSwtuB7fJt8niZjTQQIDjArKonH9rhXEqnE4xvjCLB5PoQKigHA4+nTitT6HST/DI0GYAgmWjkzGSCtRvkwt7Uy4OZcGLOFTlLKgBZsdypcSsOx7g32Sv4sFkeLRPU0Qsr5FQzRsayoxAGhRZbXUKvK1OdC4pUhghPVqF0ImnvWPQskgy1WgQ0hLwi3XHRrL6dcvyo4ZXSEWf6RPzwLG8zx5buYz+6elpd/+8hd8/OUX/OKPQGM8pSqWLwruuzKVkjaeyqHPxY+u9yxjIsdGkCSQGIpEUDaDEpDdAQst4c2hDXhgeZqsqCurIojb0E8YqjEAO2gYLddmfViW918CDE27lpVTiw63J3y0CXcuI7TIm2GmI5/WXRp/I3hXCAFvHlH5DLM97yzxZ5jPE9rIPJvA9aOod9gGtiGadSuliG2ATUso16FjIfd49mSlvxLCHjST8pPQ7FCwvnePnfDKnMoK3u5keuSz9R5U57zvjHgdYVWgDQbmI1d9JNWBNSIfZ+ri9LNOh0ZB+1++fMHHrx9Z7TzOcxk0EF8Ao8/OcYj+keAxcwwJZ03ZPp+DqMCbwmgNESoQEs67A/MMaJF+E6jtSOjCRyvkkzXVd+DXFo7zGAGnzAF7DNihczYMMB7ZbMX8CUf2y7RTuwlOO98+9EOJZ4x51mE7LRBgUK2WEZNBTW6zNCJMyEGV7J6iXPxf64X8aLqSErqPus9B+6X13b2NH0DVqjUqwNgWBKAsuCjuWlflpyngEWBLmrgzm6q81x6c0d+BkCYzVI4SGA10SY/zos+2fmpFViiwYAh/Od4VW5PGGOGaeESh54wMqTkn/vjj7zBEqqIDeHz5wJePD3z88gvsWKsC9G12bUBcwvLm5gRNErE1peWEh/1Y8jt/+dX1xqL/mcIIs7B4jF7GOYYzNRSie7dFrBwGHCxhn+UZmsVa/vwgwnrCjjgvHTYy+qiyFTrjfbHeC8RC93bqurmnxeRWK5RXpLbVZwatc/YXBGMwTaorDOcED8OcIwW3rHkP7DQ2prBkL1IQFKX1/D3u1TyE5jaesjSfp6BFvEsKDEySp7Uod6DRAc5q33YRxOoX72/Hkl9RWsBJEDpHO2fd3xQA4K1UaSlbd+D5nPj6fOLbt28YNAg2Bo7HwHEc+Mu//JaK1I4BexxVnJnWUUnpawmOhl5adUC05Y89aAm6cNH1lYf+dGH080mTTe0H5jFSAFTBrGdhRP/Xjb3OU3LzkE/tmoa0yAmHYRwHCWRwHFAZf3dgjLJuTl9gzonz+cTzeeI4RpqRVHYmdh0lsBciSWQ1CcidC9xZGNayZQGl5psKsgCZRZTWMZsHGAF2ePlVra3oqNZM2VXTP9VQZ9q4URUA6r6gjVEgPf3e8HG0gyP8We+EshLl+peQr5cqufPfANh4gMl0G3X1PgVaikfE+IGUtZQSOv70mWcmfv0a+1ePw3AcBx7D8OW3XzCO4D13D7fVKmZgY+AYj6r+Tj9SmctXIHWPrGo+i8fzrJBEPUr8cGhL4J8ujNYmy3S4prS1afAj/Qmjts+0oZaiFgSyLImu+xVO1/0RGQ2Bd4tBns9nVOoyyxOc3OnvqDwk19iWQIbL7nXrVxHPBb6imES5sJCP5h6weYx4nwUDYTyoFCPFToC3nyacsIZCCfM8MiD/7dA1zwDpUBxkWlXYM2TWUZPPnDd9sEBZMU/dlZBkXoWr398Vaz+UVBYmZoCFtgTnsh8lnN3i6vk5n3g+v7FM4oDZrMNlRli7L18euQSzKDJarSlBI+KK+a6UTEFZ+Gz9t60vPXuJbbXEjcu9KEQG8WJXaH/20oaZckmbhjbH8AMKh45jFKQS/LSRWiKZ3NRTCrbnLsS4zw1Rq9Rji4/X2R7yLfbjmR2NcOmryQrp1SWEEDOE+QMwc6+ahGjJfczTtUasUZqFdclkcUZuE4o+yPfrAShaWyV+4j1UOK5JtNZXIREpdJ73MASnJtHCSDp0wSm+2a1YnYZMKWr8tGpya7Cy4LDV8+pmQxxxvMGZy0l5BPecmZESu+sfAJgQbw98+RhEDDPhY7Rfa6hZBqXengJjbnWsgMWRBVL4kVr4XPzjvpxRA5YwI/3XOsbhWs0hKWel5vX+5nb/8HojN/WRnc4FUjKfSWtTMwacDEGY8xt3L6juJXvnke3xfD6rXcLf83zGfkKT1rdaE0No0VVT6Z8Scmm7KwTRmma7zGAmaEV/NNsvCDKLW6MVRR8DKJfR4TqT/Kt4xGu7oBhn1HpiZfS0nnUt3/osJuh5uhnCz1eUFfSM9lV0F0CG5FP42n7GIk1DC/m+vrm2wTBGh+Z8MilbFiXSFa3NSRQFe8Dk97tq5QyuW3PLEq2iptbB4NnmrooHvCGIsFJP9ql2dahK+BJhXwa9WkdZ1S74uyBe3A0aJ6Gnz1yfryh+9EVpp2K3PHE2O6KdzvOZDGKwynmU7ibMOmdYl8E0M5n8FLDkMRGgM0PSisQOCGcm3N4GQCGQmws4V/ybgAMtTSwNBir047klJm6bXLoJYUzHHzMZ11mtW+ucxWTxWBlNa0IEQitPBbEGF8LyKACh56WQUthIHFNKX0c1GllaxG5N9fv6uYhXSxJtHZglQMxQGUng9iI4HkehFW0cpuhmQMwSLbQxeIEsfaDqCrGkZfm72qudFLslb3PBDcK7ggOQvJNpfNZRV3VKyCONUUc0m3X9zPV5YXx8aRrUpaL4XtWKjHXFro2M97tzSwwA6FxGi4Vum3EIi9bXxKjLYEig0kabEqCmFuHLVaocRSSExWJEG8+VH5v/xTHYgxKo5OqEboNbZqbaq+TqtW+bGDQf1KytUja4GALqK6N4+LyrQir3wNTPDXbpXc5WtZPdhxLawePOJGy1KTZ3bcwSRG9CasxY0jKGtiZVAGdd583gXhM6k99azlYSRIp5+hrNDcU9IMjpWaYl/YAmhBEgSsSTKAG14doVuJvJC1X+UyYIyPjuCFpmkK7B9GEWecma609cb6wzFiNn+tJJAuEbhdGyjIOCK1XwSQzIxVmj1ZjN9ohBgfIhOY6x7JDvWjuEotbEtNZX1qf8Pmm42qJkZSqzjwvjOxnXaPVT442YDUS+IvCk1h3Lznz5R7E+J+aMBeXanJyjyt9eFTGqdbv+ZDF594N0qdhSHyOkCK0EwcxwPr/hPLkwb5MZMoFyPh4f+Hg8IjaQwlWCHxbRuXEaDZYWVO8CmQpD2tEBcGlmtc4IvhBNTGt5gKrAZyDM+0P1nkvQZTomzvxuPyj17rr1pe9Qrl3n4zOX+WfF9uf18/p5/Zden8/V+Xn9vH5e/6XXT2H8ef28/kmun8L48/p5/ZNcP4Xx5/Xz+ie5fgrjz+vn9U9y/RTGn9fP65/k+imMP6+f1z/J9VMYf14/r3+S66cw/rx+Xv8k1/8HZQxsFHwuXTwAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "path_of_image = os.path.join(\"..\", \"resources\", \"Columbus.jpeg\")\n", + "\n", + "image = Image.open(path_of_image)\n", + "plt.imshow(image)\n", + "plt.axis(\"off\")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 114, + "id": "b6474ea0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Response status code: 200\n" + ] + } + ], + "source": [ + "with open(path_of_image, \"rb\") as f:\n", + " files = {\n", + " \"image\": (path_of_image, f, \"image/jpeg\") # <--- WICHTIG: Filename + MIME-Type\n", + " }\n", + " response = requests.post(url, files=files)\n", + " print(f\"Response status code: {response.status_code}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 115, + "id": "9f04d32e", + "metadata": {}, + "outputs": [], + "source": [ + "class_labels_path = os.path.join(\"..\", \"resources\", \"imagenet_classes.txt\")\n", + "with open(class_labels_path) as f:\n", + " class_labels = [line.strip() for line in f.readlines()]\n", + "\n", + "def get_class_name(response, class_labels):\n", + " class_label_response = response.json()\n", + " class_index = class_label_response[\"predicted_class_index\"]\n", + " class_name = class_labels[class_index]\n", + " return class_name, class_index" + ] + }, + { + "cell_type": "code", + "execution_count": 116, + "id": "0e258583", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "('tabby', 281)" + ] + }, + "execution_count": 116, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_class_name(response, class_labels)" + ] + }, + { + "cell_type": "markdown", + "id": "cece33fe", + "metadata": {}, + "source": [ + "(kann auch nochmal [hier](https://gist.github.com/yrevar/942d3a0ac09ec9e5eb3a) verglichen werden)" + ] + }, + { + "cell_type": "markdown", + "id": "e61b39be", + "metadata": {}, + "source": [ + "Bemerkenswert ist, dass dies nun auch auf der **GPU** läuft, falls verfügbar." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dsai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.23" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/06_NN/code/nn_6_flask_app.py b/06_NN/code/nn_6_flask_app.py new file mode 100644 index 0000000..c7c96de --- /dev/null +++ b/06_NN/code/nn_6_flask_app.py @@ -0,0 +1,70 @@ +import io +import torch +import torch.nn as nn +import torchvision.models as models +import torchvision.transforms as transforms +from PIL import Image +from flask import Flask, request, jsonify +import os +import json +import urllib.request + + +device = torch.device('cuda') if torch.cuda.is_available() else torch.device('mps') if torch.backends.mps.is_available() else torch.device('cpu') +print(device) + +model = models.resnet18(weights = models.ResNet18_Weights.IMAGENET1K_V1) +model.eval() +model.to(device) + +transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize( + mean=[0.485, 0.456, 0.406], # ImageNet Normalisierung + std=[0.229, 0.224, 0.225] + ) +]) + +app = Flask(__name__) + +@app.route("/predict", methods=["POST"]) +def predict(): + """ + Erwartet eine Bilddatei ('image') im POST-Request. + Beispiel (Postman): POST -> http://127.0.0.1:5665/predict + Body -> form-data -> key='image', value= + """ + if "image" not in request.files: + return jsonify({"error": "Kein Bild hochgeladen!"}), 400 + + file = request.files["image"] + img_bytes = file.read() + + try: + image = Image.open(io.BytesIO(img_bytes)).convert("RGB") + except Exception as e: + return jsonify({"error": f"Fehler beim Öffnen des Bildes: {str(e)}"}), 400 + + # Preprocessing + input_tensor = transform(image).unsqueeze(0).to(device) + + # Inferenz + with torch.no_grad(): + outputs = model(input_tensor) + _, predicted = outputs.max(1) + + # Ergebnis zurückgeben + return jsonify({ + "predicted_class_index": predicted.item() + }) + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=5665, debug=True) + + + + + + diff --git a/06_NN/code/nn_8_example_classification_solution.ipynb b/06_NN/code/nn_8_example_classification_solution.ipynb new file mode 100644 index 0000000..49068ab --- /dev/null +++ b/06_NN/code/nn_8_example_classification_solution.ipynb @@ -0,0 +1,867 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2580d14d", + "metadata": {}, + "source": [ + "
\n", + "\n", + "

Neural Networks: Ein Beispiel (Klassifikation) (Lösung)

\n", + "

DSAI

\n", + "

Jakob Eggl

\n", + "\n", + "
\n", + " \"Logo\"\n", + "
\n", + " © 2025/26 Jakob Eggl. Nutzung oder Verbreitung nur mit ausdrücklicher Genehmigung des Autors.\n", + "
\n", + "
\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "e1a0eaf8", + "metadata": {}, + "source": [ + "Wir wollen nun auch ein neuronales Netzwerk für die Klassifizierung bauen. Dabei wollen wir ein sehr bekanntes Dataset verwenden (MNIST). Es gibt es in vielen Variationen (zum Beispiel auch mit Kleidung (Fashion-MNIST)) und ist gratis. \n", + "\n", + "Zuerst wollen wir das normale MNIST Dataset verwenden. Es beinhaltet die handgeschriebenen Zahlen von $0$ bis $9$. Ziel ist es die richtige Zahl zu erkennen." + ] + }, + { + "cell_type": "markdown", + "id": "cc8846a2", + "metadata": {}, + "source": [ + "![MNIST](../resources/MNIST.png)\n", + "\n", + "(von https://de.wikipedia.org/wiki/MNIST-Datenbank)" + ] + }, + { + "cell_type": "markdown", + "id": "51c9fecc", + "metadata": {}, + "source": [ + "Insgesamt hat das MNIST Dataset $60\\mathrm k$ Trainingsbilder und $10\\mathrm k$ Testbilder. Die Klassen sind dabei ziemlich gleichverteilt, sprich es gibt in etwa gleich viele Bilder mit Label \"1\", Label \"2\", usw." + ] + }, + { + "cell_type": "markdown", + "id": "e054c9dc", + "metadata": {}, + "source": [ + "# Lösung" + ] + }, + { + "cell_type": "markdown", + "id": "7b7c7bd4", + "metadata": {}, + "source": [ + "Zu Beginn wollen wir sicherstellen, dass jede und jeder das MNIST Dataset heruntergeladen hat. Der Pfad der folgenden Methode kann, wenn nötig, angepasst werden." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d2fdca5", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "from torch.utils.data import DataLoader, Dataset, random_split\n", + "from torchvision import datasets, transforms\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.metrics import confusion_matrix\n", + "import seaborn as sns" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38992064", + "metadata": {}, + "outputs": [], + "source": [ + "data_path = os.path.join(\"..\", \"..\", \"_data\", \"mnist_data\")\n", + "\n", + "train_dataset = datasets.MNIST(root=data_path, train=True, download=True, transform=transforms.ToTensor()) # ToTensor makes images [0, 1] instead of {1,2,...,255}\n", + "test_dataset = datasets.MNIST(root=data_path, train=False, download=True, transform=transforms.ToTensor())\n", + "\n", + "test_size = len(test_dataset) // 2\n", + "valid_size = len(test_dataset) - test_size\n", + "\n", + "test_dataset, valid_dataset = random_split(test_dataset, [test_size, valid_size])" + ] + }, + { + "cell_type": "markdown", + "id": "7e4570f5", + "metadata": {}, + "source": [ + "Mit der obigen Methode haben wir direkt ein Torch Dataset erhalten und müssen nur mehr später den Dataloader erstellen." + ] + }, + { + "cell_type": "markdown", + "id": "2fae606c", + "metadata": {}, + "source": [ + "Kurze **Wiederholung**: *Wie erstellt man sein eigenes Dataset*?" + ] + }, + { + "cell_type": "markdown", + "id": "3504b37b", + "metadata": {}, + "source": [ + "Zum Beispiel so:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c5ae338", + "metadata": {}, + "outputs": [], + "source": [ + "class MyDataSetThatIsNeverUsed(Dataset): \n", + " def __init__(self, transform=None):\n", + " super().__init__()\n", + " self.transform = transform\n", + "\n", + " def __len__(self):\n", + " return 0\n", + "\n", + " def __getitem__(self, idx):\n", + " # here is place for the transformation. Returns input and label\n", + " return torch.tensor([]), torch.tensor(0)" + ] + }, + { + "cell_type": "markdown", + "id": "1068fe47", + "metadata": {}, + "source": [ + "Ansonsten starten wir wieder mit dem device (Prinzipiell eine gute Gewohnheit, dies einmalig am Anfang zu definieren)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e5a91e4", + "metadata": {}, + "outputs": [], + "source": [ + "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('mps') if torch.backends.mps.is_available() else torch.device('cpu')\n", + "print(device)" + ] + }, + { + "cell_type": "markdown", + "id": "b992eade", + "metadata": {}, + "source": [ + "Nachdem wir die Datasets schon haben, wollen wir nun die Dataloader definieren." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01f002c0", + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 64\n", + "\n", + "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", + "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n", + "valid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "id": "6d262f12", + "metadata": {}, + "source": [ + "Wir wollen uns nun auch noch ein paar Bilder aus dem Trainingsset ansehen." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "661246a3", + "metadata": {}, + "outputs": [], + "source": [ + "examples = enumerate(train_loader)\n", + "batch_idx, (example_data, example_targets) = next(examples)\n", + "\n", + "plt.figure(figsize=(8, 3))\n", + "for i in range(6):\n", + " plt.subplot(1, 6, i+1)\n", + " plt.tight_layout()\n", + " plt.imshow(example_data[i][0], cmap='gray', interpolation='none')\n", + " plt.title(f\"{example_targets[i]}\")\n", + " plt.xticks([])\n", + " plt.yticks([])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "c14a7e3c", + "metadata": {}, + "source": [ + "Als nächstes definieren wir uns das Netzwerk. Auf was müssen wir nun acht geben im Vergleich zur Regression?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2d9cd26", + "metadata": {}, + "outputs": [], + "source": [ + "class MNISTClassifier(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.layers = nn.Sequential(\n", + " nn.Flatten(), # Very important! Why? -> We will see that for CNN's we don't need this flattening!\n", + " nn.Linear(28*28, 256),\n", + " nn.ReLU(),\n", + " nn.Linear(256, 128),\n", + " nn.ReLU(),\n", + " nn.Linear(128,64),\n", + " nn.ReLU(),\n", + " nn.Linear(64, 10),\n", + " )\n", + " def forward(self, x):\n", + " return self.layers(x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e413ca3", + "metadata": {}, + "outputs": [], + "source": [ + "model = MNISTClassifier().to(device)" + ] + }, + { + "cell_type": "markdown", + "id": "cf9deca1", + "metadata": {}, + "source": [ + "Welchen Loss wollen wir verwenden? Welchen Optimizer?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe340898", + "metadata": {}, + "outputs": [], + "source": [ + "lr = 0.001\n", + "\n", + "criterion = nn.CrossEntropyLoss()\n", + "optimizer = optim.Adam(model.parameters(), lr=lr)" + ] + }, + { + "cell_type": "markdown", + "id": "2b93a21c", + "metadata": {}, + "source": [ + "Kommen wir nun zur Trainingsmethode. Wir machen diese dieses Mal als eigene Methode. Ebenso machen wir das mit der Evaluierungsmethode. (Grund für die umgekehrte Reihenfolge ist, weil die Trainingsmethode eine Evaluierungsmethode beinhaltet.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4da656f8", + "metadata": {}, + "outputs": [], + "source": [ + "def evaluate_model(model, data_loader, criterion):\n", + " model.eval()\n", + " loss_total = 0.0\n", + " correct = 0\n", + " total = 0\n", + " \n", + " with torch.no_grad():\n", + " for data, target in data_loader:\n", + " data, target = data.to(device), target.to(device)\n", + " output = model(data)\n", + " loss = criterion(output, target)\n", + " loss_total += loss.item() * data.size(0)\n", + " \n", + " _, predicted = torch.max(output.data, 1)\n", + " total += target.size(0)\n", + " correct += (predicted == target).sum().item()\n", + " \n", + " avg_loss = loss_total / total\n", + " accuracy = 100.0 * correct / total\n", + " return avg_loss, accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "be0815fe", + "metadata": {}, + "outputs": [], + "source": [ + "def train_model(model, train_loader, valid_loader, criterion, optimizer, save_path:str=None,\n", + " epochs=20, validate_at=1, print_at=100, patience=3):\n", + " \n", + " if save_path is None:\n", + " save_path = os.path.join(\"..\", \"models\", \"nn_8_best_model.pth\")\n", + "\n", + " best_loss = float(\"inf\")\n", + " patience_counter = 0\n", + "\n", + " for epoch in range(1, epochs+1):\n", + " model.train()\n", + " running_loss = 0.0\n", + "\n", + " for batch_idx, (data, target) in enumerate(train_loader):\n", + " data, target = data.to(device), target.to(device)\n", + "\n", + " optimizer.zero_grad()\n", + " output = model(data)\n", + " loss = criterion(output, target)\n", + " loss.backward()\n", + " optimizer.step()\n", + " \n", + " running_loss += loss.item()\n", + " \n", + " if (batch_idx+1) % print_at == 0:\n", + " print(f\"Epoch [{epoch}/{epochs}], Step [{batch_idx+1}/{len(train_loader)}], Loss: {loss.item():.4f}\")\n", + "\n", + " if epoch % validate_at == 0:\n", + " val_loss, val_acc = evaluate_model(model, valid_loader, criterion)\n", + " print(f\"Epoch [{epoch}/{epochs}] - Validation Loss: {val_loss:.4f}, Validation Accuracy: {val_acc:.2f}%\")\n", + "\n", + " if val_loss < best_loss:\n", + " best_loss = val_loss\n", + " patience_counter = 0\n", + " torch.save(model.state_dict(), save_path)\n", + " print(f\">>> Found a better model and saved it at '{save_path}'\")\n", + " else:\n", + " patience_counter += 1\n", + " print(f\"No Improvement. Early Stopping Counter: {patience_counter}/{patience}\")\n", + " if patience_counter >= patience:\n", + " print(\"Early Stopping triggered.\")\n", + " break" + ] + }, + { + "cell_type": "markdown", + "id": "3afc2ae3", + "metadata": {}, + "source": [ + "Last but not least wollen wir nun das Modell trainieren. Dazu definieren wir uns die Hyperparameter zuerst (manche sind der Form halber jetzt doppelt)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "32230cc3", + "metadata": {}, + "outputs": [], + "source": [ + "### HYPERPARAMETER ###\n", + "\n", + "model = MNISTClassifier().to(device)\n", + "criterion = nn.CrossEntropyLoss()\n", + "lr = 0.001\n", + "optimizer = optim.Adam(model.parameters(), lr=lr)\n", + "epochs = 20\n", + "validate_at = 1\n", + "print_at = 200\n", + "early_stopping_patience = 3\n", + "save_path = os.path.join(\"..\", \"models\", \"nn_8_best_model_mnist.pth\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "453d2b87", + "metadata": {}, + "outputs": [], + "source": [ + "train_model(model, train_loader, valid_loader, criterion, optimizer, epochs=epochs, validate_at=validate_at, print_at=print_at, patience=early_stopping_patience, save_path=save_path)" + ] + }, + { + "cell_type": "markdown", + "id": "fd829890", + "metadata": {}, + "source": [ + "Am Schluss evaluieren wir noch das beste Modell:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d288c09d", + "metadata": {}, + "outputs": [], + "source": [ + "model.load_state_dict(torch.load(save_path))\n", + "test_loss, test_acc = evaluate_model(model, test_loader, criterion)\n", + "print(f\"Finaler Test Loss: {test_loss:.4f}\")\n", + "print(f\"Finale Test Accuracy: {test_acc:.2f}%\")" + ] + }, + { + "cell_type": "markdown", + "id": "60aa40ec", + "metadata": {}, + "source": [ + "___" + ] + }, + { + "cell_type": "markdown", + "id": "35559661", + "metadata": {}, + "source": [ + "Sind wir zufrieden? Was könnte man verbessern?" + ] + }, + { + "cell_type": "markdown", + "id": "e2a63701", + "metadata": {}, + "source": [ + "Man könnte eine (andere) Transformation verwenden." + ] + }, + { + "cell_type": "markdown", + "id": "cc538713", + "metadata": {}, + "source": [ + "Berechnen wir dazu mal den Mean und die Varianz (bzw. Standardabweichung der Trainingsdaten)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40ad9243", + "metadata": {}, + "outputs": [], + "source": [ + "mean = 0.\n", + "std = 0.\n", + "for imgs, _ in train_loader:\n", + " mean += imgs.mean()\n", + " std += imgs.std()\n", + "\n", + "mean /= len(train_loader)\n", + "std /= len(train_loader)\n", + "print(mean.item(), std.item())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1cf0477", + "metadata": {}, + "outputs": [], + "source": [ + "train_transform = transforms.Compose([\n", + " transforms.RandomRotation(10), # small data augmentation\n", + " transforms.ToTensor(),\n", + " transforms.Normalize((0.1307,), (0.3081,))\n", + "])\n", + "\n", + "test_transform = transforms.Compose([\n", + " transforms.ToTensor(),\n", + " transforms.Normalize((0.1307,), (0.3081,))\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7d0e06f", + "metadata": {}, + "outputs": [], + "source": [ + "data_path = os.path.join(\"..\", \"..\", \"_data\", \"mnist_data\")\n", + "\n", + "train_dataset = datasets.MNIST(root=data_path, train=True, download=True, transform=train_transform) # ToTensor makes images [0, 1] instead of {1,2,...,255}\n", + "test_dataset = datasets.MNIST(root=data_path, train=False, download=True, transform=test_transform)\n", + "\n", + "test_size = len(test_dataset) // 2\n", + "valid_size = len(test_dataset) - test_size\n", + "\n", + "test_dataset, valid_dataset = random_split(test_dataset, [test_size, valid_size])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f8f37e5", + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 64\n", + "\n", + "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", + "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n", + "valid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cf9c9c5c", + "metadata": {}, + "outputs": [], + "source": [ + "### HYPERPARAMETER ###\n", + "\n", + "model = MNISTClassifier().to(device)\n", + "criterion = nn.CrossEntropyLoss()\n", + "lr = 0.001\n", + "optimizer = optim.Adam(model.parameters(), lr=lr)\n", + "epochs = 10\n", + "validate_at = 1\n", + "print_at = 200\n", + "early_stopping_patience = 3\n", + "save_path = os.path.join(\"..\", \"models\", \"nn_8_best_model_mnist_transform.pth\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eab17123", + "metadata": {}, + "outputs": [], + "source": [ + "train_model(model, train_loader, valid_loader, criterion, optimizer, epochs=epochs, validate_at=validate_at, print_at=print_at, patience=early_stopping_patience, save_path=save_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "247a1257", + "metadata": {}, + "outputs": [], + "source": [ + "model.load_state_dict(torch.load(save_path))\n", + "test_loss, test_acc = evaluate_model(model, test_loader, criterion)\n", + "print(f\"Finaler Test Loss: {test_loss:.4f}\")\n", + "print(f\"Finale Test Accuracy: {test_acc:.2f}%\")" + ] + }, + { + "cell_type": "markdown", + "id": "7e0cfe7b", + "metadata": {}, + "source": [ + "Man könnte nun natürlich auch noch weitere Epochen, ein noch größeres Netzwerk, andere Learning Rate, anderer Optimierer etc. verwenden. Wir sehen aber davon ab." + ] + }, + { + "cell_type": "markdown", + "id": "2c35fdc1", + "metadata": {}, + "source": [ + "___" + ] + }, + { + "cell_type": "markdown", + "id": "ddb2a810", + "metadata": {}, + "source": [ + "Wir verwenden jetzt das **Fashion-MNIST** Dataset und führen alles nochmal aus." + ] + }, + { + "cell_type": "markdown", + "id": "6b8e57aa", + "metadata": {}, + "source": [ + "Es besteht nun aus Kleidungsstücken und dazugehörig 10 Labels. Wir müssen also unser Modell in erster Linie nicht anpassen. " + ] + }, + { + "cell_type": "markdown", + "id": "06c551b9", + "metadata": {}, + "source": [ + "Auch hier gibt es $60\\, \\mathrm k$ Trainingsbilder und $10\\, \\mathrm k$ Testbilder." + ] + }, + { + "cell_type": "markdown", + "id": "f80dc124", + "metadata": {}, + "source": [ + "Wir kopieren nun die wichtigsten Dinge und ändern sie leicht ab." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06ed22e0", + "metadata": {}, + "outputs": [], + "source": [ + "data_path = os.path.join(\"..\", \"..\", \"_data\", \"fashion_mnist_data\")\n", + "\n", + "train_dataset = datasets.FashionMNIST(root=data_path, train=True, download=True, transform=transforms.ToTensor()) # ToTensor makes images [0, 1] instead of {1,2,...,255}\n", + "test_dataset = datasets.FashionMNIST(root=data_path, train=False, download=True, transform=transforms.ToTensor())\n", + "\n", + "test_size = len(test_dataset) // 2\n", + "valid_size = len(test_dataset) - test_size\n", + "\n", + "test_dataset, valid_dataset = random_split(test_dataset, [test_size, valid_size])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11e3c83a", + "metadata": {}, + "outputs": [], + "source": [ + "batch_size = 64\n", + "\n", + "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", + "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n", + "valid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dac37e73", + "metadata": {}, + "outputs": [], + "source": [ + "examples = enumerate(train_loader)\n", + "batch_idx, (example_data, example_targets) = next(examples)\n", + "\n", + "label_dict = {\n", + " 0: \"T-Shirt\",\n", + " 1: \"Trouser\",\n", + " 2: \"Pullover\",\n", + " 3: \"Dress\",\n", + " 4: \"Coat\",\n", + " 5: \"Sandal\",\n", + " 6: \"Shirt\",\n", + " 7: \"Sneaker\",\n", + " 8: \"Bag\",\n", + " 9: \"Ankle Boot\"\n", + "}\n", + "\n", + "plt.figure(figsize=(8, 3))\n", + "for i in range(6):\n", + " plt.subplot(1, 6, i+1)\n", + " plt.tight_layout()\n", + " plt.imshow(example_data[i][0], cmap='gray', interpolation='none')\n", + " plt.title(f\"{label_dict[example_targets[i].item()]}\")\n", + " plt.xticks([])\n", + " plt.yticks([])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "7950b546", + "metadata": {}, + "source": [ + "Wir verwenden nun das gleiche Modell wie vorher, ändern aber den Klassennamen." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25451f9e", + "metadata": {}, + "outputs": [], + "source": [ + "class FashionMNISTClassifier(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.layers = nn.Sequential(\n", + " nn.Flatten(), # Very important! Why?\n", + " nn.Linear(28*28, 256),\n", + " nn.ReLU(),\n", + " nn.Linear(256, 128),\n", + " nn.ReLU(),\n", + " nn.Linear(128,64),\n", + " nn.ReLU(),\n", + " nn.Linear(64, 10),\n", + " )\n", + " def forward(self, x):\n", + " return self.layers(x)" + ] + }, + { + "cell_type": "markdown", + "id": "e60d3519", + "metadata": {}, + "source": [ + "Nun trainieren wir das Modell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12b4a9d7", + "metadata": {}, + "outputs": [], + "source": [ + "### HYPERPARAMETER ###\n", + "\n", + "model = FashionMNISTClassifier().to(device)\n", + "criterion = nn.CrossEntropyLoss()\n", + "lr = 0.001\n", + "optimizer = optim.Adam(model.parameters(), lr=lr)\n", + "epochs = 20\n", + "validate_at = 1\n", + "print_at = 200\n", + "early_stopping_patience = 3\n", + "save_path = os.path.join(\"..\", \"models\", \"nn_8_best_model_fashion_mnist.pth\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eecd887f", + "metadata": {}, + "outputs": [], + "source": [ + "train_model(model, train_loader, valid_loader, criterion, optimizer, epochs=epochs, validate_at=validate_at, print_at=print_at, patience=early_stopping_patience, save_path=save_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b63239a2", + "metadata": {}, + "outputs": [], + "source": [ + "model.load_state_dict(torch.load(save_path))\n", + "test_loss, test_acc = evaluate_model(model, test_loader, criterion)\n", + "print(f\"Finaler Test Loss: {test_loss:.4f}\")\n", + "print(f\"Finale Test Accuracy: {test_acc:.2f}%\")" + ] + }, + { + "cell_type": "markdown", + "id": "cfb071d1", + "metadata": {}, + "source": [ + "Diese Performance ist nicht wirklich gut. Für 10 Klassen bedeutet das, dass wir im Mittel 1 von 10 Klassen falsch zuordnen. Wir betrachten noch kurz die Confusion-Matrix." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4be3ab7", + "metadata": {}, + "outputs": [], + "source": [ + "# Confusion Matrix of FashionMNIST model\n", + "\n", + "test_data = test_loader.dataset.dataset.data[test_loader.dataset.indices]\n", + "test_targets = test_loader.dataset.dataset.targets[test_loader.dataset.indices]\n", + "\n", + "pred = model(test_data.unsqueeze(1).float().to(device))\n", + "cm = confusion_matrix(test_targets.cpu(), pred.argmax(dim=1).cpu())\n", + "\n", + "plt.figure(figsize=(8, 6))\n", + "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=label_dict.values(), yticklabels=label_dict.values())\n", + "plt.title(\"Confusion Matrix for FashionMNIST Classification\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "2f6df1f8", + "metadata": {}, + "source": [ + "(zur Erinnerung, $y$-Achse entspricht der Ground-Truth und $x$-Achse entspricht der Vorhersage)" + ] + }, + { + "cell_type": "markdown", + "id": "c792fa9a", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "ef946e2e", + "metadata": {}, + "source": [ + "## Reicht also immer ein Fully-Connected Neuronal Netzwerk aus?" + ] + }, + { + "cell_type": "markdown", + "id": "f438ffcc", + "metadata": {}, + "source": [ + "**Nein!**\n", + "\n", + "Es gibt viele Probleme, die andere Architekturen erwarten. Auch, wenn man in gewissen Situationen vielleicht mit so einer Performance zufrieden ist, werden wir, insbesondere, wenn wir uns später zum Beispiel der **Image-Inpainting** Challenge widmen, sehen, dass wir andere Architekturen brauchen, da diese viel besser funktionieren." + ] + }, + { + "cell_type": "markdown", + "id": "43a29933", + "metadata": {}, + "source": [ + "Als nächstes werden wir also eine neue Architektur kennenlernen, welche mit Bildern noch viel besser umgehen kann, als Feed-Forward Neuronal Netzwerke. Die Rede ist von sogenannten ***CNN's*** (*Convolutional Neuronal Networks*)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dsai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/06_NN/code/nn_9_cnn_1.ipynb b/06_NN/code/nn_9_cnn_1.ipynb new file mode 100644 index 0000000..07a438c --- /dev/null +++ b/06_NN/code/nn_9_cnn_1.ipynb @@ -0,0 +1,822 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a83eef2d", + "metadata": {}, + "source": [ + "
\n", + "\n", + "

Neural Networks: Convolutional Neural Networks (1)

\n", + "

DSAI

\n", + "

Jakob Eggl

\n", + "\n", + "
\n", + " \"Logo\"\n", + "
\n", + " © 2025/26 Jakob Eggl. Nutzung oder Verbreitung nur mit ausdrücklicher Genehmigung des Autors.\n", + "
\n", + "
\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "83a52169", + "metadata": {}, + "source": [ + "Nachdem wir bisher nur von `Linear`-Layers gesprochen haben, möchten wir uns jetzt auf eine **neue Architektur** fokusieren. Diese ist besonders geeignet für Bilder und hat eine gravierend andere funktionsweise als bisherige Layers.\n", + "\n", + "Die Rede ist von sogenannten **Convolution**-Layers, welche dann die sogenannten Convolutional Neural Networks (CNN's) bilden.\n", + "\n", + "In diesem Notebook wollen wir:\n", + "1) Zuerst die Convolution-Operation kennenlernen und deren Anwendung in der klassischen Bildbearbeitung betrachten bevor wir\n", + "2) Uns das Convolution-Layer in PyTorch ansehen und uns am Schluss\n", + "3) Über die weiteren damit verbundenen Layers wie Pooling befassen." + ] + }, + { + "cell_type": "markdown", + "id": "f13607fc", + "metadata": {}, + "source": [ + "**Hinweis:** *Convolution* ist der englische Begriff für *Faltung*." + ] + }, + { + "cell_type": "markdown", + "id": "8dd66ee2", + "metadata": {}, + "source": [ + "**Hinweis:** Die Faltung ist eine mathematische Operation, welche [hier](https://de.wikipedia.org/wiki/Faltung_(Mathematik)) nachgelesen werden *kann*." + ] + }, + { + "cell_type": "markdown", + "id": "cef6ffb9", + "metadata": {}, + "source": [ + "## Die Convolution-Operation" + ] + }, + { + "cell_type": "markdown", + "id": "27ba7dd3", + "metadata": {}, + "source": [ + "Zu Beginn wollen wir uns die Faltungsoperation auf Bildern ansehen.\n", + "\n", + "In diesem Notebook haben wir dabei immer folgendes Setup:\n", + "* Wir haben ein Bild, sprich mathematisch gesehen einen $(3, \\text{Hoehe}, \\text{Breite})$ Tensor, also dreimal eine Matrix der Größe $\\text{Hoehe} \\times \\text{Breite}$.\n", + "* Wir haben einen Filter $K$ (auch Kernel genannt), mathematisch gesehen ist das auch ein Tensor, der Einfachkeit halber (und das trifft auch meistens zu) denken wir einfach an eine Matrix.\n", + "* Wir wollen den Filter auf unser Bild anwenden und dabei ein neues Bild, also eine neue Matrix generieren." + ] + }, + { + "cell_type": "markdown", + "id": "e14cbfd5", + "metadata": {}, + "source": [ + "Betrachten wir hier zuerst mal ein paar Beispiele." + ] + }, + { + "cell_type": "markdown", + "id": "8c32d03d", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von https://www.youtube.com/watch?v=KuXjwB4LzSA)" + ] + }, + { + "cell_type": "markdown", + "id": "f1c431d9", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von https://www.youtube.com/watch?v=KuXjwB4LzSA)" + ] + }, + { + "cell_type": "markdown", + "id": "8ba45f6c", + "metadata": {}, + "source": [ + "Hier ist bei beiden Bildern rechts oben bzw. auch links das große Bild das Ausgangsbild. Rechts oben (blau umrahmt) ist der Filter zu sehen.\n", + "\n", + "Das Ergebnis der Faltungsoperation (Convolution) ist unten rechts zu sehen:\n", + "Im Super-Mario Bild ist das ein Unschärfe Filter, im zweiten Bild mehr oder weniger das Gegenteil, sprich ein Filter zum Schärfen des Bildes." + ] + }, + { + "cell_type": "markdown", + "id": "1d2ec712", + "metadata": {}, + "source": [ + "**Aber was passiert da genau?**" + ] + }, + { + "cell_type": "markdown", + "id": "8fcde416", + "metadata": {}, + "source": [ + "Wir betrachten dazu ein **schwarz-weiß Bild**, also eine Matrix als unseren Input.\n", + "\n", + "\n", + "In unserem Fall ist das die **Matrix**:\n", + "$$X=\\begin{pmatrix}\n", + " 0 & 0 & 0 & 0 & 0 & 0\\\\\n", + " 0 & 1.0 & 0 & 0 & 0.4 & 0\\\\\n", + " 0 & 0 & 0 & 0.4 & 0 & 0\\\\\n", + " 0 & 0 & 0 & 0 & 0.4 & 0\\\\\n", + " 0 & 0 & 0 & 0.4 & 0 & 0\\\\\n", + " 0 & 0 & 0 & 0 & 0.4 & 0\n", + "\\end{pmatrix}$$\n", + "\n", + "Dazu wollen wir als **Kernel (=Filter)** folgende Matrix verwenden\n", + "\n", + "$$K = \\begin{pmatrix}\n", + " 0 & 0 & 0.5 \\\\\n", + " 0 & 0.5 & 0 \\\\\n", + " 0 & 0 & 0.5\n", + "\\end{pmatrix}$$" + ] + }, + { + "cell_type": "markdown", + "id": "688d77f4", + "metadata": {}, + "source": [ + "Bei der Faltung wird nun der Filter über den Input $X$ gelegt, und das in jeder möglichen Kombination.\n", + "\n", + "Das sieht in unserem Fall dann so aus für den ersten Eintrag des Ergebnisses." + ] + }, + { + "cell_type": "markdown", + "id": "945e1193", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "54827b86", + "metadata": {}, + "source": [ + "**Was ist hier passiert?** Wir haben den Filter (rot) über unser Bild (blau) gelegt und danach einfach Elementweise Matrix-Multipliziert und das Ergebnis zusammen gezählt.\n", + "\n", + "Dies wiederholen wir jetzt auch für die anderen Möglichkeiten. Also wir shiften unseren Kern um 1 Spalte nach rechts." + ] + }, + { + "cell_type": "markdown", + "id": "e9e68bb1", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "2551d80d", + "metadata": {}, + "source": [ + "Dies wiederholen wir jetzt solange, bis wir alle Möglichkeiten durch haben." + ] + }, + { + "cell_type": "markdown", + "id": "bdf22f6b", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "03f12803", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "029b40ce", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "7c0443f6", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "baee7070", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "d2f31397", + "metadata": {}, + "source": [ + "**Wichtig:** Wie wir sehen ist also die Faltung quasi (mathematisch sehr ungenau) wie ein 2D-Skalarprodukt." + ] + }, + { + "cell_type": "markdown", + "id": "3e3fa120", + "metadata": {}, + "source": [ + "Somit können wir auch erklären, warum beim Super-Mario Bild vorher ein unscharfer Output erzeugt wird. Der Grund ist, weil einfach der neue Pixel-Wert der Durchschnitt von allen umliegenden Pixelwerten ist." + ] + }, + { + "cell_type": "markdown", + "id": "1eaf4b05", + "metadata": {}, + "source": [ + "**Wichtig:** Beim Super-Mario und Kirby Bild ist die Faltung jeweils auf jeden Channel ausgeführt worden. Dabei wurde jedes mal der gleiche Filter (Kernel) verwendet." + ] + }, + { + "cell_type": "markdown", + "id": "cc828e9e", + "metadata": {}, + "source": [ + "Wir wollen nun ein paar weitere Filter ausprobieren bei eigenen Bildern. Dazu nutzen wir die `cv2` Bibliothek. Das geht (falls noch nicht vorhanden mit dem `pip install opencv-python` Command)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc849a62", + "metadata": {}, + "outputs": [], + "source": [ + "import cv2\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import os\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b743c95", + "metadata": {}, + "outputs": [], + "source": [ + "image_path = os.path.join(\"..\", \"resources\", \"Chemnitz_Hauptplatz.jpg\")\n", + "image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)\n", + "\n", + "fig, ax = plt.subplots(figsize=(3, 6))\n", + "ax.imshow(image, cmap=\"gray\", vmin=0, vmax=255)\n", + "\n", + "ax.grid(False)\n", + "ax.set_xticks([])\n", + "ax.set_yticks([])\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "39e9c012", + "metadata": {}, + "source": [ + "Nun definieren wir unsere Kernel. Beispiele sind zum Beispiel die Sobel Filter." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47df8c04", + "metadata": {}, + "outputs": [], + "source": [ + "sobel_x = np.array([\n", + " [ -1, 0, 1],\n", + " [ -2, 0, 2],\n", + " [ -1, 0, 1]\n", + "])\n", + "sobel_y = np.array([\n", + " [ -1, -2, -1],\n", + " [ 0, 0, 0],\n", + " [ 1, 2, 1]\n", + "])\n", + "\n", + "\n", + "mean_blur = 1/400 * np.ones((20, 20))\n", + "\n", + "edge_detection = sobel_x + sobel_y\n", + "\n", + "gaussian_blur = 1/36 * np.array([\n", + " [1,4,1],\n", + " [4,16,4],\n", + " [1,4,1]\n", + "]) \n", + "\n", + "filter = edge_detection\n", + "\n", + "filtered_image = cv2.filter2D(image, -1, filter)\n", + "\n", + "\n", + "fig, ax = plt.subplots(figsize=(3, 6))\n", + "ax.imshow(filtered_image, cmap=\"gray\", vmin=0, vmax=255)\n", + "ax.grid(False)\n", + "ax.set_xticks([])\n", + "ax.set_yticks([])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "8b70388a", + "metadata": {}, + "source": [ + "> **Übung:** Suche im Internet nach dem **Edge-Detection**, **Mean-Blurr**, **Gaussian-Blurr** und einem Schärfefilter und implementiere diese oben. " + ] + }, + { + "cell_type": "markdown", + "id": "edc12eaa", + "metadata": {}, + "source": [ + "**Hinweis:** Der Filter hat normalerweise nur positive Einträge (falls negativ muss man sich überlegen, wie das interpretiert werden soll) und diese Einträge sollen in Summe 1 Ergeben, sodass die (Pixel)Werte des Outputs im gleichen Bereich bleiben." + ] + }, + { + "cell_type": "markdown", + "id": "4dac2efe", + "metadata": {}, + "source": [ + "> **Übung:** Was ändert sich beim Output neben den eigentlichen Pixel Werten noch? *Tipp:* Betrachte nochmal die genaue Berechnung im vorigen Beispiel, welches in vielen Schritten detailiert gezeigt wurde." + ] + }, + { + "cell_type": "markdown", + "id": "4ce7aa1e", + "metadata": {}, + "source": [ + "Wie wir bereits bemerkt haben, ist die Output Matrix etwas kleiner als der Input. Die genaue Größe kann folgendermaßen berechnet werden:\n", + "\n", + "$$\\begin{align*}\n", + " X_{\\text{new}} &= \\left\\lfloor \\frac{X-K_x+2P_x}{S_x} + 1 \\right\\rfloor \\\\\n", + " Y_{\\text{new}} &= \\left\\lfloor \\frac{Y-K_y+2P_y}{S_y} + 1 \\right\\rfloor\n", + "\\end{align*}$$" + ] + }, + { + "cell_type": "markdown", + "id": "54cb3aba", + "metadata": {}, + "source": [ + "Dabei ist:\n", + "\n", + "* $X_{\\text{new}}, Y_{\\text{new}}$ die Größe der neuen Matrix\n", + "* $K_x$, $K_y$ die Größe des Kernels ($K_x$ ist die Anzahl der Spalten)\n", + "* $P_x, P_y$ steht für das Padding. Dieser Parameter erlaubt uns, einen gleich großen Output wie vorher der Input zu haben. Er beschreibt wie viele Pixel wir rund um unsere Matrix noch hinzufügen. Sprich $P_x=1$ heißt, wir fügen an der linken und an der rechten Seite noch eine Spalte hinzu. Als Wert wird dabei normalerweise die $0$ verwendet (\"**Zero-Padding**\"), es gibt aber auch andere Möglichkeiten.\n", + "* $S_x$, $S_y$ steht für den Stride. Dieser steht für die Anzahl an Pixel, die wir jedes mal nach $x$ bzw. nach $y$ \"rutschen\". Standard ist $1$, also wir bewegen uns immer nur 1 Pixel." + ] + }, + { + "cell_type": "markdown", + "id": "9d043f88", + "metadata": {}, + "source": [ + "Sehr empfehlenswert ist hier dieses [GitHub Repository](https://github.com/vdumoulin/conv_arithmetic), da es sehr viele Visualisierungen beinhaltet zu den einzelnen Convolution Typen." + ] + }, + { + "cell_type": "markdown", + "id": "05233ef0", + "metadata": {}, + "source": [ + "**Visualisierung Stride=3**" + ] + }, + { + "cell_type": "markdown", + "id": "8c53d073", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "ed3967e3", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "90b7012e", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "1ff2b1c2", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "e9872f85", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "4b24fa63", + "metadata": {}, + "source": [ + "**Visualisierung Padding=1**" + ] + }, + { + "cell_type": "markdown", + "id": "0f9ad91a", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "cc9bedc9", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "80d1cf5b", + "metadata": {}, + "source": [ + "## Der CNN-Layer" + ] + }, + { + "cell_type": "markdown", + "id": "4b6f59df", + "metadata": {}, + "source": [ + "Nachdem wir nun Bescheid wissen, wie die Faltung auf Bildern allgemein funktioniert, betrachten wir nun das CNN-Layer in PyTorch." + ] + }, + { + "cell_type": "markdown", + "id": "a623b878", + "metadata": {}, + "source": [ + "Prinzipiell ist es immer gut, einen Blick in die Dokumentation zu werfen. Deswegen werden wir zuerst [hier](https://docs.pytorch.org/docs/stable/generated/torch.nn.Conv2d.html) kurz schauen. Wir betrachten dabei das `nn.Conv2d()`-Layer, da wir es hauptsächlich auf Bildern (2D) anwenden werden." + ] + }, + { + "cell_type": "markdown", + "id": "5b0d46c4", + "metadata": {}, + "source": [ + "`class torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride=1, padding=0, ..., padding_mode='zeros')`" + ] + }, + { + "cell_type": "markdown", + "id": "9a6ab52b", + "metadata": {}, + "source": [ + "Dies ist der (wichtige Teil vom) Konstruktor des `Conv2d` Layers. Wir gehen nun die Parameter durch:\n", + "\n", + "* `in_channels`: Die Anzahl der Input Channels. Für ein RGB Bild ist dies 3.\n", + "* `out_channels`: Anzahl der Kernels, die wir durch das Netzwerk schicken. Dies ist dann die Anzahl der **Output Channels** und somit auch gleich `in_channels`, falls danach eine weitere `nn.Conv2d` Schicht kommt.\n", + "* `kernel_size`: Größe des Kernels (in Tupelform, also $(3,5)$ oder in Integerform, also $3$ für einen quadratischen Kernel (in diesem Fall $3\\times 3$))\n", + "* `stride`: Größe des Strides (erneut entweder Tupelform oder Integer bei gleichem Stride)\n", + "* `padding`: Größe des Paddings (Tupel oder Integer)\n", + "* `padding_mode`: Bestimmt die Art des Paddings. Übergeben wird ein String: 'zeros', 'reflect', 'replicate' oder 'circular'. Standard ist 'zeros'. " + ] + }, + { + "cell_type": "markdown", + "id": "429f1226", + "metadata": {}, + "source": [ + "> **Übung:** Was sind nun die Parameter die gelernt werden sollen vom Netzwerk?" + ] + }, + { + "cell_type": "markdown", + "id": "d15c7721", + "metadata": {}, + "source": [ + "Wir wollen also die Werte des Filters (Kernel) lernen." + ] + }, + { + "cell_type": "markdown", + "id": "4d73d1e2", + "metadata": {}, + "source": [ + "> **Übung:** Hat ein CNN mehr oder weniger Parameter als ein Fully-Connected Neural Network?" + ] + }, + { + "cell_type": "markdown", + "id": "1c80ff0d", + "metadata": {}, + "source": [ + "Wir visualisieren nochmal kurz die Parameter des Netzwerkes." + ] + }, + { + "cell_type": "markdown", + "id": "9d23fb13", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "ab74f54c", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "4563f40a", + "metadata": {}, + "source": [ + "Wir sehen also, dass zum Beispiel für einen RGB Input der Kernel dann auch einfach 3 Channels hat (3 **verschiedene** Matrizen). Dabei wird jeder Layer des Kernels auf das entsprechende Layer im Bild angewendet. Am Ende werden alle Schichten zu **einem Channel** zusammen **addiert**." + ] + }, + { + "cell_type": "markdown", + "id": "05ecc94a", + "metadata": {}, + "source": [ + "Beim zweiten Bild ist nun die Anzahl an `out_channels` auf 2 gesetzt. Das heißt wir haben 2 **verschiedene** Kernels mit je 3 Channels. " + ] + }, + { + "cell_type": "markdown", + "id": "46485235", + "metadata": {}, + "source": [ + "**Was ist nun das Ziel von den Kernels?**\n", + "\n", + "Die Kernels versuchen im Lernprozess Werte in die einzelnen Einträge zu schreiben, sodass wir aus unseren Trainingsbildern Schritt für Schritt möglichst gute Informationen extrahieren können." + ] + }, + { + "cell_type": "markdown", + "id": "ca3e12ed", + "metadata": {}, + "source": [ + "Prinzipiell haben wir dadurch schon verstanden wie ein CNN-Layer in PyTorch funktioniert. Was gibt es jetzt noch zu beachten?" + ] + }, + { + "cell_type": "markdown", + "id": "e836a6a5", + "metadata": {}, + "source": [ + "* Wir müssen nach jedem CNN-Layer berechnen, wie groß unser Ergebnis-Bild ist. Der Grund ist, dass wir nach unseren CNN-Layers eine fixe Output Größe brauchen (siehe nächster Punkt).\n", + "* Der Output von einem Neuronalen Netzwerk, welches CNN-Schichten beinhaltet ist vielfältig:\n", + " * In den meisten Fällen wechseln wir nach einigen CNN-Layern zu einem Fully-Connected Neuronal Network, welches später die Klassifikation bzw. Regression basierend auf den Ergebnissen der Faltung(en) erledigt. Dabei müssen wir wissen, wie groß der Output nach dem letzten CNN-Layer ist. Ist dieser dann zum Beispiel ein $3\\times 3$-Bild, dann würden wir dieses `flatten()` und erhalten einen $9$-Vektor.\n", + " * Es gibt aber auch Fälle (zum Beispiel bei unserer Image Inpainting Challenge), bei der wir als Output wirklich Bilder wollen, sprich wir beenden unser Netzwerk auch mit einem CNN-Layer. Auch hier müssen wir sicherstellen, dass dieser Output dann zum Beispiel genauso groß ist wie der Input." + ] + }, + { + "cell_type": "markdown", + "id": "3a7a1438", + "metadata": {}, + "source": [ + "> **Übung:** Wie können wir das mit CNN's realisieren, dass am Ende unser Bild genauso groß ist wie vorher?" + ] + }, + { + "cell_type": "markdown", + "id": "9ceb3b52", + "metadata": {}, + "source": [ + "Am Schluss wollen wir uns noch einem weiteren, sehr ähnlichen, Konzept widmen. Die Rede ist von den sogenannten **Pooling-Layers**." + ] + }, + { + "cell_type": "markdown", + "id": "a82b2e59", + "metadata": {}, + "source": [ + "## Pooling" + ] + }, + { + "cell_type": "markdown", + "id": "369e8557", + "metadata": {}, + "source": [ + "Die Idee vom **Pooling** ist, dass wir unser Bild *downsamplen*, sprich ein kleineres Bild produzieren. Dabei gibt es mehrere Möglichkeiten, wie wir dieses Downsampling realisieren können:\n", + "* **Average-Pooling:** Wir nehmen den Mittelwert von $k\\times k$ Werten\n", + "* **Max Pooling:** Wir nehmen den Maximalwert von $k\\times k$ Werten (wird meistens verwendet)" + ] + }, + { + "cell_type": "markdown", + "id": "41d13d86", + "metadata": {}, + "source": [ + "Wir visualisieren kurz den Effekt von Max-Pooling, in diesem Fall mit $k=2$. " + ] + }, + { + "cell_type": "markdown", + "id": "5278cd51", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "4a898651", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "f460b87e", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "fda20b59", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "a74487e7", + "metadata": {}, + "source": [ + "\n", + "\n", + "(von Dr. Andreas Schörgenhumer; Hands-On AI1 WS2021)" + ] + }, + { + "cell_type": "markdown", + "id": "0fef4fb6", + "metadata": {}, + "source": [ + "**Wichtig:** Ein Pooling Layer beinhaltet also keine Parameter, er reduziert nur unsere Datengröße deutlich und führt somit auch natürlich zu einem (großen) Informationsverlust." + ] + }, + { + "cell_type": "markdown", + "id": "583d3484", + "metadata": {}, + "source": [ + "In PyTorch können wir so ein Verhalten auch ganz einfach mit einem Pooling Layer erreichen, welches wir mit `nn.MaxPool2d()` ganz einfach hinzufügen können. Die Dokumentation finden wir [hier](https://docs.pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html)." + ] + }, + { + "cell_type": "markdown", + "id": "6fd908f0", + "metadata": {}, + "source": [ + "Betrachten wir kurz (den wichtigsten Teil davon) den Konstruktor.\n", + "\n", + "`class torch.nn.MaxPool2d(kernel_size, stride=None, padding=0, ...)`\n", + "\n", + "Der einzige Wert im Konstruktor, der übergeben werden muss ist `kernel_size`, erneut als Tupel oder Integer. Als `stride` ist standardmäßig `None` übergeben, was bedeutet, dass wir pro Schritt den \"Filter\" um die `kernel_size` verschieben. Ebenso ist kein standardmäßig kein Padding vorgesehen." + ] + }, + { + "cell_type": "markdown", + "id": "04818d04", + "metadata": {}, + "source": [ + "**Hinweis:** Die Größe der Daten kann nach einem Pooling Layer mit der gleichen Formel wie vorher berechnet werden. In vielen Fällen (`stride=kernel_size` und ohne Padding) ist das (bis auf das Verhalten am Rand) die Berechnung natürlich ohne Formel auch sehr leicht." + ] + }, + { + "cell_type": "markdown", + "id": "f97b5dbe", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "480c0c1d", + "metadata": {}, + "source": [ + "Damit haben wir die Grundlagen eines CNN-Layers verstanden. Wir werden im nächsten Notebook ein großes Beispiel machen, welches uns die Details und die Funktionen in einem praktischen Setting nochmal näher bringt." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dsai", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/06_NN/models/best_model_cnn_fashion_mnist.pt b/06_NN/models/best_model_cnn_fashion_mnist.pt new file mode 100644 index 0000000..fa76245 Binary files /dev/null and b/06_NN/models/best_model_cnn_fashion_mnist.pt differ diff --git a/06_NN/models/best_model_cnn_mnist.pt b/06_NN/models/best_model_cnn_mnist.pt new file mode 100644 index 0000000..3e21d0b Binary files /dev/null and b/06_NN/models/best_model_cnn_mnist.pt differ diff --git a/06_NN/models/best_model_sophisticated_cnn_fashion_mnist.pt b/06_NN/models/best_model_sophisticated_cnn_fashion_mnist.pt new file mode 100644 index 0000000..e5f6131 Binary files /dev/null and b/06_NN/models/best_model_sophisticated_cnn_fashion_mnist.pt differ diff --git a/06_NN/models/nn_6_simple_regressor.onnx b/06_NN/models/nn_6_simple_regressor.onnx new file mode 100644 index 0000000..0798cac Binary files /dev/null and b/06_NN/models/nn_6_simple_regressor.onnx differ diff --git a/06_NN/models/nn_6_simple_regressor_scripted.pt b/06_NN/models/nn_6_simple_regressor_scripted.pt new file mode 100644 index 0000000..a81fbfa Binary files /dev/null and b/06_NN/models/nn_6_simple_regressor_scripted.pt differ diff --git a/06_NN/models/nn_6_simple_regressor_state_dict.pth b/06_NN/models/nn_6_simple_regressor_state_dict.pth new file mode 100644 index 0000000..e4c9764 Binary files /dev/null and b/06_NN/models/nn_6_simple_regressor_state_dict.pth differ diff --git a/06_NN/resources/Biological_Neuron_vs_Artificial_Neuron.png b/06_NN/resources/Biological_Neuron_vs_Artificial_Neuron.png new file mode 100644 index 0000000..9cc8893 Binary files /dev/null and b/06_NN/resources/Biological_Neuron_vs_Artificial_Neuron.png differ diff --git a/06_NN/resources/CNN_Input_1 (1).jpg b/06_NN/resources/CNN_Input_1 (1).jpg new file mode 100644 index 0000000..95ea44d Binary files /dev/null and b/06_NN/resources/CNN_Input_1 (1).jpg differ diff --git a/06_NN/resources/CNN_Input_1.jpg b/06_NN/resources/CNN_Input_1.jpg new file mode 100644 index 0000000..95ea44d Binary files /dev/null and b/06_NN/resources/CNN_Input_1.jpg differ diff --git a/06_NN/resources/CNN_Input_2 (1).jpg b/06_NN/resources/CNN_Input_2 (1).jpg new file mode 100644 index 0000000..6028cd0 Binary files /dev/null and b/06_NN/resources/CNN_Input_2 (1).jpg differ diff --git a/06_NN/resources/CNN_Input_2.jpg b/06_NN/resources/CNN_Input_2.jpg new file mode 100644 index 0000000..6028cd0 Binary files /dev/null and b/06_NN/resources/CNN_Input_2.jpg differ diff --git a/06_NN/resources/Chemnitz_Hauptplatz (1).jpg b/06_NN/resources/Chemnitz_Hauptplatz (1).jpg new file mode 100644 index 0000000..f285794 Binary files /dev/null and b/06_NN/resources/Chemnitz_Hauptplatz (1).jpg differ diff --git a/06_NN/resources/Chemnitz_Hauptplatz.jpg b/06_NN/resources/Chemnitz_Hauptplatz.jpg new file mode 100644 index 0000000..f285794 Binary files /dev/null and b/06_NN/resources/Chemnitz_Hauptplatz.jpg differ diff --git a/06_NN/resources/Cifar10 (1).jpg b/06_NN/resources/Cifar10 (1).jpg new file mode 100644 index 0000000..ad24731 Binary files /dev/null and b/06_NN/resources/Cifar10 (1).jpg differ diff --git a/06_NN/resources/Cifar10.jpg b/06_NN/resources/Cifar10.jpg new file mode 100644 index 0000000..ad24731 Binary files /dev/null and b/06_NN/resources/Cifar10.jpg differ diff --git a/06_NN/resources/Columbus (1).jpeg b/06_NN/resources/Columbus (1).jpeg new file mode 100644 index 0000000..2a10f82 Binary files /dev/null and b/06_NN/resources/Columbus (1).jpeg differ diff --git a/06_NN/resources/Columbus.jpeg b/06_NN/resources/Columbus.jpeg new file mode 100644 index 0000000..2a10f82 Binary files /dev/null and b/06_NN/resources/Columbus.jpeg differ diff --git a/06_NN/resources/Columbus_Tensor.png b/06_NN/resources/Columbus_Tensor.png new file mode 100644 index 0000000..6a559af Binary files /dev/null and b/06_NN/resources/Columbus_Tensor.png differ diff --git a/06_NN/resources/Convolution_Concept1 (1).jpg b/06_NN/resources/Convolution_Concept1 (1).jpg new file mode 100644 index 0000000..b7d41a0 Binary files /dev/null and b/06_NN/resources/Convolution_Concept1 (1).jpg differ diff --git a/06_NN/resources/Convolution_Concept1.jpg b/06_NN/resources/Convolution_Concept1.jpg new file mode 100644 index 0000000..b7d41a0 Binary files /dev/null and b/06_NN/resources/Convolution_Concept1.jpg differ diff --git a/06_NN/resources/Convolution_Concept2 (1).jpg b/06_NN/resources/Convolution_Concept2 (1).jpg new file mode 100644 index 0000000..9d74d05 Binary files /dev/null and b/06_NN/resources/Convolution_Concept2 (1).jpg differ diff --git a/06_NN/resources/Convolution_Concept2.jpg b/06_NN/resources/Convolution_Concept2.jpg new file mode 100644 index 0000000..9d74d05 Binary files /dev/null and b/06_NN/resources/Convolution_Concept2.jpg differ diff --git a/06_NN/resources/Convolution_Concept3 (1).jpg b/06_NN/resources/Convolution_Concept3 (1).jpg new file mode 100644 index 0000000..8548335 Binary files /dev/null and b/06_NN/resources/Convolution_Concept3 (1).jpg differ diff --git a/06_NN/resources/Convolution_Concept3.jpg b/06_NN/resources/Convolution_Concept3.jpg new file mode 100644 index 0000000..8548335 Binary files /dev/null and b/06_NN/resources/Convolution_Concept3.jpg differ diff --git a/06_NN/resources/Convolution_Concept4 (1).jpg b/06_NN/resources/Convolution_Concept4 (1).jpg new file mode 100644 index 0000000..ed3d045 Binary files /dev/null and b/06_NN/resources/Convolution_Concept4 (1).jpg differ diff --git a/06_NN/resources/Convolution_Concept4.jpg b/06_NN/resources/Convolution_Concept4.jpg new file mode 100644 index 0000000..ed3d045 Binary files /dev/null and b/06_NN/resources/Convolution_Concept4.jpg differ diff --git a/06_NN/resources/Convolution_Concept5 (1).jpg b/06_NN/resources/Convolution_Concept5 (1).jpg new file mode 100644 index 0000000..aadc1d7 Binary files /dev/null and b/06_NN/resources/Convolution_Concept5 (1).jpg differ diff --git a/06_NN/resources/Convolution_Concept5.jpg b/06_NN/resources/Convolution_Concept5.jpg new file mode 100644 index 0000000..aadc1d7 Binary files /dev/null and b/06_NN/resources/Convolution_Concept5.jpg differ diff --git a/06_NN/resources/Convolution_Concept6 (1).jpg b/06_NN/resources/Convolution_Concept6 (1).jpg new file mode 100644 index 0000000..4741c9e Binary files /dev/null and b/06_NN/resources/Convolution_Concept6 (1).jpg differ diff --git a/06_NN/resources/Convolution_Concept6.jpg b/06_NN/resources/Convolution_Concept6.jpg new file mode 100644 index 0000000..4741c9e Binary files /dev/null and b/06_NN/resources/Convolution_Concept6.jpg differ diff --git a/06_NN/resources/Convolution_Concept7 (1).jpg b/06_NN/resources/Convolution_Concept7 (1).jpg new file mode 100644 index 0000000..773ca1f Binary files /dev/null and b/06_NN/resources/Convolution_Concept7 (1).jpg differ diff --git a/06_NN/resources/Convolution_Concept7.jpg b/06_NN/resources/Convolution_Concept7.jpg new file mode 100644 index 0000000..773ca1f Binary files /dev/null and b/06_NN/resources/Convolution_Concept7.jpg differ diff --git a/06_NN/resources/Convolution_Stride3_1 (1).jpg b/06_NN/resources/Convolution_Stride3_1 (1).jpg new file mode 100644 index 0000000..5110795 Binary files /dev/null and b/06_NN/resources/Convolution_Stride3_1 (1).jpg differ diff --git a/06_NN/resources/Convolution_Stride3_1.jpg b/06_NN/resources/Convolution_Stride3_1.jpg new file mode 100644 index 0000000..5110795 Binary files /dev/null and b/06_NN/resources/Convolution_Stride3_1.jpg differ diff --git a/06_NN/resources/Convolution_Stride3_2 (1).jpg b/06_NN/resources/Convolution_Stride3_2 (1).jpg new file mode 100644 index 0000000..56359c7 Binary files /dev/null and b/06_NN/resources/Convolution_Stride3_2 (1).jpg differ diff --git a/06_NN/resources/Convolution_Stride3_2.jpg b/06_NN/resources/Convolution_Stride3_2.jpg new file mode 100644 index 0000000..56359c7 Binary files /dev/null and b/06_NN/resources/Convolution_Stride3_2.jpg differ diff --git a/06_NN/resources/Convolution_Stride3_3 (1).jpg b/06_NN/resources/Convolution_Stride3_3 (1).jpg new file mode 100644 index 0000000..a737eb0 Binary files /dev/null and b/06_NN/resources/Convolution_Stride3_3 (1).jpg differ diff --git a/06_NN/resources/Convolution_Stride3_3.jpg b/06_NN/resources/Convolution_Stride3_3.jpg new file mode 100644 index 0000000..a737eb0 Binary files /dev/null and b/06_NN/resources/Convolution_Stride3_3.jpg differ diff --git a/06_NN/resources/Convolution_Stride3_4 (1).jpg b/06_NN/resources/Convolution_Stride3_4 (1).jpg new file mode 100644 index 0000000..3c0e38c Binary files /dev/null and b/06_NN/resources/Convolution_Stride3_4 (1).jpg differ diff --git a/06_NN/resources/Convolution_Stride3_4.jpg b/06_NN/resources/Convolution_Stride3_4.jpg new file mode 100644 index 0000000..3c0e38c Binary files /dev/null and b/06_NN/resources/Convolution_Stride3_4.jpg differ diff --git a/06_NN/resources/Convolution_Stride3_5 (1).jpg b/06_NN/resources/Convolution_Stride3_5 (1).jpg new file mode 100644 index 0000000..5acfda9 Binary files /dev/null and b/06_NN/resources/Convolution_Stride3_5 (1).jpg differ diff --git a/06_NN/resources/Convolution_Stride3_5.jpg b/06_NN/resources/Convolution_Stride3_5.jpg new file mode 100644 index 0000000..5acfda9 Binary files /dev/null and b/06_NN/resources/Convolution_Stride3_5.jpg differ diff --git a/06_NN/resources/Convolution_Zero_Padding_1 (1).jpg b/06_NN/resources/Convolution_Zero_Padding_1 (1).jpg new file mode 100644 index 0000000..8e82b5c Binary files /dev/null and b/06_NN/resources/Convolution_Zero_Padding_1 (1).jpg differ diff --git a/06_NN/resources/Convolution_Zero_Padding_1.jpg b/06_NN/resources/Convolution_Zero_Padding_1.jpg new file mode 100644 index 0000000..8e82b5c Binary files /dev/null and b/06_NN/resources/Convolution_Zero_Padding_1.jpg differ diff --git a/06_NN/resources/Convolution_Zero_Padding_2 (1).jpg b/06_NN/resources/Convolution_Zero_Padding_2 (1).jpg new file mode 100644 index 0000000..c875068 Binary files /dev/null and b/06_NN/resources/Convolution_Zero_Padding_2 (1).jpg differ diff --git a/06_NN/resources/Convolution_Zero_Padding_2.jpg b/06_NN/resources/Convolution_Zero_Padding_2.jpg new file mode 100644 index 0000000..c875068 Binary files /dev/null and b/06_NN/resources/Convolution_Zero_Padding_2.jpg differ diff --git a/06_NN/resources/Dropout_Visualized (1).png b/06_NN/resources/Dropout_Visualized (1).png new file mode 100644 index 0000000..7374ec6 Binary files /dev/null and b/06_NN/resources/Dropout_Visualized (1).png differ diff --git a/06_NN/resources/Dropout_Visualized.png b/06_NN/resources/Dropout_Visualized.png new file mode 100644 index 0000000..7374ec6 Binary files /dev/null and b/06_NN/resources/Dropout_Visualized.png differ diff --git a/06_NN/resources/Gradient_Descent_Intiution_Derivative.png b/06_NN/resources/Gradient_Descent_Intiution_Derivative.png new file mode 100644 index 0000000..eca7c7a Binary files /dev/null and b/06_NN/resources/Gradient_Descent_Intiution_Derivative.png differ diff --git a/06_NN/resources/Gradient_Descent_Local_Minima.png b/06_NN/resources/Gradient_Descent_Local_Minima.png new file mode 100644 index 0000000..68588e8 Binary files /dev/null and b/06_NN/resources/Gradient_Descent_Local_Minima.png differ diff --git a/06_NN/resources/Gradient_Descent_Selfmade.png b/06_NN/resources/Gradient_Descent_Selfmade.png new file mode 100644 index 0000000..9b7f1b6 Binary files /dev/null and b/06_NN/resources/Gradient_Descent_Selfmade.png differ diff --git a/06_NN/resources/Gradient_Descent_Too_Big_LR.png b/06_NN/resources/Gradient_Descent_Too_Big_LR.png new file mode 100644 index 0000000..911d972 Binary files /dev/null and b/06_NN/resources/Gradient_Descent_Too_Big_LR.png differ diff --git a/06_NN/resources/Gradient_Descent_Too_Small_LR.png b/06_NN/resources/Gradient_Descent_Too_Small_LR.png new file mode 100644 index 0000000..bd475e2 Binary files /dev/null and b/06_NN/resources/Gradient_Descent_Too_Small_LR.png differ diff --git a/06_NN/resources/Instagram_Reel_First_Try_Suspicious.mp4 b/06_NN/resources/Instagram_Reel_First_Try_Suspicious.mp4 new file mode 100644 index 0000000..ca21cf3 Binary files /dev/null and b/06_NN/resources/Instagram_Reel_First_Try_Suspicious.mp4 differ diff --git a/06_NN/resources/Instagram_Reel_NaNs.mp4 b/06_NN/resources/Instagram_Reel_NaNs.mp4 new file mode 100644 index 0000000..84897c3 Binary files /dev/null and b/06_NN/resources/Instagram_Reel_NaNs.mp4 differ diff --git a/06_NN/resources/Kirby_Convolution (1).jpg b/06_NN/resources/Kirby_Convolution (1).jpg new file mode 100644 index 0000000..c121ecf Binary files /dev/null and b/06_NN/resources/Kirby_Convolution (1).jpg differ diff --git a/06_NN/resources/Kirby_Convolution.jpg b/06_NN/resources/Kirby_Convolution.jpg new file mode 100644 index 0000000..c121ecf Binary files /dev/null and b/06_NN/resources/Kirby_Convolution.jpg differ diff --git a/06_NN/resources/Loss_Landscape.png b/06_NN/resources/Loss_Landscape.png new file mode 100644 index 0000000..062b608 Binary files /dev/null and b/06_NN/resources/Loss_Landscape.png differ diff --git a/06_NN/resources/Loss_Landscape_Path.jpeg b/06_NN/resources/Loss_Landscape_Path.jpeg new file mode 100644 index 0000000..0492f4e Binary files /dev/null and b/06_NN/resources/Loss_Landscape_Path.jpeg differ diff --git a/06_NN/resources/MNIST.png b/06_NN/resources/MNIST.png new file mode 100644 index 0000000..af8c966 Binary files /dev/null and b/06_NN/resources/MNIST.png differ diff --git a/06_NN/resources/Mario_Convolution (1).png b/06_NN/resources/Mario_Convolution (1).png new file mode 100644 index 0000000..caf4d06 Binary files /dev/null and b/06_NN/resources/Mario_Convolution (1).png differ diff --git a/06_NN/resources/Mario_Convolution.png b/06_NN/resources/Mario_Convolution.png new file mode 100644 index 0000000..caf4d06 Binary files /dev/null and b/06_NN/resources/Mario_Convolution.png differ diff --git a/06_NN/resources/Max_Pooling_1 (1).jpg b/06_NN/resources/Max_Pooling_1 (1).jpg new file mode 100644 index 0000000..8fc1065 Binary files /dev/null and b/06_NN/resources/Max_Pooling_1 (1).jpg differ diff --git a/06_NN/resources/Max_Pooling_1.jpg b/06_NN/resources/Max_Pooling_1.jpg new file mode 100644 index 0000000..8fc1065 Binary files /dev/null and b/06_NN/resources/Max_Pooling_1.jpg differ diff --git a/06_NN/resources/Max_Pooling_2 (1).jpg b/06_NN/resources/Max_Pooling_2 (1).jpg new file mode 100644 index 0000000..1b6ecad Binary files /dev/null and b/06_NN/resources/Max_Pooling_2 (1).jpg differ diff --git a/06_NN/resources/Max_Pooling_2.jpg b/06_NN/resources/Max_Pooling_2.jpg new file mode 100644 index 0000000..1b6ecad Binary files /dev/null and b/06_NN/resources/Max_Pooling_2.jpg differ diff --git a/06_NN/resources/Max_Pooling_3 (1).jpg b/06_NN/resources/Max_Pooling_3 (1).jpg new file mode 100644 index 0000000..c37a42c Binary files /dev/null and b/06_NN/resources/Max_Pooling_3 (1).jpg differ diff --git a/06_NN/resources/Max_Pooling_3.jpg b/06_NN/resources/Max_Pooling_3.jpg new file mode 100644 index 0000000..c37a42c Binary files /dev/null and b/06_NN/resources/Max_Pooling_3.jpg differ diff --git a/06_NN/resources/Max_Pooling_4 (1).jpg b/06_NN/resources/Max_Pooling_4 (1).jpg new file mode 100644 index 0000000..85af3d2 Binary files /dev/null and b/06_NN/resources/Max_Pooling_4 (1).jpg differ diff --git a/06_NN/resources/Max_Pooling_4.jpg b/06_NN/resources/Max_Pooling_4.jpg new file mode 100644 index 0000000..85af3d2 Binary files /dev/null and b/06_NN/resources/Max_Pooling_4.jpg differ diff --git a/06_NN/resources/Max_Pooling_5 (1).jpg b/06_NN/resources/Max_Pooling_5 (1).jpg new file mode 100644 index 0000000..c38af43 Binary files /dev/null and b/06_NN/resources/Max_Pooling_5 (1).jpg differ diff --git a/06_NN/resources/Max_Pooling_5.jpg b/06_NN/resources/Max_Pooling_5.jpg new file mode 100644 index 0000000..c38af43 Binary files /dev/null and b/06_NN/resources/Max_Pooling_5.jpg differ diff --git a/06_NN/resources/NN_Function_Approximation.png b/06_NN/resources/NN_Function_Approximation.png new file mode 100644 index 0000000..e2bc183 Binary files /dev/null and b/06_NN/resources/NN_Function_Approximation.png differ diff --git a/06_NN/resources/Overfitting_Underfitting_Loss_Curve (1).png b/06_NN/resources/Overfitting_Underfitting_Loss_Curve (1).png new file mode 100644 index 0000000..d3a74fe Binary files /dev/null and b/06_NN/resources/Overfitting_Underfitting_Loss_Curve (1).png differ diff --git a/06_NN/resources/Overfitting_Underfitting_Loss_Curve.png b/06_NN/resources/Overfitting_Underfitting_Loss_Curve.png new file mode 100644 index 0000000..d3a74fe Binary files /dev/null and b/06_NN/resources/Overfitting_Underfitting_Loss_Curve.png differ diff --git a/06_NN/resources/Perzeptron.png b/06_NN/resources/Perzeptron.png new file mode 100644 index 0000000..157c10c Binary files /dev/null and b/06_NN/resources/Perzeptron.png differ diff --git a/06_NN/resources/Perzeptron_zugeschnitten.png b/06_NN/resources/Perzeptron_zugeschnitten.png new file mode 100644 index 0000000..94689e9 Binary files /dev/null and b/06_NN/resources/Perzeptron_zugeschnitten.png differ diff --git a/06_NN/resources/SGD_Local_Minima_Medal.jpg b/06_NN/resources/SGD_Local_Minima_Medal.jpg new file mode 100644 index 0000000..8bdf675 Binary files /dev/null and b/06_NN/resources/SGD_Local_Minima_Medal.jpg differ diff --git a/06_NN/resources/SGD_vs_GD.png b/06_NN/resources/SGD_vs_GD.png new file mode 100644 index 0000000..56585e5 Binary files /dev/null and b/06_NN/resources/SGD_vs_GD.png differ diff --git a/06_NN/resources/Sigmoid_plus_Derivative.png b/06_NN/resources/Sigmoid_plus_Derivative.png new file mode 100644 index 0000000..d388ba6 Binary files /dev/null and b/06_NN/resources/Sigmoid_plus_Derivative.png differ diff --git a/06_NN/resources/Skip_Connections (1).png b/06_NN/resources/Skip_Connections (1).png new file mode 100644 index 0000000..cd48c45 Binary files /dev/null and b/06_NN/resources/Skip_Connections (1).png differ diff --git a/06_NN/resources/Skip_Connections.png b/06_NN/resources/Skip_Connections.png new file mode 100644 index 0000000..cd48c45 Binary files /dev/null and b/06_NN/resources/Skip_Connections.png differ diff --git a/06_NN/resources/Tensor_1.png b/06_NN/resources/Tensor_1.png new file mode 100644 index 0000000..ceca6c4 Binary files /dev/null and b/06_NN/resources/Tensor_1.png differ diff --git a/06_NN/resources/Tensor_2.png b/06_NN/resources/Tensor_2.png new file mode 100644 index 0000000..025b441 Binary files /dev/null and b/06_NN/resources/Tensor_2.png differ diff --git a/06_NN/resources/Tensors_Everywhere.jpg b/06_NN/resources/Tensors_Everywhere.jpg new file mode 100644 index 0000000..1ca048d Binary files /dev/null and b/06_NN/resources/Tensors_Everywhere.jpg differ diff --git a/06_NN/resources/Vectored.png b/06_NN/resources/Vectored.png new file mode 100644 index 0000000..81cf181 Binary files /dev/null and b/06_NN/resources/Vectored.png differ diff --git a/06_NN/resources/Wait_Always_Has_Been.jpg b/06_NN/resources/Wait_Always_Has_Been.jpg new file mode 100644 index 0000000..acfacf0 Binary files /dev/null and b/06_NN/resources/Wait_Always_Has_Been.jpg differ diff --git a/06_NN/resources/gradient_descent_ball.jpg b/06_NN/resources/gradient_descent_ball.jpg new file mode 100644 index 0000000..85dad97 Binary files /dev/null and b/06_NN/resources/gradient_descent_ball.jpg differ diff --git a/06_NN/resources/gradient_descent_mountain.png b/06_NN/resources/gradient_descent_mountain.png new file mode 100644 index 0000000..4a400ab Binary files /dev/null and b/06_NN/resources/gradient_descent_mountain.png differ diff --git a/06_NN/resources/gradient_descent_mountain_2.png b/06_NN/resources/gradient_descent_mountain_2.png new file mode 100644 index 0000000..cf19647 Binary files /dev/null and b/06_NN/resources/gradient_descent_mountain_2.png differ diff --git a/06_NN/resources/imagenet_classes (1).txt b/06_NN/resources/imagenet_classes (1).txt new file mode 100644 index 0000000..888d6f5 --- /dev/null +++ b/06_NN/resources/imagenet_classes (1).txt @@ -0,0 +1,1000 @@ +tench +goldfish +great white shark +tiger shark +hammerhead +electric ray +stingray +cock +hen +ostrich +brambling +goldfinch +house finch +junco +indigo bunting +robin +bulbul +jay +magpie +chickadee +water ouzel +kite +bald eagle +vulture +great grey owl +European fire salamander +common newt +eft +spotted salamander +axolotl +bullfrog +tree frog +tailed frog +loggerhead +leatherback turtle +mud turtle +terrapin +box turtle +banded gecko +common iguana +American chameleon +whiptail +agama +frilled lizard +alligator lizard +Gila monster +green lizard +African chameleon +Komodo dragon +African crocodile +American alligator +triceratops +thunder snake +ringneck snake +hognose snake +green snake +king snake +garter snake +water snake +vine snake +night snake +boa constrictor +rock python +Indian cobra +green mamba +sea snake +horned viper +diamondback +sidewinder +trilobite +harvestman +scorpion +black and gold garden spider +barn spider +garden spider +black widow +tarantula +wolf spider +tick +centipede +black grouse +ptarmigan +ruffed grouse +prairie chicken +peacock +quail +partridge +African grey +macaw +sulphur-crested cockatoo +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +drake +red-breasted merganser +goose +black swan +tusker +echidna +platypus +wallaby +koala +wombat +jellyfish +sea anemone +brain coral +flatworm +nematode +conch +snail +slug +sea slug +chiton +chambered nautilus +Dungeness crab +rock crab +fiddler crab +king crab +American lobster +spiny lobster +crayfish +hermit crab +isopod +white stork +black stork +spoonbill +flamingo +little blue heron +American egret +bittern +crane +limpkin +European gallinule +American coot +bustard +ruddy turnstone +red-backed sandpiper +redshank +dowitcher +oystercatcher +pelican +king penguin +albatross +grey whale +killer whale +dugong +sea lion +Chihuahua +Japanese spaniel +Maltese dog +Pekinese +Shih-Tzu +Blenheim spaniel +papillon +toy terrier +Rhodesian ridgeback +Afghan hound +basset +beagle +bloodhound +bluetick +black-and-tan coonhound +Walker hound +English foxhound +redbone +borzoi +Irish wolfhound +Italian greyhound +whippet +Ibizan hound +Norwegian elkhound +otterhound +Saluki +Scottish deerhound +Weimaraner +Staffordshire bullterrier +American Staffordshire terrier +Bedlington terrier +Border terrier +Kerry blue terrier +Irish terrier +Norfolk terrier +Norwich terrier +Yorkshire terrier +wire-haired fox terrier +Lakeland terrier +Sealyham terrier +Airedale +cairn +Australian terrier +Dandie Dinmont +Boston bull +miniature schnauzer +giant schnauzer +standard schnauzer +Scotch terrier +Tibetan terrier +silky terrier +soft-coated wheaten terrier +West Highland white terrier +Lhasa +flat-coated retriever +curly-coated retriever +golden retriever +Labrador retriever +Chesapeake Bay retriever +German short-haired pointer +vizsla +English setter +Irish setter +Gordon setter +Brittany spaniel +clumber +English springer +Welsh springer spaniel +cocker spaniel +Sussex spaniel +Irish water spaniel +kuvasz +schipperke +groenendael +malinois +briard +kelpie +komondor +Old English sheepdog +Shetland sheepdog +collie +Border collie +Bouvier des Flandres +Rottweiler +German shepherd +Doberman +miniature pinscher +Greater Swiss Mountain dog +Bernese mountain dog +Appenzeller +EntleBucher +boxer +bull mastiff +Tibetan mastiff +French bulldog +Great Dane +Saint Bernard +Eskimo dog +malamute +Siberian husky +dalmatian +affenpinscher +basenji +pug +Leonberg +Newfoundland +Great Pyrenees +Samoyed +Pomeranian +chow +keeshond +Brabancon griffon +Pembroke +Cardigan +toy poodle +miniature poodle +standard poodle +Mexican hairless +timber wolf +white wolf +red wolf +coyote +dingo +dhole +African hunting dog +hyena +red fox +kit fox +Arctic fox +grey fox +tabby +tiger cat +Persian cat +Siamese cat +Egyptian cat +cougar +lynx +leopard +snow leopard +jaguar +lion +tiger +cheetah +brown bear +American black bear +ice bear +sloth bear +mongoose +meerkat +tiger beetle +ladybug +ground beetle +long-horned beetle +leaf beetle +dung beetle +rhinoceros beetle +weevil +fly +bee +ant +grasshopper +cricket +walking stick +cockroach +mantis +cicada +leafhopper +lacewing +dragonfly +damselfly +admiral +ringlet +monarch +cabbage butterfly +sulphur butterfly +lycaenid +starfish +sea urchin +sea cucumber +wood rabbit +hare +Angora +hamster +porcupine +fox squirrel +marmot +beaver +guinea pig +sorrel +zebra +hog +wild boar +warthog +hippopotamus +ox +water buffalo +bison +ram +bighorn +ibex +hartebeest +impala +gazelle +Arabian camel +llama +weasel +mink +polecat +black-footed ferret +otter +skunk +badger +armadillo +three-toed sloth +orangutan +gorilla +chimpanzee +gibbon +siamang +guenon +patas +baboon +macaque +langur +colobus +proboscis monkey +marmoset +capuchin +howler monkey +titi +spider monkey +squirrel monkey +Madagascar cat +indri +Indian elephant +African elephant +lesser panda +giant panda +barracouta +eel +coho +rock beauty +anemone fish +sturgeon +gar +lionfish +puffer +abacus +abaya +academic gown +accordion +acoustic guitar +aircraft carrier +airliner +airship +altar +ambulance +amphibian +analog clock +apiary +apron +ashcan +assault rifle +backpack +bakery +balance beam +balloon +ballpoint +Band Aid +banjo +bannister +barbell +barber chair +barbershop +barn +barometer +barrel +barrow +baseball +basketball +bassinet +bassoon +bathing cap +bath towel +bathtub +beach wagon +beacon +beaker +bearskin +beer bottle +beer glass +bell cote +bib +bicycle-built-for-two +bikini +binder +binoculars +birdhouse +boathouse +bobsled +bolo tie +bonnet +bookcase +bookshop +bottlecap +bow +bow tie +brass +brassiere +breakwater +breastplate +broom +bucket +buckle +bulletproof vest +bullet train +butcher shop +cab +caldron +candle +cannon +canoe +can opener +cardigan +car mirror +carousel +carpenter's kit +carton +car wheel +cash machine +cassette +cassette player +castle +catamaran +CD player +cello +cellular telephone +chain +chainlink fence +chain mail +chain saw +chest +chiffonier +chime +china cabinet +Christmas stocking +church +cinema +cleaver +cliff dwelling +cloak +clog +cocktail shaker +coffee mug +coffeepot +coil +combination lock +computer keyboard +confectionery +container ship +convertible +corkscrew +cornet +cowboy boot +cowboy hat +cradle +crane +crash helmet +crate +crib +Crock Pot +croquet ball +crutch +cuirass +dam +desk +desktop computer +dial telephone +diaper +digital clock +digital watch +dining table +dishrag +dishwasher +disk brake +dock +dogsled +dome +doormat +drilling platform +drum +drumstick +dumbbell +Dutch oven +electric fan +electric guitar +electric locomotive +entertainment center +envelope +espresso maker +face powder +feather boa +file +fireboat +fire engine +fire screen +flagpole +flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster +freight car +French horn +frying pan +fur coat +garbage truck +gasmask +gas pump +goblet +go-kart +golf ball +golfcart +gondola +gong +gown +grand piano +greenhouse +grille +grocery store +guillotine +hair slide +hair spray +half track +hammer +hamper +hand blower +hand-held computer +handkerchief +hard disc +harmonica +harp +harvester +hatchet +holster +home theater +honeycomb +hook +hoopskirt +horizontal bar +horse cart +hourglass +iPod +iron +jack-o'-lantern +jean +jeep +jersey +jigsaw puzzle +jinrikisha +joystick +kimono +knee pad +knot +lab coat +ladle +lampshade +laptop +lawn mower +lens cap +letter opener +library +lifeboat +lighter +limousine +liner +lipstick +Loafer +lotion +loudspeaker +loupe +lumbermill +magnetic compass +mailbag +mailbox +maillot +maillot +manhole cover +maraca +marimba +mask +matchstick +maypole +maze +measuring cup +medicine chest +megalith +microphone +microwave +military uniform +milk can +minibus +miniskirt +minivan +missile +mitten +mixing bowl +mobile home +Model T +modem +monastery +monitor +moped +mortar +mortarboard +mosque +mosquito net +motor scooter +mountain bike +mountain tent +mouse +mousetrap +moving van +muzzle +nail +neck brace +necklace +nipple +notebook +obelisk +oboe +ocarina +odometer +oil filter +organ +oscilloscope +overskirt +oxcart +oxygen mask +packet +paddle +paddlewheel +padlock +paintbrush +pajama +palace +panpipe +paper towel +parachute +parallel bars +park bench +parking meter +passenger car +patio +pay-phone +pedestal +pencil box +pencil sharpener +perfume +Petri dish +photocopier +pick +pickelhaube +picket fence +pickup +pier +piggy bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate +pitcher +plane +planetarium +plastic bag +plate rack +plow +plunger +Polaroid camera +pole +police van +poncho +pool table +pop bottle +pot +potter's wheel +power drill +prayer rug +printer +prison +projectile +projector +puck +punching bag +purse +quill +quilt +racer +racket +radiator +radio +radio telescope +rain barrel +recreational vehicle +reel +reflex camera +refrigerator +remote control +restaurant +revolver +rifle +rocking chair +rotisserie +rubber eraser +rugby ball +rule +running shoe +safe +safety pin +saltshaker +sandal +sarong +sax +scabbard +scale +school bus +schooner +scoreboard +screen +screw +screwdriver +seat belt +sewing machine +shield +shoe shop +shoji +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +ski mask +sleeping bag +slide rule +sliding door +slot +snorkel +snowmobile +snowplow +soap dispenser +soccer ball +sock +solar dish +sombrero +soup bowl +space bar +space heater +space shuttle +spatula +speedboat +spider web +spindle +sports car +spotlight +stage +steam locomotive +steel arch bridge +steel drum +stethoscope +stole +stone wall +stopwatch +stove +strainer +streetcar +stretcher +studio couch +stupa +submarine +suit +sundial +sunglass +sunglasses +sunscreen +suspension bridge +swab +sweatshirt +swimming trunks +swing +switch +syringe +table lamp +tank +tape player +teapot +teddy +television +tennis ball +thatch +theater curtain +thimble +thresher +throne +tile roof +toaster +tobacco shop +toilet seat +torch +totem pole +tow truck +toyshop +tractor +trailer truck +tray +trench coat +tricycle +trimaran +tripod +triumphal arch +trolleybus +trombone +tub +turnstile +typewriter keyboard +umbrella +unicycle +upright +vacuum +vase +vault +velvet +vending machine +vestment +viaduct +violin +volleyball +waffle iron +wall clock +wallet +wardrobe +warplane +washbasin +washer +water bottle +water jug +water tower +whiskey jug +whistle +wig +window screen +window shade +Windsor tie +wine bottle +wing +wok +wooden spoon +wool +worm fence +wreck +yawl +yurt +web site +comic book +crossword puzzle +street sign +traffic light +book jacket +menu +plate +guacamole +consomme +hot pot +trifle +ice cream +ice lolly +French loaf +bagel +pretzel +cheeseburger +hotdog +mashed potato +head cabbage +broccoli +cauliflower +zucchini +spaghetti squash +acorn squash +butternut squash +cucumber +artichoke +bell pepper +cardoon +mushroom +Granny Smith +strawberry +orange +lemon +fig +pineapple +banana +jackfruit +custard apple +pomegranate +hay +carbonara +chocolate sauce +dough +meat loaf +pizza +potpie +burrito +red wine +espresso +cup +eggnog +alp +bubble +cliff +coral reef +geyser +lakeside +promontory +sandbar +seashore +valley +volcano +ballplayer +groom +scuba diver +rapeseed +daisy +yellow lady's slipper +corn +acorn +hip +buckeye +coral fungus +agaric +gyromitra +stinkhorn +earthstar +hen-of-the-woods +bolete +ear +toilet tissue \ No newline at end of file diff --git a/06_NN/resources/imagenet_classes.txt b/06_NN/resources/imagenet_classes.txt new file mode 100644 index 0000000..888d6f5 --- /dev/null +++ b/06_NN/resources/imagenet_classes.txt @@ -0,0 +1,1000 @@ +tench +goldfish +great white shark +tiger shark +hammerhead +electric ray +stingray +cock +hen +ostrich +brambling +goldfinch +house finch +junco +indigo bunting +robin +bulbul +jay +magpie +chickadee +water ouzel +kite +bald eagle +vulture +great grey owl +European fire salamander +common newt +eft +spotted salamander +axolotl +bullfrog +tree frog +tailed frog +loggerhead +leatherback turtle +mud turtle +terrapin +box turtle +banded gecko +common iguana +American chameleon +whiptail +agama +frilled lizard +alligator lizard +Gila monster +green lizard +African chameleon +Komodo dragon +African crocodile +American alligator +triceratops +thunder snake +ringneck snake +hognose snake +green snake +king snake +garter snake +water snake +vine snake +night snake +boa constrictor +rock python +Indian cobra +green mamba +sea snake +horned viper +diamondback +sidewinder +trilobite +harvestman +scorpion +black and gold garden spider +barn spider +garden spider +black widow +tarantula +wolf spider +tick +centipede +black grouse +ptarmigan +ruffed grouse +prairie chicken +peacock +quail +partridge +African grey +macaw +sulphur-crested cockatoo +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +drake +red-breasted merganser +goose +black swan +tusker +echidna +platypus +wallaby +koala +wombat +jellyfish +sea anemone +brain coral +flatworm +nematode +conch +snail +slug +sea slug +chiton +chambered nautilus +Dungeness crab +rock crab +fiddler crab +king crab +American lobster +spiny lobster +crayfish +hermit crab +isopod +white stork +black stork +spoonbill +flamingo +little blue heron +American egret +bittern +crane +limpkin +European gallinule +American coot +bustard +ruddy turnstone +red-backed sandpiper +redshank +dowitcher +oystercatcher +pelican +king penguin +albatross +grey whale +killer whale +dugong +sea lion +Chihuahua +Japanese spaniel +Maltese dog +Pekinese +Shih-Tzu +Blenheim spaniel +papillon +toy terrier +Rhodesian ridgeback +Afghan hound +basset +beagle +bloodhound +bluetick +black-and-tan coonhound +Walker hound +English foxhound +redbone +borzoi +Irish wolfhound +Italian greyhound +whippet +Ibizan hound +Norwegian elkhound +otterhound +Saluki +Scottish deerhound +Weimaraner +Staffordshire bullterrier +American Staffordshire terrier +Bedlington terrier +Border terrier +Kerry blue terrier +Irish terrier +Norfolk terrier +Norwich terrier +Yorkshire terrier +wire-haired fox terrier +Lakeland terrier +Sealyham terrier +Airedale +cairn +Australian terrier +Dandie Dinmont +Boston bull +miniature schnauzer +giant schnauzer +standard schnauzer +Scotch terrier +Tibetan terrier +silky terrier +soft-coated wheaten terrier +West Highland white terrier +Lhasa +flat-coated retriever +curly-coated retriever +golden retriever +Labrador retriever +Chesapeake Bay retriever +German short-haired pointer +vizsla +English setter +Irish setter +Gordon setter +Brittany spaniel +clumber +English springer +Welsh springer spaniel +cocker spaniel +Sussex spaniel +Irish water spaniel +kuvasz +schipperke +groenendael +malinois +briard +kelpie +komondor +Old English sheepdog +Shetland sheepdog +collie +Border collie +Bouvier des Flandres +Rottweiler +German shepherd +Doberman +miniature pinscher +Greater Swiss Mountain dog +Bernese mountain dog +Appenzeller +EntleBucher +boxer +bull mastiff +Tibetan mastiff +French bulldog +Great Dane +Saint Bernard +Eskimo dog +malamute +Siberian husky +dalmatian +affenpinscher +basenji +pug +Leonberg +Newfoundland +Great Pyrenees +Samoyed +Pomeranian +chow +keeshond +Brabancon griffon +Pembroke +Cardigan +toy poodle +miniature poodle +standard poodle +Mexican hairless +timber wolf +white wolf +red wolf +coyote +dingo +dhole +African hunting dog +hyena +red fox +kit fox +Arctic fox +grey fox +tabby +tiger cat +Persian cat +Siamese cat +Egyptian cat +cougar +lynx +leopard +snow leopard +jaguar +lion +tiger +cheetah +brown bear +American black bear +ice bear +sloth bear +mongoose +meerkat +tiger beetle +ladybug +ground beetle +long-horned beetle +leaf beetle +dung beetle +rhinoceros beetle +weevil +fly +bee +ant +grasshopper +cricket +walking stick +cockroach +mantis +cicada +leafhopper +lacewing +dragonfly +damselfly +admiral +ringlet +monarch +cabbage butterfly +sulphur butterfly +lycaenid +starfish +sea urchin +sea cucumber +wood rabbit +hare +Angora +hamster +porcupine +fox squirrel +marmot +beaver +guinea pig +sorrel +zebra +hog +wild boar +warthog +hippopotamus +ox +water buffalo +bison +ram +bighorn +ibex +hartebeest +impala +gazelle +Arabian camel +llama +weasel +mink +polecat +black-footed ferret +otter +skunk +badger +armadillo +three-toed sloth +orangutan +gorilla +chimpanzee +gibbon +siamang +guenon +patas +baboon +macaque +langur +colobus +proboscis monkey +marmoset +capuchin +howler monkey +titi +spider monkey +squirrel monkey +Madagascar cat +indri +Indian elephant +African elephant +lesser panda +giant panda +barracouta +eel +coho +rock beauty +anemone fish +sturgeon +gar +lionfish +puffer +abacus +abaya +academic gown +accordion +acoustic guitar +aircraft carrier +airliner +airship +altar +ambulance +amphibian +analog clock +apiary +apron +ashcan +assault rifle +backpack +bakery +balance beam +balloon +ballpoint +Band Aid +banjo +bannister +barbell +barber chair +barbershop +barn +barometer +barrel +barrow +baseball +basketball +bassinet +bassoon +bathing cap +bath towel +bathtub +beach wagon +beacon +beaker +bearskin +beer bottle +beer glass +bell cote +bib +bicycle-built-for-two +bikini +binder +binoculars +birdhouse +boathouse +bobsled +bolo tie +bonnet +bookcase +bookshop +bottlecap +bow +bow tie +brass +brassiere +breakwater +breastplate +broom +bucket +buckle +bulletproof vest +bullet train +butcher shop +cab +caldron +candle +cannon +canoe +can opener +cardigan +car mirror +carousel +carpenter's kit +carton +car wheel +cash machine +cassette +cassette player +castle +catamaran +CD player +cello +cellular telephone +chain +chainlink fence +chain mail +chain saw +chest +chiffonier +chime +china cabinet +Christmas stocking +church +cinema +cleaver +cliff dwelling +cloak +clog +cocktail shaker +coffee mug +coffeepot +coil +combination lock +computer keyboard +confectionery +container ship +convertible +corkscrew +cornet +cowboy boot +cowboy hat +cradle +crane +crash helmet +crate +crib +Crock Pot +croquet ball +crutch +cuirass +dam +desk +desktop computer +dial telephone +diaper +digital clock +digital watch +dining table +dishrag +dishwasher +disk brake +dock +dogsled +dome +doormat +drilling platform +drum +drumstick +dumbbell +Dutch oven +electric fan +electric guitar +electric locomotive +entertainment center +envelope +espresso maker +face powder +feather boa +file +fireboat +fire engine +fire screen +flagpole +flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster +freight car +French horn +frying pan +fur coat +garbage truck +gasmask +gas pump +goblet +go-kart +golf ball +golfcart +gondola +gong +gown +grand piano +greenhouse +grille +grocery store +guillotine +hair slide +hair spray +half track +hammer +hamper +hand blower +hand-held computer +handkerchief +hard disc +harmonica +harp +harvester +hatchet +holster +home theater +honeycomb +hook +hoopskirt +horizontal bar +horse cart +hourglass +iPod +iron +jack-o'-lantern +jean +jeep +jersey +jigsaw puzzle +jinrikisha +joystick +kimono +knee pad +knot +lab coat +ladle +lampshade +laptop +lawn mower +lens cap +letter opener +library +lifeboat +lighter +limousine +liner +lipstick +Loafer +lotion +loudspeaker +loupe +lumbermill +magnetic compass +mailbag +mailbox +maillot +maillot +manhole cover +maraca +marimba +mask +matchstick +maypole +maze +measuring cup +medicine chest +megalith +microphone +microwave +military uniform +milk can +minibus +miniskirt +minivan +missile +mitten +mixing bowl +mobile home +Model T +modem +monastery +monitor +moped +mortar +mortarboard +mosque +mosquito net +motor scooter +mountain bike +mountain tent +mouse +mousetrap +moving van +muzzle +nail +neck brace +necklace +nipple +notebook +obelisk +oboe +ocarina +odometer +oil filter +organ +oscilloscope +overskirt +oxcart +oxygen mask +packet +paddle +paddlewheel +padlock +paintbrush +pajama +palace +panpipe +paper towel +parachute +parallel bars +park bench +parking meter +passenger car +patio +pay-phone +pedestal +pencil box +pencil sharpener +perfume +Petri dish +photocopier +pick +pickelhaube +picket fence +pickup +pier +piggy bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate +pitcher +plane +planetarium +plastic bag +plate rack +plow +plunger +Polaroid camera +pole +police van +poncho +pool table +pop bottle +pot +potter's wheel +power drill +prayer rug +printer +prison +projectile +projector +puck +punching bag +purse +quill +quilt +racer +racket +radiator +radio +radio telescope +rain barrel +recreational vehicle +reel +reflex camera +refrigerator +remote control +restaurant +revolver +rifle +rocking chair +rotisserie +rubber eraser +rugby ball +rule +running shoe +safe +safety pin +saltshaker +sandal +sarong +sax +scabbard +scale +school bus +schooner +scoreboard +screen +screw +screwdriver +seat belt +sewing machine +shield +shoe shop +shoji +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +ski mask +sleeping bag +slide rule +sliding door +slot +snorkel +snowmobile +snowplow +soap dispenser +soccer ball +sock +solar dish +sombrero +soup bowl +space bar +space heater +space shuttle +spatula +speedboat +spider web +spindle +sports car +spotlight +stage +steam locomotive +steel arch bridge +steel drum +stethoscope +stole +stone wall +stopwatch +stove +strainer +streetcar +stretcher +studio couch +stupa +submarine +suit +sundial +sunglass +sunglasses +sunscreen +suspension bridge +swab +sweatshirt +swimming trunks +swing +switch +syringe +table lamp +tank +tape player +teapot +teddy +television +tennis ball +thatch +theater curtain +thimble +thresher +throne +tile roof +toaster +tobacco shop +toilet seat +torch +totem pole +tow truck +toyshop +tractor +trailer truck +tray +trench coat +tricycle +trimaran +tripod +triumphal arch +trolleybus +trombone +tub +turnstile +typewriter keyboard +umbrella +unicycle +upright +vacuum +vase +vault +velvet +vending machine +vestment +viaduct +violin +volleyball +waffle iron +wall clock +wallet +wardrobe +warplane +washbasin +washer +water bottle +water jug +water tower +whiskey jug +whistle +wig +window screen +window shade +Windsor tie +wine bottle +wing +wok +wooden spoon +wool +worm fence +wreck +yawl +yurt +web site +comic book +crossword puzzle +street sign +traffic light +book jacket +menu +plate +guacamole +consomme +hot pot +trifle +ice cream +ice lolly +French loaf +bagel +pretzel +cheeseburger +hotdog +mashed potato +head cabbage +broccoli +cauliflower +zucchini +spaghetti squash +acorn squash +butternut squash +cucumber +artichoke +bell pepper +cardoon +mushroom +Granny Smith +strawberry +orange +lemon +fig +pineapple +banana +jackfruit +custard apple +pomegranate +hay +carbonara +chocolate sauce +dough +meat loaf +pizza +potpie +burrito +red wine +espresso +cup +eggnog +alp +bubble +cliff +coral reef +geyser +lakeside +promontory +sandbar +seashore +valley +volcano +ballplayer +groom +scuba diver +rapeseed +daisy +yellow lady's slipper +corn +acorn +hip +buckeye +coral fungus +agaric +gyromitra +stinkhorn +earthstar +hen-of-the-woods +bolete +ear +toilet tissue \ No newline at end of file diff --git a/06_NN/resources/loss_car_example.jpeg b/06_NN/resources/loss_car_example.jpeg new file mode 100644 index 0000000..70a018f Binary files /dev/null and b/06_NN/resources/loss_car_example.jpeg differ diff --git a/06_NN/resources/loss_car_example_2.jpeg b/06_NN/resources/loss_car_example_2.jpeg new file mode 100644 index 0000000..4395b41 Binary files /dev/null and b/06_NN/resources/loss_car_example_2.jpeg differ diff --git a/06_NN/resources/mlp.png b/06_NN/resources/mlp.png new file mode 100644 index 0000000..43b382b Binary files /dev/null and b/06_NN/resources/mlp.png differ diff --git a/06_NN/resources/mlp_matrix_mul.png b/06_NN/resources/mlp_matrix_mul.png new file mode 100644 index 0000000..387b901 Binary files /dev/null and b/06_NN/resources/mlp_matrix_mul.png differ diff --git a/06_NN/resources/pytorch_computation_graph.svg b/06_NN/resources/pytorch_computation_graph.svg new file mode 100644 index 0000000..539f758 --- /dev/null +++ b/06_NN/resources/pytorch_computation_graph.svg @@ -0,0 +1,7 @@ + + + + +
x
x
2
2
a
a
b
b
c
c
3
3
y
y
Viewer does not support full SVG 1.1
\ No newline at end of file diff --git a/06_NN/resources/single_layer_perceptron.jpg b/06_NN/resources/single_layer_perceptron.jpg new file mode 100644 index 0000000..4c54e0a Binary files /dev/null and b/06_NN/resources/single_layer_perceptron.jpg differ diff --git a/06_NN/resources/softmax_example1.png b/06_NN/resources/softmax_example1.png new file mode 100644 index 0000000..aa497c5 Binary files /dev/null and b/06_NN/resources/softmax_example1.png differ diff --git a/06_NN/resources/softmax_example2.png b/06_NN/resources/softmax_example2.png new file mode 100644 index 0000000..a76971f Binary files /dev/null and b/06_NN/resources/softmax_example2.png differ diff --git a/_data/fashion_mnist_data/FashionMNIST/raw/t10k-images-idx3-ubyte b/_data/fashion_mnist_data/FashionMNIST/raw/t10k-images-idx3-ubyte new file mode 100644 index 0000000..37bac79 Binary files /dev/null and b/_data/fashion_mnist_data/FashionMNIST/raw/t10k-images-idx3-ubyte differ diff --git a/_data/fashion_mnist_data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz b/_data/fashion_mnist_data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz new file mode 100644 index 0000000..667844f Binary files /dev/null and b/_data/fashion_mnist_data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz differ diff --git a/_data/fashion_mnist_data/FashionMNIST/raw/t10k-labels-idx1-ubyte b/_data/fashion_mnist_data/FashionMNIST/raw/t10k-labels-idx1-ubyte new file mode 100644 index 0000000..2195a4d Binary files /dev/null and b/_data/fashion_mnist_data/FashionMNIST/raw/t10k-labels-idx1-ubyte differ diff --git a/_data/fashion_mnist_data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz b/_data/fashion_mnist_data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz new file mode 100644 index 0000000..abdddb8 Binary files /dev/null and b/_data/fashion_mnist_data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz differ diff --git a/_data/fashion_mnist_data/FashionMNIST/raw/train-images-idx3-ubyte b/_data/fashion_mnist_data/FashionMNIST/raw/train-images-idx3-ubyte new file mode 100644 index 0000000..ff2f5a9 Binary files /dev/null and b/_data/fashion_mnist_data/FashionMNIST/raw/train-images-idx3-ubyte differ diff --git a/_data/fashion_mnist_data/FashionMNIST/raw/train-images-idx3-ubyte.gz b/_data/fashion_mnist_data/FashionMNIST/raw/train-images-idx3-ubyte.gz new file mode 100644 index 0000000..e6ee0e3 Binary files /dev/null and b/_data/fashion_mnist_data/FashionMNIST/raw/train-images-idx3-ubyte.gz differ diff --git a/_data/fashion_mnist_data/FashionMNIST/raw/train-labels-idx1-ubyte b/_data/fashion_mnist_data/FashionMNIST/raw/train-labels-idx1-ubyte new file mode 100644 index 0000000..30424ca Binary files /dev/null and b/_data/fashion_mnist_data/FashionMNIST/raw/train-labels-idx1-ubyte differ diff --git a/_data/fashion_mnist_data/FashionMNIST/raw/train-labels-idx1-ubyte.gz b/_data/fashion_mnist_data/FashionMNIST/raw/train-labels-idx1-ubyte.gz new file mode 100644 index 0000000..9c4aae2 Binary files /dev/null and b/_data/fashion_mnist_data/FashionMNIST/raw/train-labels-idx1-ubyte.gz differ diff --git a/_data/mnist_data/MNIST/raw/t10k-images-idx3-ubyte b/_data/mnist_data/MNIST/raw/t10k-images-idx3-ubyte new file mode 100644 index 0000000..1170b2c Binary files /dev/null and b/_data/mnist_data/MNIST/raw/t10k-images-idx3-ubyte differ diff --git a/_data/mnist_data/MNIST/raw/t10k-images-idx3-ubyte.gz b/_data/mnist_data/MNIST/raw/t10k-images-idx3-ubyte.gz new file mode 100644 index 0000000..5ace8ea Binary files /dev/null and b/_data/mnist_data/MNIST/raw/t10k-images-idx3-ubyte.gz differ diff --git a/_data/mnist_data/MNIST/raw/t10k-labels-idx1-ubyte b/_data/mnist_data/MNIST/raw/t10k-labels-idx1-ubyte new file mode 100644 index 0000000..d1c3a97 Binary files /dev/null and b/_data/mnist_data/MNIST/raw/t10k-labels-idx1-ubyte differ diff --git a/_data/mnist_data/MNIST/raw/t10k-labels-idx1-ubyte.gz b/_data/mnist_data/MNIST/raw/t10k-labels-idx1-ubyte.gz new file mode 100644 index 0000000..a7e1415 Binary files /dev/null and b/_data/mnist_data/MNIST/raw/t10k-labels-idx1-ubyte.gz differ diff --git a/_data/mnist_data/MNIST/raw/train-images-idx3-ubyte b/_data/mnist_data/MNIST/raw/train-images-idx3-ubyte new file mode 100644 index 0000000..bbce276 Binary files /dev/null and b/_data/mnist_data/MNIST/raw/train-images-idx3-ubyte differ diff --git a/_data/mnist_data/MNIST/raw/train-images-idx3-ubyte.gz b/_data/mnist_data/MNIST/raw/train-images-idx3-ubyte.gz new file mode 100644 index 0000000..b50e4b6 Binary files /dev/null and b/_data/mnist_data/MNIST/raw/train-images-idx3-ubyte.gz differ diff --git a/_data/mnist_data/MNIST/raw/train-labels-idx1-ubyte b/_data/mnist_data/MNIST/raw/train-labels-idx1-ubyte new file mode 100644 index 0000000..d6b4c5d Binary files /dev/null and b/_data/mnist_data/MNIST/raw/train-labels-idx1-ubyte differ diff --git a/_data/mnist_data/MNIST/raw/train-labels-idx1-ubyte.gz b/_data/mnist_data/MNIST/raw/train-labels-idx1-ubyte.gz new file mode 100644 index 0000000..707a576 Binary files /dev/null and b/_data/mnist_data/MNIST/raw/train-labels-idx1-ubyte.gz differ diff --git a/image-inpainting/.gitignore b/image-inpainting/.gitignore new file mode 100644 index 0000000..0584c7c --- /dev/null +++ b/image-inpainting/.gitignore @@ -0,0 +1,3 @@ +data/* +*.zip +*.jpg \ No newline at end of file diff --git a/image-inpainting/src/__pycache__/architecture.cpython-314.pyc b/image-inpainting/src/__pycache__/architecture.cpython-314.pyc new file mode 100644 index 0000000..eabdd34 Binary files /dev/null and b/image-inpainting/src/__pycache__/architecture.cpython-314.pyc differ diff --git a/image-inpainting/src/__pycache__/datasets.cpython-314.pyc b/image-inpainting/src/__pycache__/datasets.cpython-314.pyc new file mode 100644 index 0000000..26497b0 Binary files /dev/null and b/image-inpainting/src/__pycache__/datasets.cpython-314.pyc differ diff --git a/image-inpainting/src/__pycache__/train.cpython-314.pyc b/image-inpainting/src/__pycache__/train.cpython-314.pyc new file mode 100644 index 0000000..4d47be3 Binary files /dev/null and b/image-inpainting/src/__pycache__/train.cpython-314.pyc differ diff --git a/image-inpainting/src/__pycache__/utils.cpython-314.pyc b/image-inpainting/src/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000..ee868f4 Binary files /dev/null and b/image-inpainting/src/__pycache__/utils.cpython-314.pyc differ diff --git a/image-inpainting/src/architecture.py b/image-inpainting/src/architecture.py new file mode 100644 index 0000000..ba1f092 --- /dev/null +++ b/image-inpainting/src/architecture.py @@ -0,0 +1,11 @@ +""" + Author: Your Name + HTL-Grieskirchen 5. Jahrgang, Schuljahr 2025/26 + architecture.py +""" + +import torch + +class MyModel(torch.nn.Module): + # TODO: Implement the model architecture. + pass \ No newline at end of file diff --git a/image-inpainting/src/datasets.py b/image-inpainting/src/datasets.py new file mode 100644 index 0000000..e809603 --- /dev/null +++ b/image-inpainting/src/datasets.py @@ -0,0 +1,43 @@ +""" + Author: Your Name + HTL-Grieskirchen 5. Jahrgang, Schuljahr 2025/26 + datasets.py +""" + +import torch +import numpy as np +import random +import glob +import os +from PIL import Image + +IMAGE_DIMENSION = 100 + + +def create_arrays_from_image(image_array: np.ndarray, offset: tuple, spacing: tuple) -> tuple[np.ndarray, np.ndarray]: + image_array, known_array = None, None + + # TODO: Implement the logic to create input and known arrays based on offset and spacing + + return image_array, known_array + +def resize(img: Image): + pass +def preprocess(input_array: np.ndarray): + pass + +class ImageDataset(torch.utils.data.Dataset): + """ + Dataset class for loading images from a folder + """ + + def __init__(self, datafolder: str): + self.imagefiles = sorted(glob.glob(os.path.join(datafolder,"**","*.jpg"),recursive=True)) + + def __len__(self): + return len(self.imagefiles) + + def __getitem__(self, idx:int): + pass + + # TODO: Implement the __init__, __len__, and __getitem__ methods \ No newline at end of file diff --git a/image-inpainting/src/main.py b/image-inpainting/src/main.py new file mode 100644 index 0000000..1316cae --- /dev/null +++ b/image-inpainting/src/main.py @@ -0,0 +1,49 @@ +""" + Author: Your Name + HTL-Grieskirchen 5. Jahrgang, Schuljahr 2025/26 + main.py +""" + +import os +from utils import create_predictions + + +from train import train + + +if __name__ == '__main__': + config_dict = dict() + + config_dict['seed'] = 42 + config_dict['testset_ratio'] = 0.1 + config_dict['validset_ratio'] = 0.1 + config_dict['results_path'] = os.path.join("results") + config_dict['data_path'] = os.path.join("data", "dataset") + config_dict['device'] = None + config_dict['learningrate'] = 1e-3 + config_dict['weight_decay'] = 1e-5 # default is 0 + config_dict['n_updates'] = 50000 + config_dict['batchsize'] = 32 + config_dict['early_stopping_patience'] = 3 + config_dict['use_wandb'] = False + + config_dict['print_train_stats_at'] = 10 + config_dict['print_stats_at'] = 100 + config_dict['plot_at'] = 100 + config_dict['validate_at'] = 100 + + network_config = { + 'n_in_channels': 4 + } + + config_dict['network_config'] = network_config + + train(**config_dict) + + testset_path = os.path.join("data", "challenge_testset.npz") + state_dict_path = os.path.join(config_dict['results_path'], "best_model.pt") + save_path = os.path.join(config_dict['results_path'], "testset", "my_submission_name.npz") + plot_path = os.path.join(config_dict['results_path'], "testset", "plots") + + # Comment out, if predictions are required + create_predictions(config_dict['network_config'], state_dict_path, testset_path, None, save_path, plot_path, plot_at=20) diff --git a/image-inpainting/src/train.py b/image-inpainting/src/train.py new file mode 100644 index 0000000..d4b88df --- /dev/null +++ b/image-inpainting/src/train.py @@ -0,0 +1,166 @@ +""" + Author: Your Name + HTL-Grieskirchen 5. Jahrgang, Schuljahr 2025/26 + train.py +""" + +import datasets +from architecture import MyModel +from utils import plot, evaluate_model + +import torch +import numpy as np +import os + +from torch.utils.data import DataLoader +from torch.utils.data import Subset + +import wandb + + +def train(seed, testset_ratio, validset_ratio, data_path, results_path, early_stopping_patience, device, learningrate, + weight_decay, n_updates, use_wandb, print_train_stats_at, print_stats_at, plot_at, validate_at, batchsize, + network_config: dict): + np.random.seed(seed=seed) + torch.manual_seed(seed=seed) + + if device is None: + device = torch.device( + "cuda:0" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu") + + if isinstance(device, str): + device = torch.device(device) + + if use_wandb: + wandb.login() + wandb.init(project="image_inpainting", config={ + "learning_rate": learningrate, + "weight_decay": weight_decay, + "n_updates": n_updates, + "batch_size": batchsize, + "validation_ratio": validset_ratio, + "testset_ratio": testset_ratio, + "early_stopping_patience": early_stopping_patience, + }) + + # Prepare a path to plot to + plotpath = os.path.join(results_path, "plots") + os.makedirs(plotpath, exist_ok=True) + + image_dataset = datasets.ImageDataset(datafolder=data_path) + + n_total = len(image_dataset) + n_test = int(n_total * testset_ratio) + n_valid = int(n_total * validset_ratio) + n_train = n_total - n_test - n_valid + indices = np.random.permutation(n_total) + dataset_train = Subset(image_dataset, indices=indices[0:n_train]) + dataset_valid = Subset(image_dataset, indices=indices[n_train:n_train + n_valid]) + dataset_test = Subset(image_dataset, indices=indices[n_train + n_valid:n_total]) + + assert len(image_dataset) == len(dataset_train) + len(dataset_test) + len(dataset_valid) + + del image_dataset + + dataloader_train = DataLoader(dataset=dataset_train, batch_size=batchsize, + num_workers=0, shuffle=True) + dataloader_valid = DataLoader(dataset=dataset_valid, batch_size=1, + num_workers=0, shuffle=False) + dataloader_test = DataLoader(dataset=dataset_test, batch_size=1, + num_workers=0, shuffle=False) + + # initializing the model + network = MyModel(**network_config) + network.to(device) + network.train() + + # defining the loss + mse_loss = torch.nn.MSELoss() + + # defining the optimizer + optimizer = torch.optim.Adam(network.parameters(), lr=learningrate, weight_decay=weight_decay) + + if use_wandb: + wandb.watch(network, mse_loss, log="all", log_freq=10) + + i = 0 + counter = 0 + best_validation_loss = np.inf + loss_list = [] + + saved_model_path = os.path.join(results_path, "best_model.pt") + + print(f"Started training on device {device}") + + while i < n_updates: + + for input, target in dataloader_train: + + input, target = input.to(device), target.to(device) + + if (i + 1) % print_train_stats_at == 0: + print(f'Update Step {i + 1} of {n_updates}: Current loss: {loss_list[-1]}') + + optimizer.zero_grad() + + output = network(input) + + loss = mse_loss(output, target) + + loss.backward() + + optimizer.step() + + loss_list.append(loss.item()) + + # writing the stats to wandb + if use_wandb and (i+1) % print_stats_at == 0: + wandb.log({"training/loss_per_batch": loss.item()}, step=i) + + # plotting + if (i + 1) % plot_at == 0: + print(f"Plotting images, current update {i + 1}") + plot(input.cpu().numpy(), target.detach().cpu().numpy(), output.detach().cpu().numpy(), plotpath, i) + + # evaluating model every validate_at sample + if (i + 1) % validate_at == 0: + print(f"Evaluation of the model:") + val_loss, val_rmse = evaluate_model(network, dataloader_valid, mse_loss, device) + print(f"val_loss: {val_loss}") + print(f"val_RMSE: {val_rmse}") + + if use_wandb: + wandb.log({"validation/loss": val_loss, + "validation/RMSE": val_rmse}, step=i) + # wandb histogram + + # Save best model for early stopping + if val_loss < best_validation_loss: + best_validation_loss = val_loss + torch.save(network.state_dict(), saved_model_path) + print(f"Saved new best model with val_loss: {best_validation_loss}") + counter = 0 + else: + counter += 1 + + if counter >= early_stopping_patience: + print("Stopped training because of early stopping") + i = n_updates + break + + i += 1 + if i >= n_updates: + print("Finished training because maximum number of updates reached") + break + + print("Evaluating the self-defined testset") + network.load_state_dict(torch.load(saved_model_path)) + testset_loss, testset_rmse = evaluate_model(network=network, dataloader=dataloader_test, loss_fn=mse_loss, + device=device) + + print(f'testset_loss of model: {testset_loss}, RMSE = {testset_rmse}') + + if use_wandb: + wandb.summary["testset/loss"] = testset_loss + wandb.summary["testset/RMSE"] = testset_rmse + wandb.finish() diff --git a/image-inpainting/src/utils.py b/image-inpainting/src/utils.py new file mode 100644 index 0000000..27603fd --- /dev/null +++ b/image-inpainting/src/utils.py @@ -0,0 +1,133 @@ +""" + Author: Your Name + HTL-Grieskirchen 5. Jahrgang, Schuljahr 2025/26 + utils.py +""" + +import torch +import numpy as np +import os +from matplotlib import pyplot as plt + +from architecture import MyModel + + +def plot(inputs, targets, predictions, path, update): + """Plotting the inputs, targets and predictions to file `path`""" + + os.makedirs(path, exist_ok=True) + fig, axes = plt.subplots(ncols=3, figsize=(15, 5)) + + for i in range(len(inputs)): + for ax, data, title in zip(axes, [inputs, targets, predictions], ["Input", "Target", "Prediction"]): + ax.clear() + ax.set_title(title) + img = data[i:i + 1:, 0:3, :, :] + img = np.squeeze(img) + img = np.transpose(img, (1, 2, 0)) + img = np.clip(img, 0, 1) + ax.imshow(img) + ax.set_axis_off() + fig.savefig(os.path.join(path, f"{update + 1:07d}_{i + 1:02d}.jpg")) + + plt.close(fig) + + +def testset_plot(input_array, output_array, path, index): + """Plotting the inputs, targets and predictions to file `path` for testset (no targets available)""" + + os.makedirs(path, exist_ok=True) + fig, axes = plt.subplots(ncols=2, figsize=(10, 5)) + + for ax, data, title in zip(axes, [input_array, output_array], ["Input", "Prediction"]): + ax.clear() + ax.set_title(title) + img = data[0:3, :, :] + img = np.squeeze(img) + img = np.transpose(img, (1, 2, 0)) + img = np.clip(img, 0, 1) + ax.imshow(img) + ax.set_axis_off() + fig.savefig(os.path.join(path, f"testset_{index + 1:07d}.jpg")) + + plt.close(fig) + + +def evaluate_model(network: torch.nn.Module, dataloader: torch.utils.data.DataLoader, loss_fn, device: torch.device): + """Returnse MSE and RMSE of the model on the provided dataloader""" + network.eval() + loss = 0.0 + with torch.no_grad(): + for data in dataloader: + input_array, target = data + input_array = input_array.to(device) + target = target.to(device) + + outputs = network(input_array) + + loss += loss_fn(outputs, target).item() + + loss = loss / len(dataloader) + + network.train() + + return loss, 255.0 * np.sqrt(loss) + + +def read_compressed_file(file_path: str): + with np.load(file_path) as data: + input_arrays = data['input_arrays'] + known_arrays = data['known_arrays'] + return input_arrays, known_arrays + + +def create_predictions(model_config, state_dict_path, testset_path, device, save_path, plot_path, plot_at=20): + """ + Here, one might needs to adjust the code based on the used preprocessing + """ + + if device is None: + device = torch.device( + "cuda:0" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu") + + if isinstance(device, str): + device = torch.device(device) + + model = MyModel(**model_config) + model.load_state_dict(torch.load(state_dict_path)) + model.to(device) + model.eval() + + input_arrays, known_arrays = read_compressed_file(testset_path) + + known_arrays = known_arrays.astype(np.float32) + + input_arrays = input_arrays.astype(np.float32) / 255.0 + + input_arrays = np.concatenate((input_arrays, known_arrays), axis=1) + + predictions = list() + + with torch.no_grad(): + for i in range(len(input_arrays)): + print(f"Processing image {i + 1}/{len(input_arrays)}") + input_array = torch.from_numpy(input_arrays[i]).to( + device) + output = model(input_array) + output = output.cpu().numpy() + predictions.append(output) + + if (i + 1) % plot_at == 0: + testset_plot(input_array.cpu().numpy(), output, plot_path, i) + + predictions = np.stack(predictions, axis=0) + + predictions = (np.clip(predictions, 0, 1) * 255.0).astype(np.uint8) + + data = { + "predictions": predictions + } + + np.savez_compressed(save_path, **data) + + print(f"Predictions saved at {save_path}")