Datasets
Models
Diff of
/development/qa-server/compare_models.ipynb
[000000]
..
[507a54]
Switch to side-by-side view
--- a +++ b/development/qa-server/compare_models.ipynb @@ -0,0 +1,1621 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "compare_models.ipynb", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "CcJoBmV-Jh84", + "outputId": "952dcdd3-4d26-44ef-bcdc-23092a8bb973" + }, + "source": [ + "!pip install transformers[sentencepiece] spacy rouge" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: transformers[sentencepiece] in /usr/local/lib/python3.7/dist-packages (4.10.2)\n", + "Requirement already satisfied: spacy in /usr/local/lib/python3.7/dist-packages (2.2.4)\n", + "Requirement already satisfied: rouge in /usr/local/lib/python3.7/dist-packages (1.0.1)\n", + "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.5)\n", + "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.4.1)\n", + "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (4.62.0)\n", + "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.8.2)\n", + "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.5)\n", + "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.1.3)\n", + "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (7.4.0)\n", + "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.19.5)\n", + "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.0)\n", + "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.23.0)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy) (57.4.0)\n", + "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (3.0.5)\n", + "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.0.5)\n", + "Requirement already satisfied: importlib-metadata>=0.20 in /usr/local/lib/python3.7/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy) (4.6.4)\n", + "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20->catalogue<1.1.0,>=0.0.7->spacy) (3.7.4.3)\n", + "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20->catalogue<1.1.0,>=0.0.7->spacy) (3.5.0)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.24.3)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2021.5.30)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n", + "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n", + "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from rouge) (1.15.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (5.4.1)\n", + "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (21.0)\n", + "Requirement already satisfied: huggingface-hub>=0.0.12 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (0.0.16)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (2019.12.20)\n", + "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (0.0.45)\n", + "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (0.10.3)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (3.0.12)\n", + "Requirement already satisfied: protobuf in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (3.17.3)\n", + "Requirement already satisfied: sentencepiece==0.1.91 in /usr/local/lib/python3.7/dist-packages (from transformers[sentencepiece]) (0.1.91)\n", + "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers[sentencepiece]) (2.4.7)\n", + "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers[sentencepiece]) (1.0.1)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers[sentencepiece]) (7.1.2)\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "FA_vVNqQJonA", + "outputId": "25fe6864-6422-4ba7-dc09-633d8b57011f" + }, + "source": [ + "!python -m spacy download en_core_web_lg" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting en_core_web_lg==2.2.5\n", + " Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-2.2.5/en_core_web_lg-2.2.5.tar.gz (827.9 MB)\n", + "\u001b[K |████████████████████████████████| 827.9 MB 1.6 MB/s \n", + "\u001b[?25hRequirement already satisfied: spacy>=2.2.2 in /usr/local/lib/python3.7/dist-packages (from en_core_web_lg==2.2.5) (2.2.4)\n", + "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (2.0.5)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (57.4.0)\n", + "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (1.1.3)\n", + "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (1.0.5)\n", + "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (7.4.0)\n", + "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (3.0.5)\n", + "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (0.4.1)\n", + "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (1.0.0)\n", + "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (2.23.0)\n", + "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (0.8.2)\n", + "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (1.0.5)\n", + "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (1.19.5)\n", + "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (4.62.0)\n", + "Requirement already satisfied: importlib-metadata>=0.20 in /usr/local/lib/python3.7/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_lg==2.2.5) (4.6.4)\n", + "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_lg==2.2.5) (3.7.4.3)\n", + "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_lg==2.2.5) (3.5.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_lg==2.2.5) (2021.5.30)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_lg==2.2.5) (3.0.4)\n", + "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_lg==2.2.5) (2.10)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_lg==2.2.5) (1.24.3)\n", + "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n", + "You can now load the model via spacy.load('en_core_web_lg')\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "7DDkPtP1JqWr" + }, + "source": [ + "import torch\n", + "import re\n", + "import spacy\n", + "import nltk\n", + "import json\n", + "import en_core_web_lg\n", + "import pandas as pd\n", + "from tqdm import tqdm\n", + "from rouge import Rouge \n", + "from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction\n", + "from transformers import AutoModelForMaskedLM, AutoModel, AutoTokenizer, pipeline" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "uHlKEnVyJvgd", + "outputId": "792bdca5-b082-44f9-e26e-a70f8ddf95eb" + }, + "source": [ + "nlp = en_core_web_lg.load()\n", + "nltk.download('punkt')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[nltk_data] Downloading package punkt to /root/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "True" + ] + }, + "metadata": {}, + "execution_count": 97 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "giOXiAl3Jvz7", + "outputId": "eb2fa4d7-bd35-48d9-98c6-f38c5e337e3a" + }, + "source": [ + "%cd drive/MyDrive/" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[Errno 2] No such file or directory: 'drive/MyDrive/'\n", + "/content/drive/My Drive\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "urhqR1nTKw3U" + }, + "source": [ + "f = open ('qa.json', \"r\")\n", + "qa = json.loads(f.read())" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Ssq5_8g5UB4P", + "outputId": "d67e5b4b-efae-4ac6-ba9f-637cb8013fb2" + }, + "source": [ + "qa['data'][0]" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "{'context': \"Use acetaminophen exactly as directed on the label, or as prescribed by your doctor. Do not use in larger or smaller amounts or for longer than recommended.\\nDo not take more of this medication than is recommended. An overdose of acetaminophen can damage your liver or cause death.\\nAdults and teenagers who weigh at least 110 pounds (50 kilograms): Do not take more than 1000 milligrams (mg) at one time. Do not take more than 4000 mg in 24 hours.\\nChildren younger than 12 years old: Do not take more than 5 doses of acetaminophen in 24 hours. Use only the number of milligrams per dose that is recommended for the child's weight and age. Use exactly as directed on the label.\\nAvoid also using other medicines that contain acetaminophen, or you could have a fatal overdose.\\nIf you are treating a child, use a pediatric form of acetaminophen. Use only the special dose-measuring dropper or oral syringe that comes with the specific pediatric form you are using. Carefully follow the dosing directions on the medicine label.\\nMeasure liquid medicinewith the dosing syringe provided, or with a special dose-measuring spoon or medicine cup. If you do not have a dose-measuring device, ask your pharmacist for one.\\nAcetaminophen made for infants is available in two different dose concentrations, and each concentration comes with its own medicine dropper or oral syringe. These dosing devices are not equal between the different concentrations. Using the wrong device may cause you to give your child an overdose of acetaminophen. Never mix and match dosing devices between infant formulations of acetaminophen.\\nYou may need to shake the liquid before each use. Follow the directions on the medicine label.\\nThe chewable tablet must be chewed thoroughly before you swallow it.\\nMake sure your hands are dry when handling the acetaminophen disintegrating tablet. Place the tablet on your tongue. It will begin to dissolve right away. Do not swallow the tablet whole. Allow it to dissolve in your mouth without chewing.\\nTo use the acetaminophen effervescent granules, dissolve one packet of the granules in at least 4 ounces of water. Stir this mixture and drink all of it right away. To make sure you get the entire dose, add a little more water to the same glass, swirl gently and drink right away.\\nThe oral powder should be placed directly on the tongue and swallowed.\\nStop taking acetaminophen and call your doctor if:\\nyou still have a sore throat after 2 days of use;\\nyou still have a fever after 3 days of use;\\nyou still have pain after 7 days of use (or 5 days if treating a child);\\nyou have a skin rash, ongoing headache, nausea, vomiting, or any redness or swelling; or\\nif your symptoms get worse, or if you have any new symptoms.\\nThis medication can cause unusual results with certain lab tests for glucose (sugar) in the urine. Tell any doctor who treats you that you are using acetaminophen.\\nStore at room temperature away from heat and moisture.\\nDetailed Acetaminophen dosage information\\nWhat happens if I miss a dose?\\nSince acetaminophen is taken as needed, you may not be on a dosing schedule. If you are taking the medication regularly, take the missed dose as soon as you remember. Skip the missed dose if it is almost time for your next scheduled dose. Do not take extra medicine to make up the missed dose.\\nWhat happens if I overdose?\\nSeek emergency medical attention or call the Poison Help line at 1-800-222-1222. An overdose of acetaminophen can be fatal.\\nThe first signs of an acetaminophen overdose include loss of appetite, nausea, vomiting, stomach pain, sweating, and confusion or weakness. Later symptoms may include pain in your upper stomach, dark urine, and yellowing of your skin or the whites of your eyes.\",\n", + " 'qas': [{'answers': ['Do not take more than 5 doses of acetaminophen in 24 hours.',\n", + " \"Use only the number of milligrams per dose that is recommended for the child's weight and age.\",\n", + " 'Use exactly as directed on the label',\n", + " 'If you are treating a child, use a pediatric form of acetaminophen.',\n", + " 'Use only the special dose-measuring dropper or oral syringe that comes with the specific pediatric form you are using.',\n", + " 'Carefully follow the dosing directions on the medicine label.'],\n", + " 'question': 'Can you give dosage information acetaminophen for children?'},\n", + " {'answers': ['Do not take more than 1000 milligrams (mg) at one time.',\n", + " 'Do not take more than 4000 mg in 24 hours.'],\n", + " 'question': 'Can you give dosage information acetaminophen for adults?'},\n", + " {'answers': ['Since acetaminophen is taken as needed, you may not be on a dosing schedule.',\n", + " 'If you are taking the medication regularly, take the missed dose as soon as you remember.',\n", + " 'Skip the missed dose if it is almost time for your next scheduled dose.',\n", + " 'Do not take extra medicine to make up the missed dose.'],\n", + " 'question': 'What happens if I miss a dose?'},\n", + " {'answers': ['The first signs of an acetaminophen overdose include loss of appetite, nausea, vomiting, stomach pain, sweating, and confusion or weakness.',\n", + " 'Later symptoms may include pain in your upper stomach, dark urine, and yellowing of your skin or the whites of your eyes.'],\n", + " 'question': 'What is the symptoms of acetaminophen overdose?'}]}" + ] + }, + "metadata": {}, + "execution_count": 139 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "biW6mnlTLMdc" + }, + "source": [ + "n_top = 3\n", + "rouge = Rouge()\n", + "smoothie = SmoothingFunction().method4\n", + "bleu_scores = []\n", + "rouge1_scores = []\n", + "rouge2_scores = []\n", + "rougel_scores = []\n", + "\n", + "\n", + "model_names = {'bert-large-uncased-whole-word-masking-finetuned-squad':'question_answering',\n", + " 'ktrapeznikov/albert-xlarge-v2-squad-v2':'question_answering',\n", + " 'allenai/longformer-large-4096-finetuned-triviaqa':'question_answering',\n", + " 'deepset/roberta-large-squad2':'question_answering',\n", + " 'mrm8488/longformer-base-4096-finetuned-squadv2':'question_answering',\n", + " 'mrm8488/squeezebert-finetuned-squadv2':'question_answering',\n", + " 'bigwiz83/sapbert-from-pubmedbert-squad2':'question_answering',\n", + " 'vicgalle/xlm-roberta-large-xnli-anli':'similarity',\n", + " 'joeddav/xlm-roberta-large-xnli':'similarity',\n", + " 'valhalla/distilbart-mnli-12-3':'similarity',\n", + " 'BaptisteDoyen/camembert-base-xnli':'similarity',\n", + " 'typeform/mobilebert-uncased-mnli':'similarity',\n", + " 'valhalla/distilbart-mnli-12-1':'similarity',\n", + " 'valhalla/distilbart-mnli-12-9':'similarity',\n", + " 'cross-encoder/nli-distilroberta-base':'similarity',\n", + " 'bert-large-uncased':'similarity',\n", + " 'bert-large-cased':'similarity',\n", + " 'bert-base-uncased':'similarity',\n", + " 'bert-base-cased':'similarity',\n", + " 'bert-large-finetuned':'similarity',\n", + " 'bert-base-finetuned':'similarity'}" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "8wi60ZiZLRT0" + }, + "source": [ + "class SentenceSimilarity():\n", + " def __init__(self, model_name=None, device='cuda'):\n", + " self.tokenizer = AutoTokenizer.from_pretrained(model_name)\n", + " self.model = AutoModel.from_pretrained(model_name).eval()\n", + "\n", + " def __call__(self, text):\n", + " tokens = self.tokenizer(text, padding=True, return_tensors='pt', truncation=True)\n", + " with torch.no_grad():\n", + " embeddings = self.model(**tokens).last_hidden_state\n", + "\n", + " mask = tokens['attention_mask'].unsqueeze(-1).expand(embeddings.shape).float()\n", + " sentence_embeddings = torch.sum(embeddings * mask, dim=1) / torch.clamp(mask.sum(1), min=1e-9) \n", + " return sentence_embeddings\n", + "\n", + " def compute_label_embedding(self, labels):\n", + " self.label_embeds = self(labels)\n", + " \n", + " def similarity(self, example):\n", + " return torch.cosine_similarity(self(example), self.label_embeds).tolist()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "VUPsEEr9LUFb" + }, + "source": [ + "def get_answer(sentences, question, model, model_type, n_top=None):\n", + " if model_type == 'similarity':\n", + " scores = model.similarity(question)\n", + " dic = {}\n", + "\n", + " for i, ex in enumerate(sentences):\n", + " dic[ex] = scores[i]\n", + "\n", + " s = pd.Series(dic, name='Similarity')\n", + " s.index.name = 'Sentences'\n", + " df = s.reset_index()\n", + " df.sort_values(by='Similarity', ascending=False, inplace=True)\n", + " answer = \" \".join(df.iloc[:n_top]['Sentences'])\n", + "\n", + " elif model_type == 'question_answering':\n", + " context = \" \".join(sentences)\n", + " answer = model(question=question, context=context)['answer']\n", + "\n", + " return answer" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "NW6RmKZGLcCf" + }, + "source": [ + "def get_score(token_reference, token_candidate, sf):\n", + " bleu_score = sentence_bleu(token_reference, token_candidate, smoothing_function=sf, weights=(1, 0, 0, 0))\n", + " rouge_score = rouge.get_scores(candidate, reference)\n", + " return bleu_score, rouge_score[0]['rouge-1']['f'], rouge_score[0]['rouge-2']['f'], rouge_score[0]['rouge-l']['f']" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Q07h24hZLesx" + }, + "source": [ + "def load_model(model_name, model_type):\n", + " if model_type == 'similarity':\n", + " model = SentenceSimilarity(model_name)\n", + "\n", + " elif model_type == 'question_answering':\n", + " model = pipeline(model=model_name, tokenizer=model_name, task=\"question-answering\")\n", + " \n", + " return model" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 708 + }, + "id": "TYHoCqWyL3YO", + "outputId": "57eea346-f28f-4dfd-ce5a-8d7f1cd5d76d" + }, + "source": [ + "df = pd.DataFrame(columns=['Type', 'BLEU', 'ROUGE_1', 'ROUGE_2', 'ROUGE_l'], index=list(model_names.keys()))\n", + "df" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>Type</th>\n", + " <th>BLEU</th>\n", + " <th>ROUGE_1</th>\n", + " <th>ROUGE_2</th>\n", + " <th>ROUGE_l</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>bert-large-uncased-whole-word-masking-finetuned-squad</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>ktrapeznikov/albert-xlarge-v2-squad-v2</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>allenai/longformer-large-4096-finetuned-triviaqa</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>deepset/roberta-large-squad2</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>mrm8488/longformer-base-4096-finetuned-squadv2</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>mrm8488/squeezebert-finetuned-squadv2</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bigwiz83/sapbert-from-pubmedbert-squad2</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>vicgalle/xlm-roberta-large-xnli-anli</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>joeddav/xlm-roberta-large-xnli</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>valhalla/distilbart-mnli-12-3</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>BaptisteDoyen/camembert-base-xnli</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>typeform/mobilebert-uncased-mnli</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>valhalla/distilbart-mnli-12-1</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>valhalla/distilbart-mnli-12-9</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>cross-encoder/nli-distilroberta-base</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bert-large-uncased</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bert-large-cased</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bert-base-uncased</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bert-base-cased</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bert-large-finetuned</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bert-base-finetuned</th>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " <td>NaN</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " Type BLEU ... ROUGE_2 ROUGE_l\n", + "bert-large-uncased-whole-word-masking-finetuned... NaN NaN ... NaN NaN\n", + "ktrapeznikov/albert-xlarge-v2-squad-v2 NaN NaN ... NaN NaN\n", + "allenai/longformer-large-4096-finetuned-triviaqa NaN NaN ... NaN NaN\n", + "deepset/roberta-large-squad2 NaN NaN ... NaN NaN\n", + "mrm8488/longformer-base-4096-finetuned-squadv2 NaN NaN ... NaN NaN\n", + "mrm8488/squeezebert-finetuned-squadv2 NaN NaN ... NaN NaN\n", + "bigwiz83/sapbert-from-pubmedbert-squad2 NaN NaN ... NaN NaN\n", + "vicgalle/xlm-roberta-large-xnli-anli NaN NaN ... NaN NaN\n", + "joeddav/xlm-roberta-large-xnli NaN NaN ... NaN NaN\n", + "valhalla/distilbart-mnli-12-3 NaN NaN ... NaN NaN\n", + "BaptisteDoyen/camembert-base-xnli NaN NaN ... NaN NaN\n", + "typeform/mobilebert-uncased-mnli NaN NaN ... NaN NaN\n", + "valhalla/distilbart-mnli-12-1 NaN NaN ... NaN NaN\n", + "valhalla/distilbart-mnli-12-9 NaN NaN ... NaN NaN\n", + "cross-encoder/nli-distilroberta-base NaN NaN ... NaN NaN\n", + "bert-large-uncased NaN NaN ... NaN NaN\n", + "bert-large-cased NaN NaN ... NaN NaN\n", + "bert-base-uncased NaN NaN ... NaN NaN\n", + "bert-base-cased NaN NaN ... NaN NaN\n", + "bert-large-finetuned NaN NaN ... NaN NaN\n", + "bert-base-finetuned NaN NaN ... NaN NaN\n", + "\n", + "[21 rows x 5 columns]" + ] + }, + "metadata": {}, + "execution_count": 105 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4mHGKWNaLhJn", + "outputId": "250361e5-eddb-4cfc-859d-bb8ac06319c9" + }, + "source": [ + "for model_name in tqdm(model_names):\n", + " model_type = model_names[model_name]\n", + " df['Type'][model_name] = model_type\n", + "\n", + " model = load_model(model_name, model_type)\n", + "\n", + " for data in qa['data']:\n", + " context = data['context']\n", + " doc = nlp(context)\n", + " sentences = [str(sentence) for sentence in list(doc.sents)]\n", + "\n", + " if model_type == 'similarity':\n", + " model.compute_label_embedding(sentences)\n", + " \n", + " for q_a in data['qas']:\n", + " question = q_a['question']\n", + " reference = \"\".join(q_a['answers'])\n", + "\n", + " candidate = get_answer(sentences, question, model, model_type, n_top)\n", + " token_reference = nltk.word_tokenize(reference)\n", + " token_candidate = nltk.word_tokenize(candidate)\n", + "\n", + " bleu_score, rouge1_score, rouge2_score, rougel_score = get_score(\n", + " token_reference, token_candidate, smoothie)\n", + "\n", + " bleu_scores.append(bleu_score)\n", + " rouge1_scores.append(rouge1_score)\n", + " rouge2_scores.append(rouge2_score)\n", + " rougel_scores.append(rougel_score)\n", + "\n", + " df['BLEU'][model_name] = sum(bleu_scores)/len(bleu_scores)\n", + " df['ROUGE_1'][model_name] = sum(rouge1_scores)/len(rouge1_scores)\n", + " df['ROUGE_2'][model_name] = sum(rouge2_scores)/len(rouge2_scores)\n", + " df['ROUGE_l'][model_name] = sum(rougel_scores)/len(rougel_scores)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + " 33%|███▎ | 7/21 [50:52<1:02:07, 266.25s/it]Some weights of the model checkpoint at vicgalle/xlm-roberta-large-xnli-anli were not used when initializing XLMRobertaModel: ['classifier.out_proj.weight', 'classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.dense.weight']\n", + "- This IS expected if you are initializing XLMRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing XLMRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of XLMRobertaModel were not initialized from the model checkpoint at vicgalle/xlm-roberta-large-xnli-anli and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", + " 38%|███▊ | 8/21 [54:26<54:05, 249.67s/it] Some weights of the model checkpoint at joeddav/xlm-roberta-large-xnli were not used when initializing XLMRobertaModel: ['classifier.out_proj.weight', 'classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.dense.weight']\n", + "- This IS expected if you are initializing XLMRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing XLMRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + " 43%|████▎ | 9/21 [57:59<47:38, 238.19s/it]Some weights of the model checkpoint at valhalla/distilbart-mnli-12-3 were not used when initializing BartModel: ['classification_head.out_proj.weight', 'classification_head.dense.bias', 'classification_head.dense.weight', 'classification_head.out_proj.bias']\n", + "- This IS expected if you are initializing BartModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BartModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + " 48%|████▊ | 10/21 [1:00:11<37:37, 205.20s/it]Some weights of the model checkpoint at BaptisteDoyen/camembert-base-xnli were not used when initializing CamembertModel: ['classifier.out_proj.weight', 'classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.dense.weight']\n", + "- This IS expected if you are initializing CamembertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing CamembertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of CamembertModel were not initialized from the model checkpoint at BaptisteDoyen/camembert-base-xnli and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", + " 52%|█████▏ | 11/21 [1:01:28<27:39, 165.99s/it]Some weights of the model checkpoint at typeform/mobilebert-uncased-mnli were not used when initializing MobileBertModel: ['classifier.bias', 'classifier.weight']\n", + "- This IS expected if you are initializing MobileBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing MobileBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n", + " 57%|█████▋ | 12/21 [1:01:57<18:38, 124.33s/it]Some weights of the model checkpoint at valhalla/distilbart-mnli-12-1 were not used when initializing BartModel: ['classification_head.out_proj.weight', 'classification_head.dense.bias', 'classification_head.dense.weight', 'classification_head.out_proj.bias']\n", + "- This IS expected if you are initializing BartModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BartModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + " 62%|██████▏ | 13/21 [1:03:42<15:49, 118.73s/it]Some weights of the model checkpoint at valhalla/distilbart-mnli-12-9 were not used when initializing BartModel: ['classification_head.out_proj.weight', 'classification_head.dense.bias', 'classification_head.dense.weight', 'classification_head.out_proj.bias']\n", + "- This IS expected if you are initializing BartModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BartModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + " 67%|██████▋ | 14/21 [1:06:40<15:55, 136.56s/it]Some weights of the model checkpoint at cross-encoder/nli-distilroberta-base were not used when initializing RobertaModel: ['classifier.out_proj.weight', 'classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.dense.weight']\n", + "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of RobertaModel were not initialized from the model checkpoint at cross-encoder/nli-distilroberta-base and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", + " 71%|███████▏ | 15/21 [1:07:18<10:41, 106.91s/it]Some weights of the model checkpoint at bert-large-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias']\n", + "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + " 76%|███████▌ | 16/21 [1:10:23<10:50, 130.16s/it]Some weights of the model checkpoint at bert-large-cased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias']\n", + "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + " 81%|████████ | 17/21 [1:13:30<09:49, 147.36s/it]Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias']\n", + "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + " 86%|████████▌ | 18/21 [1:14:30<06:03, 121.20s/it]Some weights of the model checkpoint at bert-base-cased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias']\n", + "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + " 90%|█████████ | 19/21 [1:15:32<03:26, 103.31s/it]Some weights of the model checkpoint at bert-large-finetuned were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias']\n", + "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of BertModel were not initialized from the model checkpoint at bert-large-finetuned and are newly initialized: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", + " 95%|█████████▌| 20/21 [1:18:23<02:03, 123.62s/it]Some weights of the model checkpoint at bert-base-finetuned were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias']\n", + "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of BertModel were not initialized from the model checkpoint at bert-base-finetuned and are newly initialized: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", + "100%|██████████| 21/21 [1:19:14<00:00, 226.43s/it]\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WX5ytzRqD10j" + }, + "source": [ + "```\n", + " Model name: bert-large-uncased-whole-word-masking-finetuned-squad \n", + " Bleu Scores: 0.04904401154401154 \n", + " Rouge_1 Scores average 0.31644641517394717 \n", + " Rouge_2 Scores average 0.2724766888228745 \n", + " Rouge_l Scores average 0.31644641517394717 \n", + "```\n", + "--- \n", + "```\n", + " Model name: ktrapeznikov/albert-xlarge-v2-squad-v2 \n", + " Bleu Scores: 0.06971486420716247 \n", + " Rouge_1 Scores average 0.29757417507940026 \n", + " Rouge_2 Scores average 0.2536624654219619 \n", + " Rouge_l Scores average 0.29757417507940026 \n", + "```\n", + "--- \n", + "```\n", + " Model name: allenai/longformer-large-4096-finetuned-triviaqa \n", + " Bleu Scores: 0.046476576138108315 \n", + " Rouge_1 Scores average 0.22187419078216236 \n", + " Rouge_2 Scores average 0.17127281242571119 \n", + " Rouge_l Scores average 0.22187419078216236 \n", + "```\n", + "---\n", + "``` \n", + " Model name: deepset/roberta-large-squad2 \n", + " Bleu Scores: 0.04129682604297518 \n", + " Rouge_1 Scores average 0.21955236236558998 \n", + " Rouge_2 Scores average 0.16794483315016406 \n", + " Rouge_l Scores average 0.21887393631402965 \n", + "```\n", + "---\n", + "``` \n", + " Model name: mrm8488/longformer-base-4096-finetuned-squadv2 \n", + " Bleu Scores: 0.04648984178676111 \n", + " Rouge_1 Scores average 0.2385953434228197 \n", + " Rouge_2 Scores average 0.18657712347078392 \n", + " Rouge_l Scores average 0.23769609634271227 \n", + "```\n", + "---\n", + "```\n", + " Model name: mrm8488/squeezebert-finetuned-squadv2 \n", + " Bleu Scores: 0.04722471830548441 \n", + " Rouge_1 Scores average 0.22400095940279424 \n", + " Rouge_2 Scores average 0.17202397066190891 \n", + " Rouge_l Scores average 0.22301484441179562 \n", + "```\n", + "---\n", + "``` \n", + " Model name: bigwiz83/sapbert-from-pubmedbert-squad2 \n", + " Bleu Scores: 0.049341688839488355 \n", + " Rouge_1 Scores average 0.24944919188753056 \n", + " Rouge_2 Scores average 0.19754100964430785 \n", + " Rouge_l Scores average 0.2486039504666746 \n", + "```\n", + "---\n", + "``` \n", + " Model name: vicgalle/xlm-roberta-large-xnli-anli \n", + " Bleu Scores: 0.04740734658870999 \n", + " Rouge_1 Scores average 0.22964171066573294 \n", + " Rouge_2 Scores average 0.17672108237430337 \n", + " Rouge_l Scores average 0.22753524253174262 \n", + "```\n", + "---\n", + "``` \n", + " Model name: joeddav/xlm-roberta-large-xnli \n", + " Bleu Scores: 0.047315274217411704 \n", + " Rouge_1 Scores average 0.21945454186019484 \n", + " Rouge_2 Scores average 0.1635738573008003 \n", + " Rouge_l Scores average 0.21642795239288626 \n", + "```\n", + "---\n", + "``` \n", + " Model name: valhalla/distilbart-mnli-12-3 \n", + " Bleu Scores: 0.04763133893108889 \n", + " Rouge_1 Scores average 0.2087123380971496 \n", + " Rouge_2 Scores average 0.1516332558133015 \n", + " Rouge_l Scores average 0.20482137932196318 \n", + "```\n", + "---\n", + "``` \n", + " Model name: BaptisteDoyen/camembert-base-xnli \n", + " Bleu Scores: 0.04592487652938288 \n", + " Rouge_1 Scores average 0.19921692307268624 \n", + " Rouge_2 Scores average 0.14318905630596554 \n", + " Rouge_l Scores average 0.19567968782251677 \n", + "```\n", + "---\n", + "``` \n", + " Model name: typeform/mobilebert-uncased-mnli \n", + " Bleu Scores: 0.046274723756052905 \n", + " Rouge_1 Scores average 0.1911579999340068 \n", + " Rouge_2 Scores average 0.13433764523023312 \n", + " Rouge_l Scores average 0.18707070849850982 \n", + "```\n", + "---\n", + "``` \n", + " Model name: valhalla/distilbart-mnli-12-1 \n", + " Bleu Scores: 0.046644305973648485 \n", + " Rouge_1 Scores average 0.18605349749283961 \n", + " Rouge_2 Scores average 0.12885250404352783 \n", + " Rouge_l Scores average 0.1814022220313078 \n", + "```\n", + "---\n", + "```\n", + " Model name: valhalla/distilbart-mnli-12-9 \n", + " Bleu Scores: 0.04613591478837148 \n", + " Rouge_1 Scores average 0.17881783397728088 \n", + " Rouge_2 Scores average 0.12270547507575051 \n", + " Rouge_l Scores average 0.17377353349872895 \n", + "```\n", + "---\n", + "``` \n", + " Model name: cross-encoder/nli-distilroberta-base \n", + " Bleu Scores: 0.04638736081156602 \n", + " Rouge_1 Scores average 0.17832898668655725 \n", + " Rouge_2 Scores average 0.12137505515814476 \n", + " Rouge_l Scores average 0.1729153552918669 \n", + "```\n", + "---\n", + "``` \n", + " Model name: bert-large-uncased \n", + " Bleu Scores: 0.04670057489523694 \n", + " Rouge_1 Scores average 0.17682194476510882 \n", + " Rouge_2 Scores average 0.11873541679211357 \n", + " Rouge_l Scores average 0.1711206526046795 \n", + "```\n", + "---\n", + "``` \n", + " Model name: bert-large-cased \n", + " Bleu Scores: 0.04653593699430314 \n", + " Rouge_1 Scores average 0.1730175007118266 \n", + " Rouge_2 Scores average 0.11344737213744338 \n", + " Rouge_l Scores average 0.1667599034710913 \n", + "```\n", + "---\n", + "``` \n", + " Model name: bert-base-uncased \n", + " Bleu Scores: 0.0466282799842496 \n", + " Rouge_1 Scores average 0.17135281702814545 \n", + " Rouge_2 Scores average 0.11131447883448312 \n", + " Rouge_l Scores average 0.16487847216815818 \n", + "```\n", + "---\n", + "``` \n", + " Model name: bert-base-cased \n", + " Bleu Scores: 0.046950278910238195 \n", + " Rouge_1 Scores average 0.1697269807317403 \n", + " Rouge_2 Scores average 0.10887071360951835 \n", + " Rouge_l Scores average 0.1631931474452232 \n", + "```\n", + "---\n", + "```\n", + " Model name: model/bert_med/checkpoint/checkpoint-final \n", + " Bleu Scores: 0.04708176338444994 \n", + " Rouge_1 Scores average 0.16765655912804828 \n", + " Rouge_2 Scores average 0.10720455061615725 \n", + " Rouge_l Scores average 0.16110637360774427 \n", + "```\n", + "---\n", + "```\n", + " Model name: checkpoint-10000 \n", + " Bleu Scores: 0.046965961369023486 \n", + " Rouge_1 Scores average 0.16574710188693653 \n", + " Rouge_2 Scores average 0.105084539300102 \n", + " Rouge_l Scores average 0.1592690830965846 \n", + " ```" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 708 + }, + "id": "Pf2xpu1oLxiH", + "outputId": "c68191a7-1282-4c61-9715-574eaa6066ea" + }, + "source": [ + "df" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>Type</th>\n", + " <th>BLEU</th>\n", + " <th>ROUGE_1</th>\n", + " <th>ROUGE_2</th>\n", + " <th>ROUGE_l</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>bert-large-uncased-whole-word-masking-finetuned-squad</th>\n", + " <td>question_answering</td>\n", + " <td>0.049044</td>\n", + " <td>0.316446</td>\n", + " <td>0.272477</td>\n", + " <td>0.316446</td>\n", + " </tr>\n", + " <tr>\n", + " <th>ktrapeznikov/albert-xlarge-v2-squad-v2</th>\n", + " <td>question_answering</td>\n", + " <td>0.0697149</td>\n", + " <td>0.297574</td>\n", + " <td>0.253662</td>\n", + " <td>0.297574</td>\n", + " </tr>\n", + " <tr>\n", + " <th>allenai/longformer-large-4096-finetuned-triviaqa</th>\n", + " <td>question_answering</td>\n", + " <td>0.0464766</td>\n", + " <td>0.221874</td>\n", + " <td>0.171273</td>\n", + " <td>0.221874</td>\n", + " </tr>\n", + " <tr>\n", + " <th>deepset/roberta-large-squad2</th>\n", + " <td>question_answering</td>\n", + " <td>0.0412968</td>\n", + " <td>0.219552</td>\n", + " <td>0.167945</td>\n", + " <td>0.218874</td>\n", + " </tr>\n", + " <tr>\n", + " <th>mrm8488/longformer-base-4096-finetuned-squadv2</th>\n", + " <td>question_answering</td>\n", + " <td>0.0464898</td>\n", + " <td>0.238595</td>\n", + " <td>0.186577</td>\n", + " <td>0.237696</td>\n", + " </tr>\n", + " <tr>\n", + " <th>mrm8488/squeezebert-finetuned-squadv2</th>\n", + " <td>question_answering</td>\n", + " <td>0.0472247</td>\n", + " <td>0.224001</td>\n", + " <td>0.172024</td>\n", + " <td>0.223015</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bigwiz83/sapbert-from-pubmedbert-squad2</th>\n", + " <td>question_answering</td>\n", + " <td>0.0493417</td>\n", + " <td>0.249449</td>\n", + " <td>0.197541</td>\n", + " <td>0.248604</td>\n", + " </tr>\n", + " <tr>\n", + " <th>vicgalle/xlm-roberta-large-xnli-anli</th>\n", + " <td>similarity</td>\n", + " <td>0.0474073</td>\n", + " <td>0.229642</td>\n", + " <td>0.176721</td>\n", + " <td>0.227535</td>\n", + " </tr>\n", + " <tr>\n", + " <th>joeddav/xlm-roberta-large-xnli</th>\n", + " <td>similarity</td>\n", + " <td>0.0473153</td>\n", + " <td>0.219455</td>\n", + " <td>0.163574</td>\n", + " <td>0.216428</td>\n", + " </tr>\n", + " <tr>\n", + " <th>valhalla/distilbart-mnli-12-3</th>\n", + " <td>similarity</td>\n", + " <td>0.0476313</td>\n", + " <td>0.208712</td>\n", + " <td>0.151633</td>\n", + " <td>0.204821</td>\n", + " </tr>\n", + " <tr>\n", + " <th>BaptisteDoyen/camembert-base-xnli</th>\n", + " <td>similarity</td>\n", + " <td>0.0459249</td>\n", + " <td>0.199217</td>\n", + " <td>0.143189</td>\n", + " <td>0.19568</td>\n", + " </tr>\n", + " <tr>\n", + " <th>typeform/mobilebert-uncased-mnli</th>\n", + " <td>similarity</td>\n", + " <td>0.0462747</td>\n", + " <td>0.191158</td>\n", + " <td>0.134338</td>\n", + " <td>0.187071</td>\n", + " </tr>\n", + " <tr>\n", + " <th>valhalla/distilbart-mnli-12-1</th>\n", + " <td>similarity</td>\n", + " <td>0.0466443</td>\n", + " <td>0.186053</td>\n", + " <td>0.128853</td>\n", + " <td>0.181402</td>\n", + " </tr>\n", + " <tr>\n", + " <th>valhalla/distilbart-mnli-12-9</th>\n", + " <td>similarity</td>\n", + " <td>0.0461359</td>\n", + " <td>0.178818</td>\n", + " <td>0.122705</td>\n", + " <td>0.173774</td>\n", + " </tr>\n", + " <tr>\n", + " <th>cross-encoder/nli-distilroberta-base</th>\n", + " <td>similarity</td>\n", + " <td>0.0463874</td>\n", + " <td>0.178329</td>\n", + " <td>0.121375</td>\n", + " <td>0.172915</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bert-large-uncased</th>\n", + " <td>similarity</td>\n", + " <td>0.0467006</td>\n", + " <td>0.176822</td>\n", + " <td>0.118735</td>\n", + " <td>0.171121</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bert-large-cased</th>\n", + " <td>similarity</td>\n", + " <td>0.0465359</td>\n", + " <td>0.173018</td>\n", + " <td>0.113447</td>\n", + " <td>0.16676</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bert-base-uncased</th>\n", + " <td>similarity</td>\n", + " <td>0.0466283</td>\n", + " <td>0.171353</td>\n", + " <td>0.111314</td>\n", + " <td>0.164878</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bert-base-cased</th>\n", + " <td>similarity</td>\n", + " <td>0.0469503</td>\n", + " <td>0.169727</td>\n", + " <td>0.108871</td>\n", + " <td>0.163193</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bert-large-finetuned</th>\n", + " <td>similarity</td>\n", + " <td>0.0470818</td>\n", + " <td>0.167657</td>\n", + " <td>0.107205</td>\n", + " <td>0.161106</td>\n", + " </tr>\n", + " <tr>\n", + " <th>bert-base-finetuned</th>\n", + " <td>similarity</td>\n", + " <td>0.046966</td>\n", + " <td>0.165747</td>\n", + " <td>0.105085</td>\n", + " <td>0.159269</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " Type ... ROUGE_l\n", + "bert-large-uncased-whole-word-masking-finetuned... question_answering ... 0.316446\n", + "ktrapeznikov/albert-xlarge-v2-squad-v2 question_answering ... 0.297574\n", + "allenai/longformer-large-4096-finetuned-triviaqa question_answering ... 0.221874\n", + "deepset/roberta-large-squad2 question_answering ... 0.218874\n", + "mrm8488/longformer-base-4096-finetuned-squadv2 question_answering ... 0.237696\n", + "mrm8488/squeezebert-finetuned-squadv2 question_answering ... 0.223015\n", + "bigwiz83/sapbert-from-pubmedbert-squad2 question_answering ... 0.248604\n", + "vicgalle/xlm-roberta-large-xnli-anli similarity ... 0.227535\n", + "joeddav/xlm-roberta-large-xnli similarity ... 0.216428\n", + "valhalla/distilbart-mnli-12-3 similarity ... 0.204821\n", + "BaptisteDoyen/camembert-base-xnli similarity ... 0.19568\n", + "typeform/mobilebert-uncased-mnli similarity ... 0.187071\n", + "valhalla/distilbart-mnli-12-1 similarity ... 0.181402\n", + "valhalla/distilbart-mnli-12-9 similarity ... 0.173774\n", + "cross-encoder/nli-distilroberta-base similarity ... 0.172915\n", + "bert-large-uncased similarity ... 0.171121\n", + "bert-large-cased similarity ... 0.16676\n", + "bert-base-uncased similarity ... 0.164878\n", + "bert-base-cased similarity ... 0.163193\n", + "bert-large-finetuned similarity ... 0.161106\n", + "bert-base-finetuned similarity ... 0.159269\n", + "\n", + "[21 rows x 5 columns]" + ] + }, + "metadata": {}, + "execution_count": 107 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "xKU5BhiGQErn" + }, + "source": [ + "df.to_csv('qa_static.csv')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "HGhoAqCPura2" + }, + "source": [ + "df = pd.read_csv('qa_static.csv')" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "lLyCMD3evZxT", + "outputId": "5f461772-8b68-4cb2-f869-94016b329bd6" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "0 bert-large-uncased-whole-word-masking-finetune...\n", + "1 ktrapeznikov/albert-xlarge-v2-squad-v2\n", + "2 allenai/longformer-large-4096-finetuned-triviaqa\n", + "3 deepset/roberta-large-squad2\n", + "4 mrm8488/longformer-base-4096-finetuned-squadv2\n", + "5 mrm8488/squeezebert-finetuned-squadv2\n", + "6 bigwiz83/sapbert-from-pubmedbert-squad2\n", + "Name: Unnamed: 0, dtype: object" + ] + }, + "metadata": {}, + "execution_count": 128 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 489 + }, + "id": "sM0g85xJNyPa", + "outputId": "2cc70e4b-95e5-4bda-dfd4-61b1ac1ef491" + }, + "source": [ + "df.iloc[7:]" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>Unnamed: 0</th>\n", + " <th>Type</th>\n", + " <th>BLEU</th>\n", + " <th>ROUGE_1</th>\n", + " <th>ROUGE_2</th>\n", + " <th>ROUGE_l</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>7</th>\n", + " <td>vicgalle/xlm-roberta-large-xnli-anli</td>\n", + " <td>similarity</td>\n", + " <td>0.047407</td>\n", + " <td>0.229642</td>\n", + " <td>0.176721</td>\n", + " <td>0.227535</td>\n", + " </tr>\n", + " <tr>\n", + " <th>8</th>\n", + " <td>joeddav/xlm-roberta-large-xnli</td>\n", + " <td>similarity</td>\n", + " <td>0.047315</td>\n", + " <td>0.219455</td>\n", + " <td>0.163574</td>\n", + " <td>0.216428</td>\n", + " </tr>\n", + " <tr>\n", + " <th>9</th>\n", + " <td>valhalla/distilbart-mnli-12-3</td>\n", + " <td>similarity</td>\n", + " <td>0.047631</td>\n", + " <td>0.208712</td>\n", + " <td>0.151633</td>\n", + " <td>0.204821</td>\n", + " </tr>\n", + " <tr>\n", + " <th>10</th>\n", + " <td>BaptisteDoyen/camembert-base-xnli</td>\n", + " <td>similarity</td>\n", + " <td>0.045925</td>\n", + " <td>0.199217</td>\n", + " <td>0.143189</td>\n", + " <td>0.195680</td>\n", + " </tr>\n", + " <tr>\n", + " <th>11</th>\n", + " <td>typeform/mobilebert-uncased-mnli</td>\n", + " <td>similarity</td>\n", + " <td>0.046275</td>\n", + " <td>0.191158</td>\n", + " <td>0.134338</td>\n", + " <td>0.187071</td>\n", + " </tr>\n", + " <tr>\n", + " <th>12</th>\n", + " <td>valhalla/distilbart-mnli-12-1</td>\n", + " <td>similarity</td>\n", + " <td>0.046644</td>\n", + " <td>0.186053</td>\n", + " <td>0.128853</td>\n", + " <td>0.181402</td>\n", + " </tr>\n", + " <tr>\n", + " <th>13</th>\n", + " <td>valhalla/distilbart-mnli-12-9</td>\n", + " <td>similarity</td>\n", + " <td>0.046136</td>\n", + " <td>0.178818</td>\n", + " <td>0.122705</td>\n", + " <td>0.173774</td>\n", + " </tr>\n", + " <tr>\n", + " <th>14</th>\n", + " <td>cross-encoder/nli-distilroberta-base</td>\n", + " <td>similarity</td>\n", + " <td>0.046387</td>\n", + " <td>0.178329</td>\n", + " <td>0.121375</td>\n", + " <td>0.172915</td>\n", + " </tr>\n", + " <tr>\n", + " <th>15</th>\n", + " <td>bert-large-uncased</td>\n", + " <td>similarity</td>\n", + " <td>0.046701</td>\n", + " <td>0.176822</td>\n", + " <td>0.118735</td>\n", + " <td>0.171121</td>\n", + " </tr>\n", + " <tr>\n", + " <th>16</th>\n", + " <td>bert-large-cased</td>\n", + " <td>similarity</td>\n", + " <td>0.046536</td>\n", + " <td>0.173018</td>\n", + " <td>0.113447</td>\n", + " <td>0.166760</td>\n", + " </tr>\n", + " <tr>\n", + " <th>17</th>\n", + " <td>bert-base-uncased</td>\n", + " <td>similarity</td>\n", + " <td>0.046628</td>\n", + " <td>0.171353</td>\n", + " <td>0.111314</td>\n", + " <td>0.164878</td>\n", + " </tr>\n", + " <tr>\n", + " <th>18</th>\n", + " <td>bert-base-cased</td>\n", + " <td>similarity</td>\n", + " <td>0.046950</td>\n", + " <td>0.169727</td>\n", + " <td>0.108871</td>\n", + " <td>0.163193</td>\n", + " </tr>\n", + " <tr>\n", + " <th>19</th>\n", + " <td>bert-large-finetuned</td>\n", + " <td>similarity</td>\n", + " <td>0.047082</td>\n", + " <td>0.167657</td>\n", + " <td>0.107205</td>\n", + " <td>0.161106</td>\n", + " </tr>\n", + " <tr>\n", + " <th>20</th>\n", + " <td>bert-base-finetuned</td>\n", + " <td>similarity</td>\n", + " <td>0.046966</td>\n", + " <td>0.165747</td>\n", + " <td>0.105085</td>\n", + " <td>0.159269</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " Unnamed: 0 Type ... ROUGE_2 ROUGE_l\n", + "7 vicgalle/xlm-roberta-large-xnli-anli similarity ... 0.176721 0.227535\n", + "8 joeddav/xlm-roberta-large-xnli similarity ... 0.163574 0.216428\n", + "9 valhalla/distilbart-mnli-12-3 similarity ... 0.151633 0.204821\n", + "10 BaptisteDoyen/camembert-base-xnli similarity ... 0.143189 0.195680\n", + "11 typeform/mobilebert-uncased-mnli similarity ... 0.134338 0.187071\n", + "12 valhalla/distilbart-mnli-12-1 similarity ... 0.128853 0.181402\n", + "13 valhalla/distilbart-mnli-12-9 similarity ... 0.122705 0.173774\n", + "14 cross-encoder/nli-distilroberta-base similarity ... 0.121375 0.172915\n", + "15 bert-large-uncased similarity ... 0.118735 0.171121\n", + "16 bert-large-cased similarity ... 0.113447 0.166760\n", + "17 bert-base-uncased similarity ... 0.111314 0.164878\n", + "18 bert-base-cased similarity ... 0.108871 0.163193\n", + "19 bert-large-finetuned similarity ... 0.107205 0.161106\n", + "20 bert-base-finetuned similarity ... 0.105085 0.159269\n", + "\n", + "[14 rows x 6 columns]" + ] + }, + "metadata": {}, + "execution_count": 115 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "6P-PK_r8OQQl", + "outputId": "865b5255-c7ba-4571-f7b9-52c74bc24cfd" + }, + "source": [ + "list(df.iloc[7:]['Unnamed: 0'])" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "['vicgalle/xlm-roberta-large-xnli-anli',\n", + " 'joeddav/xlm-roberta-large-xnli',\n", + " 'valhalla/distilbart-mnli-12-3',\n", + " 'BaptisteDoyen/camembert-base-xnli',\n", + " 'typeform/mobilebert-uncased-mnli',\n", + " 'valhalla/distilbart-mnli-12-1',\n", + " 'valhalla/distilbart-mnli-12-9',\n", + " 'cross-encoder/nli-distilroberta-base',\n", + " 'bert-large-uncased',\n", + " 'bert-large-cased',\n", + " 'bert-base-uncased',\n", + " 'bert-base-cased',\n", + " 'bert-large-finetuned',\n", + " 'bert-base-finetuned']" + ] + }, + "metadata": {}, + "execution_count": 120 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "_vAb9djfOLGi" + }, + "source": [ + "index = list(df.iloc[7:]['Unnamed: 0'])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 479 + }, + "id": "Ys5wOUp_3p44", + "outputId": "44d1e19b-91ec-41d0-9ae2-4b6179a7bc96" + }, + "source": [ + "df.iloc[:6].plot.bar(rot=0, subplots=True, figsize=(8, 6))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "array([<matplotlib.axes._subplots.AxesSubplot object at 0x7fbfb85536d0>,\n", + " <matplotlib.axes._subplots.AxesSubplot object at 0x7fbfb8453f50>,\n", + " <matplotlib.axes._subplots.AxesSubplot object at 0x7fbfb8765510>,\n", + " <matplotlib.axes._subplots.AxesSubplot object at 0x7fbfb8381750>],\n", + " dtype=object)" + ] + }, + "metadata": {}, + "execution_count": 136 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfAAAAF1CAYAAAAX0biNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3df5RV9X3w+/enI2Tkh0UHbK0TnanxeQoGRCW2ROXp8ldN0wdMYlJ9jMYVe43rAsnVpixc5Rrjso0SvSZpiLdek9R4a4hJrOGxNj4VYm6owTAYfAoSWuQBHWoMTkiEhkkAP/ePs6HDOANn4Jw5s2fer7XOcu/v/u7v/pyvZ/ic/f3uvU9kJpIkqVx+rdEBSJKkgTOBS5JUQiZwSZJKyAQuSVIJmcAlSSohE7gkSSVkApckqYRM4NIIEhFbImJ3ROyKiB0R8fcR8dZi299ExB397JcR8e/FfvtfC/rbLyLain2Oqf+7kkYmE7g08vzXzBwHnAS8CvxVlfudmZnjerwW1y9ESYdjApdGqMzsBr4BTGl0LJIGzgQujVARMQb4Y2BVo2ORNHDOT0kjz2MRsRcYC2wH/qDK/Z6LiDd6rP9xZj5Z8+gkVcUELo08l2fmUxHRBMwBvhsR1Qyjn52Zm/oo3wuM6lU2CnijeEmqA4fQpREqM/dl5qPAPuD8o2jqJaCtV1k78HJmmsClOjGBSyNUVMwBjgc2FMVNEdHc4zW6iqa+Cbw7Ii6NiKaI+C1gEbC0TqFLwgQujUT/PSJ2Aa8DfwF8KDPXF9sWArt7vFb02O/5XveBfwag2Pcq4FPAT4HvA88CnxyUdyONUJGZjY5BkiQNkGfgkiSVkAlckqQSMoFLklRCJnBJkkrIBC5JUgmV6klsEydOzLa2tkaHIUnSoFizZs1rmTmpr22lSuBtbW10dHQ0OgxJkgZFRGztb5tD6JIklVCpzsA1+NoW/n2jQ3iTLXe+u9EhSFLDeQYuSVIJeQYuSRpS9uzZQ2dnJ93d3Y0OZdA0NzfT2trKqFG9f5m3fyZwSdKQ0tnZyfjx42lrayMiGh1O3WUmXV1ddHZ20t7eXvV+DqFLkoaU7u5uWlpaRkTyBogIWlpaBjziYAKXJA05IyV573ck79cELklSL01NTUyfPp0zzzyTs88+m2eeeQaALVu28Pa3v/1N9a+77jra29uZPn0606dP553vfCcAt912G3ffffdBddva2njttdeOOkbnwCVJQ1qtb2et5lbUY489lrVr1wLw5JNPcsstt/Dd7373kPt8+tOf5oorrqhJjNXwDFySpEN4/fXXOf744xsdxpt4Bi5JUi+7d+9m+vTpdHd388orr7BixYrD7vNnf/Zn3HHHHQCcccYZ/O3f/m1dYzSBS5LUS88h9O9///tce+21rFu37pD79DWE3t/FabW4SM8hdEmSDmHmzJm89tprbN++fcD7trS0sGPHjoPKdu7cyYQJE446LhO4JEmH8KMf/Yh9+/bR0tIy4H1nzZrFsmXL2LlzJwCPPvooZ555Jk1NTUcdl0PokiT1sn8OHCpPSnvwwQcPJN2NGzfS2tp6oO69994LHDwHDvCDH/yAadOmMW/ePM4//3wighNPPJEHHnigJjGawCVJQ1ojfoFw3759fZa3tbWxZ8+eN5W///3v77etj3zkI3zkIx+pWWz7OYQuSVIJVZXAI+KyiNgYEZsiYmEf298SEV8rtj8bEW1FeVtE7I6ItcXr/+6xzzkR8c/FPp+LkfbcPEmSjsJhE3hENAFLgHcBU4CrImJKr2rXAzsy823AvcBdPba9mJnTi9eNPcrvA/434PTiddmRvw1JkkaWas7AzwU2ZebmzPwVsBSY06vOHODBYvkbwEWHOqOOiJOA4zJzVWYm8BXg8gFHL0kaliqpYeQ4kvdbTQI/GXi5x3pnUdZnnczcC/wc2H+9fXtE/DAivhsRF/So33mYNgGIiBsioiMiOo7kHjxJUrk0NzfT1dU1YpL4/t8Db25uHtB+9b4K/RXglMzsiohzgMci4oyBNJCZ9wP3A8yYMaMm/zdr/WD8WmnElZaSNNS0trbS2dl5RA9OKavm5uaDbk2rRjUJfBvw1h7rrUVZX3U6I+IY4NeBrmJ4/JcAmbkmIl4E/lNRv2ekfbUpSRqBRo0aRXt7e6PDGPKqSeCrgdMjop1Kkr0S+G+96iwDPgR8H7gCWJGZGRGTgJ9m5r6I+G0qF6ttzsyfRsTrEfF7wLPAtcBf1eYtSZJGiqE4ojpYo6mHTeCZuTci5gFPAk3AlzJzfUTcDnRk5jLgi8BDEbEJ+CmVJA8wC7g9IvYAbwA3ZuZPi23/O/A3wLHAPxQvqZRG8j8iqg8/UzqcqubAM/MJ4IleZbf2WO4G3vQYmsz8JvDNftrsAN4+kGAlSVKFT2KTJKmETOCSJJWQCVySpBLy18gkDSovzpJqwzNwSZJKyAQuSVIJmcAlSSohE7gkSSVkApckqYRM4JIklZAJXJKkEjKBS5JUQiZwSZJKyAQuSVIJmcAlSSqhqhJ4RFwWERsjYlNELOxj+1si4mvF9mcjoq0ovyQi1kTEPxf/vbDHPk8Xba4tXifW6k1JkjTcHfbHTCKiCVgCXAJ0AqsjYllmvtCj2vXAjsx8W0RcCdwF/DHwGvBfM/PfIuLtwJPAyT32uzozO2r0XiRJGjGqOQM/F9iUmZsz81fAUmBOrzpzgAeL5W8AF0VEZOYPM/PfivL1wLER8ZZaBC5J0khWTQI/GXi5x3onB59FH1QnM/cCPwdaetV5H/BcZv6yR9mXi+Hz/zMioq+DR8QNEdERER3bt2+vIlxJkoa/QbmILSLOoDKs/pEexVdn5lTgguJ1TV/7Zub9mTkjM2dMmjSp/sFKklQC1STwbcBbe6y3FmV91omIY4BfB7qK9Vbg74BrM/PF/Ttk5rbivzuBh6kM1UuSpCpUk8BXA6dHRHtEjAauBJb1qrMM+FCxfAWwIjMzIiYAfw8szMx/2l85Io6JiInF8ijgj4B1R/dWJEkaOQ6bwIs57XlUriDfADySmesj4vaImF1U+yLQEhGbgJuB/beazQPeBtza63axtwBPRsT/BNZSOYP/f2r5xiRJGs4OexsZQGY+ATzRq+zWHsvdwPv72O8O4I5+mj2n+jAlSVJPPolNkqQSMoFLklRCJnBJkkrIBC5JUgmZwCVJKiETuCRJJWQClySphEzgkiSVkAlckqQSMoFLklRCJnBJkkrIBC5JUgmZwCVJKiETuCRJJWQClySphKpK4BFxWURsjIhNEbGwj+1viYivFdufjYi2HttuKco3RsQfVNumJEnq32ETeEQ0AUuAdwFTgKsiYkqvatcDOzLzbcC9wF3FvlOAK4EzgMuAL0REU5VtSpKkflRzBn4usCkzN2fmr4ClwJxedeYADxbL3wAuiogoypdm5i8z838Bm4r2qmlTkiT1o5oEfjLwco/1zqKszzqZuRf4OdByiH2raVOSJPXjmEYHcDgRcQNwQ7G6KyI2NjKePkwEXqtFQ3FXLVoZsuyn6lXbV1OBUUACb1D54vxSsUzcxVgqX4zHFnV2Ufmy3F3s31Icq/ff1FRgC7CzWB8D/BYwrljfA/wM+DGwr2inbf9xe1hX1O3PpOL4xwI/LY45EH6mqleTvrKfqlPjfjq1vw3VJPBtwFt7rLcWZX3V6YyIY4BfB7oOs+/h2gQgM+8H7q8izoaIiI7MnNHoOIY6+6l61fZVRGwB/iQzn4qI3wSeBP4hM/88ImYC/wj8OfAlKon+ZmAucE5mbo6I64r9z++j3blFu+8E/gfwF8CXMvPViDiFynUv38nMp/trp4r430sl6f8BcGxmXjfA/f1MVcm+qk7Z+qmaIfTVwOkR0R4Ro6lclLasV51lwIeK5SuAFZmZRfmVxVXq7cDpwA+qbFNSlTLzx1QS+PSiaDHwlcz8bGbuzMyfZuYiYBVw2wCaXgx8OTM/lZmvFsd6KTM/kZlPH2XMj2bmY1S+7EsaoMMm8GJOex6Vfxw2AI9k5vqIuD0iZhfVvgi0RMQmKt/yFxb7rgceAV4Avk3lW/2+/tqs7VuTRo6IaKVyV8emiBgDvBP4eh9VHwEuqbLNscBM4Ju1ilNS7VQ1B56ZTwBP9Cq7tcdyN/D+fvb9CyrDb4dts6SG7PD+EGM/VW8gffVYRCSV+ekVwCeAE6h8OX+lj/qvUJnnq8bxRTs/3l8QEYupXJMyCvhUZt5RbPq9iPhZj327MvO0AbyPI+Fnqnr2VXVK1U8+ie0oFXP0Ogz7qXoD7KvLM3M88PvA71BJzjuozC2f1Ef9k/iPi3T2UknEvY2icvHZm9rJzAWZOQH4Ow4+AViVmRN6vOqdvP1MDYB9VZ2y9ZMJXBoGMvO7wN8Ad2fmvwPfp+9RsQ8Ay4vll4BTimc2AFAMv58IbC3aeRZ4bx1Dl3SEhvxtZJKq9hlgS0ScSeU6lCcj4kfAl6n8rf8plTntdxT1n6VyS9nCiLgXaAI+BXQAW4s6C4p2tlG5Cv0nxXx7O5UHMx2x4o6VY4rjNkVEM7C3uEZG0mF4Bi4NE5m5HfgKcGtmrqRye9Z7qcx7bwXOAs7PzH8t6v8SeDeV4fdOYDOV+70/UNxFQtHOhcAs4F+Kee5vA08Df9Xj8DMjYlev1zs4tEXAbipfNj5YLC86qk6QRpAo/k4lSVKJeAYuSVIJOQcuqS6KJ7a90M/mKZn50mDGIw03DqFLklRCDqFLklRCpRpCnzhxYra1tTU6DEmSBsWaNWtey8xJfW0rVQJva2ujo6Oj0WFIkjQoImJrf9scQpckqYRKdQZeM7f9eqMj6NttP290BJKkkvAMXJKkEhqZZ+CSpJrYs2cPnZ2ddHd3NzqUUmtubqa1tZVRo/r6gcC+mcAlSUess7OT8ePH09bWRo8fttMAZCZdXV10dnbS3t5e9X4mcB3aULxewGsFpCGju7vb5H2UIoKWlha2b98+oP2cA5ckHRWT99E7kj40gUuSVEIOoUuSaqfW025VTJk1NTUxdepU9u7dS3t7Ow899BATJkwAYP369cyfP59t27bxxhtvcO2117Jo0SIigttuu41x48bx8Y9//EBb+x8YNnHiRF599VVuuukmVq1axfHHH8/o0aNZsGAB73nPe3j66aeZM2fOQXPWd999NxdffHGfMX74wx/m8ccf58QTT2TdunVH2SkVnoFLkkrt2GOPZe3ataxbt44TTjiBJUuWALB7925mz57NwoUL2bhxI88//zzPPPMMX/jCFw7bZmZy+eWXM2vWLDZv3syaNWtYunQpnZ2dB+pccMEFrF279sCrv+QNcN111/Htb3/76N9sDyZwSdKwMXPmTLZt2wbAww8/zHnnncell14KwJgxY/j85z/PnXfeedh2VqxYwejRo7nxxhsPlJ166qnMnz//iOKaNWsWJ5xwwhHt2x8TuCRpWNi3bx/Lly9n9uzZQGX4/JxzzjmozmmnncauXbt4/fXXD9nW+vXrOfvssw9Z53vf+x7Tp08/8HrxxReP7g0MkHPgkqRS2717N9OnT2fbtm1MnjyZSy65pKr9+rvyu6/yuXPnsnLlSkaPHs3q1auByhD6448/fuSBHyXPwCVJpbZ/Dnzr1q1k5oE58ClTprBmzZqD6m7evJlx48Zx3HHH0dLSwo4dOw7avnPnTiZMmMAZZ5zBc889d6B8yZIlLF++fMD3ateTCVySNCyMGTOGz33uc9xzzz3s3buXq6++mpUrV/LUU08BlTP1j370oyxYsACozEsvW7aMnTt3AvDoo49y5pln0tTUxIUXXkh3dzf33XffgfZ/8YtfDP6bOgSH0CVJtdPgJyWeddZZTJs2ja9+9atcc801fOtb32L+/PnMnTuXffv2cc011zBv3jwApk2bxrx58zj//POJCE488UQeeOABoDKM/thjj3HTTTexePFiJk2axNixY7nrrrsOHGv/HPh+ixYt4oorrugzrquuuoqnn36a1157jdbWVj75yU9y/fXXH9V7jcw8qgYG04wZM7Kjo+PoGxqKjweFhn/w+zQU+2oo9pM0Qm3YsIHJkyc3Ooxhoa++jIg1mTmjr/oOoUuSVEIOoUvSUOToV+l0dXVx0UUXval8+fLltLS01Px4JnBJkmqgpaWFtWvXDtrxHEKXJB2VMl1LNVQdSR96Bi7VgsOdGqGam5vp6uqipaWlMT8r+m8/HPxjHs5vnTWg6plJV1cXzc3NA9rPBC5pcPllZ1hpbW2ls7OzcQ84+dlPGnPcQ/n5hgHv0tzcTGtr64D2qUsCj4jLgM8CTcADmXlnr+03A38C7AW2Ax/OzK31iEWSVD+jRo066Cc1B91tv9e4Y/dnkL4Q1nwOPCKagCXAu4ApwFURMaVXtR8CMzJzGvANYHGt45AkaTirx0Vs5wKbMnNzZv4KWArM6VkhM7+TmfufSbcKGNi4gSRJI1w9EvjJwMs91juLsv5cD/xDfxsj4oaI6IiIjqH0EHlJkhqpobeRRcQHgRnAp/urk5n3Z+aMzJwxadKkwQtOkqQhrB4XsW0D3tpjvbUoO0hEXAz8OfBfMvOXdYhDkqRhqx5n4KuB0yOiPSJGA1cCy3pWiIizgL8GZmfmELwHQJKkoa3mCTwz9wLzgCeBDcAjmbk+Im6PiNlFtU8D44CvR8TaiFjWT3OSJKkPdbkPPDOfAJ7oVXZrj+WL63FcSZJGCp+FLklSCZnAJUkqIRO4JEklZAKXJKmETOCSJJWQCVySpBIygUuSVEImcEmSSsgELklSCZnAJUkqIRO4JEklZAKXJKmETOCSJJWQCVySpBIygUuSVEImcEmSSsgELklSCZnAJUkqIRO4JEklZAKXJKmETOCSJJWQCVySpBIygUuSVEImcEmSSsgELklSCZnAJUkqIRO4JEklZAKXJKmETOCSJJWQCVySpBIygUuSVEImcEmSSsgELklSCdUtgUfEZRGxMSI2RcTCPra/JSK+Vmx/NiLa6hWLJEnDTV0SeEQ0AUuAdwFTgKsiYkqvatcDOzLzbcC9wF31iEWSpOGoXmfg5wKbMnNzZv4KWArM6VVnDvBgsfwN4KKIiDrFI0nSsFKvBH4y8HKP9c6irM86mbkX+DnQUqd4JEkaVo5pdACHExE3ADcUq7siYmMj4+nDROC1mrT0yWE9AGE/Va/avpoKjAISeIPKl+CXimX4ZIyl8kV5bFFnF5Uv093F/i3FsXr/TU0FtgA7i/UxwG8B44r1PcDPgB8D+4p22g4c9z+sK+r2JYBTgOOo/Dv0yyK21w/3pnvwM1W92vSV/VSd2vbTqf1uycyav4CZwJM91m8BbulV50lgZrF8DJVOi3rEU88X0NHoGMrwsp9q31dUkuzFxfJvAs8Df1Gsz6SSsD8GjAdOAO4AdgC/XdS5Dlh5mHbfWbRzC/AbRdkpwCeB3z9UO4eJfSxwG5XE/2vAH1H5wtDmZ6pxn6mR/ipbP9VrCH01cHpEtEfEaOBKYFmvOsuADxXLVwArsuhBSQOTmT+m8qV4elG0GPhKZn42M3dm5k8zcxGwikrirNZi4MuZ+anMfLU41kuZ+YnMfPoo4v33zLwtM7dk5huZ+Tjwv4BzjrRNaaSpSwLPypz2PCr/oGwAHsnM9RFxe0TMLqp9EWiJiE3AzcCbbjWTVJ2IaKVy18emiBhD5cz5631UfQS4pMo2x1I5k/9mreI8xLF+A/hPwPp6H0saLuo2B56ZTwBP9Cq7tcdyN/D+eh1/EN3f6ABKwn6q3kD66rGISCrz0yuAT1AZLv814JU+6r9CZZ6vGscX7fx4f0FELKZyTcoo4FOZeUex6fci4mc99u3KzNOqOUhEjAL+FngwM39UZWzgZ2og7KvqlKqffBLbUcrMUv0PbxT7qXoD7KvLM3M88PvA71BJzjuoXFB2Uh/1T+I/LtLZSyUR9zaKysVnb2onMxdk5gTg7zj4BGBVZk7o8ao2ef8a8BDwKyqjdlXzM1U9+6o6ZesnE7g0DGTmd4G/Ae7OzH8Hvk/fI1wfAJYXyy8Bp/R8/kIx/H4isLVo51ngvfWIuTjuF4HfAN6Xmf1dsS6pD0P+NjJJVfsMsCUizqRyTcmTEfEj4MtU/tb/lMqc9juK+s9SuaVsYUTcCzQBnwI6gK1FnQVFO9uAL2XmT4r59nZg01HGex8wmcoV77uPsi1pxPEMXBomMnM78BXg1sxcCfwBlbPnV6gk5LOA8zPzX4v6vwTeTWX4vRPYTOV+7w/svyOkaOdCYBbwL8U897eBp4G/6nH4mRGxq9frHfQjIk4FPkLlqvkf99jn6tr0hjT8hXduSZJUPp6BS5JUQs6BS6qLiDgFeKGfzVMy86XBjEcabhxClySphBxClySphEo1hD5x4sRsa2trdBiSJA2KNWvWvJaZk/raVqoE3tbWRkdHR6PDkCRpUETE1v62OYQuSVIJleoMvFamPji10SH06Z8/9M+NDkGSVBKegUuSVEIj8gxcklQbe/bsobOzk+7u7kaHUmrNzc20trYyalRfPxDYNxO4JOmIdXZ2Mn78eNra2ujxw3YagMykq6uLzs5O2tvbq97PIXRJ0hHr7u6mpaXF5H0UIoKWlpYBj2J4Bq5DGooX/HmxnzS0mLyP3pH0oWfgkiSVkGfgkqSaqfWoXTUjbk1NTUydOpW9e/fS3t7OQw89xIQJEwBYv3498+fPZ9u2bbzxxhtce+21LFq0iIjgtttuY9y4cXz84x8/0Nb+B4ZNnDiRV199lZtuuolVq1Zx/PHHM3r0aBYsWMB73vMenn76aebMmXPQnPXdd9/NxRdf/Kb4Xn75Za699lpeffVVIoIbbriBj33sY0fdN56BS5JK7dhjj2Xt2rWsW7eOE044gSVLlgCwe/duZs+ezcKFC9m4cSPPP/88zzzzDF/4whcO22ZmcvnllzNr1iw2b97MmjVrWLp0KZ2dnQfqXHDBBaxdu/bAq6/kDXDMMcdwzz338MILL7Bq1SqWLFnCCy/090N91TOBS5KGjZkzZ7Jt2zYAHn74Yc477zwuvfRSAMaMGcPnP/957rzzzsO2s2LFCkaPHs2NN954oOzUU09l/vz5A47ppJNO4uyzzwZg/PjxTJ48+UCMR8MELkkaFvbt28fy5cuZPXs2UBk+P+eccw6qc9ppp7Fr1y5ef/31Q7a1fv36A0m3P9/73veYPn36gdeLL7542Bi3bNnCD3/4Q373d3/3sHUPxzlwSVKp7d69m+nTp7Nt2zYmT57MJZdcUtV+/V353Vf53LlzWblyJaNHj2b16tVAZQj98ccfrzrOXbt28b73vY/PfOYzHHfccVXv1x/PwCVJpbZ/Dnzr1q1k5oE58ClTprBmzZqD6m7evJlx48Zx3HHH0dLSwo4dOw7avnPnTiZMmMAZZ5zBc889d6B8yZIlLF++nO3btx9RjHv27OF973sfV199Ne9973uPqI3eTOCSpGFhzJgxfO5zn+Oee+5h7969XH311axcuZKnnnoKqJypf/SjH2XBggUAzJo1i2XLlrFz504AHn30Uc4880yampq48MIL6e7u5r777jvQ/i9+8Ysjiiszuf7665k8eTI333zzUb7L/+AQuiSpZhr9oKWzzjqLadOm8dWvfpVrrrmGb33rW8yfP5+5c+eyb98+rrnmGubNmwfAtGnTmDdvHueffz4RwYknnsgDDzwAVIbRH3vsMW666SYWL17MpEmTGDt2LHfdddeBY+2fA99v0aJFXHHFFW+K6Z/+6Z946KGHmDp16oH6f/mXf8kf/uEfHtV7jcw8qgb6bDTiMuCzQBPwQGbe2Wv7zcCfAHuB7cCHM7PfHy3fb8aMGdnR0XHU8Q3Fp4tB4z/4fRmKfTUU+0kaqTZs2MDkyZMbHcaw0FdfRsSazJzRV/2aD6FHRBOwBHgXMAW4KiKm9Kr2Q2BGZk4DvgEsrnUckiQNZ/UYQj8X2JSZmwEiYikwBzhw13pmfqdH/VXAB+sQhyRJg6arq4uLLrroTeXLly+npaWl5serRwI/GXi5x3oncKgb3q4H/qG/jRFxA3ADwCmnnFKL+CRJqrmWlhbWrl07aMdr6FXoEfFBYAbw6f7qZOb9mTkjM2dMmjRp8IKTJFWlHtdSjTRH0of1SODbgLf2WG8tyg4SERcDfw7Mzsxf1iEOSVKdNTc309XVZRI/CplJV1cXzc3NA9qvHkPoq4HTI6KdSuK+EvhvPStExFnAXwOXZeZP6hCDJJVaWe4AaW1tpbOz84gfcKKK5uZmWltbB7RPzRN4Zu6NiHnAk1RuI/tSZq6PiNuBjsxcRmXIfBzw9eKRdS9l5uxaxyJJqq9Ro0Yd9JOaGjx1eZBLZj4BPNGr7NYey33/5pokSaqKj1KVJKmEfJSqVANlma+UhpuR/LdnApc0qEbyP7hSLTmELklSCZnAJUkqIRO4JEklZAKXJKmETOCSJJWQCVySpBIygUuSVEImcEmSSsgELklSCZnAJUkqIRO4JEklZAKXJKmE6pLAI+KyiNgYEZsiYmEf22dFxHMRsTcirqhHDJIkDWc1T+AR0QQsAd4FTAGuiogpvaq9BFwHPFzr40uSNBLU4+dEzwU2ZeZmgIhYCswBXthfITO3FNveqMPxJUka9uoxhH4y8HKP9c6iTJIk1ciQv4gtIm6IiI6I6Ni+fXujw5EkaUioRwLfBry1x3prUXZEMvP+zJyRmTMmTZp01MFJkjQc1COBrwZOj4j2iBgNXAksq8NxJEkasWqewDNzLzAPeBLYADySmesj4vaImA0QEe+IiE7g/cBfR8T6WschSdJwVo+r0MnMJ4AnepXd2mN5NZWhdUmSdASG/EVskiTpzUzgkiSVkAlckqQSMoFLklRCJnBJkkrIBC5JUgmZwCVJKiETuCRJJWQClySphEzgkiSVkAlckqQSMoFLklRCJnBJkkrIBC5JUgmZwCVJKiETuCRJJVS3BB4Rl0XExojYFBEL+9j+loj4WrH92Yhoq1cskiQNN3VJ4BHRBCwB3gVMAa6KiCm9ql0P7MjMtwH3AnfVIxZJkoajep2BnwtsyszNmfkrYCkwp1edOcCDxfI3gIsiIuoUjyRJw0q9EuOp2OgAAA2OSURBVPjJwMs91juLsj7rZOZe4OdAS53ikTREbfzTjexav6vRYUilc0yjAziciLgBuKFY3RURGxsZTx8mAq/VoqG4blgPQNhP1RtoX00FRgEJvEHly/BLcV28UWwfS+UL89iizi4qX6q7i+0txTF7/21NBbYAO4v1McBvAeOK9T3Az4AfA/uKdtqKGHpaV9TtN/4tn97S8zjV8jNVvZr0lf1UnRr306n9bsnMmr+AmcCTPdZvAW7pVedJYGaxfAyVTot6xFPPF9DR6BjK8LKf6tdXVJLsxcXybwLPA39RrM+kkrA/BowHTgDuAHYAv13UuQ5YeZh231m0cwvwG0XZKcAngd8/VDsDid/P1ND4TI3UV9n6qV5D6KuB0yOiPSJGA1cCy3rVWQZ8qFi+AliRRQ9KOjKZ+WMqX46nF0WLga9k5mczc2dm/jQzFwGrgNsG0PRi4MuZ+anMfLU41kuZ+YnMfLp270BSteqSwLMypz2Pyj8kG4BHMnN9RNweEbOLal8EWiJiE3Az8KZbzSQNTES0Urn7Y1NEjKFy5vz1Pqo+AlxSZZtjqZzJf7NWcUo6enWbA8/MJ4AnepXd2mO5G3h/vY4/iO5vdAAlYT9V70j66rGISCrz0yuAT1AZLv814JU+6r9CZb6vGscX7fx4f0FELKZybcoo4FOZeUex6fci4mc99u3KzNMG8kYGwM9U9eyr6pSqn3wS21HKzFL9D28U+6l6R9hXl2fmeOD3gd+hkpx3ULmg7KQ+6p/Ef1yss5dKIu5tFJWLz97UTmYuyMwJwN9x8InAqsyc0ONVr+TtZ2oA7KvqlK2fTODSMJKZ3wX+Brg7M/8d+D59j3R9AFheLL8EnNLzOQzF8PuJwNainWeB99YxdEkDNORvI5M0YJ8BtkTEmVSuLXkyIn4EfJnK3/yfUpnTfkdR/1kqt5QtjIh7gSbgU0AHsLWos6BoZxvwpcz8STHf3g5sGpy3Jaknz8ClYSYztwNfAW7NzJXAH1A5e36FSkI+Czg/M/+1qP9L4N1Uht87gc1U7vf+wP47Q4p2LgRmAf9SzHN/G3ga+Kseh58ZEbt6vd6BpJoL79ySJKl8PAOXJKmEnAOXVFcRcQrwQj+bp2TmS4MZjzRcOIQuSVIJOYQuSVIJlWoIfeLEidnW1tboMCRJGhRr1qx5LTMn9bWtVAm8ra2Njo6ORochSdKgiIit/W1zCF2SpBIq1Rl4rWz4ncmNDqFPk3+0odEhSJJKwjNwSZJKaESegUuSam/Pnj10dnbS3d3d6FBKp7m5mdbWVkaN6uuHAftmApck1URnZyfjx4+nra2NHj9up8PITLq6uujs7KS9vb3q/UzgOqSheL2A1wpIQ1N3d7fJ+whEBC0tLWzfvn1A+zkHLkmqGZP3kTmSfjOBS5JUQg6hS5LqotZTcNVMnzU1NTF16lT27t1Le3s7Dz30EBMmTABg/fr1zJ8/n23btvHGG29w7bXXsmjRIiKC2267jXHjxvHxj3/8QFv7Hx42ceJEXn31VW666SZWrVrF8ccfz+jRo1mwYAHvec97ePrpp5kzZ85B89d33303F198cZ8xjhs3jl27dh1lb3gGLkkaRo499ljWrl3LunXrOOGEE1iyZAkAu3fvZvbs2SxcuJCNGzfy/PPP88wzz/CFL3zhsG1mJpdffjmzZs1i8+bNrFmzhqVLl9LZ2XmgzgUXXMDatWsPvPpL3rVkApckDUszZ85k27ZtADz88MOcd955XHrppQCMGTOGz3/+89x5552HbWfFihWMHj2aG2+88UDZqaeeyvz58+sTeJVM4JKkYWffvn0sX76c2bNnA5Xh83POOeegOqeddhq7du3i9ddfP2Rb69ev5+yzzz5kne9973tMnz79wOvFF188ujdQBefAJUnDxu7du5k+fTrbtm1j8uTJXHLJJVXt199V4H2Vz507l5UrVzJ69GhWr14NVIbQH3/88SMP/Ah4Bi5JGjb2z4Fv3bqVzDwwBz5lyhTWrFlzUN3Nmzczbtw4jjvuOFpaWtixY8dB23fu3MmECRM444wzeO655w6UL1myhOXLlw/4vu1aM4FLkoadMWPG8LnPfY577rmHvXv3cvXVV7Ny5UqeeuopoHKm/tGPfpQFCxYAMGvWLJYtW8bOnTsBePTRRznzzDNpamriwgsvpLu7m/vuu+9A+7/4xS8G/0314hC6JKkuGv3UxLPOOotp06bx1a9+lWuuuYZvfetbzJ8/n7lz57Jv3z6uueYa5s2bB8C0adOYN28e559/PhHBiSeeyAMPPABUhtEfe+wxbrrpJhYvXsykSZMYO3Ysd91114Fj7Z8D32/RokVcccUVdX1/kZl1PUAtzZgxIzs6Oo66naH4eFBo/Ie9L0Oxr4ZiP0mCDRs2MHny0Ps3oyz66r+IWJOZM/qq7xC6JEkl5BC6JA1Bjn6VW1dXFxdddNGbypcvX05LS0tNjmEClySpxlpaWli7dm1dj+EQuiSpZsp0XdVQciT95hm4VAMOd0rQ3NxMV1cXLS0tg/azorvXrRuU4wzEsW9/+4DqZyZdXV00NzcPaD8TuKRB5Zed4au1tZXOzs5BfcDJnldfHbRjVWtUU9OA92lubqa1tXVA+9QlgUfEZcBngSbggcy8s9f2m4E/AfYC24EPZ+bWesQiSRoco0aNOugnNQfDhve8d1CPV43B+kJY8znwiGgClgDvAqYAV0XElF7VfgjMyMxpwDeAxbWOQ5Kk4aweF7GdC2zKzM2Z+StgKTCnZ4XM/E5m7n8O3SpgYOMGkiSNcPVI4CcDL/dY7yzK+nM98A/9bYyIGyKiIyI6Gv3geEmShoqG3kYWER8EZgCf7q9OZt6fmTMyc8akSZMGLzhJkoawelzEtg14a4/11qLsIBFxMfDnwH/JzF/WIQ5JkoatepyBrwZOj4j2iBgNXAks61khIs4C/hqYnZk/qUMMkiQNazVP4Jm5F5gHPAlsAB7JzPURcXtEzC6qfRoYB3w9ItZGxLJ+mpMkSX2oy33gmfkE8ESvslt7LF9cj+NKkjRS+Cx0SZJKyAQuSVIJmcAlSSohE7gkSSVkApckqYRM4JIklZAJXJKkEjKBS5JUQiZwSZJKyAQuSVIJmcAlSSohE7gkSSVkApckqYRM4JIklZAJXJKkEjKBS5JUQiZwSZJKyAQuSVIJmcAlSSohE7gkSSVkApckqYRM4JIklZAJXJKkEjKBS5JUQiZwSZJKyAQuSVIJmcAlSSohE7gkSSVkApckqYRM4JIklZAJXJKkEjKBS5JUQiZwSZJKqG4JPCIui4iNEbEpIhb2sf0tEfG1YvuzEdFWr1gkSRpu6pLAI6IJWAK8C5gCXBURU3pVux7YkZlvA+4F7qpHLJIkDUf1OgM/F9iUmZsz81fAUmBOrzpzgAeL5W8AF0VE1CkeSZKGlXol8JOBl3usdxZlfdbJzL3Az4GWOsUjSdKwckyjAziciLgBuKFY3RURGxsZTx8mAq/VpKXhPQBhP1WvNn1lP1XPvqqO/VSd2vbTqf1tqFcC3wa8tcd6a1HWV53OiDgG+HWgq3dDmXk/cH+d4jxqEdGRmTMaHcdQZz9Vz76qjv1UPfuqOmXrp3oNoa8GTo+I9ogYDVwJLOtVZxnwoWL5CmBFZmad4pEkaVipyxl4Zu6NiHnAk0AT8KXMXB8RtwMdmbkM+CLwUERsAn5KJclLkqQq1G0OPDOfAJ7oVXZrj+Vu4P31Ov4gGrLD+0OM/VQ9+6o69lP17KvqlKqfwlFrSZLKx0epSpJUQibwI3S4R8WqIiK+FBE/iYh1jY5lKIuIt0bEdyLihYhYHxEfa3RMQ1VENEfEDyLi+aKvPtnomIayiGiKiB9GxOONjmUoi4gtEfHPEbE2IjoaHU81HEI/AsWjYv8FuITKQ2pWA1dl5gsNDWwIiohZwC7gK5n59kbHM1RFxEnASZn5XESMB9YAl/uZerPiiY1jM3NXRIwCVgIfy8xVDQ5tSIqIm4EZwHGZ+UeNjmeoiogtwIzMrM2zBQaBZ+BHpppHxQrIzP+Pyl0GOoTMfCUznyuWdwIbePPTCwVkxa5idVTx8kykDxHRCrwbeKDRsaj2TOBHpppHxUpHpPhlvrOAZxsbydBVDAuvBX4C/GNm2ld9+wywAHij0YGUQAL/IyLWFE8AHfJM4NIQEhHjgG8C/0dmvt7oeIaqzNyXmdOpPOXx3IhweqaXiPgj4CeZuabRsZTE+Zl5NpVf0ZxbTP8NaSbwI1PNo2KlASnmc78J/G1mPtroeMogM38GfAe4rNGxDEHnAbOLud2lwIUR8f82NqShKzO3Ff/9CfB3VKZKhzQT+JGp5lGxUtWKC7O+CGzIzP+r0fEMZRExKSImFMvHUrmY9EeNjWroycxbMrM1M9uo/Bu1IjM/2OCwhqSIGFtcPEpEjAUuBYb8nTMm8CNQ/Pzp/kfFbgAeycz1jY1qaIqIrwLfB/5zRHRGxPWNjmmIOg+4hspZ0tri9YeNDmqIOgn4TkT8Typfpv8xM71FSkfjN4CVEfE88APg7zPz2w2O6bC8jUySpBLyDFySpBIygUuSVEImcEmSSsgELklSCZnAJUkqIRO4JEklZAKXJKmETOCSJJXQ/w/AkK+e9GIlsAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "<Figure size 576x432 with 4 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 588 + }, + "id": "j3I3FobLNgJ3", + "outputId": "20495d22-44dc-4c18-affb-a71da40e4ba6" + }, + "source": [ + "df.iloc[7:].plot.bar(rot=0, subplots=True, figsize=(16, 8))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "array([<matplotlib.axes._subplots.AxesSubplot object at 0x7fbfba576150>,\n", + " <matplotlib.axes._subplots.AxesSubplot object at 0x7fbfba179b10>,\n", + " <matplotlib.axes._subplots.AxesSubplot object at 0x7fbfc1e30c10>,\n", + " <matplotlib.axes._subplots.AxesSubplot object at 0x7fbfba167210>],\n", + " dtype=object)" + ] + }, + "metadata": {}, + "execution_count": 132 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6gAAAHiCAYAAADlHeELAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdf5RV9X3v/+e7A2QkoOiAt8ZRZ2JsIoYfCpoSlbpUDDbfgIkm1Vh/rJqFaYF0mfrlkoavtSZNlJqb1Iq2Vs0P7zXEGqPEmtgrXvKNJloGg2nQ0iIBHZIYnKBCBCP4/v5xDnwPkxnmMGeG2TPn+VjrLM/+7M/+7M971DXzWvuz947MRJIkSZKkgfY7Az0BSZIkSZLAgCpJkiRJKggDqiRJkiSpEAyokiRJkqRCMKBKkiRJkgrBgCpJkiRJKgQDqiRJkiSpEAyokiT1gYjYEBHbI2JbRGyJiH+JiKPK+74SEZ/t5riMiF+Xj9v9WdDdcRHRUj5mWP9XJUnSgWVAlSSp73wgM0cBRwAvAn9f5XGTMnNUxWdx/01RkqTiMqBKktTHMnMHcC8wfqDnIknSYGJAlSSpj0XESOCPgCcGei6SJA0m3r8iSVLfuT8idgJvBTYD76vyuKci4s2K7T/KzIf7fHaSJBWcAVWSpL5zXmY+EhENwGzgexFRzTLfkzJzXRftO4HhndqGA2+WP5IkDSku8ZUkqY9l5q7MvA/YBZxWw1DPAy2d2lqBFzLTgCpJGnIMqJIk9bEomQ0cCjxbbm6IiMaKz4gqhvom8P6IOCciGiLibcAiYGk/TV2SpAFlQJUkqe98OyK2Aa8CfwNclplryvsWAtsrPo9WHPd0p/egfgmgfOxFwOeBXwE/BJ4E/vqAVCNJ0gEWmTnQc5AkSZIkySuokiRJkqRiMKBKkiRJkgrBgCpJkiRJKgQDqiRJkiSpEAyokiRJkqRCGDbQE+hs7Nix2dLSMtDTkCRJkiT1g1WrVr2UmeO62le4gNrS0kJbW9tAT0OSJEmS1A8iYmN3+1ziK0mSJEkqBAOqJEmSJKkQCrfEV6pFy8J/GdDzb7j+/QN6fkmSJGkwG3IBtd4DSr3XL0mSpPow2P7ufeONN2hvb2fHjh39NKPiaWxspLm5meHDh1d9zJALqJJUjwbbL2mpLw3kf/8D/d9+vf+/b/31Xf9g097ezujRo2lpaSEiahrrx+0v99Gsemdi85ge+2QmHR0dtLe309raWvXYBlRJQ4K/pOtbvf/7r/f6JWkw2LFjR5+E08EiImhqamLz5s37dZwBVRpC/CNVkiSpuOolnO7Wm3oNqJIkSZJUB048ponj3jWezOR3Ghr41GcWM3nqe9j0wvPMv/yPuG/5D/fq//9c9We0Pfk4o0cfDEDjQQfxtfv/lWuvvZZRo0Zx9dVX7+nb0tJCW1sbY8eOrWmOBlRJkiRJOsD6euXbsnmn9tjnLY0Hcc/D3wfg8RXLuen667jz3n3P45Ofvo4Z75/dJ3Oshu9BlSRJkqQ68+ttWzn4kJ4fdnSgeQVVkiRJkurA6zu285H3nc7rr7/OS7/8Bf+0dFmPx/yPv7mGf7rpRgCO/b138fm//6d+naMBVZIkSZLqQOUS36dX/RufvupPue+RH+zzmK6W+Hb38KO+eAiUS3wlSZIkqc5MmnIKL/+qg191vLTfxzY1NbFly5a92rZu3cqYMbUvGa4qoEbEzIhYGxHrImJhF/vfEhHfKO9/MiJaOu0/OiK2RcTVnY+VJEmSJB1YP133n7y5axdjDj1sv4+dPn06y5YtY+vWrQDcd999TJo0iYaGhprn1eMS34hoAJYAM4B2YGVELMvMZyq6XQFsycx3RMSFwA3AH1Xs/x/Ad2qerSRJkiSpV3bfgwqQmXzmi7fsCZUb169jxskn7On7f//V3wB734MK8L++vZwpEycyb948TjvtNCKCww8/nNtvv71P5ljNPainAOsycz1ARCwFZgOVAXU2cG35+73AzRERmZkRcR7wU+DXfTJjSZIkSRrkNlz//l4f++P2l3t13I82dnTZfuRRR7Pqp5t/q/2c/+u8bse68sorufLKK3s1j32pZonvkcALFdvt5bYu+2TmTuAVoCkiRgH/Hfjr2qcqSZIkSRrK+vshSdcCX8zMbfvqFBFzIqItIto2b/7t5C5JkiRJGvqqWeK7CTiqYru53NZVn/aIGAYcAnQA7wEuiIjFwBjgzYjYkZk3Vx6cmbcBtwFMnTo1e1OIJEmSJGlwqyagrgSOi4hWSkH0QuCjnfosAy4DfghcADyamQmcvrtDRFwLbOscTiVJkiSpHmRmn7wrdLAoRcL90+MS3/I9pfOAh4FngXsyc01EXBcRs8rd7qB0z+k64JPAb72KRpIkSZLqVWNjIx0dHb0KbYNRZtLR0UFjY+N+HVfNFVQy8yHgoU5t11R83wF8uIcxrt2vmUmSJEnSENHc3Ex7ezt98cydF7ds74MZ9d6zWw+qql9jYyPNzc37NXZVAVWSJEmS1HvDhw+ntbW1T8Y6d+G/9Mk4vVXLK3J60t9P8ZUkSZIkqSoGVEmSJElSIRhQJUmSJEmFYECVJEmSJBWCAVWSJEmSVAgGVEmSJElSIRhQJUmSJEmFYECVJEmSJBWCAVWSJEmSVAgGVEmSJElSIRhQJUmSJEmFYECVJEmSJBWCAVWSJEmSVAgGVEmSJElSIRhQJUmSJEmFYECVJEmSJBWCAVWSJEmSVAgGVEmSJElSIRhQJUmSJEmFYECVJEmSJBWCAVWSJEmSVAgGVEmSJElSIRhQJUmSJEmFYECVJEmSJBWCAVWSJEmSVAgGVEmSJElSIRhQJUmSJEmFYECVJEmSJBVCVQE1ImZGxNqIWBcRC7vY/5aI+EZ5/5MR0VJunxERqyLi38v/PLNvpy9JkiRJGip6DKgR0QAsAc4FxgMXRcT4Tt2uALZk5juALwI3lNtfAj6QmROAy4C7+mrikiRJkqShpZorqKcA6zJzfWb+BlgKzO7UZzbw1fL3e4GzIiIy80eZ+bNy+xrgoIh4S19MXJIkSZI0tFQTUI8EXqjYbi+3ddknM3cCrwBNnfqcDzyVma/3bqqSJEmSpKFs2IE4SUScQGnZ7znd7J8DzAE4+uijD8SUJEmSJEkFU80V1E3AURXbzeW2LvtExDDgEKCjvN0MfAu4NDOf6+oEmXlbZk7NzKnjxo3bvwokSZIkSUNCNQF1JXBcRLRGxAjgQmBZpz7LKD0ECeAC4NHMzIgYA/wLsDAzH++rSUuSJEmShp4eA2r5ntJ5wMPAs8A9mbkmIq6LiFnlbncATRGxDvgksPtVNPOAdwDXRMTq8ufwPq9CkiRJkjToVXUPamY+BDzUqe2aiu87gA93cdxngc/WOEdJkiRJUh2oZomvJEmSJEn9zoAqSZIkSSoEA6okSZIkqRAMqJIkSZKkQjCgSpIkSZIKwYAqSZIkSSoEA6okSZIkqRAMqJIkSZKkQjCgSpIkSZIKwYAqSZIkSSoEA6okSZIkqRAMqJIkSZKkQjCgSpIkSZIKwYAqSZIkSSoEA6okSZIkqRAMqJIkSZKkQjCgSpIkSZIKwYAqSZIkSSoEA6okSZIkqRAMqJIkSZKkQjCgSpIkSZIKwYAqSZIkSSoEA6okSZIkqRAMqJIkSZKkQjCgSpIkSZIKwYAqSZIkSSoEA6okSZIkqRAMqJIkSZKkQjCgSpIkSZIKoaqAGhEzI2JtRKyLiIVd7H9LRHyjvP/JiGip2PepcvvaiHhf301dkiRJkjSU9BhQI6IBWAKcC4wHLoqI8Z26XQFsycx3AF8EbigfOx64EDgBmAncUh5PkiRJkqS9VHMF9RRgXWauz8zfAEuB2Z36zAa+Wv5+L3BWRES5fWlmvp6ZPwXWlceTJEmSJGkvkZn77hBxATAzMz9W3r4EeE9mzqvo85Nyn/by9nPAe4BrgScy83+W2+8AvpOZ93Y6xxxgTnnzncDa2kvrtbHASwN4/oFm/dZfr/XXc+1g/dZv/fVafz3XDtZv/fVb/0DXfkxmjutqx7ADPZOuZOZtwG0DPQ+AiGjLzKkDPY+BYv3WX6/113PtYP3Wb/31Wn891w7Wb/31W3+Ra69mie8m4KiK7eZyW5d9ImIYcAjQUeWxkiRJkiRVFVBXAsdFRGtEjKD00KNlnfosAy4rf78AeDRLa4eXAReWn/LbChwH/FvfTF2SJEmSNJT0uMQ3M3dGxDzgYaABuDMz10TEdUBbZi4D7gDuioh1wK8ohVjK/e4BngF2AnMzc1c/1dJXCrHUeABZf32r5/rruXawfuuvb/Vcfz3XDtZv/fWrsLX3+JAkSZIkSZIOhGqW+EqSVHciYkNEbI+IbRHxi4j4SkSMqtj/3oh4NCK2RsQrEfHtyveER8TlEfFYN+OeXbE9NSIejIgtEfFyRDwTEX8TEYdWjLOrPI/Kz9t6mP+8iGiLiNcj4it98kORJKmfGVAlSereBzJzFDAZOBH4FEBETAP+FXgAeBvQCjwNPB4Rb6928Ih4L7ACeBx4V2aOAWZSui1mUkXXH2bmqE6fn/Uw/M+AzwJ3VjsfSZIGWiFeMyNJUpFl5i8i4mFKQRVgMfC1zPy7im6LImIKpXeAX1rl0IuBL2fm5yvO9TzwV30w5/ugdIWW0lP0JUkqPK+gSpLUg4hoBs4F1kXESOC9wD930fUeYEaVY74VmAZ8s6/mKUnSYGdAlSSpe/dHxFbgBeCXlK5sHkbp9+fPu+j/c2BslWMfWh7nF7sbImJx+T7UX0fEooq+v19u3/15rjfFSJJUdAZUSZK6d15mjgbOAN5FKXxuAd4Ejuii/xHAS+XvO4HhXfQZDrzR1TiZuaB8H+q32Ps2nCcyc0zF59iaqpIkqaAMqJIk9SAzvwd8BbgxM38N/BD4cBddPwIsL39/Hjg6ImL3zvLy4MOBjeVxngQ+1I9TlyRpUPEhSZIkVedLwIaImAQsBB6OiP8Avkzp9+lfULqn9ORy/yeBHcDCiPgi0AB8HmgDNpb7LCiPswm4MzN/Wb7ftRVYV8tkI2JYeV4NQENENAI7M3NnLeNKktSfvIIqSVIVMnMz8DXgmsx8DHgfpaufP6cUOE8ETsvM/yr3fx14P6Xlwe3AekqvpPlIZma5z2PAmcB04D8j4mXgu5RePfP3Faef1sV7UE9m3xYB2ymF6T8uf1+0zyMkSRpgUf4dKUmSJEnSgPIKqiRJkiSpELwHVZKkQSgijgae6Wb3+Mx8/kDOR5KkvuASX0mSJElSIRTuCurYsWOzpaVloKchSZIkSeoHq1ateikzx3W1r3ABtaWlhba2toGehiRJkiSpH0TExu72+ZAkSZIkSVIhGFAlSZIkSYVgQJUkSZIkFULh7kGt2bWHDPD5XxnY80uSJEmqyRtvvEF7ezs7duwY6KkMao2NjTQ3NzN8+PCqjxl6AVWSJEmSatDe3s7o0aNpaWkhIgZ6OoNSZtLR0UF7ezutra1VH+cSX0mSJEmqsGPHDpqamgynNYgImpqa9vsqtAFVkiRJkjoxnNauNz9Dl/gONd6DK0mSJGmQMqBKkiRJ0r709UUgL+p0yyW+kiRJklQwDQ0NTJ48mXe/+9184AMf4OWXX96zb82aNZx55pm8853v5LjjjuMzn/kMmQnAtddey4033rjXWC0tLbz00ksAvPjii3z0ox/l7W9/O1OmTGHatGl861vfAmDFihUccsghTJ48ec/nkUce6XaOf/Inf8Lhhx/Ou9/97j6r24AqSZIkSQVz0EEHsXr1an7yk59w2GGHsWTJEgC2b9/OrFmzWLhwIWvXruXpp5/mBz/4AbfcckuPY2Ym5513HtOnT2f9+vWsWrWKpUuX0t7evqfP6aefzurVq/d8zj777G7Hu/zyy/nud79be7EValriGxEzgb8DGoDbM/P6Tvs/CXwM2AlsBv4kMzfWck5pn7wHV5IkSUPMtGnT+PGPfwzA3Xffzamnnso555wDwMiRI7n55ps544wzmDt37j7HefTRRxkxYgQf//jH97Qdc8wxzJ8/v1fzmj59Ohs2bOjVsd3p9RXUiGgAlgDnAuOBiyJifKduPwKmZuZE4F5gcW/PJ0mSJEn1ZteuXSxfvpxZs2YBpeW9U6ZM2avPsccey7Zt23j11Vf3OdaaNWs46aST9tnn+9///l5LfJ977rnaCthPtVxBPQVYl5nrASJiKTAbeGZ3h8z8PxX9nwD+uIbzSZIkSVJd2L59O5MnT2bTpk0cf/zxzJgxo6rjunu1S1ftc+fO5bHHHmPEiBGsXLkSKC3xffDBB3s/8RrVcg/qkcALFdvt5bbuXAF8p6sdETEnItoiom3z5s01TEmSJEmSBr/d96Bu3LiRzNxzD+r48eNZtWrVXn3Xr1/PqFGjOPjgg2lqamLLli177d+6dStjxozhhBNO4KmnntrTvmTJEpYvX06RMtgBec1MRPwxMBX4g672Z+ZtwG0AU6dOzQMxJ2lI8h5cSZKkvjeAf+OMHDmSm266ifPOO48/+7M/4+KLL+Zzn/scjzzyCGeffTbbt2/nE5/4BAsWLABK94VefPHFLFy4kNGjR3PfffcxadIkGhoaOPPMM/nLv/xLbr31Vv70T/8UgNdee23AautKLVdQNwFHVWw3l9v2EhFnA58GZmXm6zWcT5IkSZLqzoknnsjEiRP5+te/zkEHHcQDDzzAZz/7Wd75zncyYcIETj75ZObNmwfAxIkTmTdvHqeddhqTJ0/mH/7hH7j99tuB0jLf+++/n+9973u0trZyyimncNlll3HDDTfsOVfne1Dvvffebud10UUXMW3aNNauXUtzczN33HFHzbXG7vfl7PeBEcOA/wTOohRMVwIfzcw1FX1OpPRwpJmZ+V/VjDt16tRsa2vr1ZwAryBZ/wCf3/olSZIGu2effZbjjz9+oKcxJHT1s4yIVZk5tav+vV7im5k7I2Ie8DCl18zcmZlrIuI6oC0zlwF/C4wC/rl8U+7zmTmrt+eUpG4ZziVJkga9mu5BzcyHgIc6tV1T8b37t7pKkiRJkgqto6ODs84667faly9fTlNTU5+f74A8JEmS1M+8gixJUp/KzG5f2VJPmpqaWL16da+O7c3tpLU8JEmSJEmShpzGxkY6Ojp6FbBUkpl0dHTQ2Ni4X8d5BVWSJEmSKjQ3N9Pe3l6o94MORo2NjTQ3N+/XMQZUSdLgV+9LnOu9fknqY8OHD6e1tXWgp1GXDKiSJGlwG8iAbjiXpD5lQJUkSRqsvHouaYgxoEqSJGlwqveAXu/1a0gyoEqSJEkafAzoQ5IBVZIkSZIGkyEczn0PqiRJkiSpEAyokiRJkqRCMKBKkiRJkgrBgCpJkiRJKgQDqiRJkiSpEAyokiRJkqRCMKBKkiRJkgrBgCpJkiRJKgQDqiRJkiSpEAyokiRJkqRCMKBKkiRJkgrBgCpJkiRJKgQDqiRJkiSpEAyokiRJkqRCMKBKkiRJkgrBgCpJkiRJKgQDqiRJkiSpEAyokiRJkqRCMKBKkiRJkgrBgCpJkiRJKgQDqiRJkiSpEGoKqBExMyLWRsS6iFjYxf7pEfFUROyMiAtqOZckSZIkaWjrdUCNiAZgCXAuMB64KCLGd+r2PHA5cHdvzyNJkiRJqg/Dajj2FGBdZq4HiIilwGzgmd0dMnNDed+bNZxHkiRJklQHalnieyTwQsV2e7lNkiRJkqT9VoiHJEXEnIhoi4i2zZs3D/R0JEmSJEkDoJaAugk4qmK7udy23zLztsycmplTx40bV8OUJEmSJEmDVS0BdSVwXES0RsQI4EJgWd9MS5IkSZJUb3odUDNzJzAPeBh4FrgnM9dExHURMQsgIk6OiHbgw8A/RsSavpi0JEmSJGnoqeUpvmTmQ8BDndquqfi+ktLSX0mSJEmS9qkQD0mSJEmSJMmAKkmSJEkqBAOqJEmSJKkQDKiSJEmSpEIwoEqSJEmSCsGAKkmSJEkqBAOqJEmSJKkQDKiSJEmSpEIwoEqSJEmSCsGAKkmSJEkqBAOqJEmSJKkQDKiSJEmSpEIwoEqSJEmSCsGAKkmSJEkqBAOqJEmSJKkQDKiSJEmSpEIwoEqSJEmSCsGAKkmSJEkqBAOqJEmSJKkQDKiSJEmSpEIwoEqSJEmSCsGAKkmSJEkqBAOqJEmSJKkQDKiSJEmSpEIwoEqSJEmSCsGAKkmSJEkqBAOqJEmSJKkQDKiSJEmSpEIwoEqSJEmSCsGAKkmSJEkqhJoCakTMjIi1EbEuIhZ2sf8tEfGN8v4nI6KllvNJkiRJkoauXgfUiGgAlgDnAuOBiyJifKduVwBbMvMdwBeBG3p7PkmSJEnS0FbLFdRTgHWZuT4zfwMsBWZ36jMb+Gr5+73AWRERNZxTkiRJkjRE1RJQjwReqNhuL7d12SczdwKvAE01nFOSJEmSNERFZvbuwIgLgJmZ+bHy9iXAezJzXkWfn5T7tJe3nyv3eanTWHOAOeXNdwJrezWpvjEWeKnHXkOX9Vt/vdZfz7WD9Vu/9ddr/fVcO1i/9ddv/QNd+zGZOa6rHcNqGHQTcFTFdnO5ras+7RExDDgE6Og8UGbeBtxWw1z6TES0ZebUgZ7HQLF+66/X+uu5drB+67f+eq2/nmsH67f++q2/yLXXssR3JXBcRLRGxAjgQmBZpz7LgMvK3y8AHs3eXrKVJEmSJA1pvb6Cmpk7I2Ie8DDQANyZmWsi4jqgLTOXAXcAd0XEOuBXlEKsJEmSJEm/pZYlvmTmQ8BDndquqfi+A/hwLecYAIVYajyArL++1XP99Vw7WL/117d6rr+eawfrt/76Vdjae/2QJEmShrKI2AD8N2AXsA34LjAvM7eV978X+CxwMvAm8P8C/z0znynvvxz4WGae1sW4H8vMR8rbU4FrgVOBAH4GfAu4MTO3lMe5A9jeaYq/l5k/62bubwFuAc4GDgOeAz6Vmd/p1Q9DkqQDpJZ7UCVJGuo+kJmjgMnAicCnACJiGvCvwAPA24BW4Gng8Yh4e7WDl0PuCuBx4F2ZOQaYCewEJlV0/WFmjur06TKclg2j9Jq3P6D0gMJFwD0R0VLt3CRJGgg1LfGVJKkeZOYvIuJhSkEVYDHwtcz8u4puiyJiCqWroZdWOfRi4MuZ+fmKcz0P/FWN8/11eR67PRgRPwWmABtqGVuSpP7kFVRJknoQEc3AucC6iBgJvBf45y663gPMqHLMtwLTgG/21Tz3ca7/BvwesKa/zyVJUi0MqJIkde/+iNhKabnsLyld2TyM0u/Pn3fR/+eUXn5ejUPL4/xid0NELI6IlyPi1xGxqKLv75fbd3+eq7aAiBgO/C/gq5n5H9UeJ0nSQDCgSpLUvfMyczRwBvAuSuFzC6WHIh3RRf8jgJfK33cCw7voMxx4o6txMnNB+T7Ub7H3bThPZOaYis+x1Uw+In4HuAv4DTCvmmMkSRpIBlRJknqQmd8DvkLpybq/Bn5I169R+wiwvPz9eeDoiIjdO8vLgw8HNpbHeRL4UH/MuXzeOyg9ifj8zHyjP84jSVJf8iFJkiRV50vAhoiYBCwEHo6I/wC+TOn36V9Quqf05HL/J4EdwMKI+CLQAHweaAM2lvssKI+zCbgzM39Zvt+1FVhX43xvBY4Hzs7Mzq+okSSpkLyCKklSFTJzM/A14JrMfAx4H6Wrnz+nFDhPBE7LzP8q938deD+l5cHtwHpKr6T5SJZfQl4e50xgOvCfEfEypfetrgD+vuL00yJiW6fPyXQjIo4BrqT01OFfVBxzcd/8NCRJ6h9R/h0pSZIkSdKA8gqqJEmSJKkQvAdVkqRBKCKOBp7pZvf4zHz+QM5HkqS+4BJfSZIkSVIhuMRXkiRJklQIhVviO3bs2GxpaRnoaUiSJEmS+sGqVateysxxXe0rXEBtaWmhra1toKchSZIkSeoHEbGxu30u8ZUkSZIkFYIBVZIkSZJUCIVb4lurCV+dMKDn//fL/n1Azy9JkiRJg9WQC6iSJEmSVIs33niD9vZ2duzYMdBTGdQaGxtpbm5m+PDhVR9jQJUkSZKkCu3t7YwePZqWlhYiYqCnMyhlJh0dHbS3t9Pa2lr1cd6DKkmSJEkVduzYQVNTk+G0BhFBU1PTfl+FNqBKkiRJUieG09r15mdoQJUkSZIkFYL3oEqSJEnSPvT1m0KqefNHQ0MDEyZMYOfOnbS2tnLXXXcxZswYANasWcP8+fPZtGkTb775JpdeeimLFi0iIrj22msZNWoUV1999Z6xWlpaaGtrY+zYsbz44otcddVVPPHEExx66KGMGDGCBQsW8MEPfpAVK1Ywe/bsve4ZvfHGGzn77LN/a34vvPACl156KS+++CIRwZw5c/jzP//zmn82BtQhxtfsSJIkSYPfQQcdxOrVqwG47LLLWLJkCZ/+9KfZvn07s2bN4tZbb+Wcc87htdde4/zzz+eWW25h7ty5+xwzMznvvPO47LLLuPvuuwHYuHEjy5Yt29Pn9NNP58EHH+xxfsOGDeMLX/gCJ510Elu3bmXKlCnMmDGD8ePH11C1S3wlSZIkqdCmTZvGpk2bALj77rs59dRTOeeccwAYOXIkN998M9dff32P4zz66KOMGDGCj3/843vajjnmGObPn7/fczriiCM46aSTABg9ejTHH3/8njnWwoAqSZIkSQW1a9culi9fzqxZs4DS8t4pU6bs1efYY49l27ZtvPrqq/sca82aNXtCZXe+//3vM3ny5D2f5557rsc5btiwgR/96Ee85z3v6bFvT6oKqBExMyLWRsS6iFjYxf7pEfFUROyMiAs67dsVEavLn2Wdj5UkSZIk7W379u1MnjyZ3/3d3+XFF19kxowZVR3X3ZNzu2qfO3cukyZN4uSTT97Tdvrpp7N69eo9n2OPPXaf59u2bRvnn38+X/rSlzj44IOrmuO+9BhQI6IBWAKcC4wHLoqIzguLnwcuB+7uYojtmTm5/JlV43wlSZIkacjbfQ/qxo0byUyWLFkCwPjx41m1atVefdevX8+oUaM4+OCDaWpqYsuWLXvt37p1K2PGjOGEE07gqaee2tO+ZMkSli9fzubNm3s1xzfeeIPzzz+fiy++mA996EO9GkLopBoAABzLSURBVKOzaq6gngKsy8z1mfkbYCkwu7JDZm7IzB8Db/bJrCRJkiRJjBw5kptuuokvfOEL7Ny5k4svvpjHHnuMRx55BChdaf3EJz7BggULAJg+fTrLli1j69atANx3331MmjSJhoYGzjzzTHbs2MGtt966Z/zXXnutV/PKTK644gqOP/54PvnJT9ZY5f+vmqf4Hgm8ULHdDuzP4uLGiGgDdgLXZ+b9+3GstF98irEkSZL62kD/jXfiiScyceJEvv71r3PJJZfwwAMPMH/+fObOncuuXbu45JJLmDdvHgATJ05k3rx5nHbaaUQEhx9+OLfffjtQWuZ7//33c9VVV7F48WLGjRvHW9/6Vm644YY959p9D+puixYt4oILLqCzxx9/nLvuuosJEybs6f+5z32OP/zDP6yp1gPxmpljMnNTRLwdeDQi/j0z97rTNiLmAHMAjj766AMwJUmSJEkqrm3btu21/e1vf3vP9wkTJrBixYpuj73yyiu58soru9x3xBFHsHTp0i73nXHGGbzyyitVze+0004jM6vquz+qWeK7CTiqYru53FaVzNxU/ud6YAVwYhd9bsvMqZk5ddy4cdUOLUmSJEkaQqq5groSOC4iWikF0wuBj1YzeEQcCryWma9HxFjgVGBxbycrSZIkSTpwOjo6OOuss36rffny5TQ1NfX5+XoMqJm5MyLmAQ8DDcCdmbkmIq4D2jJzWUScDHwLOBT4QET8dWaeABwP/GNEvEnpau31mflMn1chCfAeXEmSJPWtpqYmVq9efcDOV9U9qJn5EPBQp7ZrKr6vpLT0t/NxPwAG9i9mSZIkSdpPmdntO0VVnd7co1rNPaiSJEmSVDcaGxvp6Ojol4cA1YvMpKOjg8bGxv067kA8xVeSJEmSBo3m5mba29vZvHnzQE9lUGtsbKS5+bcW2u6TAVWSJEmSKgwfPpzW1taBnkZdMqBKGhJ8QJQkSdLg5z2okiRJkqRCMKBKkiRJkgrBJb6SNAS4xFmSJA0FXkGVJEmSJBWCAVWSJEmSVAgu8ZUkDXoucZYkaWgwoEqSNMgZ0CVJQ4UBVZIkDWoDGdAN55LUt7wHVZIkSZJUCAZUSZIkSVIhuMRXkiRpkPL+Y0lDjQFVkiRJg5IBXRp6DKiSJEnSIGRA11BUVUCNiJnA3wENwO2ZeX2n/dOBLwETgQsz896KfZcBi8qbn83Mr/bFxCVJkiTVLwP60NRjQI2IBmAJMANoB1ZGxLLMfKai2/PA5cDVnY49DPgrYCqQwKrysVv6ZvqSJEmSVF+Gcjiv5im+pwDrMnN9Zv4GWArMruyQmRsy88fAm52OfR/wvzPzV+VQ+r+BmX0wb0mSJEnSEFNNQD0SeKFiu73cVo2qjo2IORHRFhFtmzdvrnJoSZIkSdJQUoj3oGbmbZk5NTOnjhs3bqCnI0mSJEkaANUE1E3AURXbzeW2atRyrCRJkiSpjlQTUFcCx0VEa0SMAC4EllU5/sPAORFxaEQcCpxTbpMkSZIkaS89BtTM3AnMoxQsnwXuycw1EXFdRMwCiIiTI6Id+DDwjxGxpnzsr4DPUAq5K4Hrym2SJEmSJO2lqvegZuZDwEOd2q6p+L6S0vLdro69E7izhjlKkiRJkupAIR6SJEmSJEmSAVWSJEmSVAgGVEmSJElSIRhQJUmSJEmFYECVJEmSJBWCAVWSJEmSVAgGVEmSJElSIRhQJUmSJEmFYECVJEmSJBWCAVWSJEmSVAgGVEmSJElSIRhQJUmSJEmFYECVJEmSJBWCAVWSJEmSVAgGVEmSJElSIRhQJUmSJEmFUFVAjYiZEbE2ItZFxMIu9r8lIr5R3v9kRLSU21siYntErC5//qFvpy9JkiRJGiqG9dQhIhqAJcAMoB1YGRHLMvOZim5XAFsy8x0RcSFwA/BH5X3PZebkPp63JEmSJGmIqeYK6inAusxcn5m/AZYCszv1mQ18tfz9XuCsiIi+m6YkSZIkaairJqAeCbxQsd1ebuuyT2buBF4Bmsr7WiPiRxHxvYg4vcb5SpIkSZKGqB6X+Nbo58DRmdkREVOA+yPihMx8tbJTRMwB5gAcffTR/TwlSZIkSVIRVXMFdRNwVMV2c7mtyz4RMQw4BOjIzNczswMgM1cBzwG/1/kEmXlbZk7NzKnjxo3b/yokSZIkSYNeNQF1JXBcRLRGxAjgQmBZpz7LgMvK3y8AHs3MjIhx5YcsERFvB44D1vfN1CVJkiRJQ0mPS3wzc2dEzAMeBhqAOzNzTURcB7Rl5jLgDuCuiFgH/IpSiAWYDlwXEW8AbwIfz8xf9UchkiRJkqTBrap7UDPzIeChTm3XVHzfAXy4i+O+CXyzxjlKkiRJkupANUt8JUmSJEnqdwZUSZIkSVIhGFAlSZIkSYVgQJUkSZIkFYIBVZIkSZJUCAZUSZIkSVIhGFAlSZIkSYVgQJUkSZIkFYIBVZIkSZJUCAZUSZIkSVIhGFAlSZIkSYVgQJUkSZIkFYIBVZIkSZJUCAZUSZIkSVIhGFAlSZIkSYVgQJUkSZIkFYIBVZIkSZJUCFUF1IiYGRFrI2JdRCzsYv9bIuIb5f1PRkRLxb5PldvXRsT7+m7qkiRJkqShpMeAGhENwBLgXGA8cFFEjO/U7QpgS2a+A/gicEP52PHAhcAJwEzglvJ4kiRJkiTtpZorqKcA6zJzfWb+BlgKzO7UZzbw1fL3e4GzIiLK7Usz8/XM/CmwrjyeJEmSJEl7iczcd4eIC4CZmfmx8vYlwHsyc15Fn5+U+7SXt58D3gNcCzyRmf+z3H4H8J3MvLfTOeYAc8qb7wTW1l5ar40FXhrA8w8067f+eq2/nmsH67d+66/X+uu5drB+66/f+ge69mMyc1xXO4Yd6Jl0JTNvA24b6HkARERbZk4d6HkMFOu3/nqtv55rB+u3fuuv1/rruXawfuuv3/qLXHs1S3w3AUdVbDeX27rsExHDgEOAjiqPlSRJkiSpqoC6EjguIlojYgSlhx4t69RnGXBZ+fsFwKNZWju8DLiw/JTfVuA44N/6ZuqSJEmSpKGkxyW+mbkzIuYBDwMNwJ2ZuSYirgPaMnMZcAdwV0SsA35FKcRS7ncP8AywE5ibmbv6qZa+UoilxgPI+utbPddfz7WD9Vt/favn+uu5drB+669fha29x4ckSZKkwSsiNgAfy8xHBnoukiT1pJolvpIk1a2I2BAR2yNiW0T8IiK+EhGjKva/NyIejYitEfFKRHy78n3hEXF5RDzWzbhnV2xPjYgHI2JLRLwcEc9ExN9ExKEV4+wqz6Py87b+/hlIknSgGFAlSerZBzJzFDAZOBH4FEBETAP+FXgAeBvQCjwNPB4Rb6928Ih4L7ACeBx4V2aOAWZSuj1mUkXXH2bmqE6fn9VcnSRJBVGI18xIkjQYZOYvIuJhSkEVYDHwtcz8u4puiyJiCqV3gV9a5dCLgS9n5ucrzvU88Fe1z1qSpMHDK6iSJFUpIpqBc4F1ETESeC/wz110vQeYUeWYbwWmAd/sq3lKkjRYGVAlSerZ/RGxFXgB+CWlK5uHUfo9+vMu+v8cGFvl2IeWx/nF7oaIWFy+D/XXEbGoou/vl9t3f57rTTGSJBWVAVWSpJ6dl5mjgTOAd1EKn1uAN4Ejuuh/BPBS+ftOYHgXfYYDb3Q1TmYuKN+H+i32vh3nicwcU/E5tqaqJEkqGAOqJElVyszvAV8BbszMXwM/BD7cRdePAMvL358Hjo6I2L2zvDz4cGBjeZwngQ/149QlSRoUfEiSJEn750vAhoiYBCwEHo6I/wC+TOn36l9Quqf05HL/J4EdwMKI+CLQAHweaAM2lvssKI+zCbgzM39Zvt+1FVh3YMqSJGngeQVVkqT9kJmbga8B12TmY8D7KF39/DmlwHkicFpm/le5/+vA+yktD24H1lN6Jc1HMjPLfR4DzgSmA/8ZES8D36X06pm/rzj9tC7eg3oykiQNEVH+3ShJkiRJ0oDyCqokSZIkqRC8B1WSpEEsIo4Gnulm9/jMfP5AzkeSpFq4xFeSJEmSVAiFu4I6duzYbGlpGehpSJIkSZL6wapVq17KzHFd7StcQG1paaGtrW2gpyFJkiRJ6gcRsbG7fT4kSZIkSZJUCAZUSZIkSVIhGFAlSZIkSYVQuHtQa/Xsu44f0PMf/x/PDuj5JUmSJPWdN954g/b2dnbs2DHQUxl0GhsbaW5uZvjw4VUfM+QCqiRJkiT1lfb2dkaPHk1LSwsRMdDTGTQyk46ODtrb22ltba36OJf4SpIkSVI3duzYQVNTk+F0P0UETU1N+33l2YAqSZIkSftgOO2d3vzcXOI7xHgPriRJkqTByoAqSZIkSVXq6wtC1VzgaWhoYMKECezcuZPW1lbuuusuxowZA8CaNWuYP38+mzZt4s033+TSSy9l0aJFRATXXnsto0aN4uqrr94zVktLC21tbYwdO5YXX3yRq666iieeeIJDDz2UESNGsGDBAj74wQ+yYsUKZs+evdf9ozfeeCNnn312l3McNWoU27Ztq/Gn4RJfSZIkSSq0gw46iNWrV/OTn/yEww47jCVLlgCwfft2Zs2axcKFC1m7di1PP/00P/jBD7jlllt6HDMzOe+885g+fTrr169n1apVLF26lPb29j19Tj/9dFavXr3n01047Us1BdSImBkRayNiXUQs7GL/JyPimYj4cUQsj4hjajmfJEmSJNWzadOmsWnTJgDuvvtuTj31VM455xwARo4cyc0338z111/f4ziPPvooI0aM4OMf//ietmOOOYb58+f3z8Sr1OuAGhENwBLgXGA8cFFEjO/U7UfA1MycCNwLLO7t+SRJkiSpnu3atYvly5cza9YsoLS8d8qUKXv1OfbYY9m2bRuvvvrqPsdas2YNJ5100j77fP/732fy5Ml7Ps8991xtBVShlntQTwHWZeZ6gIhYCswGntndITP/T0X/J4A/ruF8Uo98SJQkSZKGmu3btzN58mQ2bdrE8ccfz4wZM6o6rrun6HbVPnfuXB577DFGjBjBypUrgdIS3wcffLD3E++FWpb4Hgm8ULHdXm7rzhXAd2o4nyRJkiTVnd33oG7cuJHM3HMP6vjx41m1atVefdevX8+oUaM4+OCDaWpqYsuWLXvt37p1K2PGjOGEE07gqaee2tO+ZMkSli9fzubNm/u/oH04IA9Jiog/BqYCf9vN/jkR0RYRbQP9A5EkSZKkIho5ciQ33XQTX/jCF9i5cycXX3wxjz32GI888ghQutL6iU98ggULFgAwffp0li1bxtatWwG47777mDRpEg0NDZx55pns2LGDW2+9dc/4r7322oEvqpNalvhuAo6q2G4ut+0lIs4GPg38QWa+3tVAmXkbcBvA1KlTs4Y5SXXNJc6SJEn9a6D/3jnxxBOZOHEiX//617nkkkt44IEHmD9/PnPnzmXXrl1ccsklzJs3D4CJEycyb948TjvtNCKCww8/nNtvvx0oLfO9//77ueqqq1i8eDHjxo3jrW99KzfccMOec+2+B3W3RYsWccEFF/RrfbUE1JXAcRHRSimYXgh8tLJDRJwI/CMwMzN/WcO5JEmSJKkudX6/6Le//e093ydMmMCKFSu6PfbKK6/kyiuv7HLfEUccwdKlS7vcd8YZZ/DKK6/0eo691eslvpm5E5gHPAw8C9yTmWsi4rqImFXu9rfAKOCfI2J1RCyrecaSJEmSpCGpliuoZOZDwEOd2q6p+N7/b3KVJEmSJPW7jo4OzjrrrN9qX758OU1NTX1yjpoCqiQVhfffSpIk9a+mpiZWr17dr+c4IE/xlSRJkqTBKtPnuPZGb35uXkGVpCHAK8iSJPWPxsZGOjo6aGpqIiIGejqDRmbS0dFBY2Pjfh1nQJUkSZKkbjQ3N9Pe3s7mzZsHeiqDTmNjI83Nzft1jAFVkiRJkroxfPhwWltbB3oadcOAKkka9Op9iXO91y9JGjoMqJIkaVAbyIBuOJekvuVTfCVJkiRJheAVVEmSpEHK5d2ShhoDqiRJkgaleg/o9V6/hiYDqiRJkqRBx4A+NBlQJUmSJGkQGcrh3IckSZIkSZIKwYAqSZIkSSoEA6okSZIkqRAMqJIkSZKkQjCgSpIkSZIKwYAqSZIkSSoEA6okSZIkqRAMqJIkSZKkQjCgSpIkSZIKwYAqSZIkSSoEA6okSZIkqRAMqJIkSZKkQjCgSpIkSZIKwYAqSZIkSSoEA6okSZIkqRAMqJIkSZKkQjCgSpIkSZIKwYAqSZIkSSoEA6okSZIkqRBqCqgRMTMi1kbEuohY2MX+6RHxVETsjIgLajmXJEmSJGlo63VAjYgGYAlwLjAeuCgixnfq9jxwOXB3b88jSZIkSaoPw2o49hRgXWauB4iIpcBs4JndHTJzQ3nfmzWcR5IkSZJUB2pZ4nsk8ELFdnu5bb9FxJyIaIuIts2bN9cwJUmSJEnSYFWIhyRl5m2ZOTUzp44bN26gpyNJkiRJGgC1BNRNwFEV283lNkmSJEmS9lstAXUlcFxEtEbECOBCYFnfTEuSJEmSVG96HVAzcycwD3gYeBa4JzPXRMR1ETELICJOjoh24MPAP0bEmr6YtCRJkiRp6KnlKb5k5kPAQ53arqn4vpLS0l9JkiRJkvapEA9JkiRJkiTJgCpJkiRJKgQDqiRJkiSpEAyokiRJkqRCMKBKkiRJkgrBgCpJkiRJKgQDqiRJkiSpEAyokiRJkqRCMKBKkiRJkv6/9u491rKyvOP491cRFWy9gFrFUbyAKaJgUWKaeoUgEON4v0QTjLZeqUKtBtRQbdNExUqMMRp1yCEGES+oaLxAsFH/kFHEUWaEUbSAII5VKWqI4sjjH2tN3D3nbP+Yw17v6l7fTzLZt3XOfn6z9tn7fdZ619qjYIMqSZIkSRoFG1RJkiRJ0ijYoEqSJEmSRsEGVZIkSZI0CjaokiRJkqRRsEGVJEmSJI2CDaokSZIkaRRsUCVJkiRJo2CDKkmSJEkaBRtUSZIkSdIo2KBKkiRJkkbBBlWSJEmSNAo2qJIkSZKkUbBBlSRJkiSNgg2qJEmSJGkUbFAlSZIkSaNggypJkiRJGgUbVEmSJEnSKNigSpIkSZJGwQZVkiRJkjQKNqiSJEmSpFHYUIOa5PgkO5NcneS0dR6/U5Lz+8e3Jjl4I88nSZIkSVpee92gJrkD8F7gBOAw4AVJDlu12EuBm6rqocBZwNv39vkkSZIkScttI3tQjwaurqofVdWtwEeBzauW2Qyc01//BHBMkmzgOSVJkiRJS2ojDepBwI9nbl/f37fuMlW1G7gZOGADzylJkiRJWlL7tC4AIMnLgJf1N3+TZGfDcg4Efr7XP/3/fwex+c0/1fxTzg7mN7/59y7/lLOD+c1/+1XSxpTzt87+wHkPbKRBvQHYNHP7/v196y1zfZJ9gLsBv1j9i6rqA8AHNlDL7SbJZVX16NZ1tGJ+8081/5Szg/nNb/6p5p9ydjC/+aebf8zZNzLF95vAIUkelGRf4PnAhauWuRA4qb/+bODLVVUbeE5JkiRJ0pLa6z2oVbU7ycnAl4A7AGdX1Y4k/wZcVlUXAluADye5GvglXRMrSZIkSdIaGzoGtao+D3x+1X1nzFz/LfCcjTxHA6OYatyQ+adtyvmnnB3Mb/5pm3L+KWcH85t/ukabPc64lSRJkiSNwUaOQZUkSZIk6XZjg9pL8rAk22b+/SrJKa3rGlKSU5PsSLI9yXlJ7ty6pqEkeW2fe8cU1nuSs5P8LMn2mfvumeTiJD/oL+/RssZFmpP/Of36vy3JKM9qd3uZk//MJFcl+W6STyW5e8saF2lO/n/vs29LclGS+7WscZHWyz/z2OuSVJIDW9S2aHPW/VuS3DDz+X9iyxoXad66T/JP/d//jiTvaFXfos1Z/+fPrPtrkmxrWeMizcl/ZJJL+/yXJTm6ZY2LMif7EUm+nuSKJJ9N8lcta1ykJJuS/FeS7/V/56/t7x/l2M8GtVdVO6vqyKo6EjgKuAX4VOOyBpPkIOA1wKOr6nC6E19N4qRWSQ4H/hE4GjgCeGqSh7atauFWgONX3XcacElVHQJc0t9eViuszb8deCbw1cGrGd4Ka/NfDBxeVY8Evg+cPnRRA1phbf4zq+qR/WfA54Az1vzU8lhhbX6SbAKOA64buqABrbBOduCsPWOA/vway2qFVfmTPAnYDBxRVQ8H3tmgrqGssCp/VT1vZvz3SeCCFoUNZIW1r/93AG/t85/R315GK6zN/iHgtKp6BN2Y//VDFzWg3cDrquow4LHAq5McxkjHfjao6zsG+GFVXdu6kIHtA9yl/87a/YCfNK5nKH8DbK2qW6pqN/AVukZlaVXVV+nOrD1rM3BOf/0c4OmDFjWg9fJX1ZVVtbNRSYOak/+i/vUPcCndd1svpTn5fzVzc39gaU/QMOfvH+As4A1MM/skzMn/SuBtVfW7fpmfDV7YQP7c+k8S4LnAeYMWNaA5+QvYs+fwbizp2G9O9kP500bpi4FnDVrUgKrqxqq6vL/+a+BK4CBGOvazQV3f81niN6j1VNUNdFtNrwNuBG6uqovaVjWY7cDjkhyQZD/gRGBT45pauE9V3dhf/ylwn5bFqKmXAF9oXcTQkvxHkh8DL2S596CukWQzcENVfad1LY2c3E/xPnssU9wGdCjdZ+DWJF9J8pjWBTXyOGBXVf2gdSEDOwU4s3/veyfLPXtmtR10DRp03zoyibFfkoOBRwFbGenYzwZ1lST7Ak8DPt66liH1H8ibgQcB9wP2T/KitlUNo6quBN4OXAR8EdgG/KFpUY1Vd3rvpd2LovmSvIluKtC5rWsZWlW9qao20WU/uXU9Q+k3zL2RiTXlM94HPAQ4km4D7X+2LWdw+wD3pJv293rgY/3exKl5ARPbOdF7JXBq/953KrClcT1DegnwqiTfAv4SuLVxPQuX5K50U9lPWTVzaFRjPxvUtU4ALq+qXa0LGdixwH9X1f9U1e/pjsH4u8Y1DaaqtlTVUVX1eOAmumPwpmZXkvsC9JdLO81L60vyYuCpwAtr2t9Bdi5LPNVrHQ+h2zj5nSTX0E3vvjzJXzetaiBVtauq/lBVtwEfpDsfwZRcD1xQnW8AtwFLeZKsefpDm54JnN+6lgZO4k/H3X6cCb3+q+qqqjquqo6i2zjxw9Y1LVKSO9I1p+dW1Z51Psqxnw3qWlPdgnYd8Ngk+/VbTo+hm58+CUnu3V8+gO5D6iNtK2riQroPKvrLzzSsRQNLcjzd8YdPq6pbWtcztCSHzNzcDFzVqpahVdUVVXXvqjq4qg6ma1j+tqp+2ri0QewZnPWeQXfYx5R8GngSQJJDgX2BnzetaHjHAldV1fWtC2ngJ8AT+utPBiYzxXlm7PcXwJuB97etaHH6sf0W4MqqetfMQ6Mc+2XaG8n/ryT70zVqD66qm1vXM7QkbwWeRze979vAP+w5acKyS/I14ADg98A/V9UljUtaqCTnAU+k20q+C/hXukHKx4AHANcCz62qpTyZyJz8vwTeA9wL+F9gW1U9pVWNizQn/+nAnYBf9ItdWlWvaFLggs3JfyLwMLq9R9cCr+iPzV866+Wvqi0zj19Dd0b3pWtS5qz7J9JN7y3gGuDlM8dkLZU5+T8MnE33f3Ar8C9V9eVWNS7SvNd+khW697ylbVBg7vrfCbybbqr3b4FXVdW3WtW4KHOy3xV4db/IBcDpyzp7KMnfA18DrqD7nIPu0I6tjHDsZ4MqSZIkSRoFp/hKkiRJkkbBBlWSJEmSNAo2qJIkSZKkUbBBlSRJkiSNgg2qJEmSJGkUbFAlSZIkSaNggypJkiRJGgUbVEmSJEnSKPwRqi+4df6nft0AAAAASUVORK5CYII=\n", + "text/plain": [ + "<Figure size 1152x576 with 4 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 588 + }, + "id": "AsCAmTELQuf4", + "outputId": "16196351-a0d4-44c3-8576-90789a1ae6fa" + }, + "source": [ + "df.plot.bar(rot=0, subplots=True, figsize=(20, 8))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "array([<matplotlib.axes._subplots.AxesSubplot object at 0x7fbfb88a6650>,\n", + " <matplotlib.axes._subplots.AxesSubplot object at 0x7fbfb8855e50>,\n", + " <matplotlib.axes._subplots.AxesSubplot object at 0x7fbfb8886b90>,\n", + " <matplotlib.axes._subplots.AxesSubplot object at 0x7fbfb87c0610>],\n", + " dtype=object)" + ] + }, + "metadata": {}, + "execution_count": 135 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABIcAAAHiCAYAAABycKzVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdfZSeZX0n8O9vQ2LAoMAQ9lgjzBRZJQoECLhRpD0IFNYj0IpdKBU8xRPdJbRH12Vxy2FZ161IadHW6JaDVmVLqcW31FLZY1hd8a0kNm4NyDZmQSb1JaQoRBMl4do/5iE7jBMymWfen8/nnOf43Pd93dfLzOF28j3Xdd3VWgsAAAAAvemfTXcHAAAAAJg+wiEAAACAHiYcAgAAAOhhwiEAAACAHiYcAgAAAOhhwiEAAACAHiYcAgAAAOhhwiEAoOdV1YNVtaOqtlfVo1X111X1gs61D1fVO/dyX6uqH3fue+pz1d7uq6r+zj0HTP6oAADGRjgEADDkNa21RUmel+T7Sf54jPed0FpbNOxzw+R1EQBg4gmHAACGaa3tTHJHkqXT3RcAgKkgHAIAGKaqDkryr5N8dbr7AgAwFax3BwAY8qmq2pXk2Um2JvmVMd739ap6ctjxv26t3TXhvQMAmCTCIQCAIRe01j5XVfOSnJ/kC1U1lqVlJ7XWNo1yfleS+SPOzU/yZOcDADAjWFYGADBMa213a+0TSXYnOa2Lqr6TpH/EuYEkD7fWhEMAwIwhHAIAGKaGnJ/k0CT3d07Pq6qFwz4LxlDVx5O8uqrOrqp5VfULSa5JcvskdR0AYFyEQwAAQ/6qqrYneSzJf01yWWttY+fa1Ul2DPvcPey+b1TV9mGf9yRJ596Lk7wryT8l+UqSryX5z1MyGgCAMarW2nT3AQAAAIBpYuYQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA87YLo7MNLhhx/e+vv7p7sbAAAAAHPG+vXrH2mtLR7t2owLh/r7+7Nu3brp7gYAAADAnFFVD+3tmmVlAAAAAD1MOAQAAADQw2bcsjL+v/6r/3pK2nnw+ldPSTsAAADAzCMcAgAAAOakJ554IoODg9m5c+d0d2XKLFy4MEuWLMn8+fPHfI9wCAAAAJiTBgcHc/DBB6e/vz9VNd3dmXSttWzbti2Dg4MZGBgY8332HAIAAADmpJ07d6avr68ngqEkqar09fXt90wp4RAAAAAwZ/VKMPSU8YxXOAQAAAAwSebNm5dly5blhBNOyEknnZQvf/nLSZIHH3wwL33pS3+u/Bve8IYMDAxk2bJlWbZsWV7+8pcnSa677rrceOONTyvb39+fRx55pOs+2nMIAAAA6AkT/Vbwsbz9+8ADD8yGDRuSJHfddVfe/va35wtf+MIz3vP7v//7ufDCCyekj2Nh5hAAAADAFHjsscdy6KGHTnc3fo6ZQwAAAACTZMeOHVm2bFl27tyZ7373u7n77rv3ec+///f/Pu985zuTJC95yUvyZ3/2Z5PaxzGFQ1V1TpL3JpmX5JbW2vUjrj8ryUeTnJxkW5J/3Vp7sHPt+CR/kuQ5SZ5Mckprbf+2zQYAAACYhYYvK/vKV76SSy+9NN/85jef8Z7RlpXtbaPpidhwe5/LyqpqXpLVSc5NsjTJxVW1dESxy5M82lp7YZKbkry7c+8BSf57kje31l6S5JeTPNF1rwEAAABmmRUrVuSRRx7J1q1b9/vevr6+PProo0879/jjj+eQQw7pul9j2XPo1CSbWmubW2s/S3J7kvNHlDk/yUc63+9I8qoaiq7OTvK/W2vfSJLW2rbW2u6uew0AAAAwy3zrW9/K7t2709fXt9/3nn766VmzZk0ef/zxJMknPvGJnHDCCZk3b17X/RrLsrLnJ3l42PFgkpftrUxrbVdV/ShJX5J/kaRV1V1JFie5vbV2Q9e9BgAAAJgFntpzKElaa/nIRz6yJ9B54IEHsmTJkj1lb7rppiRP33MoSf72b/82xx9/fFatWpXTTjstVZUjjjgit9xyy4T0cbI3pD4gyWlJTknykyRrq2p9a23t8EJVtTLJyiQ58sgjJ7lLAAAAQC8ay6vnJ9ru3aMvoOrv788TT/z8zjuve93r9lrXm970przpTW+asL49ZSzLyrYkecGw4yWdc6OW6ewz9NwMbUw9mOR/tdYeaa39JMmdSU4a2UBr7ebW2vLW2vLFixfv/ygAAAAAGJexhEP3JjmmqgaqakGSi5KsGVFmTZLLOt8vTHJ3a60luSvJcVV1UCc0+qUk901M1wEAAADo1j6XlXX2EFqVoaBnXpIPtdY2VtU7kqxrra1J8sEkt1bVpiT/lKEAKa21R6vqDzMUMLUkd7bW/nqSxgIAAADAfhrTnkOttTsztCRs+Llrh33fmWTURXGttf+eodfZAwAAAEyp1lqGXqjeG4YWcu2fsSwrAwAAAJh1Fi5cmG3bto0rMJmNWmvZtm1bFi5cuF/3TfbbygAAAACmxZIlSzI4OJitW7dOd1emzMKFC7NkyZL9ukc4BAAAAMxJ8+fPz8DAwHR3Y8azrAwAAACghwmHAAAAAHqYcAgAAACghwmHAAAAAHqYcAgAAACghwmHAAAAAHrYnHmVff/Vfz0l7Tx4/aunpB2AfZmq517i2QcAAHOZmUMAAAAAPWzOzBwCeCZm2cwec3EmqDGNn/+eAAAmn3CIKeUfE7ODIAWAmcr/RzGd5uLfssbUnakalzF1x5j2TTgEAJDZ/Qfd3szFMc1Fc/X3NBdDB4C5yp5DAAAAAD1MOAQAAADQw4RDAAAAAD1sTOFQVZ1TVQ9U1aaqunqU68+qqr/oXP9aVfWPuH5kVW2vqrdNTLcBAAAAmAj7DIeqal6S1UnOTbI0ycVVtXREscuTPNpae2GSm5K8e8T1P0zyN913FwAAAICJNJa3lZ2aZFNrbXOSVNXtSc5Pct+wMucnua7z/Y4k76uqaq21qrogyf9N8uMJ6zXMIHP1DSMAAAD0hrEsK3t+koeHHQ92zo1aprW2K8mPkvRV1aIk/yHJf+6+qwAAAABMtMnekPq6JDe11rY/U6GqWllV66pq3datWye5SwAAAAA8ZSzLyrYkecGw4yWdc6OVGayqA5I8N8m2JC9LcmFV3ZDkkCRPVtXO1tr7ht/cWrs5yc1Jsnz58jaegQAAAACw/8YSDt2b5JiqGshQCHRRkt8YUWZNksuSfCXJhUnubq21JK98qkBVXZdk+8hgCAAAAIDps89wqLW2q6pWJbkrybwkH2qtbayqdyRZ11pbk+SDSW6tqk1J/ilDARIAAAAAM9xYZg6ltXZnkjtHnLt22PedSV63jzquG0f/AAAAAJhEk70hNQAAAAAzmHAIAAAAoIcJhwAAAAB6mHAIAAAAoIcJhwAAAAB6mHAIAAAAoIcJhwAAAAB6mHAIAAAAoIcJhwAAAAB6mHAIAAAAoIcJhwAAAAB6mHAIAAAAoIcJhwAAAAB6mHAIAAAAoIcJhwAAAAB6mHAIAAAAoIcJhwAAAAB62JjCoao6p6oeqKpNVXX1KNefVVV/0bn+tarq75w/q6rWV9Xfd/73jIntPgAAAADd2Gc4VFXzkqxOcm6SpUkurqqlI4pdnuTR1toLk9yU5N2d848keU1r7bgklyW5daI6DgAAAED3xjJz6NQkm1prm1trP0tye5LzR5Q5P8lHOt/vSPKqqqrW2t+11v6xc35jkgOr6lkT0XEAAAAAujeWcOj5SR4edjzYOTdqmdbariQ/StI3osxrk3y9tfbT8XUVAAAAgIl2wFQ0UlUvydBSs7P3cn1lkpVJcuSRR05FlwAAAADI2GYObUnygmHHSzrnRi1TVQckeW6SbZ3jJUk+meTS1tq3R2ugtXZza215a2354sWL928EAAAAAIzbWMKhe5McU1UDVbUgyUVJ1owosyZDG04nyYVJ7m6ttao6JMlfJ7m6tfalieo0AAAAABNjn+FQZw+hVUnuSnJ/ko+11jZW1Tuq6rxOsQ8m6auqTUnemuSp192vSvLCJNdW1YbO54gJHwUAAAAA4zKmPYdaa3cmuXPEuWuHfd+Z5HWj3PfOJO/sso8AAAAATJKxLCsDAAAAYI4SDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8bUzhUVedU1QNVtamqrh7l+rOq6i86179WVf3Drr29c/6BqvqVies6AAAAAN3aZzhUVfOSrE5ybpKlSS6uqqUjil2e5NHW2guT3JTk3Z17lya5KMlLkpyT5P2d+gAAAACYAcYyc+jUJJtaa5tbaz9LcnuS80eUOT/JRzrf70jyqqqqzvnbW2s/ba393ySbOvUBAAAAMANUa+2ZC1RdmOSc1tobO8evT/Ky1tqqYWW+2Skz2Dn+dpKXJbkuyVdba/+9c/6DSf6mtXbHiDZWJlnZOXxRkge6H9qYHJ7kkSlqa6oYU3eem+RHU9CO39PsMRfHZUzjN1XPiMTvabYwptnDc2J2MKbZwZi6498c3ZmL45qqMR3VWls82oUDpqDxfWqt3Zzk5qlut6rWtdaWT3W7k8mYum7r5tbayn2X7Lodv6dZYi6Oy5i6amdKnhGdtvyeZgFjmj08J2YHY5odjKnrtvybowtzcVwzYUxjWVa2JckLhh0v6ZwbtUxVHZChJHTbGO+FmeSvprsDwIzmGQHsi+cEsC+eE8w4YwmH7k1yTFUNVNWCDG0wvWZEmTVJLut8vzDJ3W1ovdqaJBd13mY2kOSYJH87MV2Hidda86AG9sozAtgXzwlgXzwnmIn2uaystbarqlYluSvJvCQfaq1trKp3JFnXWluT5INJbq2qTUn+KUMBUjrlPpbkviS7klzRWts9SWMZjylfyjYFjGl2MKbZYy6Oy5hmB2OaHYxp9piL4zKm2cGYZgdjmj3m4rimfUz73JAaAAAAgLlrLMvKAABmjap6sKp2VNX2qvpeVX24qhYNu/7yqrq7qh6vqh9V1V9V1dJh199QVffspd4zhx0vr6rPVNWjVfXDqrqvqv5rVR06rJ7dnX4M//zCPvq/qqrWVdVPq+rDE/JDAQB4BsIhAGAuek1rbVGSZUlOTPL2JKmqFUn+R5JPJ/mFJANJvpHkS1X1i2OtvKpenuTzSb6U5MWttUOSnJOhZfQnDCv6ldbaohGff9xH9f+Y5J1JPjTW/gAAdGNGvMoeAGAytNa+V1V3ZSgkSpIbkny0tfbeYcWuqaqTk1yX5NIxVn1Dkj9trb1rWFvfSfKfJqDPn0iGZiZl6E2vAACTyswhAGDOqqolSc5NsqmqDkry8iR/OUrRjyU5a4x1PjvJiiQfn6h+AgBMJ+EQADAXfaqqHk/ycJIfZGhGz2EZ+tvnu6OU/26Sw8dY96Gder731ImquqGz79CPq+qaYWX/Zef8U59vj2cwAACTSTgEAMxFF7TWDk7yy0lenKHg59EkTyZ53ijln5fkkc73XUnmj1JmfpInRquntXZVZ9+hT+bpy/a/2lo7ZNjn6K5GBQAwCYRDAMCc1Vr7QpIPJ7mxtfbjJF9J8rpRiv56krWd799JcmRV1VMXO0vSjkjyUKeeryX5tUnsOgDAlLEhNQAw170nyYNVdUKSq5PcVVXfSvKnGfpb6N9laA+hUzrlv5ZkZ5Krq+qmJPOSvCvJuiQPdcpc1alnS5IPtdZ+0NnfaCDJpm46W1UHdPo1L8m8qlqYZFdrbVc39QIA7I2ZQwDAnNZa25rko0muba3dk+RXMjTr57sZCntOTHJaa+0fOuV/muTVGVqSNphkc4Zee//rrbXWKXNPkjOSnJ7k/1TVD5N8NkOvt//jYc2vqKrtIz6n5Jldk2RHhoKs3+x8v+YZ7wAA6EJ1/sYBAAAAoAeZOQQAAADQw+w5BAAwharqyCT37eXy0tbad6ayPwAAlpUBAAAA9LAZN3Po8MMPb/39/dPdDQAAAIA5Y/369Y+01haPdm3GhUP9/f1Zt27ddHcDAAAAYM6oqof2ds2G1AAAAAA9TDgEAAAA0MOEQwAAAAA9bMbtOTRu1z13itr50dS0AwAAAD3kiSeeyODgYHbu3DndXZnVFi5cmCVLlmT+/PljvqercKiqzkny3iTzktzSWrt+xPU3J7kiye4k25OsbK3d102bAAAAwNwzODiYgw8+OP39/amq6e7OrNRay7Zt2zI4OJiBgYEx3zfuZWVVNS/J6iTnJlma5OKqWjqi2G2tteNaa8uS3JDkD8fbHgAAADB37dy5M319fYKhLlRV+vr69nv2VTd7Dp2aZFNrbXNr7WdJbk9y/vACrbXHhh0+O0nroj0AAABgDhMMdW88P8NulpU9P8nDw44Hk7xsZKGquiLJW5MsSHJGF+0BAAAAMMEmfUPq1trqJKur6jeSXJPkspFlqmplkpVJcuSRR052l2YPm2wDAADQqyb638T+7btX3Swr25LkBcOOl3TO7c3tSS4Y7UJr7ebW2vLW2vLFixd30SUAAACA8Zk3b16WLVuWl770pXnNa16TH/7wh3uubdy4MWeccUZe9KIX5Zhjjsl/+S//Ja0N7Z5z3XXX5cYbb3xaXf39/XnkkUeSJN///vfzG7/xG/nFX/zFnHzyyVmxYkU++clPJkk+//nP57nPfW6WLVu25/O5z31ur338rd/6rRxxxBF56UtfOmHj7iYcujfJMVU1UFULklyUZM3wAlV1zLDDVyf5hy7aAwAAAJg0Bx54YDZs2JBvfvObOeyww7J69eokyY4dO3Leeefl6quvzgMPPJBvfOMb+fKXv5z3v//9+6yztZYLLrggp59+ejZv3pz169fn9ttvz+Dg4J4yr3zlK7Nhw4Y9nzPPPHOv9b3hDW/IZz/72e4HO8y4w6HW2q4kq5LcleT+JB9rrW2sqndU1XmdYquqamNVbcjQvkM/t6QMAAAAYKZZsWJFtmwZWiB122235RWveEXOPvvsJMlBBx2U973vfbn++uv3Wc/dd9+dBQsW5M1vfvOec0cddVSuvPLKcfXr9NNPz2GHHTaue/emqz2HWmt3JrlzxLlrh33/nW7qBwAAAJhqu3fvztq1a3P55ZcnGVpSdvLJJz+tzNFHH53t27fnscceG62KPTZu3JiTTjrpGct88YtfzLJly/Ycf/zjH8/RRx89zt7vv0nfkBoAAABgNtixY0eWLVuWLVu25Nhjj81ZZ501pvv29vr40c5fccUVueeee7JgwYLce++9SYaWlX3mM58Zf8e71M2eQwAAAABzxlN7Dj300ENpre3Zc2jp0qVZv37908pu3rw5ixYtynOe85z09fXl0Ucffdr1xx9/PIccckhe8pKX5Otf//qe86tXr87atWuzdevWyR/QGJk5BAAAAMw80/jq+YMOOih/9Ed/lAsuuCD/9t/+21xyySX5vd/7vXzuc5/LmWeemR07duS3f/u3c9VVVyUZ2gfokksuydVXX52DDz44n/jEJ3LCCSdk3rx5OeOMM/If/+N/zAc+8IH8m3/zb5IkP/nJT6ZtbKMxcwgAAABghBNPPDHHH398/vzP/zwHHnhgPv3pT+ed73xnXvSiF+W4447LKaecklWrViVJjj/++KxatSqnnXZali1blv/23/5bbrnlliRDS8s+9alP5Qtf+EIGBgZy6qmn5rLLLsu73/3uPW09tefQU5877rhjr/26+OKLs2LFijzwwANZsmRJPvjBD3Y91mqtdV3JRFq+fHlbt27d/t943XMnvjOjtjOFyeVcHBMAAACM4v7778+xxx473d2YE0b7WVbV+tba8tHKmzkEAAAA0MPsOQQAAAAwg2zbti2vetWrfu782rVr09fXN+HtCYcAAACAGaG1ttfXwveSvr6+bNiwYVz3jmf7IMvKAAAAgGm3cOHCbNu2bVzhBkNaa9m2bVsWLly4X/eZOQQwW03VpvWJjesBAJh0S5YsyeDgYLZu3TrdXZnVFi5cmCVLluzXPcIhAAAAYNrNnz8/AwMD092NnmRZGQAAAEAPM3MI+HmWKwEAAPQMM4cAAAAAephwCAAAAKCHWVYGwMwyVcsaLWkEAIAkwiGgV9hHCQAAYFTCIabWXJwRIHQA9mUuPvsAAJgz7DkEAAAA0MO6Coeq6pyqeqCqNlXV1aNcf2tV3VdV/7uq1lbVUd20BwAAAMDEGnc4VFXzkqxOcm6SpUkurqqlI4r9XZLlrbXjk9yR5IbxtgcAAADAxOtmz6FTk2xqrW1Okqq6Pcn5Se57qkBr7X8OK//VJL/ZRXsAwExhHyUAgDmjm2Vlz0/y8LDjwc65vbk8yd900R4AAAAAE2xK3lZWVb+ZZHmSX9rL9ZVJVibJkUceORVdAgB4Om+fBAB6VDczh7YkecGw4yWdc09TVWcm+d0k57XWfjpaRa21m1try1tryxcvXtxFlwAAAADYH92EQ/cmOaaqBqpqQZKLkqwZXqCqTkzyJxkKhn7QRVsAAAAATIJxLytrre2qqlVJ7koyL8mHWmsbq+odSda11tYk+f0ki5L8ZVUlyXdaa+dNQL8BANgXS+UAgDHoas+h1tqdSe4cce7aYd/P7KZ+AAB4GoEXAEy4bpaVAQAAADDLTcnbygAAgL2Yq7OhpmpcZngBdE04BAAAMBYCL2COEg4BAAD0qrkYeM3V2XgwiYRDAAAAMJPNxcBrLo5pFhMOAQAAAHRrFgde3lYGAAAA0MOEQwAAAAA9TDgEAAAA0MOEQwAAAAA9TDgEAAAA0MOEQwAAAAA9TDgEAAAA0MOEQwAAAAA9TDgEAAAA0MOEQwAAAAA9TDgEAAAA0MOEQwAAAAA9TDgEAAAA0MOEQwAAAAA9rKtwqKrOqaoHqmpTVV09yvXTq+rrVbWrqi7spi0AAAAAJt64w6GqmpdkdZJzkyxNcnFVLR1R7DtJ3pDktvG2AwAAAMDkOaCLe09Nsqm1tjlJqur2JOcnue+pAq21BzvXnuyiHQAAAAAmSTfLyp6f5OFhx4Odc/utqlZW1bqqWrd169YuugQAAADA/pgRG1K31m5urS1vrS1fvHjxdHcHAAAAoGd0Ew5tSfKCYcdLOucAAAAAmCW6CYfuTXJMVQ1U1YIkFyVZMzHdAgAAAGAqjDscaq3tSrIqyV1J7k/ysdbaxqp6R1WdlyRVdUpVDSZ5XZI/qaqNE9FpAAAAACZGN28rS2vtziR3jjh37bDv92ZouRkAAAAAM9CM2JAaAAAAgOkhHAIAAADoYcIhAAAAgB4mHAIAAADoYcIhAAAAgB4mHAIAAADoYcIhAAAAgB4mHAIAAADoYcIhAAAAgB4mHAIAAADoYcIhAAAAgB4mHAIAAADoYcIhAAAAgB4mHAIAAADoYcIhAAAAgB4mHAIAAADoYcIhAAAAgB4mHAIAAADoYcIhAAAAgB7WVThUVedU1QNVtamqrh7l+rOq6i86179WVf3dtAcAAADAxBp3OFRV85KsTnJukqVJLq6qpSOKXZ7k0dbaC5PclOTd420PAAAAgInXzcyhU5Nsaq1tbq39LMntSc4fUeb8JB/pfL8jyauqqrpoEwAAAIAJ1E049PwkDw87HuycG7VMa21Xkh8l6euiTQAAAAAmULXWxndj1YVJzmmtvbFz/PokL2utrRpW5pudMoOd4293yjwyoq6VSVZ2Dl+U5IFxdWr/HZ7kkX2Wml2MqTvPzVCIOdn8nmaPuTguYxq/qXpGJH5Ps4UxzR6eE7ODMc0OxtQd/+bozlwc11SN6ajW2uLRLhzQRaVbkrxg2PGSzrnRygxW1QEZ+o9g28iKWms3J7m5i76MS1Wta60tn+p2J5Mxdd3Wza21lfsu2XU7fk+zxFwclzF11c6UPCM6bfk9zQLGNHt4TswOxjQ7GFPXbfk3Rxfm4rhmwpi6WVZ2b5JjqmqgqhYkuSjJmhFl1iS5rPP9wiR3t/FOVYKp8VfT3QFgRvOMAPbFcwLYF88JZpxxzxxqre2qqlVJ7koyL8mHWmsbq+odSda11tYk+WCSW6tqU5J/ylCABDNWa82DGtgrzwhgXzwngH3xnGAm6mZZWVprdya5c8S5a4d935nkdd20McmmfCnbFDCm2cGYZo+5OC5jmh2MaXYwptljLo7LmGYHY5odjGn2mIvjmvYxjXtDagCAmaiqHkzyz5PsTrI9yWeTrGqtbe9cf3mSdyY5JcmTSf5Xkv/QWruvc/0NSd7YWjttlHrf2Fr7XOd4eZLrkrwiSSX5xySfTHJja+3RTj0fTLJjRBf/RWvtH/fS92cleX+SM5McluTbSd7eWvubcf0wAADGoJs9hwAAZqrXtNYWJVmW5MQkb0+SqlqR5H8k+XSSX0gykOQbSb5UVb841so7AdPnk3wpyYtba4ckOSfJriQnDCv6ldbaohGfUYOhjgOSPJzklzL0Io9rknysqvrH2jcAgP3V1bIyAICZrLX2vaq6K0MhUZLckOSjrbX3Dit2TVWdnKFZQJeOseobkvxpa+1dw9r6TpL/1GV/f9zpx1M+U1X/N8nJSR7spm4AgL0xcwgAmLOqakmSc5NsqqqDkrw8yV+OUvRjSc4aY53PTrIiyccnqp/P0NY/T/Ivkmyc7LYAgN4lHAIA5qJPVdXjGVqi9YMMzeg5LEN/+3x3lPLfTXL4GOs+tFPP9546UVU3VNUPq+rHVXXNsLL/snP+qc+3xzqAqpqf5M+SfKS19q2x3gcAsL+EQwDAXHRBa+3gJL+c5MUZCn4ezdAG1M8bpfzzkjzS+b4ryfxRysxP8sRo9bTWrursO/TJPH3Z/ldba4cM+xw9ls5X1T9LcmuSnyVZNZZ7AADGSzgEAMxZrbUvJPlwht4g9uMkX0nyulGK/nqStZ3v30lyZFXVUxc7S9KOSPJQp56vJfm1yehzp90PZuiNa69trT0xGe0AADzFhtQAwFz3niQPVtUJSa5OcldVfSvJn2bob6F/l6E9hE7plP9akp1Jrq6qm5LMS/KuJOuSPNQpc1Wnni1JPtRa+0Fnf6OBJOk4dt0AACAASURBVJu67O8Hkhyb5MzW2o4u6wIA2CczhwCAOa21tjXJR5Nc21q7J8mvZGjWz3czFPacmOS01to/dMr/NMmrM7QkbTDJ5gy99v7XW2utU+aeJGckOT3J/6mqHyb5bIZeb//Hw5pfUVXbR3xOyV5U1VFJ3pSht6t9b9g9l0zMTwMA4OdV528cAAAAAHqQmUMAAAAAPcyeQwAAU6iqjkxy314uL22tfWcq+wMAYFkZAAAAQA+zrAwAAACgh824ZWWHH3546+/vn+5uAAAAAMwZ69evf6S1tni0azMuHOrv78+6deumuxsAAAAAc0ZVPbS3a5aVAQAAAPQw4RAAAABAD5txy8rG67iPHDcl7fz9ZX8/Je0AAAAATIU5Ew4BAAAAs9cTTzyRwcHB7Ny5c7q7MqstXLgwS5Ysyfz588d8j3AIAAAAmHaDg4M5+OCD09/fn6qa7u7MSq21bNu2LYODgxkYGBjzffYcAgAAAKbdzp0709fXJxjqQlWlr69vv2dfCYcAAACAGUEw1L3x/AyFQwAAAAA9zJ5DAAAAwIwz0W8lH8vbx+fNm5fjjjsuu3btysDAQG699dYccsghSZKNGzfmyiuvzJYtW/Lkk0/m0ksvzTXXXJOqynXXXZdFixblbW972566+vv7s27duhx++OH5/ve/n7e85S356le/mkMPPTQLFizIVVddlV/91V/N5z//+Zx//vlP2yPoxhtvzJlnnvlz/Xv44Ydz6aWX5vvf/36qKitXrszv/M7vdP2zEQ7NYBP9H8LejOU/EAAAAJjrDjzwwGzYsCFJctlll2X16tX53d/93ezYsSPnnXdePvCBD+Tss8/OT37yk7z2ta/N+9///lxxxRXPWGdrLRdccEEuu+yy3HbbbUmShx56KGvWrNlT5pWvfGU+85nP7LN/BxxwQP7gD/4gJ510Uh5//PGcfPLJOeuss7J06dIuRm1ZGQAAAMDPWbFiRbZs2ZIkue222/KKV7wiZ599dpLkoIMOyvve975cf/31+6zn7rvvzoIFC/LmN795z7mjjjoqV1555X736XnPe15OOumkJMnBBx+cY489dk8fuyEcAgAAABhm9+7dWbt2bc4777wkQ0vKTj755KeVOfroo7N9+/Y89thjz1jXxo0b9wQ6e/PFL34xy5Yt2/P59re/vc8+Pvjgg/m7v/u7vOxlL9tn2X2xrAwAAAAgyY4dO7Js2bJs2bIlxx57bM4666wx3be3N4SNdv6KK67IPffckwULFuTee+9NMvZlZU/Zvn17Xvva1+Y973lPnvOc54z5vr0xcwgAAAAg/3/PoYceeiittaxevTpJsnTp0qxfv/5pZTdv3pxFixblOc95Tvr6+vLoo48+7frjjz+eQw45JC95yUvy9a9/fc/51atXZ+3atdm6deu4+vjEE0/kta99bS655JL82q/92rjqGEk4BAAAADDMQQcdlD/6oz/KH/zBH2TXrl255JJLcs899+Rzn/tckqEZRr/927+dq666Kkly+umnZ82aNXn88ceTJJ/4xCdywgknZN68eTnjjDOyc+fOfOADH9hT/09+8pNx9au1lssvvzzHHnts3vrWt3Y5yv/PsjIAAABgxpnuN2ufeOKJOf744/Pnf/7nef3rX59Pf/rTufLKK3PFFVdk9+7def3rX59Vq1YlSY4//visWrUqp512WqoqRxxxRG655ZYkQ0vLPvWpT+Utb3lLbrjhhixevDjPfvaz8+53v3tPW0/tOfSUa665JhdeeOHP9elLX/pSbr311hx33HF7yv/e7/1e/tW/+lddjbVaa11VMNGWL1/e1q1bt9/3zcXXvs/FMQEAAMBo7r///hx77LHT3Y05YbSfZVWtb60tH628ZWUAAAAAPcyyMgAAAIAZZNu2bXnVq171c+fXrl2bvr6+CW9POAQAAAAwg/T19WXDhg1T1p5lZQAAAMCMMNP2RZ6NxvMzFA4BAAAA027hwoXZtm2bgKgLrbVs27YtCxcu3K/7LCsDAAAApt2SJUsyODiYrVu3TndXZrWFCxdmyZIl+3VPV+FQVZ2T5L1J5iW5pbV2/Yjrb03yxiS7kmxN8luttYe6aRMAAACYe+bPn5+BgYHp7kZPGveysqqal2R1knOTLE1ycVUtHVHs75Isb60dn+SOJDeMtz0AAAAAJl43ew6dmmRTa21za+1nSW5Pcv7wAq21/9la+0nn8KtJ9m9eEwAAAACTqptw6PlJHh52PNg5tzeXJ/mb0S5U1cqqWldV66wtBAAAAJg6U7IhdVX9ZpLlSX5ptOuttZuT3Jwky5cvty05wBgc95Hjpqytv7/s76esLQAAYGp1Ew5tSfKCYcdLOueepqrOTPK7SX6ptfbTLtoDAAAAYIJ1s6zs3iTHVNVAVS1IclGSNcMLVNWJSf4kyXmttR900RYAAAAAk2Dc4VBrbVeSVUnuSnJ/ko+11jZW1Tuq6rxOsd9PsijJX1bVhqpas5fqAAAAAJgGXe051Fq7M8mdI85dO+z7md3UD0wPe9kAAAD0jm6WlQEAAAAwywmHAAAAAHrYlLzKHgDGaqqWNVrSCAAAQ8wcAgAAAOhhZg4xpcwIYLrYZBsAAGB0wiHoktABAACA2Uw4BACTzKxJAABmMnsOAQAAAPQw4RAAAABADxMOAQAAAPQwew4BAPvNPkoAAHOHmUMAAAAAPUw4BAAAANDDLCsDAMjULZVLLJcDAGYWM4cAAAAAephwCAAAAKCHWVYGADBHWSoHAIyFmUMAAAAAPczMIQAAZg2zoQBg4pk5BAAAANDDzBwCAIBpNFdnQ03VuMzwAuiecAgAAGAMBF7AXGVZGQAAAEAPM3MIAACgR5kNBSRdhkNVdU6S9yaZl+SW1tr1I66fnuQ9SY5PclFr7Y5u2gMAAIBnMhf38ZqLY2JmGXc4VFXzkqxOclaSwST3VtWa1tp9w4p9J8kbkrytm04CAAAAc4fAa2bpZubQqUk2tdY2J0lV3Z7k/CR7wqHW2oOda0920Q4AAADAjDabA69uNqR+fpKHhx0Pds7tt6paWVXrqmrd1q1bu+gSAAAAAPtjRrytrLV2c2tteWtt+eLFi6e7OwAAAAA9o5twaEuSFww7XtI5BwAAAMAs0U04dG+SY6pqoKoWJLkoyZqJ6RYAAAAAU2Hc4VBrbVeSVUnuSnJ/ko+11jZW1Tuq6rwkqapTqmowyeuS/ElVbZyITgMAAAAwMbp5W1laa3cmuXPEuWuHfb83Q8vNAAAAAJiBZsSG1AAAAABMD+EQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA8TDgEAAAD0MOEQAAAAQA/rKhyqqnOq6oGq2lRVV49y/VlV9Red61+rqv5u2gMAAABgYo07HKqqeUlWJzk3ydIkF1fV0hHFLk/yaGvthUluSvLu8bYHAAAAwMTrZubQqUk2tdY2t9Z+luT2JOePKHN+ko90vt+R5FVVVV20CQAAAMAEqtba+G6sujDJOa21N3aOX5/kZa21VcPKfLNTZrBz/O1OmUdG1LUyycrO4YuSPDCuTu2/w5M8ss9Ss4sxdee5SX40Be34Pc0ec3FcxjR+U/WMSPyeZgtjmj08J2YHY5odjKk7/s3Rnbk4rqka01GttcWjXThgChrfp9bazUlunup2q2pda235VLc7mYyp67Zubq2t3HfJrtvxe5ol5uK4jKmrdqbkGdFpy+9pFjCm2cNzYnYwptnBmLpuy785ujAXxzUTxtTNsrItSV4w7HhJ59yoZarqgAwlpNu6aBMm219NdweAGc0zAtgXzwlgXzwnmHG6CYfuTXJMVQ1U1YIkFyVZM6LMmiSXdb5fmOTuNt51bDAFWmse1MBeeUYA++I5AeyL5wQz0biXlbXWdlXVqiR3JZmX5EOttY1V9Y4k61pra5J8MMmtVbUpyT9lKECaSaZ8KdsUMKbZwZhmj7k4LmOaHYxpdjCm2WMujsuYZgdjmh2MafaYi+Oa9jGNe0NqAAAmT1U9mOSNrbXPTXdfAIC5rZtlZQAAM1ZVPVhVO6pqe1V9r6o+XFWLhl1/eVXdXVWPV9WPquqvqmrpsOtvqKp79lLvmcOOl1fVZ6rq0ar6YVXdV1X/taoOHVbP7k4/hn9+YbJ/BgAAYyEcAgDmste01hYlWZbkxCRvT5KqWpHkfyT5dJJfSDKQ5BtJvlRVvzjWyqvq5Uk+n+RLSV7cWjskyTlJdiU5YVjRr7TWFo34/GPXowMAmAAz4lX2AACTqbX2vaq6K0MhUZLckOSjrbX3Dit2TVWdnOS6JJeOseobkvxpa+1dw9r6TpL/1H2vAQCmhplDAMCcV1VLkpybZFNVHZTk5Un+cpSiH0ty1hjrfHaSFUk+PlH9BACYDsIhAGAu+1RVPZ7k4SQ/yNCMnsMy9DfQd0cp/90kh4+x7kM79XzvqRNVdUNn36EfV9U1w8r+y875pz7fHs9gAAAmg3AIAJjLLmitHZzkl5O8OEPBz6NJnkzyvFHKPy/JI53vu5LMH6XM/CRPjFZPa+2qzr5Dn8zTl+9/tbV2yLDP0V2NCgBgAgmHAIA5r7X2hSQfTnJja+3HSb6S5HWjFP31JGs737+T5MiqqqcudpakHZHkoU49X0vya5PYdQCASWdDagCgV7wnyYNVdUKSq5PcVVXfSvKnGfqb6N9laA+hUzrlv5ZkZ5Krq+qmJPOSvCvJuiQPdcpc1alnS5IPtdZ+0NnfaCDJpqkZFgBAd8wcAgB6Qmtta5KPJrm2tXZPkl/J0Kyf72Yo7DkxyWmttX/olP9pkldnaEnaYJLNGXrt/a+31lqnzD1JzkhyepL/U1U/TPLZDL3e/o+HNb+iqraP+JwSAIAZoDp/2wAAAADQg8wcAgAAAOhh9hwCAJgGVXVkkvv2cnlpa+07U9kfAKB3WVYGAAAA0MNm3Myhww8/vPX39093NwAAAADmjPXr1z/SWls82rUZFw719/dn3bp1090NAAAAgDmjqh7a2zUbUgMAAAD0MOEQAAAAQA8TDgEAAAD0sBm359B43f/iY6eknWO/df+UtAMAAAC96oknnsjg4GB27tw53V2ZdRYuXJglS5Zk/vz5Y76nq3Coqs5J8t4k85Lc0lq7fsT1Nye5IsnuJNuTrGyt3ddNmwAAAMDcNjg4mIMPPjj9/f2pqunuzqzRWsu2bdsyODiYgYGBMd837mVlVTUvyeok5yZZmuTiqlo6othtrbXjWmvLktyQ5A/H2x4AAADQG3bu3Jm+vj7B0H6qqvT19e33jKtu9hw6Ncmm1trm1trPktye5PzhBVprjw07fHaS1kV7AAAAQI8QDI3PeH5u3Swre36Sh4cdDyZ52chCVXVFkrcmWZDkjNEqqqqVSVYmyZFHHtlFlwAAAADYH5O+IXVrbXWS1VX1G0muSXLZKGVuTnJzkixfvtzsog6bbAMAAMDE//t4LP8OnjdvXo477rjs2rUrAwMDufXWW3PIIYckSTZu3Jgrr7wyW7ZsyZNPPplLL70011xzTaoq1113XRYtWpS3ve1te+rq7+/PunXrcvjhh+f73/9+3vKWt+SrX/1qDj300CxYsCBXXXVVfvVXfzWf//znc/755z9tv6Abb7wxZ5555qh9XLRoUbZv397lT6O7ZWVbkrxg2PGSzrm9uT3JBV20BwAAADAlDjzwwGzYsCHf/OY3c9hhh2X16tVJkh07duS8887L1VdfnQceeCDf+MY38uUvfznvf//791lnay0XXHBBTj/99GzevDnr16/P7bffnsHBwT1lXvnKV2bDhg17PnsLhiZSN+HQvUmOqaqBqlqQ5KIka4YXqKpjhh2+Osk/dNEeAAAAwJRbsWJFtmwZmg9z22235RWveEXOPvvsJMlBBx2U973vfbn++uufqYokyd13350FCxbkzW9+855zRx11VK688srJ6fgYjXtZWWttV1WtSnJXhl5l/6HW2saqekeSda21NUlWVdWZSZ5I8mhGWVIGAAAAMFPt3r07a9euzeWXX55kaEnZySef/LQyRx99dLZv357HHntstCr22LhxY0466aRnLPPFL34xy5Yt23P88Y9/PEcfffQ4ez82Xe051Fq7M8mdI85dO+z773RTPwAAAMB02LFjx/9r7/6D7ajLO46/PyZgBFRqBKsEhVFUgg2gkdqqqCAWrENaiy0MMlodaW0CSh0Z/DG02umMgNRqh6FSxDit4g9EjRQVjI4OMwX5YfwRIzVSkESUGKliGcTA0z/OBq839wbv3ZO7d895v2Yy9+yePXefJ3vunu8+5/v9LocddhibN2/m4IMP5phjjvmtXjfd3cKmWr9y5UquueYadt99d66//npgMKzsiiuumH3gs9BmWJkkSZIkSdJI2j7n0G233UZVPTjn0NKlS7nxxht/Y9tbbrmFvfbai0c96lEsXryYu+666zeev/vuu9l777055JBDuOmmmx5cf8EFF7B27Vq2bNmy6xPaCYtDkiRJkiRJ09hjjz143/vex/nnn8+2bds4+eSTueaaa/jiF78IDHoYnX766Zx55pkAHHnkkaxZs4a7774bgMsvv5xDDz2UBQsWcNRRR3Hvvfdy4YUXPvj777nnnrlPapJdfit7SZIkSZKkNn6bW8/vSocffjjLli3j0ksv5ZRTTuEzn/kMp512GitXruT+++/nlFNOYdWqVQAsW7aMVatW8bznPY8k7Lvvvlx88cXAYGjZpz/9ac444wzOPfdc9tlnH/bcc0/OOeecB/c1ec6ht7/97Zxwwgm7NL9U1S7dwUwtX768brjhhhm/bsPTD94F0exoLt+Qo5iTJEmSJEkPZcOGDRx88NxcE4+iqf7/ktxYVcun2t5hZZIkSZIkSWPMYWWSJEmSJEnz1NatWzn66KN3WL927VoWL148lH1YHJIkSZIkSZqnFi9ezLp163bpPhxWJkmSJEmS5p35NkdyX8zm/83ikCRJkiRJmlcWLVrE1q1bLRDNUFWxdetWFi1aNKPXOaxMknpqru5oCN7VUJIkSXNryZIlbNq0iS1btnQdSu8sWrSIJUuWzOg1FockSZIkSdK8sttuu3HggQd2HcbYcFiZJEmSJEnSGLPnkKQdOFxJkiRJksaHPYckSZIkSZLGmMUhSZIkSZKkMeawMknSvDJXwxod0ihJkiQNWBySNBacR0mSJEmSpmZxSHPKHgGSJEmSJM0vFoekluyRIumhWBiXJEnSfNZqQuokxya5OcnGJGdN8fzfJvlOkm8mWZvkSW32J0mSJEmSpOGadXEoyQLgAuA4YClwUpKlkzb7OrC8qpYBlwHnznZ/kiRJkiRJGr42PYeOADZW1S1VdR/wUWDFxA2q6stVdU+zeC2wpMX+JEmSJEmSNGRt5hzaD7h9wvIm4Pd3sv1rgc+12J8kSZonnEdJkiRpdMzJhNRJXgksB14wzfOnAqcCPPGJT5yLkCRJkiRJkkS74tBmYP8Jy0uadb8hyYuBtwEvqKpfTvWLquoi4CKA5cuXV4uYJEmSZsW7T0qSpHHVZs6h64GDkhyYZHfgRGDNxA2SHA68Hzi+qu5ssS9JkiRJkiTtArMuDlXVNmAV8AVgA/Dxqlqf5J1Jjm82Ow/YC/hEknVJ1kzz6yRJkiRJktSBVnMOVdWVwJWT1p094fGL2/x+SZIkzZ5D5SRJ0m9jTiakliRJkobBgpckScPXZs4hSZIkSZIk9Zw9hyRJkqQO2RtKktQ1i0OSJEmShm6uil5zWfAaxZwkCSwOSZIkSdLYsuAlCSwOSZIkSZJGyCgO1RzFnDS/WBySJEmSJElzahQLXn3OybuVSZIkSZIkjTGLQ5IkSZIkSWPM4pAkSZIkSdIYszgkSZIkSZI0xiwOSZIkSZIkjTGLQ5IkSZIkSWPM4pAkSZIkSdIYszgkSZIkSZI0xiwOSZIkSZIkjTGLQ5IkSZIkSWPM4pAkSZIkSdIYszgkSZIkSZI0xiwOSZIkSZIkjbFWxaEkxya5OcnGJGdN8fyRSW5Ksi3JCW32JUmSJEmSpOGbdXEoyQLgAuA4YClwUpKlkzb7AfBq4COz3Y8kSZIkSZJ2nYUtXnsEsLGqbgFI8lFgBfCd7RtU1a3Ncw+02I8kSZIkSZJ2kTbDyvYDbp+wvKlZN2NJTk1yQ5IbtmzZ0iIkSZIkSZIkzcS8mJC6qi6qquVVtXyfffbpOhxJkiRJkqSx0aY4tBnYf8LykmadJEmSJEmSeqJNceh64KAkBybZHTgRWDOcsCRJkiRJkjQXZl0cqqptwCrgC8AG4ONVtT7JO5McD5Dk2Uk2Aa8A3p9k/TCCliRJkiRJ0nC0uVsZVXUlcOWkdWdPeHw9g+FmkiRJkiRJmofmxYTUkiRJkiRJ6obFIUmSJEmSpDFmcUiSJEmSJGmMWRySJEmSJEkaYxaHJEmSJEmSxpjFIUmSJEmSpDFmcUiSJEmSJGmMWRySJEmSJEkaYxaHJEmSJEmSxpjFIUmSJEmSpDFmcUiSJEmSJGmMWRySJEmSJEkaYxaHJEmSJEmSxpjFIUmSJEmSpDFmcUiSJEmSJGmMWRySJEmSJEkaYxaHJEmSJEmSxpjFIUmSJEmSpDFmcUiSJEmSJGmMtSoOJTk2yc1JNiY5a4rnH57kY83z1yU5oM3+JEmSJEmSNFyzLg4lWQBcABwHLAVOSrJ00mavBe6qqqcA7wHOme3+JEmSJEmSNHxteg4dAWysqluq6j7go8CKSdusAD7UPL4MODpJWuxTkiRJkiRJQ9SmOLQfcPuE5U3Nuim3qaptwM+AxS32KUmSJEmSpCFa2HUAAElOBU5tFn+R5OY52vVjgZ/M6BXzv+OTOUGbnB7NoIi5q808J5jvx2oUcwL/pgbMabu5OkeAx2lgFHOC+Z7XKOYEnicGPE5dMCcYzZxgvp8nPJ9vZ07bPWm6J9oUhzYD+09YXtKsm2qbTUkWMvgj2Dr5F1XVRcBFLWKZlSQ3VNXyud7vrmROrfd1UVWd+tBbtt6Px6knRjEvc2q1nzk5RzT78jj1gDn1h+eJfjCnfjCn1vvymqOFUcxrPuTUZljZ9cBBSQ5MsjtwIrBm0jZrgFc1j08AvlRV1WKf0q722a4DkDSveY6Q9FA8T0h6KJ4nNO/MuudQVW1Lsgr4ArAAuKSq1id5J3BDVa0BPgD8e5KNwE8ZFJCkeauqPFFLmpbnCEkPxfOEpIfieULzUas5h6rqSuDKSevOnvD4XuAVbfaxi835ULY5YE79YE79MYp5mVM/mFM/mFN/jGJe5tQP5tQP5tQfo5hX5znFUV6SJEmSJEnjq82cQ5IkSZIkSeq5sSwOJTk2yc1JNiY5q+t4hiHJJUnuTPLtrmMZliT7J/lyku8kWZ/kDV3H1FaSRUm+luQbTU7v6DqmYUmyIMnXk1zRdSzDkOTWJN9Ksi7JDV3HMwxJ9k5yWZLvJtmQ5A+6jqmtJE9rjtH2fz9P8sau42oryRnNOeLbSS5NsqjrmNpK8oYmn/V9PUZTfdYmeUySq5N8r/n5O13GOFPT5PSK5jg9kKR3d4OZJqfzmnPfN5N8KsneXcY4G9Pk9Q9NTuuSXJXkCV3GOFM7a78meVOSSvLYLmKbrWmO098n2Tzhs+qlXcY4U9MdpySnNX9X65Oc21V8szHNcfrYhGN0a5J1XcY4U9PkdFiSa7e3Z5Mc0WWMMzVNTocm+a+mnf7ZJI/qMsaZmu4adz60J8auOJRkAXABcBywFDgpydJuoxqK1cCxXQcxZNuAN1XVUuA5wMoROFa/BI6qqkOBw4Bjkzyn45iG5Q3Ahq6DGLIXVdVhXd9WcojeC3y+qp4OHMoIHK+qurk5RocBzwLuAT7VcVitJNkPOB1YXlXPYHDTh17f0CHJM4DXAUcweO+9LMlTuo1qVlaz42ftWcDaqjoIWNss98lqdszp28DLga/OeTTDsZodc7oaeEZVLQP+G3jLXAc1BKvZMa/zqmpZcw68Ajh7h1fNb6uZov2aZH/gJcAP5jqgIVjN1G3y92z/vGrmbe2T1UzKKcmLgBXAoVV1CPDuDuJqYzWTcqqqv5jQpvgkcHkXgbWwmh3fe+cC72hyOrtZ7pPV7JjTxcBZVfV7DNp8b57roFqa7hq38/bE2BWHGDRMN1bVLVV1H/BRBie2XquqrzK4I9zIqKo7quqm5vHdDC5k9+s2qnZq4BfN4m7Nv95P/JVkCfDHDE7WmoeSPBo4ksFdJKmq+6rqf7uNauiOBr5fVbd1HcgQLAQekWQhsAfww47jaetg4LqquqeqtgFfYVB86JVpPmtXAB9qHn8I+JM5DaqlqXKqqg1VdXNHIbU2TU5XNe89gGuBJXMeWEvT5PXzCYt70rM2xU7ar+8BzqRn+cDItsmnyun1wLuq6pfNNnfOeWAt7Ow4JQnw58ClcxpUS9PkVMD2njWPpmftiWlyeiq//vLiauDP5jSolnZyjdt5e2Ici0P7AbdPWN5EzwsO4yDJAcDhwHXdRtJeBsOv1gF3AldXVe9zAv6ZQSPuga4DGaICrkpyY5JTuw5mCA4EtgAfzGD438VJ9uw6qCE7kZ415KZSVZsZfAP7A+AO4GdVdVW3UbX2beD5SRYn2QN4KbB/xzENy+Oq6o7m8Y+Ax3UZjH4rrwE+13UQw5LkH5PcDpxM/3oO7SDJCmBzVX2j61iGbFUzBPCSvg0/ncZTGZzXr0vylSTP7jqgIXo+8OOq+l7XgQzBG4HzmnPEu+lnr8nJ1vPrzh2voMftiUnXuJ23J8axOKSeSbIXg66dWpXJlQAABDxJREFUb5z0DVkvVdX9TdfOJcARzXCL3kryMuDOqrqx61iG7HlV9UwGQ1BXJjmy64BaWgg8E7iwqg4H/o/+DX+ZVpLdgeOBT3QdS1vNRcMKBgW9JwB7Jnllt1G1U1UbgHOAq4DPA+uA+zsNaheowS1ge9fTYZwkeRuDLv0f7jqWYamqt1XV/gxyWtV1PG00xeO3MgJFrkkuBJ7MYEqBO4Dzuw1nKBYCj2EwLObNwMebHjej4CRG4MumxuuBM5pzxBk0Pch77jXA3yS5EXgkcF/H8czKzq5xu2pPjGNxaDO/WV1c0qzTPJRkNwZ/NB+uqr6N+92pZkjPl+n/XFHPBY5PciuDYZpHJfmPbkNqr+m9sb2b9KcYDEnts03Apgk91S5jUCwaFccBN1XVj7sOZAheDPxPVW2pql8xmPPgDzuOqbWq+kBVPauqjgTuYjDvyyj4cZLHAzQ/ezW0YpwkeTXwMuDkpuE9aj5Mz4ZXTOHJDArj32jaFUuAm5L8bqdRtVRVP26+HHwA+Df636aAQbvi8mbKhK8x6D3eq8nDp9IM53458LGuYxmSV/HruZM+wQi896rqu1X1kqp6FoMi3ve7jmmmprnG7bw9MY7FoeuBg5Ic2HzTfCKwpuOYNIXm24cPABuq6p+6jmcYkuyT5g4pSR4BHAN8t9uo2qmqt1TVkqo6gMHf05eqqte9HJLsmeSR2x8zmBSz13cCrKofAbcneVqz6mjgOx2GNGyj9C3fD4DnJNmjOQ8ezQhMHp5k3+bnExk0vD/SbURDs4ZB45vm52c6jEXTSHIsg+HPx1fVPV3HMyxJDpqwuIL+tym+VVX7VtUBTbtiE/DM5jOst7Zf8DX+lJ63KRqfBl4EkOSpwO7ATzqNaDheDHy3qjZ1HciQ/BB4QfP4KKD3Q+UmtCceBrwd+NduI5qZnVzjdt6eWDjXO+xaVW1Lsgr4AoM7wFxSVes7Dqu1JJcCLwQem2QT8HdV1fdug88FTgG+NeFWkm/t4R0eJno88KHmrnkPAz5eVSNx6/cR8zjgU03v6IXAR6rq892GNBSnAR9uCuO3AH/ZcTxD0RTwjgH+qutYhqGqrktyGXATg+EvXwcu6jaqofhkksXAr4CVfZwQfarPWuBdDIZTvBa4jcEkpr0xTU4/Bf4F2Af4zyTrquqPuotyZqbJ6S3Aw4Grm3P7tVX1150FOQvT5PXSpuj/AIP3X+9z6nv7dZrj9MIkhzEYJnIrPfu8mianS4BLMrjF+H3Aq/rUI28n773ezl84zXF6HfDepkfUvUCv5tGcJqe9kqxsNrkc+GBH4c3WlNe4zIP2RHr0NyxJkiRJkqQhG8dhZZIkSZIkSWpYHJIkSZIkSRpjFockSZIkSZLGmMUhSZIkSZKkMWZxSJIkSZIkaYxZHJIkSZIkSRpjFockSZIkSZLGmMUhSZIkSZKkMfb/P8ImtQw4k6sAAAAASUVORK5CYII=\n", + "text/plain": [ + "<Figure size 1440x576 with 4 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "balb23Y6Sh8v" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file