--- a
+++ b/Code/Drug Discovery/ProtBert/Protein Fill-Mask, kkawchak.ipynb
@@ -0,0 +1,312 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "machine_shape": "hm"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": 8,
+      "metadata": {
+        "id": "-oTEyfkwNdmj"
+      },
+      "outputs": [],
+      "source": [
+        "# from transformers import pipeline, BertForMaskedLM, BertTokenizer, BertModel\n",
+        "# from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
+        "# import time\n",
+        "# import torch\n",
+        "# import math\n",
+        "# import re"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Ref 1: Hugging Face, Rostlab/prot_bert. https://huggingface.co/Rostlab/prot_bert\n",
+        "# Ref 2: Hayes, J. Medium, 2023 https://medium.com/labs-notebook/large-language-models-for-drug-discovery-7ddfc005e0bb\n",
+        "# Ref 3: HF Intro, AssemblyAI 2022 https://www.youtube.com/watch?v=QEaBAZQCtwE&t=4s\n",
+        "# Ref 4: ChatGPT3.5 Coding assistance 2024 https://chat.openai.com/"
+      ],
+      "metadata": {
+        "id": "jzr3h73L1w-7"
+      },
+      "execution_count": 9,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "seconds = time.time()\n",
+        "print(\"Time in seconds since beginning of run:\", seconds)\n",
+        "local_time = time.ctime(seconds)\n",
+        "print(local_time)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 0
+        },
+        "id": "2K0dcoVBuBGy",
+        "outputId": "8a904c07-4c64-41e8-94d1-0ec72d15c425"
+      },
+      "execution_count": 10,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Time in seconds since beginning of run: 1712858775.0148897\n",
+            "Thu Apr 11 18:06:15 2024\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Load tokenizer and model\n",
+        "tokenizer = BertTokenizer.from_pretrained(\"Rostlab/prot_bert\", do_lower_case=False)\n",
+        "model = BertForMaskedLM.from_pretrained(\"Rostlab/prot_bert\")\n",
+        "unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer)\n",
+        "\n",
+        "# Define the sequence with two masked tokens\n",
+        "sequence = 'FYLSITIHRPLRP[MASK]SSSSFLSLCLSLLSISIYYPS\\nLLIRRFTSISSCSSITIYHPLLYPSPSSLFLSLSHTYIYISPLHPSSLLLSISLLFYLSI\\nYIIYPLQPSSLLLSI[MASK]SLPLSISIYLSYPPLSSPSPSLSLYLTPFLLIPSLSIYLSLPFPY\\nHSYLYLRLLFHPPLPLHICHLPHSLTLFIFLLPPHLSHLPILFSRLQPFYPSTSPSSYRP\\nLPCIPSASYFSYHPLSPPPSLHPHPLSYPSVSRPSPPYLSIHLHSPPPPPPPSPFSSIHP\\nPFLSSTLPLPSSTSSLPPSSSPFSSTHLIPSPSSPPPPSLLPSSLPL'\n",
+        "\n",
+        "# Predict and print top 10 predictions for each mask\n",
+        "result = unmasker(sequence, top_k=10)\n",
+        "for predictions in result:\n",
+        "    for prediction in predictions:\n",
+        "        print(prediction)\n",
+        "    print(\"------\")"
+      ],
+      "metadata": {
+        "id": "YX05PGDZNImk",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 0
+        },
+        "outputId": "25f2ab86-fc66-4279-cf8f-a1621d267e6f"
+      },
+      "execution_count": 11,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Some weights of the model checkpoint at Rostlab/prot_bert were not used when initializing BertForMaskedLM: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
+            "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+            "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "{'score': 0.9999492168426514, 'token': 25, 'token_str': 'X', 'sequence': '[CLS] [UNK] X [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 7.70106453273911e-06, 'token': 13, 'token_str': 'R', 'sequence': '[CLS] [UNK] R [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 6.548679721163353e-06, 'token': 5, 'token_str': 'L', 'sequence': '[CLS] [UNK] L [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 4.85780628878274e-06, 'token': 19, 'token_str': 'F', 'sequence': '[CLS] [UNK] F [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 4.367872861621436e-06, 'token': 18, 'token_str': 'Q', 'sequence': '[CLS] [UNK] Q [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 4.232546871207887e-06, 'token': 7, 'token_str': 'G', 'sequence': '[CLS] [UNK] G [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 3.1130280149227474e-06, 'token': 12, 'token_str': 'K', 'sequence': '[CLS] [UNK] K [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 2.9373295546974987e-06, 'token': 16, 'token_str': 'P', 'sequence': '[CLS] [UNK] P [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 2.8243982796993805e-06, 'token': 6, 'token_str': 'A', 'sequence': '[CLS] [UNK] A [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 2.3990457975742174e-06, 'token': 15, 'token_str': 'T', 'sequence': '[CLS] [UNK] T [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "------\n",
+            "{'score': 0.9999690055847168, 'token': 25, 'token_str': 'X', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] X [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 4.743668341689045e-06, 'token': 5, 'token_str': 'L', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] L [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 4.336807705840329e-06, 'token': 13, 'token_str': 'R', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] R [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 4.116420768696116e-06, 'token': 18, 'token_str': 'Q', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] Q [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 3.6834383081441047e-06, 'token': 6, 'token_str': 'A', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] A [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 2.364193051107577e-06, 'token': 7, 'token_str': 'G', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] G [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 2.1007083432778018e-06, 'token': 16, 'token_str': 'P', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] P [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 1.6038892454162124e-06, 'token': 15, 'token_str': 'T', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] T [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 1.3503024547389941e-06, 'token': 10, 'token_str': 'S', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] S [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "{'score': 1.272862732548674e-06, 'token': 12, 'token_str': 'K', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] K [UNK] [UNK] [UNK] [UNK] [SEP]'}\n",
+            "------\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Load tokenizer and model\n",
+        "tokenizer = BertTokenizer.from_pretrained(\"Rostlab/prot_bert\", do_lower_case=False)\n",
+        "model = BertForMaskedLM.from_pretrained(\"Rostlab/prot_bert\")\n",
+        "unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer)\n",
+        "\n",
+        "# Define the sequence with two masked tokens\n",
+        "sequence = 'FYLSITIHRPLRP[MASK]SSSSFLSLCLSLLSISIYYPS\\nLLIRRFTSISSCSSITIYHPLLYPSPSSLFLSLSHTYIYISPLHPSSLLLSISLLFYLSI\\nYIIYPLQPSSLLLSI[MASK]SLPLSISIYLSYPPLSSPSPSLSLYLTPFLLIPSLSIYLSLPFPY\\nHSYLYLRLLFHPPLPLHICHLPHSLTLFIFLLPPHLSHLPILFSRLQPFYPSTSPSSYRP\\nLPCIPSASYFSYHPLSPPPSLHPHPLSYPSVSRPSPPYLSIHLHSPPPPPPPSPFSSIHP\\nPFLSSTLPLPSSTSSLPPSSSPFSSTHLIPSPSSPPPPSLLP[MASK]SSLPL'\n",
+        "\n",
+        "# Predict and print top 10 predictions for each mask\n",
+        "result = unmasker(sequence, top_k=10)\n",
+        "for predictions in result:\n",
+        "    for prediction in predictions:\n",
+        "        print(prediction)\n",
+        "    print(\"------\")"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 0
+        },
+        "id": "McDmgTTdNlA9",
+        "outputId": "551fd198-7b8b-4601-cf93-40dc5da22f95"
+      },
+      "execution_count": 12,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Some weights of the model checkpoint at Rostlab/prot_bert were not used when initializing BertForMaskedLM: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
+            "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+            "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "{'score': 0.9999243021011353, 'token': 25, 'token_str': 'X', 'sequence': '[CLS] [UNK] X [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 9.95707159745507e-06, 'token': 5, 'token_str': 'L', 'sequence': '[CLS] [UNK] L [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 7.378088412224315e-06, 'token': 7, 'token_str': 'G', 'sequence': '[CLS] [UNK] G [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 7.330473636102397e-06, 'token': 9, 'token_str': 'E', 'sequence': '[CLS] [UNK] E [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 6.249400939850602e-06, 'token': 12, 'token_str': 'K', 'sequence': '[CLS] [UNK] K [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 6.2195363170758355e-06, 'token': 19, 'token_str': 'F', 'sequence': '[CLS] [UNK] F [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 6.003328053338919e-06, 'token': 6, 'token_str': 'A', 'sequence': '[CLS] [UNK] A [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 5.373394287744304e-06, 'token': 13, 'token_str': 'R', 'sequence': '[CLS] [UNK] R [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 4.275994342606282e-06, 'token': 20, 'token_str': 'Y', 'sequence': '[CLS] [UNK] Y [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 3.627200385381002e-06, 'token': 8, 'token_str': 'V', 'sequence': '[CLS] [UNK] V [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "------\n",
+            "{'score': 0.9996041655540466, 'token': 25, 'token_str': 'X', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] X [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 5.5235712352441624e-05, 'token': 6, 'token_str': 'A', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] A [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 5.422084723250009e-05, 'token': 5, 'token_str': 'L', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] L [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 4.226986857247539e-05, 'token': 7, 'token_str': 'G', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] G [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 3.9134716644184664e-05, 'token': 13, 'token_str': 'R', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] R [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 3.062105315621011e-05, 'token': 18, 'token_str': 'Q', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] Q [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 2.8334136004559696e-05, 'token': 9, 'token_str': 'E', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] E [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 2.7488824343890883e-05, 'token': 12, 'token_str': 'K', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] K [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 1.9406317733228207e-05, 'token': 16, 'token_str': 'P', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] P [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "{'score': 1.778588557499461e-05, 'token': 19, 'token_str': 'F', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] F [UNK] [UNK] [UNK] [UNK] [MASK] [UNK] [SEP]'}\n",
+            "------\n",
+            "{'score': 0.9990819692611694, 'token': 25, 'token_str': 'X', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] X [UNK] [SEP]'}\n",
+            "{'score': 0.00016688588948454708, 'token': 6, 'token_str': 'A', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] A [UNK] [SEP]'}\n",
+            "{'score': 8.889797754818574e-05, 'token': 5, 'token_str': 'L', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] L [UNK] [SEP]'}\n",
+            "{'score': 8.658925071358681e-05, 'token': 12, 'token_str': 'K', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] K [UNK] [SEP]'}\n",
+            "{'score': 6.849779310869053e-05, 'token': 18, 'token_str': 'Q', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] Q [UNK] [SEP]'}\n",
+            "{'score': 6.579031469300389e-05, 'token': 20, 'token_str': 'Y', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] Y [UNK] [SEP]'}\n",
+            "{'score': 6.401719292625785e-05, 'token': 7, 'token_str': 'G', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] G [UNK] [SEP]'}\n",
+            "{'score': 5.076546221971512e-05, 'token': 9, 'token_str': 'E', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] E [UNK] [SEP]'}\n",
+            "{'score': 4.9771988415159285e-05, 'token': 13, 'token_str': 'R', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] R [UNK] [SEP]'}\n",
+            "{'score': 4.564020491670817e-05, 'token': 19, 'token_str': 'F', 'sequence': '[CLS] [UNK] [MASK] [UNK] [UNK] [UNK] [MASK] [UNK] [UNK] [UNK] [UNK] F [UNK] [SEP]'}\n",
+            "------\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Load tokenizer and model\n",
+        "tokenizer = BertTokenizer.from_pretrained(\"Rostlab/prot_bert\", do_lower_case=False)\n",
+        "model = BertModel.from_pretrained(\"Rostlab/prot_bert\")\n",
+        "\n",
+        "# Example protein sequence\n",
+        "sequence_Example = \"FYLSITIHRPLRPSSSSFLSLCLSLLSISIYYPS\\nLLIRRFTSISSCSSITIYHPLLYPSPSSLFLSLSHTYIYISPLHPSSLLLSISLLFYLSI\\nYIIYPLQPSSLLLSISLPLSISIYLSYPPLSSPSPSLSLYLTPFLLIPSLSIYLSLPFPY\\nHSYLYLRLLFHPPLPLHICHLPHSLTLFIFLLPPHLSHLPILFSRLQPFYPSTSPSSYRP\\nLPCIPSASYFSYHPLSPPPSLHPHPLSYPSVSRPSPPYLSIHLHSPPPPPPPSPFSSIHP\\nPFLSSTLPLPSSTSSLPPSSSPFSSTHLIPSPSSPPPPSLLPSSLPL\"\n",
+        "sequence_Example = re.sub(r\"[UZOB]\", \"X\", sequence_Example)\n",
+        "\n",
+        "# Tokenize input sequence\n",
+        "encoded_input = tokenizer(sequence_Example, return_tensors='pt')\n",
+        "\n",
+        "# Forward pass through the model\n",
+        "output = model(**encoded_input)\n",
+        "\n",
+        "print(output)\n",
+        "\n",
+        "# Extract last hidden states\n",
+        "last_hidden_state = output.last_hidden_state\n",
+        "\n",
+        "# Calculate additional metrics\n",
+        "mean_tensor = torch.mean(last_hidden_state, dim=1)  # Mean along sequence length\n",
+        "std_tensor = torch.std(last_hidden_state, dim=1)    # Standard deviation along sequence length\n",
+        "max_tensor = torch.max(last_hidden_state, dim=1).values  # Maximum value along sequence length\n",
+        "\n",
+        "# Print additional metrics\n",
+        "print(\"Mean tensor shape:\", mean_tensor.shape)\n",
+        "print(\"Standard deviation tensor shape:\", std_tensor.shape)\n",
+        "print(\"Maximum tensor shape:\", max_tensor.shape)"
+      ],
+      "metadata": {
+        "id": "tiXjSEo2MfrD",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 0
+        },
+        "outputId": "55f75b62-24fc-4524-c3fe-04c076842cdd"
+      },
+      "execution_count": 13,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "BaseModelOutputWithPoolingAndCrossAttentions(last_hidden_state=tensor([[[-0.1217,  0.0759, -0.2128,  ...,  0.0417, -0.0052,  0.1067],\n",
+            "         [-0.0694,  0.1312, -0.1388,  ..., -0.0231,  0.0595,  0.0174],\n",
+            "         [-0.1399,  0.0801, -0.1725,  ...,  0.0329, -0.0341,  0.0568],\n",
+            "         ...,\n",
+            "         [-0.1345,  0.0499, -0.1611,  ...,  0.0035, -0.0035,  0.0531],\n",
+            "         [-0.1569,  0.0764, -0.1574,  ..., -0.0122, -0.0043,  0.0533],\n",
+            "         [-0.1272,  0.0725, -0.1479,  ..., -0.0294,  0.0445, -0.0033]]],\n",
+            "       grad_fn=<NativeLayerNormBackward0>), pooler_output=tensor([[-0.1908,  0.2020, -0.1812,  ...,  0.1921,  0.1790, -0.1964]],\n",
+            "       grad_fn=<TanhBackward0>), hidden_states=None, past_key_values=None, attentions=None, cross_attentions=None)\n",
+            "Mean tensor shape: torch.Size([1, 1024])\n",
+            "Standard deviation tensor shape: torch.Size([1, 1024])\n",
+            "Maximum tensor shape: torch.Size([1, 1024])\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "seconds = time.time()\n",
+        "print(\"Time in seconds since end of run:\", seconds)\n",
+        "local_time = time.ctime(seconds)\n",
+        "print(local_time)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 0
+        },
+        "id": "TkiND_HHuERP",
+        "outputId": "c8fbb5be-e846-42ef-d6bc-6347d9a3ba87"
+      },
+      "execution_count": 14,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Time in seconds since end of run: 1712858783.8144772\n",
+            "Thu Apr 11 18:06:23 2024\n"
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file