Downloads: 0
797 lines (796 with data), 47.5 kB
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Collecting benepar\n",
" Downloading benepar-0.2.0.tar.gz (33 kB)\n",
" Preparing metadata (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25hRequirement already satisfied: nltk>=3.2 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from benepar) (3.8.1)\n",
"Collecting spacy>=2.0.9 (from benepar)\n",
" Downloading spacy-3.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.6 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.6/6.6 MB\u001b[0m \u001b[31m51.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: torch>=1.6.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from benepar) (2.0.0)\n",
"Collecting torch-struct>=0.5 (from benepar)\n",
" Downloading torch_struct-0.5-py3-none-any.whl (34 kB)\n",
"Requirement already satisfied: tokenizers>=0.9.4 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from benepar) (0.12.1)\n",
"Requirement already satisfied: transformers[tokenizers,torch]>=4.2.2 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from benepar) (4.28.0.dev0)\n",
"Requirement already satisfied: protobuf in /anaconda/envs/llava_med/lib/python3.10/site-packages (from benepar) (4.22.4)\n",
"Requirement already satisfied: sentencepiece>=0.1.91 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from benepar) (0.1.99)\n",
"Requirement already satisfied: click in /anaconda/envs/llava_med/lib/python3.10/site-packages (from nltk>=3.2->benepar) (8.1.3)\n",
"Requirement already satisfied: joblib in /anaconda/envs/llava_med/lib/python3.10/site-packages (from nltk>=3.2->benepar) (1.2.0)\n",
"Requirement already satisfied: regex>=2021.8.3 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from nltk>=3.2->benepar) (2023.5.5)\n",
"Requirement already satisfied: tqdm in /anaconda/envs/llava_med/lib/python3.10/site-packages (from nltk>=3.2->benepar) (4.65.0)\n",
"Collecting spacy-legacy<3.1.0,>=3.0.11 (from spacy>=2.0.9->benepar)\n",
" Downloading spacy_legacy-3.0.12-py2.py3-none-any.whl (29 kB)\n",
"Collecting spacy-loggers<2.0.0,>=1.0.0 (from spacy>=2.0.9->benepar)\n",
" Downloading spacy_loggers-1.0.4-py3-none-any.whl (11 kB)\n",
"Collecting murmurhash<1.1.0,>=0.28.0 (from spacy>=2.0.9->benepar)\n",
" Downloading murmurhash-1.0.9-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (21 kB)\n",
"Collecting cymem<2.1.0,>=2.0.2 (from spacy>=2.0.9->benepar)\n",
" Downloading cymem-2.0.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (34 kB)\n",
"Collecting preshed<3.1.0,>=3.0.2 (from spacy>=2.0.9->benepar)\n",
" Downloading preshed-3.0.8-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (124 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.7/124.7 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting thinc<8.2.0,>=8.1.8 (from spacy>=2.0.9->benepar)\n",
" Downloading thinc-8.1.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (913 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m913.3/913.3 kB\u001b[0m \u001b[31m29.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting wasabi<1.2.0,>=0.9.1 (from spacy>=2.0.9->benepar)\n",
" Downloading wasabi-1.1.1-py3-none-any.whl (27 kB)\n",
"Collecting srsly<3.0.0,>=2.4.3 (from spacy>=2.0.9->benepar)\n",
" Downloading srsly-2.4.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (492 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m492.2/492.2 kB\u001b[0m \u001b[31m28.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting catalogue<2.1.0,>=2.0.6 (from spacy>=2.0.9->benepar)\n",
" Downloading catalogue-2.0.8-py3-none-any.whl (17 kB)\n",
"Collecting typer<0.8.0,>=0.3.0 (from spacy>=2.0.9->benepar)\n",
" Downloading typer-0.7.0-py3-none-any.whl (38 kB)\n",
"Collecting pathy>=0.10.0 (from spacy>=2.0.9->benepar)\n",
" Downloading pathy-0.10.1-py3-none-any.whl (48 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.9/48.9 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hCollecting smart-open<7.0.0,>=5.2.1 (from spacy>=2.0.9->benepar)\n",
" Downloading smart_open-6.3.0-py3-none-any.whl (56 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.8/56.8 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: numpy>=1.15.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy>=2.0.9->benepar) (1.24.3)\n",
"Requirement already satisfied: requests<3.0.0,>=2.13.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy>=2.0.9->benepar) (2.30.0)\n",
"Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy>=2.0.9->benepar) (1.10.7)\n",
"Requirement already satisfied: jinja2 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy>=2.0.9->benepar) (3.1.2)\n",
"Requirement already satisfied: setuptools in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy>=2.0.9->benepar) (66.0.0)\n",
"Requirement already satisfied: packaging>=20.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy>=2.0.9->benepar) (23.1)\n",
"Collecting langcodes<4.0.0,>=3.2.0 (from spacy>=2.0.9->benepar)\n",
" Downloading langcodes-3.3.0-py3-none-any.whl (181 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m181.6/181.6 kB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: filelock in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (3.12.0)\n",
"Requirement already satisfied: typing-extensions in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (4.5.0)\n",
"Requirement already satisfied: sympy in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (1.11.1)\n",
"Requirement already satisfied: networkx in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (3.1)\n",
"Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (11.7.99)\n",
"Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (11.7.99)\n",
"Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (11.7.101)\n",
"Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (8.5.0.96)\n",
"Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (11.10.3.66)\n",
"Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (10.9.0.58)\n",
"Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (10.2.10.91)\n",
"Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (11.4.0.1)\n",
"Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (11.7.4.91)\n",
"Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (2.14.3)\n",
"Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (11.7.91)\n",
"Requirement already satisfied: triton==2.0.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from torch>=1.6.0->benepar) (2.0.0)\n",
"Requirement already satisfied: wheel in /anaconda/envs/llava_med/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.6.0->benepar) (0.38.4)\n",
"Requirement already satisfied: cmake in /anaconda/envs/llava_med/lib/python3.10/site-packages (from triton==2.0.0->torch>=1.6.0->benepar) (3.26.3)\n",
"Requirement already satisfied: lit in /anaconda/envs/llava_med/lib/python3.10/site-packages (from triton==2.0.0->torch>=1.6.0->benepar) (16.0.3)\n",
"Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from transformers[tokenizers,torch]>=4.2.2->benepar) (0.14.1)\n",
"Requirement already satisfied: pyyaml>=5.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from transformers[tokenizers,torch]>=4.2.2->benepar) (6.0)\n",
"Requirement already satisfied: fsspec in /anaconda/envs/llava_med/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.11.0->transformers[tokenizers,torch]>=4.2.2->benepar) (2023.4.0)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy>=2.0.9->benepar) (3.1.0)\n",
"Requirement already satisfied: idna<4,>=2.5 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy>=2.0.9->benepar) (3.4)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy>=2.0.9->benepar) (1.26.15)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy>=2.0.9->benepar) (2022.12.7)\n",
"Collecting blis<0.8.0,>=0.7.8 (from thinc<8.2.0,>=8.1.8->spacy>=2.0.9->benepar)\n",
" Downloading blis-0.7.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (10.2 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.2/10.2 MB\u001b[0m \u001b[31m65.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
"\u001b[?25hCollecting confection<1.0.0,>=0.0.1 (from thinc<8.2.0,>=8.1.8->spacy>=2.0.9->benepar)\n",
" Downloading confection-0.0.4-py3-none-any.whl (32 kB)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from jinja2->spacy>=2.0.9->benepar) (2.1.2)\n",
"Requirement already satisfied: mpmath>=0.19 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from sympy->torch>=1.6.0->benepar) (1.3.0)\n",
"Building wheels for collected packages: benepar\n",
" Building wheel for benepar (setup.py) ... \u001b[?25ldone\n",
"\u001b[?25h Created wheel for benepar: filename=benepar-0.2.0-py3-none-any.whl size=37624 sha256=91a135f874d159bf5e85a61b59f9c6fce61cb82fc2e0d8a207976b951822d911\n",
" Stored in directory: /home/chunyl/.cache/pip/wheels/8d/4d/c1/a5af726368d5dbaaaa0b2dd36ed39b9da8cec46279a49bd6db\n",
"Successfully built benepar\n",
"Installing collected packages: cymem, wasabi, typer, spacy-loggers, spacy-legacy, smart-open, murmurhash, langcodes, catalogue, blis, srsly, preshed, pathy, confection, thinc, spacy, torch-struct, benepar\n",
"Successfully installed benepar-0.2.0 blis-0.7.9 catalogue-2.0.8 confection-0.0.4 cymem-2.0.7 langcodes-3.3.0 murmurhash-1.0.9 pathy-0.10.1 preshed-3.0.8 smart-open-6.3.0 spacy-3.5.3 spacy-legacy-3.0.12 spacy-loggers-1.0.4 srsly-2.4.6 thinc-8.1.10 torch-struct-0.5 typer-0.7.0 wasabi-1.1.1\n",
"Requirement already satisfied: spacy in /anaconda/envs/llava_med/lib/python3.10/site-packages (3.5.3)\n",
"Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (3.0.12)\n",
"Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (1.0.4)\n",
"Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (1.0.9)\n",
"Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (2.0.7)\n",
"Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (3.0.8)\n",
"Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (8.1.10)\n",
"Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (1.1.1)\n",
"Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (2.4.6)\n",
"Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (2.0.8)\n",
"Requirement already satisfied: typer<0.8.0,>=0.3.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (0.7.0)\n",
"Requirement already satisfied: pathy>=0.10.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (0.10.1)\n",
"Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (6.3.0)\n",
"Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (4.65.0)\n",
"Requirement already satisfied: numpy>=1.15.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (1.24.3)\n",
"Requirement already satisfied: requests<3.0.0,>=2.13.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (2.30.0)\n",
"Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (1.10.7)\n",
"Requirement already satisfied: jinja2 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (3.1.2)\n",
"Requirement already satisfied: setuptools in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (66.0.0)\n",
"Requirement already satisfied: packaging>=20.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (23.1)\n",
"Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy) (3.3.0)\n",
"Requirement already satisfied: typing-extensions>=4.2.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy) (4.5.0)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.1.0)\n",
"Requirement already satisfied: idna<4,>=2.5 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.4)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy) (1.26.15)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2022.12.7)\n",
"Requirement already satisfied: blis<0.8.0,>=0.7.8 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from thinc<8.2.0,>=8.1.8->spacy) (0.7.9)\n",
"Requirement already satisfied: confection<1.0.0,>=0.0.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from thinc<8.2.0,>=8.1.8->spacy) (0.0.4)\n",
"Requirement already satisfied: click<9.0.0,>=7.1.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from typer<0.8.0,>=0.3.0->spacy) (8.1.3)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from jinja2->spacy) (2.1.2)\n"
]
}
],
"source": [
"!pip install benepar\n",
"!pip install spacy"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Collecting en-core-web-md==3.5.0\n",
" Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.5.0/en_core_web_md-3.5.0-py3-none-any.whl (42.8 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.8/42.8 MB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
"\u001b[?25hRequirement already satisfied: spacy<3.6.0,>=3.5.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from en-core-web-md==3.5.0) (3.5.3)\n",
"Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (3.0.12)\n",
"Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (1.0.4)\n",
"Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (1.0.9)\n",
"Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (2.0.7)\n",
"Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (3.0.8)\n",
"Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (8.1.10)\n",
"Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (1.1.1)\n",
"Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (2.4.6)\n",
"Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (2.0.8)\n",
"Requirement already satisfied: typer<0.8.0,>=0.3.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (0.7.0)\n",
"Requirement already satisfied: pathy>=0.10.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (0.10.1)\n",
"Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (6.3.0)\n",
"Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (4.65.0)\n",
"Requirement already satisfied: numpy>=1.15.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (1.24.3)\n",
"Requirement already satisfied: requests<3.0.0,>=2.13.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (2.30.0)\n",
"Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (1.10.7)\n",
"Requirement already satisfied: jinja2 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (3.1.2)\n",
"Requirement already satisfied: setuptools in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (66.0.0)\n",
"Requirement already satisfied: packaging>=20.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (23.1)\n",
"Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (3.3.0)\n",
"Requirement already satisfied: typing-extensions>=4.2.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (4.5.0)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (3.1.0)\n",
"Requirement already satisfied: idna<4,>=2.5 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (3.4)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (1.26.15)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (2022.12.7)\n",
"Requirement already satisfied: blis<0.8.0,>=0.7.8 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from thinc<8.2.0,>=8.1.8->spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (0.7.9)\n",
"Requirement already satisfied: confection<1.0.0,>=0.0.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from thinc<8.2.0,>=8.1.8->spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (0.0.4)\n",
"Requirement already satisfied: click<9.0.0,>=7.1.1 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from typer<0.8.0,>=0.3.0->spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (8.1.3)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /anaconda/envs/llava_med/lib/python3.10/site-packages (from jinja2->spacy<3.6.0,>=3.5.0->en-core-web-md==3.5.0) (2.1.2)\n",
"\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
"You can now load the package via spacy.load('en_core_web_md')\n"
]
}
],
"source": [
"import benepar, spacy\n",
"!python -m spacy download en_core_web_md\n",
"nlp = spacy.load('en_core_web_md')\n",
"doc = nlp(\"The time for action is now. It's never too late to do something.\")\n",
"\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"('write', 'story')"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def find_root_verb_and_its_dobj(tree_root):\n",
" # first check if the current node and its children satisfy the condition\n",
" if tree_root.pos_ == \"VERB\":\n",
" for child in tree_root.children:\n",
" if child.dep_ == \"dobj\" and child.pos_ == \"NOUN\":\n",
" return tree_root.lemma_, child.lemma_\n",
" return tree_root.lemma_, None\n",
" # if not, check its children\n",
" for child in tree_root.children:\n",
" return find_root_verb_and_its_dobj(child)\n",
" # if no children satisfy the condition, return None\n",
" return None, None\n",
"\n",
"def find_root_verb_and_its_dobj_in_string(s):\n",
" doc = nlp(s)\n",
" first_sent = list(doc.sents)[0]\n",
" return find_root_verb_and_its_dobj(first_sent.root)\n",
"\n",
"find_root_verb_and_its_dobj_in_string(\"Write me a story about education.\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"500000\n",
"10712\n",
"60088\n",
"60017\n"
]
}
],
"source": [
"import pandas as pd\n",
"import json\n",
"import tqdm\n",
"import os\n",
"\n",
"dataset_name_list = [\"pretrain\", \"instruct_10K\", \"instruct_60K\", \"instruct_60K_intext\"]\n",
"\n",
"folder_path = \"../../data/\"\n",
"file_path_list = [\n",
" \"2023-04-26_text_image_alignment.json\", \n",
" \"2023-04-28_conv_finetune_processed.json\", \n",
" \"2023-05-10_finetune_postprocess_caption_cleaned\", \n",
" \"2023-05-10_finetune_postprocess_caption_in_text_cleaned\"\n",
"]\n",
"file_path_list = [os.path.join(folder_path, f) for f in file_path_list]\n",
"generated_data_path = file_path_list\n",
"\n",
"dict_instruct_response = {}\n",
"\n",
"for i, ds_name in enumerate(dataset_name_list):\n",
"\n",
" with open(generated_data_path[i], 'r') as fin:\n",
" line_dict = json.load(fin)\n",
"\n",
" print(len(line_dict))\n",
" instruct_list, response_list = [], []\n",
" for line in line_dict:\n",
" try:\n",
" conversations = line['conversations']\n",
" except:\n",
" conversations = line['conversatons']\n",
"\n",
" for c in conversations:\n",
" if c['from'] == \"human\":\n",
" if \"<image>\" in c['value']:\n",
" for c1 in c['value'].split(\"\\n\"):\n",
" if \"<image>\" not in c1:\n",
" instruct_list.append(c1)\n",
" else:\n",
" instruct_list.append(c['value'])\n",
" elif c['from'] == \"gpt\":\n",
" response_list.append(c['value'])\n",
"\n",
" dict_instruct_response[ ds_name ] = {\"instruct\": instruct_list, \"response\": response_list}\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"pretrain: 500000 | 500000\n",
"instruct_10K: 52805 | 52805\n",
"instruct_60K: 171673 | 171673\n",
"instruct_60K_intext: 173696 | 173696\n"
]
}
],
"source": [
"for k, v in dict_instruct_response.items():\n",
" num_instruct, num_response = len(v['instruct']), len(v['response'])\n",
" print(f'{k}: {num_instruct} | {num_response}')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"\n",
"def compute_verb_noun_paris(sequences, csv_file_name):\n",
" # print(sequences[:3])\n",
" sequences = set(sequences) # if you are interested in studying the instructions, please change the task key\n",
" print(f'# unique sequences {len(sequences)}')\n",
"\n",
" raw_phrases = []\n",
" for seq in tqdm.tqdm(sequences):\n",
" try:\n",
" verb, noun = find_root_verb_and_its_dobj_in_string(seq)\n",
" raw_phrases.append({\n",
" \"verb\": verb,\n",
" \"noun\": noun,\n",
" \"seq\": seq\n",
" })\n",
" except Exception as e:\n",
" print(e)\n",
" print(seq)\n",
" print(f'# raw_phrases {len(raw_phrases)}')\n",
" raw_phrases = pd.DataFrame(raw_phrases)\n",
" raw_phrases.to_csv(csv_file_name) "
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"pretrain: 500000 | 500000\n",
"# unique sequences 27\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 27/27 [00:00<00:00, 240.38it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"# raw_phrases 27\n",
"# unique sequences 99748\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 0%| | 0/99748 [00:00<?, ?it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"list index out of range\n",
"\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 99748/99748 [18:59<00:00, 87.52it/s] \n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"# raw_phrases 99747\n",
"instruct_10K: 52805 | 52805\n",
"# unique sequences 19117\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 19117/19117 [01:21<00:00, 234.93it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"# raw_phrases 19117\n",
"# unique sequences 47006\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 47006/47006 [05:27<00:00, 143.41it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"# raw_phrases 47006\n",
"instruct_60K: 171673 | 171673\n",
"# unique sequences 48401\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 48401/48401 [03:06<00:00, 258.84it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"# raw_phrases 48401\n",
"# unique sequences 94892\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 94892/94892 [12:17<00:00, 128.64it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"# raw_phrases 94892\n",
"instruct_60K_intext: 173696 | 173696\n",
"# unique sequences 51578\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 51578/51578 [03:23<00:00, 253.53it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"# raw_phrases 51578\n",
"# unique sequences 95344\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 95344/95344 [11:49<00:00, 134.43it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"# raw_phrases 95344\n"
]
}
],
"source": [
"import random\n",
"\n",
"num_select = 100000\n",
"for k, v in dict_instruct_response.items():\n",
" num_instruct, num_response = len(v['instruct']), len(v['response'])\n",
" print(f'{k}: {num_instruct} | {num_response}')\n",
" \n",
" output_csv_folder_path = os.path.join(folder_path, 'csv')\n",
"\n",
" if num_instruct > num_select:\n",
" instruct = random.sample(v['instruct'], num_select)\n",
" response = random.sample(v['response'], num_select)\n",
" else:\n",
" instruct = v['instruct']\n",
" response = v['response']\n",
"\n",
" compute_verb_noun_paris(instruct, f'{output_csv_folder_path}/{k}_instruction_verb_noun.csv') \n",
" compute_verb_noun_paris(response, f'{output_csv_folder_path}/{k}_response_verb_noun.csv') \n"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"import plotly.graph_objects as go\n",
"import plotly.express as px\n",
"\n",
"folder_path = \"../../data/\"\n",
"\n",
"def visualize_verb_noun_paris(file_name, count_thredhold=10):\n",
" raw_phrases = pd.read_csv(f'{file_name}.csv')\n",
" raw_phrases = pd.DataFrame(raw_phrases)\n",
" phrases = pd.DataFrame(raw_phrases).dropna()\n",
" count_list = phrases[[\"verb\", \"noun\"]].groupby([\"verb\", \"noun\"]).size().sort_values(ascending=False)\n",
" print(len(count_list))\n",
"\n",
" top_verbs = phrases[[\"verb\"]].groupby([\"verb\"]).size().nlargest(20).reset_index()\n",
"\n",
" df = phrases[phrases[\"verb\"].isin(top_verbs[\"verb\"].tolist())]\n",
" # df = df[~df[\"noun\"].isin([\"I\", \"what\"])]\n",
" # df = phrases\n",
" # df[~df[\"verb\"].isin(top_verbs[\"verb\"].tolist())][\"verb\"] = \"other\"\n",
" # df[~df[\"verb\"].isin(top_verbs[\"verb\"].tolist())][\"noun\"] = \"other\"\n",
" df = df.groupby([\"verb\", \"noun\"]).size().reset_index().rename(columns={0: \"count\"}).sort_values(by=[\"count\"], ascending=False)\n",
" # df = df[df[\"count\"] > 10]\n",
" df = df.groupby(\"verb\").apply(lambda x: x.sort_values(\"count\", ascending=False).head(4)).reset_index(drop=True)\n",
" print(df)\n",
"\n",
" # df[\"blank\"] = \"ROOT\"\n",
" # df = phrases.groupby([\"verb\", \"noun\"]).size().sort_values(ascending=False).head(5).reset_index().rename(columns={0: \"count\"})\n",
"\n",
" df = df[df[\"count\"] > count_thredhold]\n",
" fig = px.sunburst(df, path=['verb', 'noun'], values='count')\n",
" # fig.update_layout(uniformtext=dict(minsize=10, mode='hide'))\n",
" fig.update_layout(\n",
" margin=dict(l=0, r=0, t=0, b=0),\n",
" font_family=\"Times New Roman\",\n",
" )\n",
" # fig.show()\n",
" output_html_folder_path = os.path.join(folder_path, 'output')\n",
" file_name = file_name.split(\"/\")[-1].split(\".\")[0]\n",
" html_file_path = f\"{output_html_folder_path}/{file_name}_{count_thredhold}.html\"\n",
" print(html_file_path)\n",
" fig.write_html(html_file_path)\n",
" # fig.savefig(\"output/verb_noun.pdf\")\n"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"22\n",
" verb noun count\n",
"0 analyze image 1\n",
"1 break element 1\n",
"2 characterize image 1\n",
"3 clarify content 1\n",
"4 create narrative 1\n",
"5 describe image 2\n",
"6 examine image 1\n",
"7 explain aspect 1\n",
"8 give explanation 2\n",
"9 illustrate image 1\n",
"10 offer explanation 1\n",
"11 offer analysis 1\n",
"12 portray image 1\n",
"13 present description 1\n",
"14 provide description 2\n",
"15 relay account 1\n",
"16 render summary 1\n",
"17 share rundown 1\n",
"18 share interpretation 1\n",
"19 summarize content 1\n",
"20 write depiction 1\n",
"21 write summary 1\n",
"../../data/output/pretrain_instruction_verb_noun_30.html\n",
"8626\n",
" verb noun count\n",
"0 affect expression 12\n",
"1 affect activity 8\n",
"2 affect proliferation 6\n",
"3 affect ability 5\n",
"4 alter expression 14\n",
".. ... ... ...\n",
"75 suppress activation 6\n",
"76 use method 6\n",
"77 use algorithm 4\n",
"78 use distance 3\n",
"79 use model 3\n",
"\n",
"[80 rows x 3 columns]\n",
"../../data/output/pretrain_response_verb_noun_30.html\n",
"1031\n",
" verb noun count\n",
"0 affect structure 18\n",
"1 affect appearance 17\n",
"2 affect patient 6\n",
"3 affect visibility 5\n",
"4 compare sub 1\n",
".. ... ... ...\n",
"74 stain image 2\n",
"75 tell value 3\n",
"76 tell percentage 3\n",
"77 tell number 2\n",
"78 tell size 2\n",
"\n",
"[79 rows x 3 columns]\n",
"../../data/output/instruct_10K_instruction_verb_noun_30.html\n",
"2889\n",
" verb noun count\n",
"0 appear area 54\n",
"1 appear structure 11\n",
"2 appear region 8\n",
"3 appear opacity 7\n",
"4 contain image 73\n",
".. ... ... ...\n",
"72 show scan 379\n",
"73 stain sub 14\n",
"74 stain image 14\n",
"75 stain section 10\n",
"76 stain tissue 2\n",
"\n",
"[77 rows x 3 columns]\n",
"../../data/output/instruct_10K_response_verb_noun_30.html\n",
"936\n",
" verb noun count\n",
"0 affect lung 8\n",
"1 affect structure 6\n",
"2 affect image 4\n",
"3 affect patient 4\n",
"4 cause lesion 4\n",
".. ... ... ...\n",
"75 tell location 2\n",
"76 undergo kind 4\n",
"77 undergo procedure 3\n",
"78 undergo type 2\n",
"79 undergo imaging 1\n",
"\n",
"[80 rows x 3 columns]\n",
"../../data/output/instruct_60K_instruction_verb_noun_30.html\n",
"5615\n",
" verb noun count\n",
"0 cause effect 15\n",
"1 cause compression 13\n",
"2 cause shift 9\n",
"3 cause symptom 7\n",
"4 compare group 41\n",
".. ... ... ...\n",
"73 undergo radiosurgery 6\n",
"74 use hematoxylin 86\n",
"75 use staining 56\n",
"76 use stain 55\n",
"77 use sequence 51\n",
"\n",
"[78 rows x 3 columns]\n",
"../../data/output/instruct_60K_response_verb_noun_30.html\n",
"894\n",
" verb noun count\n",
"0 affect lung 6\n",
"1 affect structure 4\n",
"2 affect number 4\n",
"3 affect patient 3\n",
"4 cause compression 3\n",
".. ... ... ...\n",
"75 undergo x 1\n",
"76 use type 2\n",
"77 use nomogram 2\n",
"78 use kind 2\n",
"79 use assay 1\n",
"\n",
"[80 rows x 3 columns]\n",
"../../data/output/instruct_60K_intext_instruction_verb_noun_30.html\n",
"6515\n",
" verb noun count\n",
"0 cause compression 18\n",
"1 cause effect 18\n",
"2 cause displacement 14\n",
"3 cause shift 8\n",
"4 compare score 33\n",
".. ... ... ...\n",
"72 undergo chemotherapy 6\n",
"73 use hematoxylin 101\n",
"74 use stain 70\n",
"75 use staining 55\n",
"76 use imaging 49\n",
"\n",
"[77 rows x 3 columns]\n",
"../../data/output/instruct_60K_intext_response_verb_noun_30.html\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Bad pipe message: %s [b\"K\\xc6\\x1b\\xda\\xfd\\x1a\\x16[s7\\xbfz%\\x11:\\x0f\\xc2%\\x00\\x00|\\xc0,\\xc00\\x00\\xa3\\x00\\x9f\\xcc\\xa9\\xcc\\xa8\\xcc\\xaa\\xc0\\xaf\\xc0\\xad\\xc0\\xa3\\xc0\\x9f\\xc0]\\xc0a\\xc0W\\xc0S\\xc0+\\xc0/\\x00\\xa2\\x00\\x9e\\xc0\\xae\\xc0\\xac\\xc0\\xa2\\xc0\\x9e\\xc0\\\\\\xc0`\\xc0V\\xc0R\\xc0$\\xc0(\\x00k\\x00j\\xc0#\\xc0'\\x00g\\x00@\\xc0\\n\\xc0\\x14\\x009\\x008\\xc0\\t\\xc0\\x13\\x003\\x002\\x00\\x9d\\xc0\\xa1\\xc0\\x9d\\xc0Q\\x00\\x9c\\xc0\\xa0\\xc0\\x9c\\xc0P\\x00=\\x00<\\x005\\x00/\"]\n",
"Bad pipe message: %s [b'\\x95\\x8a`\\xb1\\xd9]\\x93\\xcd\\xbc9aa\\x03K\\xf4\\xf7\\xa5\\xc7']\n",
"Bad pipe message: %s [b'\\xa4f\\xeb\\x91\\x87\\x02\\x17$\\x85q^\\x82\\x86\\x94\\x9c\\xa7tq\\x00\\x00\\xa2\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\x00\\x88\\x00\\x87\\x00\\x86\\x00\\x85\\xc0\\x19\\x00:\\x00\\x89\\xc0\\x0f\\xc0\\x05\\x005\\x00\\x84\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00E\\x00D\\x00C\\x00B\\xc0\\x18\\x004\\x00\\x9b\\x00F\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x96\\x00A\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x16\\x00\\x18\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\xc0\\x12\\xc0\\x08\\x00\\x16\\x00', b'\\x10\\x00\\r\\xc0\\x17\\x00\\x1b\\xc0\\r\\xc0\\x03\\x00\\n\\x00\\x15\\x00\\x12\\x00']\n",
"Bad pipe message: %s [b'\\x0c\\x00\\x1a\\x00\\t\\x00\\x14\\x00\\x11\\x00\\x19\\x00\\x08\\x00']\n",
"Bad pipe message: %s [b'\\x17\\x00\\x03\\xc0\\x10']\n",
"Bad pipe message: %s [b'\\x01\\xdd>\\x9e\\xb7\\x96[\\xc3\\x15\\xd9\\xe8M\\xe5\\xa14\\xaa\\x0e!\\x00\\x00\\xa2\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\x00\\x88\\x00\\x87\\x00\\x86\\x00\\x85\\xc0\\x19\\x00:\\x00\\x89\\xc0\\x0f\\xc0\\x05\\x005\\x00\\x84\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00E\\x00D\\x00C\\x00B\\xc0\\x18\\x004\\x00\\x9b\\x00F\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x96\\x00A\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x16\\x00\\x18\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\xc0\\x12\\xc0\\x08\\x00\\x16\\x00\\x13\\x00\\x10\\x00\\r\\xc0\\x17\\x00\\x1b\\xc0\\r\\xc0\\x03\\x00\\n\\x00\\x15\\x00\\x12\\x00\\x0f\\x00\\x0c\\x00\\x1a\\x00\\t\\x00\\x14\\x00\\x11\\x00']\n",
"Bad pipe message: %s [b'\\x08\\x00\\x06\\x00\\x17\\x00\\x03\\xc0\\x10\\xc0\\x06\\xc0\\x15\\xc0\\x0b\\xc0\\x01\\x00\\x02\\x00\\x01\\x00\\xff\\x02']\n",
"Bad pipe message: %s [b'']\n",
"Bad pipe message: %s [b'\\xba\\xf6.\\xe1\\xd4\\x15!\\xf2\\xc4jJ\\xde\\x11\\xd1^>X\\xbe\\x00\\x00>']\n",
"Bad pipe message: %s [b'J/\\xc1\\x98\\xd3\\xd3\\xe2\\xa9\\xe9\\x91#3\\x96\\xe1\\xd6']\n",
"Bad pipe message: %s [b'\\xb1\\x00\\x00\\xa2\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\x00\\x88\\x00\\x87\\x00\\x86\\x00\\x85\\xc0\\x19\\x00:\\x00\\x89\\xc0\\x0f\\xc0\\x05\\x005\\x00\\x84\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00E\\x00D\\x00C\\x00B\\xc0\\x18\\x004\\x00\\x9b\\x00F\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x96\\x00A\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x16\\x00\\x18\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\xc0\\x12\\xc0\\x08\\x00\\x16\\x00\\x13\\x00\\x10\\x00\\r\\xc0\\x17\\x00\\x1b\\xc0\\r\\xc0\\x03\\x00\\n\\x00\\x15\\x00\\x12\\x00\\x0f\\x00\\x0c\\x00\\x1a\\x00\\t\\x00\\x14\\x00\\x11\\x00\\x19\\x00\\x08\\x00', b'\\x17\\x00\\x03\\xc0\\x10']\n",
"Bad pipe message: %s [b\"\\x1b\\xc2(\\xbf\\xf0\\x126\\x88\\xd1n\\x1c\\xe6\\xe4d\\xe1\\x80\\x1cw\\x00\\x00\\xf4\\xc00\\xc0,\\xc0(\\xc0$\\xc0\\x14\\xc0\\n\\x00\\xa5\\x00\\xa3\\x00\\xa1\\x00\\x9f\\x00k\\x00j\\x00i\\x00h\\x009\\x008\\x007\\x006\\x00\\x88\\x00\\x87\\x00\\x86\\x00\\x85\\xc0\\x19\\x00\\xa7\\x00m\\x00:\\x00\\x89\\xc02\\xc0.\\xc0*\\xc0&\\xc0\\x0f\\xc0\\x05\\x00\\x9d\\x00=\\x005\\x00\\x84\\xc0/\\xc0+\\xc0'\\xc0#\\xc0\\x13\\xc0\\t\\x00\\xa4\\x00\\xa2\\x00\\xa0\\x00\\x9e\\x00g\\x00@\\x00?\\x00>\\x003\\x002\\x001\\x000\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00E\\x00D\\x00C\\x00B\\xc0\\x18\\x00\\xa6\\x00l\\x004\\x00\\x9b\\x00F\\xc01\\xc0-\\xc0)\\xc0%\\xc0\\x0e\\xc0\\x04\\x00\\x9c\\x00<\\x00/\\x00\\x96\\x00A\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x16\\x00\\x18\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\xc0\"]\n",
"Bad pipe message: %s [b'\\x08\\x00\\x16\\x00\\x13\\x00\\x10\\x00\\r\\xc0\\x17\\x00\\x1b\\xc0\\r\\xc0\\x03']\n"
]
}
],
"source": [
"c_list = [30] # [0,10,20]\n",
"for k, v in dict_instruct_response.items():\n",
" output_csv_folder_path = os.path.join(folder_path, 'csv')\n",
" for c in c_list:\n",
" visualize_verb_noun_paris(f'{output_csv_folder_path}/{k}_instruction_verb_noun', c) \n",
" visualize_verb_noun_paris(f'{output_csv_folder_path}/{k}_response_verb_noun', c) \n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "llava_med",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.11"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}