Models:
Samuel Callahan
SCallahan/
RaDialog
0
Downloads: 1
Diff of /docs/index.html [000000] .. [4abb48]

Switch to unified view

a b/docs/index.html 

  1
  
<!DOCTYPE html>





  
  

  2
  
<!-- adapted from https://github.com/eliahuhorwitz/Academic-project-page-template -->





  
  

  3
  
<html>





  
  

  4
  
<head>





  
  

  5
  
  <meta charset="utf-8">





  
  

  6
  
  <!-- Meta tags for social media banners, these should be filled in appropriatly as they are your "business card" -->





  
  

  7
  
  <!-- Replace the content tag with appropriate information -->





  
  

  8
  
  <meta name="description" content="RaDialog: A Large Vision-Language Model for Radiology Report Generation and Conversational Assistance">





  
  

  9
  
  <meta property="og:title" content="RaDialog: A Large Vision-Language Model for Radiology Report Generation and Conversational Assistance"/>





  
  

  10
  
  <meta property="og:description" content="Conversational AI tools that can generate and discuss clinically correct radiology reports for a given medical image have the potential to transform radiology. Such a human-in-the-loop radiology assistant could facilitate a collaborative diagnostic process, thus saving time and improving the quality of reports. Towards this goal, we introduce RaDialog, the first thoroughly evaluated and publicly available large vision-language model for radiology report generation and interactive dialog. RaDialog effectively integrates visual image features and structured pathology findings with a large language model (LLM) while simultaneously adapting it to a specialized domain using parameter-efficient fine-tuning. To keep the conversational abilities of the underlying LLM, we propose a comprehensive, semi-automatically labeled, image-grounded instruct dataset for chest X-ray radiology tasks. By training with this dataset, our method achieves state-of-the-art clinical correctness in report generation and shows impressive abilities in interactive tasks such as correcting reports and answering questions, serving as a foundational step toward clinical dialog systems."/>





  
  

  11
  
  <meta property="og:url" content="https://chantalmp.github.io/RaDialog/"/>





  
  

  12
  
  <!-- Path to banner image, should be in the path listed below. Optimal dimenssions are 1200X630-->





  
  

  13
  
  <meta property="og:image" content="static/images/example.png" />





  
  

  14
  
  <meta property="og:image:width" content="1462"/>





  
  

  15
  
  <meta property="og:image:height" content="1269"/>





  
  

  16
  





  
  

  17
  





  
  

  18
  
  <meta name="twitter:title" content="RaDialog: A Large Vision-Language Model for Radiology Report Generation and Conversational Assistance">





  
  

  19
  
  <meta name="twitter:description" content="Conversational AI tools that can generate and discuss clinically correct radiology reports for a given medical image have the potential to transform radiology. Such a human-in-the-loop radiology assistant could facilitate a collaborative diagnostic process, thus saving time and improving the quality of reports. Towards this goal, we introduce RaDialog, the first thoroughly evaluated and publicly available large vision-language model for radiology report generation and interactive dialog. RaDialog effectively integrates visual image features and structured pathology findings with a large language model (LLM) while simultaneously adapting it to a specialized domain using parameter-efficient fine-tuning. To keep the conversational abilities of the underlying LLM, we propose a comprehensive, semi-automatically labeled, image-grounded instruct dataset for chest X-ray radiology tasks. By training with this dataset, our method achieves state-of-the-art clinical correctness in report generation and shows impressive abilities in interactive tasks such as correcting reports and answering questions, serving as a foundational step toward clinical dialog systems.">





  
  

  20
  
  <!-- Path to banner image, should be in the path listed below. Optimal dimenssions are 1200X600-->





  
  

  21
  
  <meta name="twitter:image" content="static/images/example.png">





  
  

  22
  
  <!-- <meta name="twitter:card" content="summary_large_image"> -->





  
  

  23
  
  <!-- Keywords for your paper to be indexed by-->





  
  

  24
  
  <meta name="keywords" content="Radiology AI, Chatbot, Large Language Model, Vison-Language Model, ChatGPT, GPT4-Vision, Medical Image Analysis, AI in Healthcare, Clinical Report Generation, Radiology Report Automation, AI-Powered Diagnosis, Conversational AI in Medicine, Medical Imaging Technology, AI Radiology Assistant, Healthcare Technology Innovation, Interactive Radiology Reports, Medical Data Analysis AI, Diagnostic AI Tools, Machine Learning in Radiology">





  
  

  25
  
  <meta name="viewport" content="width=device-width, initial-scale=1">





  
  

  26
  





  
  

  27
  





  
  

  28
  
  <title>RaDialog: A Large Vision-Language Model for Radiology Report Generation and Conversational Assistance</title>





  
  

  29
  
  <link rel="icon" type="image/x-icon" href="static/images/favicon.ico">





  
  

  30
  
  <link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro"





  
  

  31
  
  rel="stylesheet">





  
  

  32
  





  
  

  33
  
  <link rel="stylesheet" href="static/css/bulma.min.css">





  
  

  34
  
  <link rel="stylesheet" href="static/css/bulma-carousel.min.css">





  
  

  35
  
  <link rel="stylesheet" href="static/css/bulma-slider.min.css">





  
  

  36
  
  <link rel="stylesheet" href="static/css/fontawesome.all.min.css">





  
  

  37
  
  <link rel="stylesheet"





  
  

  38
  
  href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css">





  
  

  39
  
  <link rel="stylesheet" href="static/css/index.css">





  
  

  40
  





  
  

  41
  
  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>





  
  

  42
  
  <script src="https://documentcloud.adobe.com/view-sdk/main.js"></script>





  
  

  43
  
  <script defer src="static/js/fontawesome.all.min.js"></script>





  
  

  44
  
  <script src="static/js/bulma-carousel.min.js"></script>





  
  

  45
  
  <script src="static/js/bulma-slider.min.js"></script>





  
  

  46
  
  <script src="static/js/index.js"></script>





  
  

  47
  
</head>





  
  

  48
  
<body>





  
  

  49
  





  
  

  50
  





  
  

  51
  
  <section class="hero">





  
  

  52
  
    <div class="hero-body">





  
  

  53
  
      <div class="container is-max-desktop">





  
  

  54
  
        <div class="columns is-centered">





  
  

  55
  
          <div class="column has-text-centered">





  
  

  56
  
            <h1 class="title is-1 publication-title">RaDialog: A Large Vision-Language Model for Radiology Report Generation and Conversational Assistance</h1>





  
  

  57
  
            <div class="is-size-5 publication-authors">





  
  

  58
  
              <!-- Paper authors -->





  
  

  59
  
              <span class="author-block">





  
  

  60
  
                <a href="https://www.cs.cit.tum.de/camp/members/chantal-pellegrini/" target="_blank">Chantal Pellegrini</a><sup>*</sup>,</span>





  
  

  61
  
                <span class="author-block">





  
  

  62
  
                  <a href="https://www.cs.cit.tum.de/camp/members/ege-oezsoy/" target="_blank">Ege Özsoy</a><sup>*</sup>,</span>





  
  

  63
  
                  <span class="author-block">





  
  

  64
  
                    <a href="https://www.cs.cit.tum.de/camp/members/benjamin-busam-1/" target="_blank">Benjamin Busam</a>,





  
  

  65
  
                  </span>





  
  

  66
  
                  <span class="author-block">





  
  

  67
  
                    <a href="https://www.cs.cit.tum.de/camp/members/cv-nassir-navab/nassir-navab/" target="_blank">Nassir Navab</a>,





  
  

  68
  
                  </span>





  
  

  69
  
                  <span class="author-block">





  
  

  70
  
                    <a href="https://www.cs.cit.tum.de/camp/members/matthias-keicher/" target="_blank">Matthias Keicher</a>





  
  

  71
  
                  </span>





  
  

  72
  
                  </div>





  
  

  73
  





  
  

  74
  
                  <div class="is-size-5 publication-authors">





  
  

  75
  
                    <span class="author-block">Computer Aided Medical Procedures, </br>Technical University of Munich</span>





  
  

  76
  
                    <span class="eql-cntrb"><small><br><sup>*</sup>Indicates equal contribution.</small></span>





  
  

  77
  
                  </div>





  
  

  78
  





  
  

  79
  
                  <div class="column has-text-centered">





  
  

  80
  
                    <div class="publication-links">





  
  

  81
  
                         <!-- Arxiv PDF link -->





  
  

  82
  
                      <span class="link-block">





  
  

  83
  
                        <a href="https://arxiv.org/pdf/2311.18681.pdf" target="_blank"





  
  

  84
  
                        class="external-link button is-normal is-rounded is-dark">





  
  

  85
  
                        <span class="icon">





  
  

  86
  
                          <i class="fas fa-file-pdf"></i>





  
  

  87
  
                        </span>





  
  

  88
  
                        <span>Paper</span>





  
  

  89
  
                      </a>





  
  

  90
  
                    </span>





  
  

  91
  





  
  

  92
  
                    <!-- Supplementary PDF link -->





  
  

  93
  
                    <!-- <span class="link-block">





  
  

  94
  
                      <a href="static/pdfs/supplementary_material.pdf" target="_blank"





  
  

  95
  
                      class="external-link button is-normal is-rounded is-dark">





  
  

  96
  
                        <span class="icon">





  
  

  97
  
                          <i class="fas fa-file-pdf"></i>





  
  

  98
  
                        </span>





  
  

  99
  
                        <span>Supplementary</span>





  
  

  100
  
                      </a>





  
  

  101
  
                    </span> -->





  
  

  102
  





  
  

  103
  
                  <!-- Github link -->





  
  

  104
  
                  <span class="link-block">





  
  

  105
  
                    <a href="https://github.com/ChantalMP/RaDialog" target="_blank"





  
  

  106
  
                    class="external-link button is-normal is-rounded is-dark">





  
  

  107
  
                    <span class="icon">





  
  

  108
  
                      <i class="fab fa-github"></i>





  
  

  109
  
                    </span>





  
  

  110
  
                    <span>Code</span>





  
  

  111
  
                  </a>





  
  

  112
  
                </span>





  
  

  113
  





  
  

  114
  
                <!-- ArXiv abstract Link -->





  
  

  115
  
                <span class="link-block">





  
  

  116
  
                  <a href="https://arxiv.org/abs/2311.18681" target="_blank"





  
  

  117
  
                  class="external-link button is-normal is-rounded is-dark">





  
  

  118
  
                  <span class="icon">





  
  

  119
  
                    <i class="ai ai-arxiv"></i>





  
  

  120
  
                  </span>





  
  

  121
  
                  <span>arXiv</span>





  
  

  122
  
                </a>





  
  

  123
  
              </span>





  
  

  124
  
            </div>





  
  

  125
  
          </div>





  
  

  126
  
        </div>





  
  

  127
  
      </div>





  
  

  128
  
    </div>





  
  

  129
  
  </div>





  
  

  130
  
</section>





  
  

  131
  





  
  

  132
  





  
  

  133
  
<!-- Teaser video-->





  
  

  134
  
<section class="hero teaser">





  
  

  135
  
  <div class="container is-max-desktop">





  
  

  136
  
    <div class="hero-body">





  
  

  137
  
      <video poster="" id="tree" autoplay controls muted loop height="100%">





  
  

  138
  
        <!-- Your video here -->





  
  

  139
  
        <source src="static/videos/RaDialog_demo.mp4"





  
  

  140
  
        type="video/mp4">





  
  

  141
  
      </video>





  
  

  142
  
      <!-- <h2 class="subtitle has-text-centered">





  
  

  143
  
        RaDialog generates the finding section for a given chest X-ray image and allows for interactive dialog.





  
  

  144
  
      </h2> -->





  
  

  145
  
    </div>





  
  

  146
  
  </div>





  
  

  147
  
</section>





  
  

  148
  
<!-- End teaser video -->





  
  

  149
  





  
  

  150
  
<!-- Paper abstract -->





  
  

  151
  
<section class="section hero is-light">





  
  

  152
  
<!-- <section class="section hero"> -->





  
  

  153
  
  <div class="container is-max-desktop">





  
  

  154
  
    <div class="columns is-centered has-text-centered">





  
  

  155
  
      <div class="column is-four-fifths">





  
  

  156
  
        <h2 class="title is-3">Abstract</h2>





  
  

  157
  
        <div class="content has-text-justified">





  
  

  158
  
          <p>





  
  

  159
  
            Conversational AI tools that can generate and discuss clinically correct radiology reports for a given medical image have the potential to transform radiology. Such a human-in-the-loop radiology assistant could facilitate a collaborative diagnostic process, thus saving time and improving the quality of reports. Towards this goal, we introduce RaDialog, the first thoroughly evaluated and publicly available large vision-language model for radiology report generation and interactive dialog. RaDialog effectively integrates visual image features and structured pathology findings with a large language model (LLM) while simultaneously adapting it to a specialized domain using parameter-efficient fine-tuning. To keep the conversational abilities of the underlying LLM, we propose a comprehensive, semi-automatically labeled, image-grounded instruct dataset for chest X-ray radiology tasks. By training with this dataset, our method achieves state-of-the-art clinical correctness in report generation and shows impressive abilities in interactive tasks such as correcting reports and answering questions, serving as a foundational step toward clinical dialog systems.





  
  

  160
  
          </p>





  
  

  161
  
        </div>





  
  

  162
  
      </div>





  
  

  163
  
    </div>





  
  

  164
  
  </div>





  
  

  165
  
</section>





  
  

  166
  
<!-- End paper abstract -->





  
  

  167
  





  
  

  168
  
<section class="section hero">





  
  

  169
  
  <div class="container is-max-desktop">





  
  

  170
  
    <div class="columns is-centered has-text-centered">





  
  

  171
  
      <div class="column is-full">





  
  

  172
  
        <h2 class="title is-3">Methodology</h2>





  
  

  173
  
        <img src="static/images/model.png"> </img>





  
  

  174
  
        <div class="content has-text-justified">





  
  

  175
  
          RaDialog integrates both image features and structured pathology labels for clinically correct report generation, along with the integration of image information with a large language model to enable conversational downstream tasks. Additionally, the LLM is adapted using parameter-efficient fine-tuning for teaching image understanding and radiological knowledge. RaDialog has achieved state-of-the-art results in the clinical correctness of report generation.





  
  

  176
  
        </div>





  
  

  177
  
        <div class="methodology-image">





  
  

  178
  
        





  
  

  179
  
        </div>





  
  

  180
  
      </div>





  
  

  181
  
    </div>





  
  

  182
  
  </div>





  
  

  183
  
</section>





  
  

  184
  





  
  

  185
  





  
  

  186
  
<section class="section hero">





  
  

  187
  
  <div class="container is-max-desktop">





  
  

  188
  
    <div class="columns is-centered has-text-centered">





  
  

  189
  
      <div class="column is-full">





  
  

  190
  
        <h2 class="title is-3">RaDialog Instruct Dataset</h2>





  
  

  191
  
        <img src="static/images/dataset.png"> </img>





  
  

  192
  
        <div class="content has-text-justified">





  
  

  193
  
          Regarding the RaDialog Instruct Dataset, it includes a proposal for a diverse instruct dataset that maintains the general capacities of LLMs while imparting radiology-specific knowledge and style. This dataset encompasses a variety of tasks ranging from report generation to report correction and question answering. It is uniquely constructed using a combination of existing datasets and LLM-generated pseudo-ground truth answers. Importantly, training with this instruct dataset has significantly enhanced RaDialog's performance in interactive downstream tasks.





  
  

  194
  
        </div>





  
  

  195
  
        <div class="methodology-image">





  
  

  196
  
        





  
  

  197
  
        </div>





  
  

  198
  
      </div>





  
  

  199
  
    </div>





  
  

  200
  
  </div>





  
  

  201
  
</section>





  
  

  202
  





  
  

  203
  
<!--BibTex citation -->





  
  

  204
  
  <section class="section" id="BibTeX">





  
  

  205
  
    <div class="container is-max-desktop content">





  
  

  206
  
      <h2 class="title">BibTeX</h2>





  
  

  207
  
      <pre><code>@misc{pellegrini2023radialog,





  
  

  208
  
      title={RaDialog: A Large Vision-Language Model for Radiology Report Generation and Conversational Assistance}, 





  
  

  209
  
      author={Chantal Pellegrini and Ege Özsoy and Benjamin Busam and Nassir Navab and Matthias Keicher},





  
  

  210
  
      year={2023},





  
  

  211
  
      eprint={2311.18681},





  
  

  212
  
      archivePrefix={arXiv},





  
  

  213
  
      primaryClass={cs.CV}





  
  

  214
  
}</code></pre>





  
  

  215
  
    </div>





  
  

  216
  
</section>





  
  

  217
  
<!--End BibTex citation -->





  
  

  218
  





  
  

  219
  





  
  

  220
  
  <footer class="footer">





  
  

  221
  
  <div class="container">





  
  

  222
  
    <div class="columns is-centered">





  
  

  223
  
      <div class="column is-8">





  
  

  224
  
        <div class="content has-text-centered is-vcentered">





  
  

  225
  
          <a class="icon-link" href="https://arxiv.org/pdf/2311.18681.pdf">





  
  

  226
  
            <span class="icon">





  
  

  227
  
              <i class="fas fa-file-pdf"></i>





  
  

  228
  
            </span>





  
  

  229
  
        </a>





  
  

  230
  
        <a class="icon-link" href="https://github.com/ChantalMP/RaDialog" disabled="">





  
  

  231
  
          <span class="icon">





  
  

  232
  
            <i class="fab fa-github"></i>





  
  

  233
  
          </span>





  
  

  234
  
        </a>





  
  

  235
  
        <!-- <a class="icon-link" href="https://arxiv.org/abs/2311.18681" disabled="">





  
  

  236
  
          <span class="icon">





  
  

  237
  
            <i class="ai ai-arxiv"></i>





  
  

  238
  
          </span>





  
  

  239
  
        </a> -->





  
  

  240
  
      </div>





  
  

  241
  
        <div class="content">





  
  

  242
  
          <p style="font-size: 9pt; text-align: center">





  
  

  243
  
            The authors gratefully acknowledge the financial support by the Federal Ministry of Education and Research of Germany (BMBF) under project DIVA (13GW0469C) and the Bavarian Ministry of Economic Affairs, Regional Development and Energy (StMWi) under project ThoraXAI (DIK-2302-0002).





  
  

  244
  
            </br></br>





  
  

  245
  
            This page was built using the <a href="https://github.com/eliahuhorwitz/Academic-project-page-template" target="_blank">Academic Project Page Template</a> which was adopted from the <a href="https://nerfies.github.io" target="_blank">Nerfies</a> project page.





  
  

  246
  
            </br>





  
  

  247
  
            This website is licensed under a <a rel="license"  href="http://creativecommons.org/licenses/by-sa/4.0/" target="_blank">Creative





  
  

  248
  
            Commons Attribution-ShareAlike 4.0 International License</a>.





  
  

  249
  
          </p>





  
  

  250
  





  
  

  251
  
        </div>





  
  

  252
  
      </div>





  
  

  253
  
    </div>





  
  

  254
  
  </div>





  
  

  255
  
</footer>





  
  

  256
  





  
  

  257
  
<!-- Statcounter tracking code -->





  
  

  258
  
  





  
  

  259
  
<!-- You can add a tracker to track page visits by creating an account at statcounter.com -->





  
  

  260
  





  
  

  261
  
    <!-- End of Statcounter Code -->





  
  

  262
  





  
  

  263
  
  </body>





  
  

  264
  
  </html>