slideflow / Git / [78ef36] /docs/uq/index.html

Models:
DanielG/
slideflow
Downloads: 1
[78ef36]: / docs / uq / index.html
History
Download this file
569 lines (413 with data), 34.0 kB

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  <meta name="generator" content="Docutils 0.18.1: http://docutils.sourceforge.net/" />

  <meta name="viewport" content="width=device-width, initial-scale=1.0">

  <title>Uncertainty Quantification &mdash; slideflow 3.0.0 documentation</title>















  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
    <link rel="index" title="Index" href="../genindex/" />
    <link rel="search" title="Search" href="../search/" />
    <link rel="next" title="Generating Features" href="../features/" />
    <link rel="prev" title="Layer Activations" href="../posthoc/" />




  <script src="../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
  <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.15.2/css/all.css" integrity="sha384-vSIIfh2YWi9wW0r9iZe7RJPrKwp6bG+s9QZMoITbCckVJqGCCRhc+ccxNcdpHuYu" crossorigin="anonymous">
  <script defer data-domain="slideflow.dev" src="https://plausible.io/js/script.js"></script>
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://slideflow.dev" aria-label="Slideflow"></a>

      <div class="main-menu">
        <ul>
          <li class="active">
            <a href="https://slideflow.dev">Docs</a>
          </li>

          <li>
            <a href="https://slideflow.dev/tutorial1/">Tutorials</a>
          </li>

          <li>
            <a href="https://github.com/slideflow/slideflow">GitHub</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>
  </div>
</div>

<body class="pytorch-body">





    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">





                <div class="version">
                  3.0
                </div>









<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search/" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>


          </div>







              <p class="caption" role="heading"><span class="caption-text">Introduction</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../installation/">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/">Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../quickstart/">Quickstart</a></li>
<li class="toctree-l1"><a class="reference internal" href="../project_setup/">Setting up a Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../datasets_and_val/">Datasets</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slide_processing/">Slide Processing</a></li>
<li class="toctree-l1"><a class="reference internal" href="../training/">Training</a></li>
<li class="toctree-l1"><a class="reference internal" href="../evaluation/">Evaluation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../posthoc/">Layer Activations</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">Uncertainty Quantification</a></li>
<li class="toctree-l1"><a class="reference internal" href="../features/">Generating Features</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mil/">Multiple-Instance Learning (MIL)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../ssl/">Self-Supervised Learning (SSL)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../stylegan/">Generative Networks (GANs)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../saliency/">Saliency Maps</a></li>
<li class="toctree-l1"><a class="reference internal" href="../segmentation/">Tissue Segmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../cellseg/">Cell Segmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../custom_loops/">Custom Training Loops</a></li>
<li class="toctree-l1"><a class="reference internal" href="../studio/">Slideflow Studio: Live Visualization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../troubleshooting/">Troubleshooting</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Developer Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tfrecords/">TFRecords: Reading and Writing</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dataloaders/">Dataloaders: Sampling and Augmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../custom_extractors/">Custom Feature Extractors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tile_labels/">Strong Supervision with Tile Labels</a></li>
<li class="toctree-l1"><a class="reference internal" href="../plugins/">Creating a Slideflow Plugin</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../slideflow/">slideflow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../project/">slideflow.Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dataset/">slideflow.Dataset</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dataset_features/">slideflow.DatasetFeatures</a></li>
<li class="toctree-l1"><a class="reference internal" href="../heatmap/">slideflow.Heatmap</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model_params/">slideflow.ModelParams</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mosaic/">slideflow.Mosaic</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slidemap/">slideflow.SlideMap</a></li>
<li class="toctree-l1"><a class="reference internal" href="../biscuit/">slideflow.biscuit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slideflow_cellseg/">slideflow.cellseg</a></li>
<li class="toctree-l1"><a class="reference internal" href="../io/">slideflow.io</a></li>
<li class="toctree-l1"><a class="reference internal" href="../io_tensorflow/">slideflow.io.tensorflow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../io_torch/">slideflow.io.torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../gan/">slideflow.gan</a></li>
<li class="toctree-l1"><a class="reference internal" href="../grad/">slideflow.grad</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mil_module/">slideflow.mil</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model/">slideflow.model</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model_tensorflow/">slideflow.model.tensorflow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model_torch/">slideflow.model.torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../norm/">slideflow.norm</a></li>
<li class="toctree-l1"><a class="reference internal" href="../simclr/">slideflow.simclr</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slide/">slideflow.slide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slide_qc/">slideflow.slide.qc</a></li>
<li class="toctree-l1"><a class="reference internal" href="../stats/">slideflow.stats</a></li>
<li class="toctree-l1"><a class="reference internal" href="../util/">slideflow.util</a></li>
<li class="toctree-l1"><a class="reference internal" href="../studio_module/">slideflow.studio</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Tutorials</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tutorial1/">Tutorial 1: Model training (simple)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial2/">Tutorial 2: Model training (advanced)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial3/">Tutorial 3: Using a custom architecture</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial4/">Tutorial 4: Model evaluation &amp; heatmaps</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial5/">Tutorial 5: Creating a mosaic map</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial6/">Tutorial 6: Custom slide filtering</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial7/">Tutorial 7: Training with custom augmentations</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial8/">Tutorial 8: Multiple-Instance Learning</a></li>
</ul>



        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">

      <li>
        <a href="../">

            Docs

        </a> &gt;
      </li>


      <li>Uncertainty Quantification</li>


      <li class="pytorch-breadcrumbs-aside">


            <a href="../_sources/uq.rst.txt" rel="nofollow"><img src="../_static/images/view-page-source-icon.svg"></a>


      </li>

  </ul>


</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">



          <div class="rst-content">

            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">

  <section id="uncertainty-quantification">
<span id="uncertainty"></span><h1>Uncertainty Quantification<a class="headerlink" href="#uncertainty-quantification" title="Permalink to this heading">¶</a></h1>
<p>Several uncertainty quantification (UQ) methods have been developed for deep learning models and tested in digital histopathology, including MC Dropout, deep ensembles, hyper-deep ensembles, and test-time augmentation.</p>
<p>Slideflow includes a dropout-based method of uncertainty estimation. MC dropout UQ methods exploit the observation that neural networks with dropout approximate sampling of the Bayesian posterior. Images undergo multiple forward passes in a dropout-enabled network during inference, which results in a distribution of predictions. The standard deviation of such a distribution represents the uncertainty estimate.</p>
<section id="training-with-uq">
<h2>Training with UQ<a class="headerlink" href="#training-with-uq" title="Permalink to this heading">¶</a></h2>
<p>Training models with UQ is straightforward, requiring only two hyperparameter settings. Dropout must be enabled (set to a nonzero value), and <code class="docutils literal notranslate"><span class="pre">uq</span></code> should be <code class="docutils literal notranslate"><span class="pre">True</span></code>:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">slideflow</span> <span class="k">as</span> <span class="nn">sf</span>

<span class="n">params</span> <span class="o">=</span> <span class="n">sf</span><span class="o">.</span><span class="n">ModelParams</span><span class="p">(</span>
  <span class="n">tile_px</span><span class="o">=</span><span class="mi">299</span><span class="p">,</span>
  <span class="n">tile_um</span><span class="o">=</span><span class="mi">302</span><span class="p">,</span>
  <span class="o">...</span><span class="p">,</span>
  <span class="n">dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
  <span class="n">uq</span><span class="o">=</span><span class="kc">True</span>
<span class="p">)</span>
</pre></div>
</div>
<p>All predictions from this model will now involve 30 forward passes through the network, with dropout always enabled. Final tile-level predictions will be the average from each of the 30 forward passes, and tile-level uncertainty will be the standard deviation of the forward passes.</p>
</section>
<section id="evaluating-with-uq">
<h2>Evaluating with UQ<a class="headerlink" href="#evaluating-with-uq" title="Permalink to this heading">¶</a></h2>
<p>Any pipeline function using a model trained with UQ will automatically estimate uncertainty, without any additional action from the user. When model predictions are saved during validation or evaluation, uncertainty estimates will be saved alongside predictions in the tile- and patient-level predictions files found in the model folder.</p>
</section>
<section id="uncertainty-heatmaps">
<h2>Uncertainty heatmaps<a class="headerlink" href="#uncertainty-heatmaps" title="Permalink to this heading">¶</a></h2>
<p>If a model was trained with UQ enabled, the <a class="reference internal" href="../project/#slideflow.Project.generate_heatmaps" title="slideflow.Project.generate_heatmaps"><code class="xref py py-meth docutils literal notranslate"><span class="pre">slideflow.Project.generate_heatmaps()</span></code></a> function will automatically create uncertainty heatmaps alongside the prediction heatmaps.</p>
</section>
<section id="uncertainty-thresholding">
<h2>Uncertainty thresholding<a class="headerlink" href="#uncertainty-thresholding" title="Permalink to this heading">¶</a></h2>
<p>Uncertainty information can be exploited to separate slide- and patient-level predictions into low- and high-confidence. We developed an uncertainty thresholding algorithm (<a class="reference external" href="https://github.com/jamesdolezal/biscuit/">BISCUIT</a>) to accomplish this task, which is available in <code class="xref py py-mod docutils literal notranslate"><span class="pre">slideflow.biscuit</span></code>. Algorithmic details and validation studies can be found in our <a class="reference external" href="https://www.nature.com/articles/s41467-022-34025-x">manuscript</a> detailing the method.</p>
<p>Here, we will run through an example of how to apply this UQ thresholding strategy for a weakly-supervised classification model. At present, <code class="docutils literal notranslate"><span class="pre">biscuit</span></code> only supports uncertainty estimation and confidence thresholding for binary classification.</p>
<section id="prepare-an-experiment">
<h3>Prepare an Experiment<a class="headerlink" href="#prepare-an-experiment" title="Permalink to this heading">¶</a></h3>
<p>Start by creating a Slideflow project and then initializing a <code class="docutils literal notranslate"><span class="pre">biscuit</span></code> experiment, including the outcome target and the two classes.  We will be training models to predict <code class="docutils literal notranslate"><span class="pre">&quot;HPV_status&quot;</span></code>, with the two classes <code class="docutils literal notranslate"><span class="pre">&quot;positive&quot;</span></code> and <code class="docutils literal notranslate"><span class="pre">&quot;negative&quot;</span></code>.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">slideflow</span> <span class="k">as</span> <span class="nn">sf</span>
<span class="kn">from</span> <span class="nn">slideflow</span> <span class="kn">import</span> <span class="n">biscuit</span>

<span class="c1"># Create a Slideflow project</span>
<span class="n">P</span> <span class="o">=</span> <span class="n">sf</span><span class="o">.</span><span class="n">Project</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>

<span class="c1"># Initialize a biscuit experiment</span>
<span class="n">experiment</span> <span class="o">=</span> <span class="n">biscuit</span><span class="o">.</span><span class="n">Experiment</span><span class="p">(</span>
    <span class="n">train_project</span><span class="o">=</span><span class="n">P</span><span class="p">,</span>
    <span class="n">outcome</span><span class="o">=</span><span class="s1">&#39;HPV_status&#39;</span><span class="p">,</span>
    <span class="n">outcome1</span><span class="o">=</span><span class="s1">&#39;negative&#39;</span><span class="p">,</span>
    <span class="n">outcome2</span><span class="o">=</span><span class="s1">&#39;positive&#39;</span>
<span class="p">)</span>
</pre></div>
</div>
<p>Next, prepare the model hyperparameters. Here, we will use the hyperparameters used in the original manuscript.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">hp</span> <span class="o">=</span> <span class="n">biscuit</span><span class="o">.</span><span class="n">hp</span><span class="o">.</span><span class="n">nature2022</span><span class="p">()</span>
</pre></div>
</div>
</section>
<section id="train-with-cross-validation">
<h3>Train with cross-validation<a class="headerlink" href="#train-with-cross-validation" title="Permalink to this heading">¶</a></h3>
<p>We’ll start by training models in cross-validation on the full dataset. We’ll use the default three-fold cross-validation strategy. We need to supply a label for experiment model tracking, which will be used for the rest of our experiments.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Train outer cross-validation models.</span>
<span class="n">experiment</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="n">hp</span><span class="o">=</span><span class="n">hp</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;HPV&#39;</span><span class="p">)</span>
</pre></div>
</div>
<p>Models will be saved in the project model folder.</p>
</section>
<section id="train-inner-cross-validation">
<h3>Train inner cross-validation<a class="headerlink" href="#train-inner-cross-validation" title="Permalink to this heading">¶</a></h3>
<p>Next, for each of the three cross-validation models trained, we will perform 5-fold nested cross-validation. Uncertainty thresholds are determined from nested cross-validation results.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Train inner, nested cross-validation models.</span>
<span class="n">experiment</span><span class="o">.</span><span class="n">train_nested_cv</span><span class="p">(</span><span class="n">hp</span><span class="o">=</span><span class="n">hp</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">&#39;HPV&#39;</span><span class="p">)</span>
</pre></div>
</div>
<p>Models will again be saved in the project model directory. We can view a summary of the results from these cross-validation studies using the <code class="xref py py-func docutils literal notranslate"><span class="pre">biscuit.find_cv()</span></code> and <code class="xref py py-func docutils literal notranslate"><span class="pre">biscuit.get_model_results()</span></code> functions.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">slideflow.biscuit</span> <span class="kn">import</span> <span class="n">find_cv</span><span class="p">,</span> <span class="n">get_model_results</span>

<span class="c1"># Print results from outer cross-validation</span>
<span class="n">cv_models</span> <span class="o">=</span> <span class="n">find_cv</span><span class="p">(</span>
    <span class="n">project</span><span class="o">=</span><span class="n">P</span><span class="p">,</span>
    <span class="n">label</span><span class="o">=</span><span class="s1">&#39;HPV&#39;</span><span class="p">,</span>
    <span class="n">outcome</span><span class="o">=</span><span class="s1">&#39;HPV_status&#39;</span>
<span class="p">)</span>
<span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">cv_models</span><span class="p">:</span>
    <span class="n">results</span> <span class="o">=</span> <span class="n">get_model_results</span><span class="p">(</span><span class="n">m</span><span class="p">,</span> <span class="n">outcome</span><span class="o">=</span><span class="s1">&#39;HPV_status&#39;</span><span class="p">,</span> <span class="n">epoch</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="n">m</span><span class="p">,</span> <span class="n">results</span><span class="p">[</span><span class="s1">&#39;pt_auc&#39;</span><span class="p">])</span>
</pre></div>
</div>
<p>Uncertainty thresholds are calculated using results from the inner cross-validation studies. <code class="xref py py-func docutils literal notranslate"><span class="pre">biscuit.Experiment.thresholds_from_nested_cv()</span></code> will calculate and return uncertainty and prediction thresholds.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Calculate uncertainty thresholds</span>
<span class="n">df</span><span class="p">,</span> <span class="n">thresh</span> <span class="o">=</span> <span class="n">experiment</span><span class="o">.</span><span class="n">thresholds_from_nested_cv</span><span class="p">(</span><span class="n">label</span><span class="o">=</span><span class="s1">&#39;HPV&#39;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">thresh</span><span class="p">)</span>
</pre></div>
</div>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>{&#39;tile_uq&#39;: 0.02726791,
 &#39;slide_uq&#39;: 0.0147878695,
 &#39;tile_pred&#39;: 0.41621968,
 &#39;slide_pred&#39;: 0.4756707}
</pre></div>
</div>
</section>
<section id="apply-thresholds-to-test-set">
<h3>Apply thresholds to test set<a class="headerlink" href="#apply-thresholds-to-test-set" title="Permalink to this heading">¶</a></h3>
<p>Finally, we can apply these thresholds to a held out test set. First, generate predictions for a held-out test set as described in <a class="reference internal" href="../evaluation/#evaluation"><span class="std std-ref">Evaluation</span></a>. Locate the parquet file containing the saved tile-level predictions and load it into a DataFrame. Rename the columns in the dataframe so that ground-truth is <code class="docutils literal notranslate"><span class="pre">y_true</span></code>, predictions are <code class="docutils literal notranslate"><span class="pre">y_pred</span></code>, and uncertainty is <code class="docutils literal notranslate"><span class="pre">uncertainty</span></code>.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>

<span class="c1"># Load tile-level predictions from a test set evaluation</span>
<span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_parquet</span><span class="p">(</span><span class="s1">&#39;/path/to/tile_predictions.parquet.gzip&#39;</span><span class="p">)</span>

<span class="c1"># Rename the columns to y_true, y_pred, and uncertainty</span>
<span class="n">df</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="p">{</span>
    <span class="s1">&#39;HPV_status-y_true&#39;</span><span class="p">:</span> <span class="s1">&#39;y_true,</span>
    <span class="s1">&#39;HPV_status-y_pred1&#39;</span><span class="p">:</span> <span class="s1">&#39;y_pred&#39;</span><span class="p">,</span>
    <span class="s1">&#39;HPV_status-uncertainty1&#39;</span><span class="p">:</span> <span class="s1">&#39;uncertainty&#39;</span>
    <span class="s1">&#39;</span>
<span class="p">})</span>
</pre></div>
</div>
<p>Use <code class="xref py py-func docutils literal notranslate"><span class="pre">biscuit.threshold.apply()</span></code> to apply the previously-determined thresholds to these predictions. This will return classifier metrics (AUROC, accuracy, sensitivity, specificity) for high-confidence predictions and a dataframe of slide-level high-confidence predictions. Slides with low-confidence predictions will be omitted. The percentage of slides with high-confidence predictions will be reported as <code class="docutils literal notranslate"><span class="pre">'percent_incl'</span></code>.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Calculate high-confidence slide-level predictions</span>
<span class="n">metrics</span><span class="p">,</span> <span class="n">high_conf_df</span> <span class="o">=</span> <span class="n">biscuit</span><span class="o">.</span><span class="n">threshold</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span>
    <span class="n">df</span><span class="p">,</span>           <span class="c1"># Dataframe of tile-level predictions</span>
    <span class="o">**</span><span class="n">thresh</span><span class="p">,</span>     <span class="c1"># Uncertainty thresholds</span>
    <span class="n">level</span><span class="o">=</span><span class="s1">&#39;slide&#39;</span> <span class="c1"># We want slide-level predictions</span>
<span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">metrics</span><span class="p">)</span>
</pre></div>
</div>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>{&#39;auc&#39;: 0.9703296703296704,
 &#39;percent_incl&#39;: 0.907051282051282,
 &#39;acc&#39;: 0.9222614840989399,
 &#39;sensitivity&#39;: 0.9230769230769231,
 &#39;specificity&#39;: 0.9214285714285714}
</pre></div>
</div>
</section>
</section>
</section>


             </article>

            </div>
            <footer>

    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">

        <a href="../features/" class="btn btn-neutral float-right" title="Generating Features" accesskey="n" rel="next">Next <img src="../_static/images/chevron-right-orange.svg" class="next-page"></a>


        <a href="../posthoc/" class="btn btn-neutral" title="Layer Activations" accesskey="p" rel="prev"><img src="../_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>

    </div>




    <hr>



  <div role="contentinfo">
    <p>
        &copy; Copyright 2023, James M Dolezal.

    </p>
  </div>

      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>


</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              <ul>
<li><a class="reference internal" href="#">Uncertainty Quantification</a><ul>
<li><a class="reference internal" href="#training-with-uq">Training with UQ</a></li>
<li><a class="reference internal" href="#evaluating-with-uq">Evaluating with UQ</a></li>
<li><a class="reference internal" href="#uncertainty-heatmaps">Uncertainty heatmaps</a></li>
<li><a class="reference internal" href="#uncertainty-thresholding">Uncertainty thresholding</a><ul>
<li><a class="reference internal" href="#prepare-an-experiment">Prepare an Experiment</a></li>
<li><a class="reference internal" href="#train-with-cross-validation">Train with cross-validation</a></li>
<li><a class="reference internal" href="#train-inner-cross-validation">Train inner cross-validation</a></li>
<li><a class="reference internal" href="#apply-thresholds-to-test-set">Apply thresholds to test set</a></li>
</ul>
</li>
</ul>
</li>
</ul>

            </div>
          </div>
        </div>
      </section>
    </div>







       <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
         <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
         <script src="../_static/doctools.js"></script>
         <script src="../_static/sphinx_highlight.js"></script>



  <script type="text/javascript" src="../_static/js/vendor/jquery-3.6.3.min.js"></script>
  <script type="text/javascript" src="../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  <!-- Begin Footer -->

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://slideflow.dev">Docs</a>
          </li>

          <li>
            <a href="https://slideflow.dev/tutorial1/">Tutorials</a>
          </li>

          <li>
            <a href="https://github.com/slideflow/slideflow">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script script type="text/javascript">
    var collapsedSections = [];
  </script>

  <script type="text/javascript" src="../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>