slideflow / Git / [78ef36] /docs/posthoc/index.html

Models:
DanielG/
slideflow
Downloads: 1
[78ef36]: / docs / posthoc / index.html
History
Download this file
638 lines (478 with data), 48.6 kB

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  <meta name="generator" content="Docutils 0.18.1: http://docutils.sourceforge.net/" />

  <meta name="viewport" content="width=device-width, initial-scale=1.0">

  <title>Layer Activations &mdash; slideflow 3.0.0 documentation</title>















  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
    <link rel="index" title="Index" href="../genindex/" />
    <link rel="search" title="Search" href="../search/" />
    <link rel="next" title="Uncertainty Quantification" href="../uq/" />
    <link rel="prev" title="Evaluation" href="../evaluation/" />




  <script src="../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
  <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.15.2/css/all.css" integrity="sha384-vSIIfh2YWi9wW0r9iZe7RJPrKwp6bG+s9QZMoITbCckVJqGCCRhc+ccxNcdpHuYu" crossorigin="anonymous">
  <script defer data-domain="slideflow.dev" src="https://plausible.io/js/script.js"></script>
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://slideflow.dev" aria-label="Slideflow"></a>

      <div class="main-menu">
        <ul>
          <li class="active">
            <a href="https://slideflow.dev">Docs</a>
          </li>

          <li>
            <a href="https://slideflow.dev/tutorial1/">Tutorials</a>
          </li>

          <li>
            <a href="https://github.com/slideflow/slideflow">GitHub</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>
  </div>
</div>

<body class="pytorch-body">





    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">





                <div class="version">
                  3.0
                </div>









<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search/" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>


          </div>







              <p class="caption" role="heading"><span class="caption-text">Introduction</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../installation/">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/">Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../quickstart/">Quickstart</a></li>
<li class="toctree-l1"><a class="reference internal" href="../project_setup/">Setting up a Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../datasets_and_val/">Datasets</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slide_processing/">Slide Processing</a></li>
<li class="toctree-l1"><a class="reference internal" href="../training/">Training</a></li>
<li class="toctree-l1"><a class="reference internal" href="../evaluation/">Evaluation</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">Layer Activations</a></li>
<li class="toctree-l1"><a class="reference internal" href="../uq/">Uncertainty Quantification</a></li>
<li class="toctree-l1"><a class="reference internal" href="../features/">Generating Features</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mil/">Multiple-Instance Learning (MIL)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../ssl/">Self-Supervised Learning (SSL)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../stylegan/">Generative Networks (GANs)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../saliency/">Saliency Maps</a></li>
<li class="toctree-l1"><a class="reference internal" href="../segmentation/">Tissue Segmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../cellseg/">Cell Segmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../custom_loops/">Custom Training Loops</a></li>
<li class="toctree-l1"><a class="reference internal" href="../studio/">Slideflow Studio: Live Visualization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../troubleshooting/">Troubleshooting</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Developer Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tfrecords/">TFRecords: Reading and Writing</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dataloaders/">Dataloaders: Sampling and Augmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../custom_extractors/">Custom Feature Extractors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tile_labels/">Strong Supervision with Tile Labels</a></li>
<li class="toctree-l1"><a class="reference internal" href="../plugins/">Creating a Slideflow Plugin</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../slideflow/">slideflow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../project/">slideflow.Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dataset/">slideflow.Dataset</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dataset_features/">slideflow.DatasetFeatures</a></li>
<li class="toctree-l1"><a class="reference internal" href="../heatmap/">slideflow.Heatmap</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model_params/">slideflow.ModelParams</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mosaic/">slideflow.Mosaic</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slidemap/">slideflow.SlideMap</a></li>
<li class="toctree-l1"><a class="reference internal" href="../biscuit/">slideflow.biscuit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slideflow_cellseg/">slideflow.cellseg</a></li>
<li class="toctree-l1"><a class="reference internal" href="../io/">slideflow.io</a></li>
<li class="toctree-l1"><a class="reference internal" href="../io_tensorflow/">slideflow.io.tensorflow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../io_torch/">slideflow.io.torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../gan/">slideflow.gan</a></li>
<li class="toctree-l1"><a class="reference internal" href="../grad/">slideflow.grad</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mil_module/">slideflow.mil</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model/">slideflow.model</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model_tensorflow/">slideflow.model.tensorflow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model_torch/">slideflow.model.torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../norm/">slideflow.norm</a></li>
<li class="toctree-l1"><a class="reference internal" href="../simclr/">slideflow.simclr</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slide/">slideflow.slide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slide_qc/">slideflow.slide.qc</a></li>
<li class="toctree-l1"><a class="reference internal" href="../stats/">slideflow.stats</a></li>
<li class="toctree-l1"><a class="reference internal" href="../util/">slideflow.util</a></li>
<li class="toctree-l1"><a class="reference internal" href="../studio_module/">slideflow.studio</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Tutorials</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tutorial1/">Tutorial 1: Model training (simple)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial2/">Tutorial 2: Model training (advanced)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial3/">Tutorial 3: Using a custom architecture</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial4/">Tutorial 4: Model evaluation &amp; heatmaps</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial5/">Tutorial 5: Creating a mosaic map</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial6/">Tutorial 6: Custom slide filtering</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial7/">Tutorial 7: Training with custom augmentations</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial8/">Tutorial 8: Multiple-Instance Learning</a></li>
</ul>



        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">

      <li>
        <a href="../">

            Docs

        </a> &gt;
      </li>


      <li>Layer Activations</li>


      <li class="pytorch-breadcrumbs-aside">


            <a href="../_sources/posthoc.rst.txt" rel="nofollow"><img src="../_static/images/view-page-source-icon.svg"></a>


      </li>

  </ul>


</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">



          <div class="rst-content">

            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">

  <section id="layer-activations">
<span id="activations"></span><h1>Layer Activations<a class="headerlink" href="#layer-activations" title="Permalink to this heading">¶</a></h1>
<p>Investigating the latent space of a neural network can provide useful insights into the structure of your data and what models have learned during training. Slideflow provides several tools for post-hoc latent space analysis of trained neural networks, primarily by calculating activations at one or more neural network layers for all images in a dataset. In the next sections, we will take a look at how these layer activations can be calculated for downstream analysis and provide examples of analyses that can be performed.</p>
<section id="calculating-layer-activations">
<h2>Calculating Layer Activations<a class="headerlink" href="#calculating-layer-activations" title="Permalink to this heading">¶</a></h2>
<p>Activations at one or more layers of a trained network can be calculated with <a class="reference internal" href="../model/#slideflow.model.Features" title="slideflow.model.Features"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.model.Features</span></code></a> and <a class="reference internal" href="../dataset_features/#slideflow.DatasetFeatures" title="slideflow.DatasetFeatures"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.DatasetFeatures</span></code></a>. The former provides an interface for calculating layer activations for a batch of images, and the latter supervises calculations across an entire dataset.</p>
<section id="batch-of-images">
<h3>Batch of images<a class="headerlink" href="#batch-of-images" title="Permalink to this heading">¶</a></h3>
<p><a class="reference internal" href="../model/#slideflow.model.Features" title="slideflow.model.Features"><code class="xref py py-class docutils literal notranslate"><span class="pre">Features</span></code></a> provides an interface for calculating layer activations and predictions on a batch of images. The following arguments are available:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">path</span></code>: Path to model, from which layer activations are calculated. Required.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">layers</span></code>: Layer(s) at which to calculate activations.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">include_preds</span></code>: Also return the final network output (predictions)</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">pooling</span></code>: Apply pooling to layer activations, to reduce dimensionality to one dimension.</p></li>
</ul>
<p>If <code class="docutils literal notranslate"><span class="pre">layers</span></code> is not supplied, activations at the post-convolutional layer will be calculated by default.</p>
<p>Once initialized, the resulting object can be called on a batch of images and will return the layer activations for all images in the batch. For example, to calculate activations at the <code class="docutils literal notranslate"><span class="pre">sep_conv_3</span></code> layer of a model while looping through a dataset:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">slideflow</span> <span class="k">as</span> <span class="nn">sf</span>

<span class="n">sepconv3</span> <span class="o">=</span> <span class="n">sf</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">Features</span><span class="p">(</span><span class="s1">&#39;model/path&#39;</span><span class="p">,</span> <span class="n">layer</span><span class="o">=</span><span class="s1">&#39;sep_conv_3&#39;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">img_batch</span> <span class="ow">in</span> <span class="n">dataset</span><span class="p">:</span>
    <span class="n">postconv_activations</span> <span class="o">=</span> <span class="n">sepconv3</span><span class="p">(</span><span class="n">img_batch</span><span class="p">)</span>
</pre></div>
</div>
<p>If <code class="docutils literal notranslate"><span class="pre">layer</span></code> is a list of layer names, activations at each layer will be calculated and concatenated. If <code class="docutils literal notranslate"><span class="pre">include_preds</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, the interface will also return the final predictions:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sepconv3_and_preds</span> <span class="o">=</span> <span class="n">sf</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">Features</span><span class="p">(</span><span class="o">...</span><span class="p">,</span> <span class="n">include_preds</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">layer_activations</span><span class="p">,</span> <span class="n">preds</span> <span class="o">=</span> <span class="n">sepconv3_and_preds</span><span class="p">(</span><span class="n">img_batch</span><span class="p">)</span>
</pre></div>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><a class="reference internal" href="../model/#slideflow.model.Features" title="slideflow.model.Features"><code class="xref py py-class docutils literal notranslate"><span class="pre">Features</span></code></a> assumes that image batches already have any necessary preprocessing already applied, including standardization and stain normalization.</p>
</div>
<p>See the API documentation for <a class="reference internal" href="../model/#slideflow.model.Features" title="slideflow.model.Features"><code class="xref py py-class docutils literal notranslate"><span class="pre">Features</span></code></a> for more information.</p>
</section>
<section id="single-slide">
<h3>Single slide<a class="headerlink" href="#single-slide" title="Permalink to this heading">¶</a></h3>
<p>Layer activations can also be calculated across an entire slide using the same <a class="reference internal" href="../model/#slideflow.model.Features" title="slideflow.model.Features"><code class="xref py py-class docutils literal notranslate"><span class="pre">Features</span></code></a> interface. Calling the object on a <code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.WSI</span></code> object will generate a grid of activations of size <code class="docutils literal notranslate"><span class="pre">(slide.grid.shape[0],</span> <span class="pre">slide.grid.shape[1],</span> <span class="pre">num_features)</span></code>:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">slideflow</span> <span class="k">as</span> <span class="nn">sf</span>

<span class="n">slide</span> <span class="o">=</span> <span class="n">sf</span><span class="o">.</span><span class="n">WSI</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="n">postconv</span> <span class="o">=</span> <span class="n">sf</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">Features</span><span class="p">(</span><span class="s1">&#39;/model/path&#39;</span><span class="p">,</span> <span class="n">layers</span><span class="o">=</span><span class="s1">&#39;postconv&#39;</span><span class="p">)</span>
<span class="n">feature_grid</span> <span class="o">=</span> <span class="n">postconv</span><span class="p">(</span><span class="n">slide</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">feature_grid</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
</pre></div>
</div>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>(50, 45, 2048)
</pre></div>
</div>
</section>
<section id="entire-dataset">
<span id="dataset-features"></span><h3>Entire dataset<a class="headerlink" href="#entire-dataset" title="Permalink to this heading">¶</a></h3>
<p>Finally, layer activations can also be calculated for an entire dataset using <a class="reference internal" href="../dataset_features/#slideflow.DatasetFeatures" title="slideflow.DatasetFeatures"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.DatasetFeatures</span></code></a>. Instancing the class supervises the calculation and caching of layer activations, which can then be used for downstream analysis. The project function <a class="reference internal" href="../project/#slideflow.Project.generate_features" title="slideflow.Project.generate_features"><code class="xref py py-func docutils literal notranslate"><span class="pre">slideflow.Project.generate_features()</span></code></a> creates and returns an instance of this class.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">dts_ftrs</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">generate_features</span><span class="p">(</span><span class="s1">&#39;/path/to/trained_model&#39;</span><span class="p">)</span>
</pre></div>
</div>
<p>Alternatively, you can create an instance of this class directly:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">slideflow</span> <span class="k">as</span> <span class="nn">sf</span>

<span class="n">dataset</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">dataset</span><span class="p">(</span><span class="n">tile_px</span><span class="o">=</span><span class="mi">299</span><span class="p">,</span> <span class="n">tile_um</span><span class="o">=</span><span class="mi">302</span><span class="p">)</span>
<span class="n">dts_ftrs</span> <span class="o">=</span> <span class="n">sf</span><span class="o">.</span><span class="n">DatasetFeatures</span><span class="p">(</span>
  <span class="n">model</span><span class="o">=</span><span class="s1">&#39;/path/to/trained_model&#39;</span><span class="p">,</span>
  <span class="n">dataset</span><span class="o">=</span><span class="n">dataset</span><span class="p">,</span>
<span class="p">)</span>
</pre></div>
</div>
<p>Tile-level feature activations for each slide can be accessed directly from <code class="docutils literal notranslate"><span class="pre">DatasetFeatures.activations</span></code>, a dict mapping slide names to numpy arrays of shape <code class="docutils literal notranslate"><span class="pre">(num_tiles,</span> <span class="pre">num_features)</span></code>. Predictions are stored in <code class="docutils literal notranslate"><span class="pre">DatasetFeatures.predictions</span></code>, a dict mapping slide names to numpy arrays of shape <code class="docutils literal notranslate"><span class="pre">(num_tiles,</span> <span class="pre">num_classes)</span></code>. Tile-level location data (coordinates from which the tiles were taken from their respective source slides) is stored in <code class="docutils literal notranslate"><span class="pre">DatasetFeatures.locations</span></code>, a dict mapping slide names to numpy arrays of shape <code class="docutils literal notranslate"><span class="pre">(num_tiles,</span> <span class="pre">2)</span></code> (<code class="docutils literal notranslate"><span class="pre">x</span></code>, <code class="docutils literal notranslate"><span class="pre">y</span></code>).</p>
<p>Activations can be exported to a Pandas DataFrame with <a class="reference internal" href="../dataset_features/#slideflow.DatasetFeatures.to_df" title="slideflow.DatasetFeatures.to_df"><code class="xref py py-meth docutils literal notranslate"><span class="pre">slideflow.DatasetFeatures.to_df()</span></code></a> or exported into PyTorch format with <a class="reference internal" href="../dataset_features/#slideflow.DatasetFeatures.to_torch" title="slideflow.DatasetFeatures.to_torch"><code class="xref py py-meth docutils literal notranslate"><span class="pre">slideflow.DatasetFeatures.to_torch()</span></code></a>. See <a class="reference internal" href="../features/#features"><span class="std std-ref">Generating Features</span></a> for more information about generating and exporting features for MIL models.</p>
<p>Read the API documentation for <a class="reference internal" href="../dataset_features/#slideflow.DatasetFeatures" title="slideflow.DatasetFeatures"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.DatasetFeatures</span></code></a> for more information.</p>
</section>
</section>
<section id="mapping-activations">
<span id="slidemap"></span><h2>Mapping Activations<a class="headerlink" href="#mapping-activations" title="Permalink to this heading">¶</a></h2>
<p>Layer activations across a dataset can be dimensionality reduced with UMAP and plotted for visualization using <a class="reference internal" href="../dataset_features/#slideflow.DatasetFeatures.map_activations" title="slideflow.DatasetFeatures.map_activations"><code class="xref py py-meth docutils literal notranslate"><span class="pre">slideflow.DatasetFeatures.map_activations()</span></code></a>. This function returns an instance of <a class="reference internal" href="../slidemap/#slideflow.SlideMap" title="slideflow.SlideMap"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.SlideMap</span></code></a>, a class that provides easy access to labeling and plotting.</p>
<p>The below example calculates layer activations at the neural network layer <code class="docutils literal notranslate"><span class="pre">sep_conv_3</span></code> for an entire dataset, and then reduces the activations into two dimensions for easy visualization using UMAP. Any valid <a class="reference external" href="https://umap-learn.readthedocs.io/en/latest/parameters.html">UMAP parameters</a> can be passed via keyword argument.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">dts_ftrs</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">generate_features</span><span class="p">(</span>
  <span class="n">model</span><span class="o">=</span><span class="s1">&#39;/path/to/trained_model&#39;</span><span class="p">,</span>
  <span class="n">layers</span><span class="o">=</span><span class="s1">&#39;sep_conv_3&#39;</span>
<span class="p">)</span>
<span class="n">slide_map</span> <span class="o">=</span> <span class="n">dts_ftrs</span><span class="o">.</span><span class="n">map_activations</span><span class="p">(</span>
    <span class="n">n_neighbors</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="c1"># UMAP parameter</span>
    <span class="n">min_dist</span><span class="o">=</span><span class="mf">0.2</span>    <span class="c1"># UMAP parameter</span>
<span class="p">)</span>
</pre></div>
</div>
<p>We can then plot the activations with <a class="reference internal" href="../slidemap/#slideflow.SlideMap.plot" title="slideflow.SlideMap.plot"><code class="xref py py-meth docutils literal notranslate"><span class="pre">slideflow.SlideMap.plot()</span></code></a>. All keyword arguments are passed to the <a class="reference external" href="https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.scatter.html">matplotlib scatter</a> function.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>

<span class="n">slide_map</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">s</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
</pre></div>
</div>
<p>We can add labels to our plot by first passing a dictionary with slide labels to the function <a class="reference internal" href="../slidemap/#slideflow.SlideMap.label_by_slide" title="slideflow.SlideMap.label_by_slide"><code class="xref py py-meth docutils literal notranslate"><span class="pre">slideflow.SlideMap.label_by_slide()</span></code></a>.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Get a dictionary mapping slide names to category labels</span>
<span class="n">dataset</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">dataset</span><span class="p">(</span><span class="n">tile_px</span><span class="o">=</span><span class="mi">299</span><span class="p">,</span> <span class="n">tile_um</span><span class="o">=</span><span class="s1">&#39;10x&#39;</span><span class="p">)</span>
<span class="n">labels</span><span class="p">,</span> <span class="n">unique_labels</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">labels</span><span class="p">(</span><span class="s1">&#39;subtype&#39;</span><span class="p">,</span> <span class="nb">format</span><span class="o">=</span><span class="s1">&#39;name&#39;</span><span class="p">)</span>

<span class="c1"># Assign the labels to the slide map, then plot</span>
<span class="n">slide_map</span><span class="o">.</span><span class="n">label_by_slide</span><span class="p">(</span><span class="n">labels</span><span class="p">)</span>
<span class="n">slide_map</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span>
</pre></div>
</div>
<img alt="../_images/umap_example.png" src="../_images/umap_example.png" />
<div class="line-block">
<div class="line"><br /></div>
</div>
<p>Finally, we can use <code class="xref py py-meth docutils literal notranslate"><span class="pre">SlideMap.umap_transform()</span></code> to project new data into two dimensions using the previously fit UMAP.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">slideflow</span> <span class="k">as</span> <span class="nn">sf</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>

<span class="c1"># Create a SlideMap using layer activations reduced with UMAP</span>
<span class="n">dts_ftrs</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">generate_features</span><span class="p">(</span>
  <span class="n">model</span><span class="o">=</span><span class="s1">&#39;/path/to/trained_model&#39;</span><span class="p">,</span>
  <span class="n">layers</span><span class="o">=</span><span class="s1">&#39;sep_conv_3&#39;</span>
<span class="p">)</span>
<span class="n">slide_map</span> <span class="o">=</span> <span class="n">dts_ftrs</span><span class="o">.</span><span class="n">map_activations</span><span class="p">()</span>

<span class="c1"># Load some dummy data.</span>
<span class="c1"># Second dimension must match size of activation vector.</span>
<span class="n">dummy</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">random</span><span class="p">((</span><span class="mi">100</span><span class="p">,</span> <span class="mi">1024</span><span class="p">))</span>

<span class="c1"># Transform the data using the already-fit UMAP.</span>
<span class="n">transformed</span> <span class="o">=</span> <span class="n">slide_map</span><span class="o">.</span><span class="n">umap_transform</span><span class="p">(</span><span class="n">dummy</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">transformed</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
</pre></div>
</div>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>(100, 2)
</pre></div>
</div>
<p>Read more about additional <a class="reference internal" href="../slidemap/#slideflow.SlideMap" title="slideflow.SlideMap"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.SlideMap</span></code></a> functions, including saving, loading, and clustering, in the linked API documentation.</p>
</section>
<section id="mosaic-maps">
<span id="mosaic-map"></span><h2>Mosaic Maps<a class="headerlink" href="#mosaic-maps" title="Permalink to this heading">¶</a></h2>
<p>Mosaic maps provide a tool for visualizing the distribution of histologic image features in a dataset through analysis of neural network layer activations. Similar to <a class="reference external" href="https://distill.pub/2019/activation-atlas/">activation atlases</a>, a mosaic map is generated by first calculating layer activations for a dataset, dimensionality reducing these activations with <a class="reference external" href="https://joss.theoj.org/papers/10.21105/joss.00861">UMAP</a>,  and then overlaying corresponding images in a grid-wise fashion.</p>
<img alt="../_images/mosaic_example.png" src="../_images/mosaic_example.png" />
<div class="line-block">
<div class="line"><br /></div>
</div>
<p>In the previous sections, we reviewed how to calculate layer activations across a dataset, and then dimensionality reduce these activations into two dimensions using UMAP. <a class="reference internal" href="../mosaic/#slideflow.Mosaic" title="slideflow.Mosaic"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.Mosaic</span></code></a> provides a tool for converting these activation maps into a grid of image tiles plotted according to their associated activation vectors.</p>
<section id="quickstart">
<h3>Quickstart<a class="headerlink" href="#quickstart" title="Permalink to this heading">¶</a></h3>
<p>The fastest way to build a mosaic map is using <a class="reference internal" href="../project/#slideflow.Project.generate_mosaic" title="slideflow.Project.generate_mosaic"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.Project.generate_mosaic</span></code></a>, which requires a <code class="docutils literal notranslate"><span class="pre">DatasetFeatures</span></code> object as its only mandatory argument and returns an instance of <a class="reference internal" href="../mosaic/#slideflow.Mosaic" title="slideflow.Mosaic"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.Mosaic</span></code></a>.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">dts_ftrs</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">generate_features</span><span class="p">(</span><span class="s1">&#39;/path/to/trained_model&#39;</span><span class="p">,</span> <span class="n">layers</span><span class="o">=</span><span class="s1">&#39;postconv&#39;</span><span class="p">)</span>
<span class="n">mosaic</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">generate_mosaic</span><span class="p">(</span><span class="n">dts_ftrs</span><span class="p">)</span>
<span class="n">mosaic</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="s1">&#39;mosaic.png&#39;</span><span class="p">)</span>
</pre></div>
</div>
<p>When created with this interface, the underlying <a class="reference internal" href="../slidemap/#slideflow.SlideMap" title="slideflow.SlideMap"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.SlideMap</span></code></a> object used to create the mosaic map is accessible via <code class="docutils literal notranslate"><span class="pre">slideflow.Mosaic.slide_map</span></code>. You could, for example, use <a class="reference internal" href="../slidemap/#slideflow.SlideMap.save" title="slideflow.SlideMap.save"><code class="xref py py-func docutils literal notranslate"><span class="pre">slideflow.SlideMap.save()</span></code></a> to save the UMAP plot:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">mosiac</span><span class="o">.</span><span class="n">slide_map</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="s1">&#39;umap.png&#39;</span><span class="p">)</span>
</pre></div>
</div>
</section>
<section id="from-a-slidemap">
<h3>From a SlideMap<a class="headerlink" href="#from-a-slidemap" title="Permalink to this heading">¶</a></h3>
<p>Any <code class="docutils literal notranslate"><span class="pre">SlideMap</span></code> can be converted to a mosaic map with <code class="xref py py-meth docutils literal notranslate"><span class="pre">slideflow.SlideMap.generate_mosaic()</span></code>.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">ftrs</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">generate_features</span><span class="p">(</span><span class="s1">&#39;/path/to/model&#39;</span><span class="p">)</span>
<span class="n">slide_map</span> <span class="o">=</span> <span class="n">ftrs</span><span class="o">.</span><span class="n">map_activations</span><span class="p">()</span>
<span class="n">mosaic</span> <span class="o">=</span> <span class="n">slide_map</span><span class="o">.</span><span class="n">generate_mosaic</span><span class="p">()</span>
<span class="n">mosaic</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="s1">&#39;mosaic.png&#39;</span><span class="p">)</span>
</pre></div>
</div>
</section>
<section id="manual-creation">
<h3>Manual creation<a class="headerlink" href="#manual-creation" title="Permalink to this heading">¶</a></h3>
<p>Mosaic maps can be flexibly created with <a class="reference internal" href="../mosaic/#slideflow.Mosaic" title="slideflow.Mosaic"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.Mosaic</span></code></a>, requiring two components: a set of images and corresponding coordinates. Images and coordinates can either be manually provided, or the mosaic can dynamically read images from TFRecords (as is done with <code class="xref py py-meth docutils literal notranslate"><span class="pre">Project.generate_mosaic()</span></code>).</p>
<p>The first argument of <a class="reference internal" href="../mosaic/#slideflow.Mosaic" title="slideflow.Mosaic"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.Mosaic</span></code></a> provides the images, and may be either of the following:</p>
<ul class="simple">
<li><p>A list or array of images (np.ndarray, HxWxC)</p></li>
<li><p>A list of tuples, containing <code class="docutils literal notranslate"><span class="pre">(slide_name,</span> <span class="pre">tfrecord_index)</span></code></p></li>
</ul>
<p>The second argument provides the coordinates:</p>
<ul class="simple">
<li><p>A list or array of (x, y) coordinates for each image</p></li>
</ul>
<p>For example, to create a mosaic map from a list of images and coordinates:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Example data (images are HxWxC, np.ndarray)</span>
<span class="n">images</span> <span class="o">=</span> <span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">(</span><span class="o">...</span><span class="p">),</span> <span class="o">...</span><span class="p">]</span>
<span class="n">coords</span> <span class="o">=</span> <span class="p">[(</span><span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">),</span> <span class="o">...</span><span class="p">]</span>

<span class="c1"># Generate the mosaic</span>
<span class="n">mosaic</span> <span class="o">=</span> <span class="n">Mosaic</span><span class="p">(</span><span class="n">images</span><span class="p">,</span> <span class="n">coordinates</span><span class="p">)</span>
<span class="n">mosaic</span><span class="o">.</span><span class="n">plot</span><span class="p">()</span>
</pre></div>
</div>
<p>You can also generate a mosaic map where the images are tuples of <cite>(tfrecord, tfrecord_index)</cite>. In this case, the mosaic map will dynamically read images from TFRecords during plotting.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Example data</span>
<span class="n">tfrecords</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;/path/to/tfrecord`.tfrecords&#39;</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span>
<span class="n">idx</span> <span class="o">=</span> <span class="p">[</span><span class="mi">253</span><span class="p">,</span> <span class="mi">112</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span>
<span class="n">coords</span> <span class="o">=</span> <span class="p">[(</span><span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">),</span> <span class="o">...</span><span class="p">]</span>

<span class="c1"># Generate mosaic map</span>
<span class="n">mosaic</span> <span class="o">=</span> <span class="n">sf</span><span class="o">.</span><span class="n">Mosaic</span><span class="p">(</span>
    <span class="n">images</span><span class="o">=</span><span class="p">[(</span><span class="n">tfr</span><span class="p">,</span> <span class="n">idx</span><span class="p">)</span> <span class="k">for</span> <span class="n">tfr</span><span class="p">,</span> <span class="n">idx</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">tfrecords</span><span class="p">,</span> <span class="n">idx</span><span class="p">)],</span>
    <span class="n">coords</span><span class="o">=</span><span class="n">coords</span>
<span class="p">)</span>
</pre></div>
</div>
<p>There are several additional arguments that can be used to customize the mosaic map plotting. Read the linked API documentation for <a class="reference internal" href="../mosaic/#slideflow.Mosaic" title="slideflow.Mosaic"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.Mosaic</span></code></a> for more information.</p>
</section>
</section>
</section>


             </article>

            </div>
            <footer>

    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">

        <a href="../uq/" class="btn btn-neutral float-right" title="Uncertainty Quantification" accesskey="n" rel="next">Next <img src="../_static/images/chevron-right-orange.svg" class="next-page"></a>


        <a href="../evaluation/" class="btn btn-neutral" title="Evaluation" accesskey="p" rel="prev"><img src="../_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>

    </div>




    <hr>



  <div role="contentinfo">
    <p>
        &copy; Copyright 2023, James M Dolezal.

    </p>
  </div>

      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>


</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              <ul>
<li><a class="reference internal" href="#">Layer Activations</a><ul>
<li><a class="reference internal" href="#calculating-layer-activations">Calculating Layer Activations</a><ul>
<li><a class="reference internal" href="#batch-of-images">Batch of images</a></li>
<li><a class="reference internal" href="#single-slide">Single slide</a></li>
<li><a class="reference internal" href="#entire-dataset">Entire dataset</a></li>
</ul>
</li>
<li><a class="reference internal" href="#mapping-activations">Mapping Activations</a></li>
<li><a class="reference internal" href="#mosaic-maps">Mosaic Maps</a><ul>
<li><a class="reference internal" href="#quickstart">Quickstart</a></li>
<li><a class="reference internal" href="#from-a-slidemap">From a SlideMap</a></li>
<li><a class="reference internal" href="#manual-creation">Manual creation</a></li>
</ul>
</li>
</ul>
</li>
</ul>

            </div>
          </div>
        </div>
      </section>
    </div>







       <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
         <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
         <script src="../_static/doctools.js"></script>
         <script src="../_static/sphinx_highlight.js"></script>



  <script type="text/javascript" src="../_static/js/vendor/jquery-3.6.3.min.js"></script>
  <script type="text/javascript" src="../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  <!-- Begin Footer -->

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://slideflow.dev">Docs</a>
          </li>

          <li>
            <a href="https://slideflow.dev/tutorial1/">Tutorials</a>
          </li>

          <li>
            <a href="https://github.com/slideflow/slideflow">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script script type="text/javascript">
    var collapsedSections = [];
  </script>

  <script type="text/javascript" src="../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>