<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="generator" content="Docutils 0.18.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Tutorial 4: Model evaluation & heatmaps — slideflow 3.0.0 documentation</title>
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<!-- <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> -->
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<link rel="index" title="Index" href="../genindex/" />
<link rel="search" title="Search" href="../search/" />
<link rel="next" title="Tutorial 5: Creating a mosaic map" href="../tutorial5/" />
<link rel="prev" title="Tutorial 3: Using a custom architecture" href="../tutorial3/" />
<script src="../_static/js/modernizr.min.js"></script>
<!-- Preload the theme fonts -->
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<!-- Preload the katex fonts -->
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.15.2/css/all.css" integrity="sha384-vSIIfh2YWi9wW0r9iZe7RJPrKwp6bG+s9QZMoITbCckVJqGCCRhc+ccxNcdpHuYu" crossorigin="anonymous">
<script defer data-domain="slideflow.dev" src="https://plausible.io/js/script.js"></script>
</head>
<div class="container-fluid header-holder tutorials-header" id="header-holder">
<div class="container">
<div class="header-container">
<a class="header-logo" href="https://slideflow.dev" aria-label="Slideflow"></a>
<div class="main-menu">
<ul>
<li class="active">
<a href="https://slideflow.dev">Docs</a>
</li>
<li>
<a href="https://slideflow.dev/tutorial1/">Tutorials</a>
</li>
<li>
<a href="https://github.com/slideflow/slideflow">GitHub</a>
</li>
</ul>
</div>
<a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
</div>
</div>
</div>
<body class="pytorch-body">
<div class="table-of-contents-link-wrapper">
<span>Table of Contents</span>
<a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
</div>
<nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
<div class="pytorch-side-scroll">
<div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<div class="pytorch-left-menu-search">
<div class="version">
3.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../search/" method="get">
<input type="text" name="q" placeholder="Search Docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<p class="caption" role="heading"><span class="caption-text">Introduction</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../installation/">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview/">Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../quickstart/">Quickstart</a></li>
<li class="toctree-l1"><a class="reference internal" href="../project_setup/">Setting up a Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../datasets_and_val/">Datasets</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slide_processing/">Slide Processing</a></li>
<li class="toctree-l1"><a class="reference internal" href="../training/">Training</a></li>
<li class="toctree-l1"><a class="reference internal" href="../evaluation/">Evaluation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../posthoc/">Layer Activations</a></li>
<li class="toctree-l1"><a class="reference internal" href="../uq/">Uncertainty Quantification</a></li>
<li class="toctree-l1"><a class="reference internal" href="../features/">Generating Features</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mil/">Multiple-Instance Learning (MIL)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../ssl/">Self-Supervised Learning (SSL)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../stylegan/">Generative Networks (GANs)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../saliency/">Saliency Maps</a></li>
<li class="toctree-l1"><a class="reference internal" href="../segmentation/">Tissue Segmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../cellseg/">Cell Segmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../custom_loops/">Custom Training Loops</a></li>
<li class="toctree-l1"><a class="reference internal" href="../studio/">Slideflow Studio: Live Visualization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../troubleshooting/">Troubleshooting</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Developer Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tfrecords/">TFRecords: Reading and Writing</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dataloaders/">Dataloaders: Sampling and Augmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../custom_extractors/">Custom Feature Extractors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tile_labels/">Strong Supervision with Tile Labels</a></li>
<li class="toctree-l1"><a class="reference internal" href="../plugins/">Creating a Slideflow Plugin</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../slideflow/">slideflow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../project/">slideflow.Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dataset/">slideflow.Dataset</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dataset_features/">slideflow.DatasetFeatures</a></li>
<li class="toctree-l1"><a class="reference internal" href="../heatmap/">slideflow.Heatmap</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model_params/">slideflow.ModelParams</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mosaic/">slideflow.Mosaic</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slidemap/">slideflow.SlideMap</a></li>
<li class="toctree-l1"><a class="reference internal" href="../biscuit/">slideflow.biscuit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slideflow_cellseg/">slideflow.cellseg</a></li>
<li class="toctree-l1"><a class="reference internal" href="../io/">slideflow.io</a></li>
<li class="toctree-l1"><a class="reference internal" href="../io_tensorflow/">slideflow.io.tensorflow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../io_torch/">slideflow.io.torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../gan/">slideflow.gan</a></li>
<li class="toctree-l1"><a class="reference internal" href="../grad/">slideflow.grad</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mil_module/">slideflow.mil</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model/">slideflow.model</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model_tensorflow/">slideflow.model.tensorflow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model_torch/">slideflow.model.torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../norm/">slideflow.norm</a></li>
<li class="toctree-l1"><a class="reference internal" href="../simclr/">slideflow.simclr</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slide/">slideflow.slide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../slide_qc/">slideflow.slide.qc</a></li>
<li class="toctree-l1"><a class="reference internal" href="../stats/">slideflow.stats</a></li>
<li class="toctree-l1"><a class="reference internal" href="../util/">slideflow.util</a></li>
<li class="toctree-l1"><a class="reference internal" href="../studio_module/">slideflow.studio</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Tutorials</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../tutorial1/">Tutorial 1: Model training (simple)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial2/">Tutorial 2: Model training (advanced)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial3/">Tutorial 3: Using a custom architecture</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">Tutorial 4: Model evaluation & heatmaps</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial5/">Tutorial 5: Creating a mosaic map</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial6/">Tutorial 6: Custom slide filtering</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial7/">Tutorial 7: Training with custom augmentations</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorial8/">Tutorial 8: Multiple-Instance Learning</a></li>
</ul>
</div>
</div>
</nav>
<div class="pytorch-container">
<div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
<div class="pytorch-breadcrumbs-wrapper">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="pytorch-breadcrumbs">
<li>
<a href="../">
Docs
</a> >
</li>
<li>Tutorial 4: Model evaluation & heatmaps</li>
<li class="pytorch-breadcrumbs-aside">
<a href="../_sources/tutorial4.rst.txt" rel="nofollow"><img src="../_static/images/view-page-source-icon.svg"></a>
</li>
</ul>
</div>
</div>
<div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
Shortcuts
</div>
</div>
<section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
<div class="pytorch-content-left">
<div class="rst-content">
<div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
<article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
<section id="tutorial-4-model-evaluation-heatmaps">
<h1>Tutorial 4: Model evaluation & heatmaps<a class="headerlink" href="#tutorial-4-model-evaluation-heatmaps" title="Permalink to this heading">¶</a></h1>
<p>In this tutorial, we will evaluate a trained model on a separate, held-out evaluation dataset.</p>
<section id="locate-a-saved-model">
<h2>Locate a saved model<a class="headerlink" href="#locate-a-saved-model" title="Permalink to this heading">¶</a></h2>
<p>First, locate a saved model to evaluate, which will be a *.zip file for PyTorch models and a directory for Tensorflow models.
The directory structure for saved models trained with Tensorflow will look like:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>models/
├── 00001-er_status_by_ihc-HP0 /
│ ├── er_status_by_ihc-HP0-epoch1/
...
...
</pre></div>
</div>
<p>And for PyTorch:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>models/
├── 00001-er_status_by_ihc-HP0 /
│ ├── er_status_by_ihc-HP0-epoch1.zip
...
...
</pre></div>
</div>
<p>For this tutorial, we will assume that the final trained model is stored at <code class="docutils literal notranslate"><span class="pre">/home/er_project/models/00003-er_status_by_ihc/er_status_by_ihc_epoch3</span></code>.
The results we report are a continuation of the project described in <a class="reference internal" href="../tutorial1/#tutorial1"><span class="std std-ref">Tutorial 1: Model training (simple)</span></a>.</p>
</section>
<section id="evaluation">
<h2>Evaluation<a class="headerlink" href="#evaluation" title="Permalink to this heading">¶</a></h2>
<p>To evaluate a saved model, use the <a class="reference internal" href="../project/#slideflow.Project.evaluate" title="slideflow.Project.evaluate"><code class="xref py py-meth docutils literal notranslate"><span class="pre">slideflow.Project.evaluate()</span></code></a> function:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="s1">'/home/er_project/models/00003-er_status_by_ihc/er_status_by_ihc_epoch3'</span>
<span class="k">def</span> <span class="nf">main</span><span class="p">(</span><span class="n">P</span><span class="p">):</span>
<span class="n">P</span><span class="o">.</span><span class="n">evaluate</span><span class="p">(</span>
<span class="n">model</span><span class="p">,</span>
<span class="s1">'er_status_by_ihc'</span><span class="p">,</span>
<span class="n">filters</span><span class="o">=</span><span class="p">{</span><span class="s1">'dataset'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'eval'</span><span class="p">],</span>
<span class="s1">'er_status_by_ihc'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'Positive'</span><span class="p">,</span> <span class="s1">'Negative'</span><span class="p">]}</span>
<span class="p">)</span>
</pre></div>
</div>
<p>This function should yield the following final results:</p>
<table class="docutils align-default">
<tbody>
<tr class="row-odd"><td><figure class="align-default">
<img alt="../_images/val_er_roc_tile.png" src="../_images/val_er_roc_tile.png" />
</figure>
<p>Tile-level receiver operator curve</p>
</td>
<td><figure class="align-default">
<img alt="../_images/val_er_roc_patient.png" src="../_images/val_er_roc_patient.png" />
</figure>
<p>Patient-level receiver operator curve</p>
</td>
</tr>
</tbody>
</table>
</section>
<section id="heatmaps">
<h2>Heatmaps<a class="headerlink" href="#heatmaps" title="Permalink to this heading">¶</a></h2>
<p>Predictive heatmaps can be generated for either individuals slides or for all slides in a dataset. We will give examples for both, starting with automatic heatmap generation for all slides in the evaluation dataset.</p>
<section id="from-a-dataset">
<h3>From a dataset<a class="headerlink" href="#from-a-dataset" title="Permalink to this heading">¶</a></h3>
<p>The <a class="reference internal" href="../project/#slideflow.Project.generate_heatmaps" title="slideflow.Project.generate_heatmaps"><code class="xref py py-meth docutils literal notranslate"><span class="pre">slideflow.Project.generate_heatmaps()</span></code></a> can be used to easily and quickly create predictive heatmaps for a dataset. The filters argument is used to specify which slides should be included.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="s1">'/home/er_project/models/00003-er_status_by_ihc/er_status_by_ihc_epoch3'</span>
<span class="k">def</span> <span class="nf">main</span><span class="p">(</span><span class="n">P</span><span class="p">):</span>
<span class="n">P</span><span class="o">.</span><span class="n">generate_heatmaps</span><span class="p">(</span>
<span class="n">model</span><span class="p">,</span>
<span class="n">filters</span><span class="o">=</span><span class="p">{</span><span class="s1">'dataset'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'eval'</span><span class="p">]},</span>
<span class="n">resolution</span><span class="o">=</span><span class="s1">'high'</span> <span class="c1"># options include "low", "medium" and "high"</span>
<span class="p">)</span>
</pre></div>
</div>
<p>If the referenced model was trained with digital stain normalization, this will be automaticaly detected and used at the time of heatmap calculation. The heatmap thumbnail image will be displayed without normalization, however.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">resolution</span></code> parameter indicates the stride at which tiles should be extracted from slides to generate predictions. <code class="docutils literal notranslate"><span class="pre">"low"</span></code> resolution yields predictions on non-overlapping slides (stride_div=1). <code class="docutils literal notranslate"><span class="pre">"medium"</span></code> resolutions uses tiles with 50% overlap (stide_div=2), and <code class="docutils literal notranslate"><span class="pre">"high"</span></code> resolution uses tiles with 75% overlap (stride_div=4).</p>
<p>Heatmaps are colored and scaled in a manner optimized for categorical outcomes, with the colorscale 0 (blue) -> 0.5 (white) -> 1.0 (red). To change this colorscaling (particularly important for regression outcomes), set <code class="docutils literal notranslate"><span class="pre">vmin</span></code>, <code class="docutils literal notranslate"><span class="pre">vcenter</span></code>, and <code class="docutils literal notranslate"><span class="pre">vmax</span></code> accordingly.</p>
<p>Heatmaps are displayed without any color interpolation by default. To generate a smoothed heatmap, interpolate colors with any strategy supported by matplotlib (including, for example, “bicubic”, “nearest”, “bilnear”, and many more) with the argument <code class="docutils literal notranslate"><span class="pre">interpolation</span></code>.</p>
<p>Heatmaps are saved by default in the <code class="docutils literal notranslate"><span class="pre">project_folder/heatmaps</span></code> folder. A separate image is saved for each outcome category, as well as a base non-annotated slide thumbnail and a thumbnail with an ROI overlay (if ROIs are available). If the model had three outcomes, the heatmaps folder would contain:</p>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>heatmaps/
├──<span class="w"> </span>er_status_by_ihc-HP0-epoch1<span class="w"> </span>/
│<span class="w"> </span>├──<span class="w"> </span>slide_name-raw.png
│<span class="w"> </span>├──<span class="w"> </span>slide_name-raw+roi.png
│<span class="w"> </span>├──<span class="w"> </span>slide_name-0.png
│<span class="w"> </span>├──<span class="w"> </span>slide_name-1.png
│<span class="w"> </span>├──<span class="w"> </span>slide_name-2.png
<span class="w"> </span>...
...
</pre></div>
</div>
<img alt="../_images/heatmap_example.png" src="../_images/heatmap_example.png" />
</section>
<section id="from-a-single-slide">
<h3>From a single slide<a class="headerlink" href="#from-a-single-slide" title="Permalink to this heading">¶</a></h3>
<p>Heatmaps can also be manually created with the <a class="reference internal" href="../heatmap/#slideflow.Heatmap" title="slideflow.Heatmap"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.Heatmap</span></code></a> class. To create a heatmap from a slide located at ‘/home/user/example.svs’, use the following syntax:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span> <span class="nn">slideflow</span> <span class="k">as</span> <span class="nn">sf</span>
<span class="gp">>>> </span><span class="n">heatmap</span> <span class="o">=</span> <span class="n">sf</span><span class="o">.</span><span class="n">Heatmap</span><span class="p">(</span>
<span class="gp">... </span> <span class="s1">'/home/user/example.svs'</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">model</span><span class="o">=</span><span class="s1">'/path/to/model'</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">stride_div</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="c1"># optional, defaults to 1</span>
<span class="gp">... </span> <span class="n">num_threads</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="c1"># optional, defaults to 8</span>
<span class="gp">... </span> <span class="n">buffer</span><span class="o">=</span><span class="s1">'/mnt/ramdisk'</span><span class="p">,</span> <span class="c1"># optional</span>
<span class="gp">... </span><span class="p">)</span>
</pre></div>
</div>
<p>To calculate heatmaps only within areas of an annotated ROI, you can pass the directory in which ROIs are contained to <code class="docutils literal notranslate"><span class="pre">roi_dir</span></code>, or pass the path (or list of ROI paths) to <code class="docutils literal notranslate"><span class="pre">rois</span></code>. The easiest way to do this is to let the <a class="reference internal" href="../dataset/#slideflow.Dataset" title="slideflow.Dataset"><code class="xref py py-class docutils literal notranslate"><span class="pre">slideflow.Dataset</span></code></a> object handle this for you:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span> <span class="nn">slideflow</span> <span class="k">as</span> <span class="nn">sf</span>
<span class="gp">>>> </span><span class="n">P</span> <span class="o">=</span> <span class="n">sf</span><span class="o">.</span><span class="n">Project</span><span class="p">(</span><span class="s1">'/project/path'</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">P</span><span class="o">.</span><span class="n">dataset</span><span class="p">(</span><span class="mi">299</span><span class="p">,</span> <span class="mi">302</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">heatmap</span> <span class="o">=</span> <span class="n">sf</span><span class="o">.</span><span class="n">Heatmap</span><span class="p">(</span>
<span class="gp">... </span> <span class="s1">'/home/user/example.svs'</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">model</span><span class="o">=</span><span class="s1">'/path/to/model'</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">rois</span><span class="o">=</span><span class="n">dataset</span><span class="o">.</span><span class="n">rois</span><span class="p">(),</span>
<span class="gp">... </span> <span class="o">...</span>
<span class="gp">... </span><span class="p">)</span>
</pre></div>
</div>
<p>The heatmap can then be saved with <a class="reference internal" href="../heatmap/#slideflow.Heatmap.save" title="slideflow.Heatmap.save"><code class="xref py py-meth docutils literal notranslate"><span class="pre">slideflow.Heatmap.save()</span></code></a>, which accepts a directory location as its first positional argument:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">heatmap</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="s1">'/save_dir'</span><span class="p">)</span>
</pre></div>
</div>
<p>Heatmap interpolation and colorscale settings are passed as additional arguments to the <a class="reference internal" href="../heatmap/#slideflow.Heatmap.save" title="slideflow.Heatmap.save"><code class="xref py py-meth docutils literal notranslate"><span class="pre">slideflow.Heatmap.save()</span></code></a> function, and <code class="docutils literal notranslate"><span class="pre">show_roi</span></code> can be used to indicate whether ROIs should be shown on the thumbnail images:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">heatmap</span><span class="o">.</span><span class="n">save</span><span class="p">(</span>
<span class="gp">... </span> <span class="s1">'/save_dir'</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">interpolation</span><span class="o">=</span><span class="s1">'bicubic'</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">vmin</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">vcenter</span><span class="o">=</span><span class="mi">15</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">vmax</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">show_roi</span><span class="o">=</span><span class="kc">False</span>
<span class="gp">... </span><span class="p">)</span>
</pre></div>
</div>
</section>
</section>
</section>
</article>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="../tutorial5/" class="btn btn-neutral float-right" title="Tutorial 5: Creating a mosaic map" accesskey="n" rel="next">Next <img src="../_static/images/chevron-right-orange.svg" class="next-page"></a>
<a href="../tutorial3/" class="btn btn-neutral" title="Tutorial 3: Using a custom architecture" accesskey="p" rel="prev"><img src="../_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>
</div>
<hr>
<div role="contentinfo">
<p>
© Copyright 2023, James M Dolezal.
</p>
</div>
<div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</div>
</footer>
</div>
</div>
<div class="pytorch-content-right" id="pytorch-content-right">
<div class="pytorch-right-menu" id="pytorch-right-menu">
<div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
<ul>
<li><a class="reference internal" href="#">Tutorial 4: Model evaluation & heatmaps</a><ul>
<li><a class="reference internal" href="#locate-a-saved-model">Locate a saved model</a></li>
<li><a class="reference internal" href="#evaluation">Evaluation</a></li>
<li><a class="reference internal" href="#heatmaps">Heatmaps</a><ul>
<li><a class="reference internal" href="#from-a-dataset">From a dataset</a></li>
<li><a class="reference internal" href="#from-a-single-slide">From a single slide</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
</div>
</section>
</div>
<script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
<script src="../_static/doctools.js"></script>
<script src="../_static/sphinx_highlight.js"></script>
<script type="text/javascript" src="../_static/js/vendor/jquery-3.6.3.min.js"></script>
<script type="text/javascript" src="../_static/js/vendor/popper.min.js"></script>
<script type="text/javascript" src="../_static/js/vendor/bootstrap.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
<script type="text/javascript" src="../_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
<!-- Begin Footer -->
<!-- End Footer -->
<!-- Begin Mobile Menu -->
<div class="mobile-main-menu">
<div class="container-fluid">
<div class="container">
<div class="mobile-main-menu-header-container">
<a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
<a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
</div>
</div>
</div>
<div class="mobile-main-menu-links-container">
<div class="main-menu">
<ul>
<li>
<a href="https://slideflow.dev">Docs</a>
</li>
<li>
<a href="https://slideflow.dev/tutorial1/">Tutorials</a>
</li>
<li>
<a href="https://github.com/slideflow/slideflow">Github</a>
</li>
</ul>
</div>
</div>
</div>
<!-- End Mobile Menu -->
<script script type="text/javascript">
var collapsedSections = [];
</script>
<script type="text/javascript" src="../_static/js/vendor/anchor.min.js"></script>
<script type="text/javascript">
$(document).ready(function() {
mobileMenu.bind();
mobileTOC.bind();
pytorchAnchors.bind();
sideMenus.bind();
scrollToAnchor.bind();
highlightNavigation.bind();
mainMenuDropdown.bind();
filterTags.bind();
// Add class to links that have code blocks, since we cannot create links in code blocks
$("article.pytorch-article a span.pre").each(function(e) {
$(this).closest("a").addClass("has-code");
});
})
</script>
</body>
</html>
Datasets
Models
