<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>pathflowai.datasets — PathFlowAI 0.1 documentation</title>
<script type="text/javascript" src="../../_static/js/modernizr.min.js"></script>
<script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
<script type="text/javascript" src="../../_static/jquery.js"></script>
<script type="text/javascript" src="../../_static/underscore.js"></script>
<script type="text/javascript" src="../../_static/doctools.js"></script>
<script type="text/javascript" src="../../_static/language_data.js"></script>
<script async="async" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
<script type="text/javascript" src="../../_static/js/theme.js"></script>
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="../../index.html" class="icon icon-home"> PathFlowAI
</a>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<!-- Local TOC -->
<div class="local-toc"></div>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../index.html">PathFlowAI</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../index.html">Docs</a> »</li>
<li><a href="../index.html">Module code</a> »</li>
<li>pathflowai.datasets</li>
<li class="wy-breadcrumbs-aside">
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<h1>Source code for pathflowai.datasets</h1><div class="highlight"><pre>
<span></span><span class="sd">"""</span>
<span class="sd">datasets.py</span>
<span class="sd">=======================</span>
<span class="sd">Houses the DynamicImageDataset class, also functions to help with image color channel normalization, transformers, etc..</span>
<span class="sd">"""</span>
<span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">from</span> <span class="nn">torchvision</span> <span class="k">import</span> <span class="n">transforms</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">dask</span>
<span class="c1">#from dask.distributed import Client; Client()</span>
<span class="kn">import</span> <span class="nn">dask.array</span> <span class="k">as</span> <span class="nn">da</span><span class="o">,</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span><span class="o">,</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">from</span> <span class="nn">pathflowai.utils</span> <span class="k">import</span> <span class="o">*</span>
<span class="kn">import</span> <span class="nn">pysnooper</span>
<span class="kn">import</span> <span class="nn">nonechucks</span> <span class="k">as</span> <span class="nn">nc</span>
<span class="kn">from</span> <span class="nn">torch.utils.data</span> <span class="k">import</span> <span class="n">Dataset</span><span class="p">,</span> <span class="n">DataLoader</span>
<span class="kn">import</span> <span class="nn">random</span>
<span class="kn">import</span> <span class="nn">albumentations</span> <span class="k">as</span> <span class="nn">alb</span>
<span class="kn">import</span> <span class="nn">copy</span>
<span class="kn">from</span> <span class="nn">albumentations</span> <span class="k">import</span> <span class="n">pytorch</span> <span class="k">as</span> <span class="n">albtorch</span>
<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="k">import</span> <span class="n">LabelBinarizer</span>
<span class="kn">from</span> <span class="nn">sklearn.utils.class_weight</span> <span class="k">import</span> <span class="n">compute_class_weight</span>
<span class="kn">from</span> <span class="nn">pathflowai.losses</span> <span class="k">import</span> <span class="n">class2one_hot</span>
<div class="viewcode-block" id="RandomRotate90"><a class="viewcode-back" href="../../index.html#pathflowai.datasets.RandomRotate90">[docs]</a><span class="k">def</span> <span class="nf">RandomRotate90</span><span class="p">():</span>
<span class="sd">"""Transformer for random 90 degree rotation image.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> function</span>
<span class="sd"> Transformer function for operation.</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="p">(</span><span class="k">lambda</span> <span class="n">img</span><span class="p">:</span> <span class="n">img</span><span class="o">.</span><span class="n">rotate</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">sample</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">90</span><span class="p">,</span> <span class="mi">180</span><span class="p">,</span> <span class="mi">270</span><span class="p">],</span> <span class="n">k</span><span class="o">=</span><span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]))</span></div>
<div class="viewcode-block" id="get_data_transforms"><a class="viewcode-back" href="../../index.html#pathflowai.datasets.get_data_transforms">[docs]</a><span class="k">def</span> <span class="nf">get_data_transforms</span><span class="p">(</span><span class="n">patch_size</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">mean</span><span class="o">=</span><span class="p">[],</span> <span class="n">std</span><span class="o">=</span><span class="p">[],</span> <span class="n">resize</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">transform_platform</span><span class="o">=</span><span class="s1">'torch'</span><span class="p">,</span> <span class="n">elastic</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="sd">"""Get data transformers for training test and validation sets.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> patch_size:int</span>
<span class="sd"> Original patch size being transformed.</span>
<span class="sd"> mean:list of float</span>
<span class="sd"> Mean RGB</span>
<span class="sd"> std:list of float</span>
<span class="sd"> Std RGB</span>
<span class="sd"> resize:int</span>
<span class="sd"> Which patch size to resize to.</span>
<span class="sd"> transform_platform:str</span>
<span class="sd"> Use pytorch or albumentation transforms.</span>
<span class="sd"> elastic:bool</span>
<span class="sd"> Whether to add elastic deformations from albumentations.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> dict</span>
<span class="sd"> Transformers.</span>
<span class="sd"> """</span>
<span class="n">data_transforms</span> <span class="o">=</span> <span class="p">{</span> <span class="s1">'torch'</span><span class="p">:</span> <span class="p">{</span>
<span class="s1">'train'</span><span class="p">:</span> <span class="n">transforms</span><span class="o">.</span><span class="n">Compose</span><span class="p">([</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">ToPILImage</span><span class="p">(),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">Resize</span><span class="p">((</span><span class="n">patch_size</span><span class="p">,</span><span class="n">patch_size</span><span class="p">)),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">CenterCrop</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span> <span class="c1"># if not resize else</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">ColorJitter</span><span class="p">(</span><span class="n">brightness</span><span class="o">=</span><span class="mf">0.8</span><span class="p">,</span> <span class="n">contrast</span><span class="o">=</span><span class="mf">0.8</span><span class="p">,</span> <span class="n">saturation</span><span class="o">=</span><span class="mf">0.8</span><span class="p">,</span> <span class="n">hue</span><span class="o">=</span><span class="mf">0.5</span><span class="p">),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">RandomHorizontalFlip</span><span class="p">(),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">RandomVerticalFlip</span><span class="p">(),</span>
<span class="n">RandomRotate90</span><span class="p">(),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">ToTensor</span><span class="p">(),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">Normalize</span><span class="p">(</span><span class="n">mean</span> <span class="k">if</span> <span class="n">mean</span> <span class="k">else</span> <span class="p">[</span><span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.6</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">],</span> <span class="n">std</span> <span class="k">if</span> <span class="n">std</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">[</span><span class="mf">0.15</span><span class="p">,</span> <span class="mf">0.15</span><span class="p">,</span> <span class="mf">0.15</span><span class="p">])</span> <span class="c1">#mean and standard deviations for lung adenocarcinoma resection slides</span>
<span class="p">]),</span>
<span class="s1">'val'</span><span class="p">:</span> <span class="n">transforms</span><span class="o">.</span><span class="n">Compose</span><span class="p">([</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">ToPILImage</span><span class="p">(),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">Resize</span><span class="p">((</span><span class="n">patch_size</span><span class="p">,</span><span class="n">patch_size</span><span class="p">)),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">CenterCrop</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">ToTensor</span><span class="p">(),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">Normalize</span><span class="p">(</span><span class="n">mean</span> <span class="k">if</span> <span class="n">mean</span> <span class="k">else</span> <span class="p">[</span><span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.6</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">],</span> <span class="n">std</span> <span class="k">if</span> <span class="n">std</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">[</span><span class="mf">0.15</span><span class="p">,</span> <span class="mf">0.15</span><span class="p">,</span> <span class="mf">0.15</span><span class="p">])</span>
<span class="p">]),</span>
<span class="s1">'test'</span><span class="p">:</span> <span class="n">transforms</span><span class="o">.</span><span class="n">Compose</span><span class="p">([</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">ToPILImage</span><span class="p">(),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">Resize</span><span class="p">((</span><span class="n">patch_size</span><span class="p">,</span><span class="n">patch_size</span><span class="p">)),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">CenterCrop</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">ToTensor</span><span class="p">(),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">Normalize</span><span class="p">(</span><span class="n">mean</span> <span class="k">if</span> <span class="n">mean</span> <span class="k">else</span> <span class="p">[</span><span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.6</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">],</span> <span class="n">std</span> <span class="k">if</span> <span class="n">std</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">[</span><span class="mf">0.15</span><span class="p">,</span> <span class="mf">0.15</span><span class="p">,</span> <span class="mf">0.15</span><span class="p">])</span>
<span class="p">]),</span>
<span class="s1">'pass'</span><span class="p">:</span> <span class="n">transforms</span><span class="o">.</span><span class="n">Compose</span><span class="p">([</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">ToPILImage</span><span class="p">(),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">CenterCrop</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">ToTensor</span><span class="p">(),</span>
<span class="p">])</span>
<span class="p">},</span>
<span class="s1">'albumentations'</span><span class="p">:{</span>
<span class="s1">'train'</span><span class="p">:</span><span class="n">alb</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">composition</span><span class="o">.</span><span class="n">Compose</span><span class="p">([</span>
<span class="n">alb</span><span class="o">.</span><span class="n">augmentations</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">Resize</span><span class="p">(</span><span class="n">patch_size</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">),</span>
<span class="n">alb</span><span class="o">.</span><span class="n">augmentations</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">CenterCrop</span><span class="p">(</span><span class="n">patch_size</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">)</span>
<span class="p">]</span><span class="o">+</span><span class="p">([</span><span class="n">alb</span><span class="o">.</span><span class="n">augmentations</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">Flip</span><span class="p">(</span><span class="n">p</span><span class="o">=</span><span class="mf">0.5</span><span class="p">),</span>
<span class="n">alb</span><span class="o">.</span><span class="n">augmentations</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">Transpose</span><span class="p">(</span><span class="n">p</span><span class="o">=</span><span class="mf">0.5</span><span class="p">),</span>
<span class="n">alb</span><span class="o">.</span><span class="n">augmentations</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">ShiftScaleRotate</span><span class="p">(</span><span class="n">p</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)]</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">elastic</span> <span class="k">else</span> <span class="p">[</span><span class="n">alb</span><span class="o">.</span><span class="n">augmentations</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">RandomRotate90</span><span class="p">(</span><span class="n">p</span><span class="o">=</span><span class="mf">0.5</span><span class="p">),</span>
<span class="n">alb</span><span class="o">.</span><span class="n">augmentations</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">ElasticTransform</span><span class="p">(</span><span class="n">p</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)])</span><span class="o">+</span><span class="p">[</span><span class="n">albtorch</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">ToTensor</span><span class="p">(</span><span class="n">normalize</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">mean</span><span class="o">=</span><span class="n">mean</span> <span class="k">if</span> <span class="n">mean</span> <span class="k">else</span> <span class="p">[</span><span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.6</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">],</span> <span class="n">std</span><span class="o">=</span><span class="n">std</span> <span class="k">if</span> <span class="n">std</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">[</span><span class="mf">0.15</span><span class="p">,</span> <span class="mf">0.15</span><span class="p">,</span> <span class="mf">0.15</span><span class="p">]))]</span>
<span class="p">),</span>
<span class="s1">'val'</span><span class="p">:</span><span class="n">alb</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">composition</span><span class="o">.</span><span class="n">Compose</span><span class="p">([</span>
<span class="n">alb</span><span class="o">.</span><span class="n">augmentations</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">Resize</span><span class="p">(</span><span class="n">patch_size</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">),</span>
<span class="n">alb</span><span class="o">.</span><span class="n">augmentations</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">CenterCrop</span><span class="p">(</span><span class="n">patch_size</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">),</span>
<span class="n">albtorch</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">ToTensor</span><span class="p">(</span><span class="n">normalize</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">mean</span><span class="o">=</span><span class="n">mean</span> <span class="k">if</span> <span class="n">mean</span> <span class="k">else</span> <span class="p">[</span><span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.6</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">],</span> <span class="n">std</span><span class="o">=</span><span class="n">std</span> <span class="k">if</span> <span class="n">std</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">[</span><span class="mf">0.15</span><span class="p">,</span> <span class="mf">0.15</span><span class="p">,</span> <span class="mf">0.15</span><span class="p">]))</span>
<span class="p">]),</span>
<span class="s1">'test'</span><span class="p">:</span><span class="n">alb</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">composition</span><span class="o">.</span><span class="n">Compose</span><span class="p">([</span>
<span class="n">alb</span><span class="o">.</span><span class="n">augmentations</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">Resize</span><span class="p">(</span><span class="n">patch_size</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">),</span>
<span class="n">alb</span><span class="o">.</span><span class="n">augmentations</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">CenterCrop</span><span class="p">(</span><span class="n">patch_size</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">),</span>
<span class="n">albtorch</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">ToTensor</span><span class="p">(</span><span class="n">normalize</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">mean</span><span class="o">=</span><span class="n">mean</span> <span class="k">if</span> <span class="n">mean</span> <span class="k">else</span> <span class="p">[</span><span class="mf">0.7</span><span class="p">,</span> <span class="mf">0.6</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">],</span> <span class="n">std</span><span class="o">=</span><span class="n">std</span> <span class="k">if</span> <span class="n">std</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">[</span><span class="mf">0.15</span><span class="p">,</span> <span class="mf">0.15</span><span class="p">,</span> <span class="mf">0.15</span><span class="p">]))</span>
<span class="p">])</span>
<span class="p">}}</span>
<span class="k">return</span> <span class="n">data_transforms</span><span class="p">[</span><span class="n">transform_platform</span><span class="p">]</span></div>
<div class="viewcode-block" id="create_transforms"><a class="viewcode-back" href="../../index.html#pathflowai.datasets.create_transforms">[docs]</a><span class="k">def</span> <span class="nf">create_transforms</span><span class="p">(</span><span class="n">mean</span><span class="p">,</span> <span class="n">std</span><span class="p">):</span>
<span class="sd">"""Create transformers.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> mean:list</span>
<span class="sd"> See get_data_transforms.</span>
<span class="sd"> std:list</span>
<span class="sd"> See get_data_transforms.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> dict</span>
<span class="sd"> Transformers.</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">get_data_transforms</span><span class="p">(</span><span class="n">patch_size</span> <span class="o">=</span> <span class="mi">224</span><span class="p">,</span> <span class="n">mean</span><span class="o">=</span><span class="n">mean</span><span class="p">,</span> <span class="n">std</span><span class="o">=</span><span class="n">std</span><span class="p">,</span> <span class="n">resize</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="get_normalizer"><a class="viewcode-back" href="../../index.html#pathflowai.datasets.get_normalizer">[docs]</a><span class="k">def</span> <span class="nf">get_normalizer</span><span class="p">(</span><span class="n">normalization_file</span><span class="p">,</span> <span class="n">dataset_opts</span><span class="p">):</span>
<span class="sd">"""Find mean and standard deviation of images in batches.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> normalization_file:str</span>
<span class="sd"> File to store normalization information.</span>
<span class="sd"> dataset_opts:type</span>
<span class="sd"> Dictionary storing information to create DynamicDataset class.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> dict</span>
<span class="sd"> Stores RGB mean, stdev.</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">normalization_file</span><span class="p">):</span>
<span class="n">norm_dict</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">normalization_file</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">norm_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'normalization_file'</span><span class="p">:</span><span class="n">normalization_file</span><span class="p">}</span>
<span class="k">if</span> <span class="s1">'normalization_file'</span> <span class="ow">in</span> <span class="n">norm_dict</span><span class="p">:</span>
<span class="n">transformers</span> <span class="o">=</span> <span class="n">get_data_transforms</span><span class="p">(</span><span class="n">patch_size</span> <span class="o">=</span> <span class="mi">224</span><span class="p">,</span> <span class="n">mean</span><span class="o">=</span><span class="p">[],</span> <span class="n">std</span><span class="o">=</span><span class="p">[],</span> <span class="n">resize</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">transform_platform</span><span class="o">=</span><span class="s1">'torch'</span><span class="p">)</span>
<span class="n">dataset_opts</span><span class="p">[</span><span class="s1">'transformers'</span><span class="p">]</span><span class="o">=</span><span class="n">transformers</span>
<span class="c1">#print(dict(pos_annotation_class=pos_annotation_class, segmentation=segmentation, patch_size=patch_size, fix_names=fix_names, other_annotations=other_annotations))</span>
<span class="n">dataset</span> <span class="o">=</span> <span class="n">DynamicImageDataset</span><span class="p">(</span><span class="o">**</span><span class="n">dataset_opts</span><span class="p">)</span><span class="c1">#nc.SafeDataset(DynamicImageDataset(**dataset_opts))</span>
<span class="k">if</span> <span class="n">dataset_opts</span><span class="p">[</span><span class="s1">'classify_annotations'</span><span class="p">]:</span>
<span class="n">dataset</span><span class="o">.</span><span class="n">binarize_annotations</span><span class="p">()</span>
<span class="n">dataloader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">num_workers</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span>
<span class="n">all_mean</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.</span><span class="p">,</span><span class="mf">0.</span><span class="p">,</span><span class="mf">0.</span><span class="p">],</span><span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float</span><span class="p">)</span><span class="c1">#[]</span>
<span class="n">all_std</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.</span><span class="p">,</span><span class="mf">0.</span><span class="p">,</span><span class="mf">0.</span><span class="p">],</span><span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float</span><span class="p">)</span>
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
<span class="n">all_mean</span><span class="o">=</span><span class="n">all_mean</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
<span class="n">all_std</span><span class="o">=</span><span class="n">all_std</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
<span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
<span class="k">for</span> <span class="n">i</span><span class="p">,(</span><span class="n">X</span><span class="p">,</span><span class="n">_</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">dataloader</span><span class="p">):</span> <span class="c1"># x,3,224,224</span>
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
<span class="n">X</span><span class="o">=</span><span class="n">X</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
<span class="n">all_mean</span> <span class="o">+=</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">3</span><span class="p">))</span>
<span class="n">all_std</span> <span class="o">+=</span> <span class="n">torch</span><span class="o">.</span><span class="n">std</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">3</span><span class="p">))</span>
<span class="n">N</span><span class="o">=</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span>
<span class="n">all_mean</span> <span class="o">/=</span> <span class="nb">float</span><span class="p">(</span><span class="n">N</span><span class="p">)</span> <span class="c1">#(np.array(all_mean).mean(axis=0)).tolist()</span>
<span class="n">all_std</span> <span class="o">/=</span> <span class="nb">float</span><span class="p">(</span><span class="n">N</span><span class="p">)</span> <span class="c1">#(np.array(all_std).mean(axis=0)).tolist()</span>
<span class="n">all_mean</span> <span class="o">=</span> <span class="n">all_mean</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
<span class="n">all_std</span> <span class="o">=</span> <span class="n">all_std</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
<span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">mean</span><span class="o">=</span><span class="n">all_mean</span><span class="p">,</span><span class="n">std</span><span class="o">=</span><span class="n">all_std</span><span class="p">),</span><span class="n">norm_dict</span><span class="p">[</span><span class="s1">'normalization_file'</span><span class="p">])</span>
<span class="n">norm_dict</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">norm_dict</span><span class="p">[</span><span class="s1">'normalization_file'</span><span class="p">])</span>
<span class="k">return</span> <span class="n">norm_dict</span></div>
<div class="viewcode-block" id="segmentation_transform"><a class="viewcode-back" href="../../index.html#pathflowai.datasets.segmentation_transform">[docs]</a><span class="k">def</span> <span class="nf">segmentation_transform</span><span class="p">(</span><span class="n">img</span><span class="p">,</span><span class="n">mask</span><span class="p">,</span> <span class="n">transformer</span><span class="p">):</span>
<span class="sd">"""Run albumentations and return an image and its segmentation mask.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> img:array</span>
<span class="sd"> Image as array</span>
<span class="sd"> mask:array</span>
<span class="sd"> Categorical pixel by pixel.</span>
<span class="sd"> transformer :</span>
<span class="sd"> Transformation object.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> tuple arrays</span>
<span class="sd"> Image and mask array.</span>
<span class="sd"> """</span>
<span class="n">res</span><span class="o">=</span><span class="n">transformer</span><span class="p">(</span><span class="kc">True</span><span class="p">,</span> <span class="n">image</span><span class="o">=</span><span class="n">img</span><span class="p">,</span> <span class="n">mask</span><span class="o">=</span><span class="n">mask</span><span class="p">)</span>
<span class="c1">#res_mask_shape = res['mask'].size()</span>
<span class="k">return</span> <span class="n">res</span><span class="p">[</span><span class="s1">'image'</span><span class="p">],</span> <span class="n">res</span><span class="p">[</span><span class="s1">'mask'</span><span class="p">]</span><span class="o">.</span><span class="n">long</span><span class="p">()</span><span class="c1">#.view(res_mask_shape[0],res_mask_shape[1],res_mask_shape[2])</span></div>
<div class="viewcode-block" id="DynamicImageDataset"><a class="viewcode-back" href="../../index.html#pathflowai.datasets.DynamicImageDataset">[docs]</a><span class="k">class</span> <span class="nc">DynamicImageDataset</span><span class="p">(</span><span class="n">Dataset</span><span class="p">):</span>
<span class="sd">"""Generate image dataset that accesses images and annotations via dask.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> dataset_df:dataframe</span>
<span class="sd"> Dataframe with WSI, which set it is in (train/test/val) and corresponding WSI labels if applicable.</span>
<span class="sd"> set:str</span>
<span class="sd"> Whether train, test, val or pass (normalization) set.</span>
<span class="sd"> patch_info_file:str</span>
<span class="sd"> SQL db with positional and annotation information on each slide.</span>
<span class="sd"> transformers:dict</span>
<span class="sd"> Contains transformers to apply on images.</span>
<span class="sd"> input_dir:str</span>
<span class="sd"> Directory where images comes from.</span>
<span class="sd"> target_names:list/str</span>
<span class="sd"> Names of initial targets, which may be modified.</span>
<span class="sd"> pos_annotation_class:str</span>
<span class="sd"> If selected and predicting on WSI, this class is labeled as a positive from the WSI, while the other classes are not.</span>
<span class="sd"> other_annotations:list</span>
<span class="sd"> Other annotations to consider from patch info db.</span>
<span class="sd"> segmentation:bool</span>
<span class="sd"> Conducting segmentation task?</span>
<span class="sd"> patch_size:int</span>
<span class="sd"> Patch size.</span>
<span class="sd"> fix_names:bool</span>
<span class="sd"> Whether to change the names of dataset_df.</span>
<span class="sd"> target_segmentation_class:list</span>
<span class="sd"> Now can be used for classification as well, matched with two below options, samples images only from this class. Can specify this and below two options multiple times.</span>
<span class="sd"> target_threshold:list</span>
<span class="sd"> Sampled only if above this threshold of occurence in the patches.</span>
<span class="sd"> oversampling_factor:list</span>
<span class="sd"> Over sample them at this amount.</span>
<span class="sd"> n_segmentation_classes:int</span>
<span class="sd"> Number classes to segment.</span>
<span class="sd"> gdl:bool</span>
<span class="sd"> Using generalized dice loss?</span>
<span class="sd"> mt_bce:bool</span>
<span class="sd"> For multi-target prediction tasks.</span>
<span class="sd"> classify_annotations:bool</span>
<span class="sd"> For classifying annotations.</span>
<span class="sd"> """</span>
<span class="c1"># when building transformers, need a resize patch size to make patches 224 by 224</span>
<span class="c1">#@pysnooper.snoop('init_data.log')</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span><span class="n">dataset_df</span><span class="p">,</span> <span class="nb">set</span><span class="p">,</span> <span class="n">patch_info_file</span><span class="p">,</span> <span class="n">transformers</span><span class="p">,</span> <span class="n">input_dir</span><span class="p">,</span> <span class="n">target_names</span><span class="p">,</span> <span class="n">pos_annotation_class</span><span class="p">,</span> <span class="n">other_annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">segmentation</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">patch_size</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">fix_names</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">target_segmentation_class</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> <span class="n">target_threshold</span><span class="o">=</span><span class="mf">0.</span><span class="p">,</span> <span class="n">oversampling_factor</span><span class="o">=</span><span class="mf">1.</span><span class="p">,</span> <span class="n">n_segmentation_classes</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">gdl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">mt_bce</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">classify_annotations</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="c1">#print('check',classify_annotations)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">transformer</span><span class="o">=</span><span class="n">transformers</span><span class="p">[</span><span class="nb">set</span><span class="p">]</span>
<span class="n">original_set</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="nb">set</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">set</span><span class="o">==</span><span class="s1">'pass'</span><span class="p">:</span>
<span class="nb">set</span><span class="o">=</span><span class="s1">'train'</span>
<span class="bp">self</span><span class="o">.</span><span class="n">targets</span> <span class="o">=</span> <span class="n">target_names</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mt_bce</span><span class="o">=</span><span class="n">mt_bce</span>
<span class="bp">self</span><span class="o">.</span><span class="n">set</span> <span class="o">=</span> <span class="nb">set</span>
<span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span> <span class="o">=</span> <span class="n">segmentation</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">)</span><span class="o">==</span><span class="mi">1</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">targets</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="n">original_set</span> <span class="o">==</span> <span class="s1">'pass'</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">transform_fn</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">,</span><span class="n">y</span><span class="p">:</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">transformer</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">1.</span><span class="p">,</span><span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">transform_fn</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">,</span><span class="n">y</span><span class="p">:</span> <span class="n">segmentation_transform</span><span class="p">(</span><span class="n">x</span><span class="p">,</span><span class="n">y</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">transformer</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="s1">'p'</span> <span class="ow">in</span> <span class="nb">dir</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">transformer</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">transform_fn</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">,</span><span class="n">y</span><span class="p">:</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">transformer</span><span class="p">(</span><span class="kc">True</span><span class="p">,</span> <span class="n">image</span><span class="o">=</span><span class="n">x</span><span class="p">)[</span><span class="s1">'image'</span><span class="p">],</span> <span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">y</span><span class="p">)</span><span class="o">.</span><span class="n">float</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">transform_fn</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">,</span><span class="n">y</span><span class="p">:</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">transformer</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">y</span><span class="p">)</span><span class="o">.</span><span class="n">float</span><span class="p">())</span>
<span class="bp">self</span><span class="o">.</span><span class="n">image_set</span> <span class="o">=</span> <span class="n">dataset_df</span><span class="p">[</span><span class="n">dataset_df</span><span class="p">[</span><span class="s1">'set'</span><span class="p">]</span><span class="o">==</span><span class="nb">set</span><span class="p">]</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="o">=</span><span class="s1">'target'</span>
<span class="bp">self</span><span class="o">.</span><span class="n">image_set</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">]</span> <span class="o">=</span> <span class="mf">1.</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span> <span class="ow">and</span> <span class="n">fix_names</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">image_set</span><span class="o">.</span><span class="n">loc</span><span class="p">[:,</span><span class="s1">'ID'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_set</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">]</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">fix_name</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">slide_info</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">image_set</span><span class="o">.</span><span class="n">set_index</span><span class="p">(</span><span class="s1">'ID'</span><span class="p">)</span><span class="o">.</span><span class="n">loc</span><span class="p">[:,</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">])</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mt_bce</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span><span class="p">:</span>
<span class="k">if</span> <span class="n">pos_annotation_class</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">targets</span> <span class="o">=</span> <span class="p">[</span><span class="n">pos_annotation_class</span><span class="p">]</span><span class="o">+</span><span class="nb">list</span><span class="p">(</span><span class="n">other_annotations</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">targets</span> <span class="o">=</span> <span class="kc">None</span>
<span class="nb">print</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">)</span>
<span class="n">IDs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">slide_info</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
<span class="n">pi_dict</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">input_info_db</span><span class="o">=</span><span class="n">patch_info_file</span><span class="p">,</span> <span class="n">slide_labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">slide_info</span><span class="p">,</span> <span class="n">pos_annotation_class</span><span class="o">=</span><span class="n">pos_annotation_class</span><span class="p">,</span> <span class="n">patch_size</span><span class="o">=</span><span class="n">patch_size</span><span class="p">,</span> <span class="n">segmentation</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span><span class="p">,</span> <span class="n">other_annotations</span><span class="o">=</span><span class="n">other_annotations</span><span class="p">,</span> <span class="n">target_segmentation_class</span><span class="o">=</span><span class="n">target_segmentation_class</span><span class="p">,</span> <span class="n">target_threshold</span><span class="o">=</span><span class="n">target_threshold</span><span class="p">,</span> <span class="n">classify_annotations</span><span class="o">=</span><span class="n">classify_annotations</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span> <span class="o">=</span> <span class="n">modify_patch_info</span><span class="p">(</span><span class="o">**</span><span class="n">pi_dict</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span> <span class="ow">and</span> <span class="n">original_set</span><span class="o">!=</span><span class="s1">'pass'</span><span class="p">:</span>
<span class="c1">#IDs = self.patch_info['ID'].unique()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">segmentation_maps</span> <span class="o">=</span> <span class="p">{</span><span class="n">slide</span><span class="p">:</span><span class="n">da</span><span class="o">.</span><span class="n">from_array</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">input_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">_mask.npy'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">slide</span><span class="p">)),</span><span class="n">mmap_mode</span><span class="o">=</span><span class="s1">'r+'</span><span class="p">))</span> <span class="k">for</span> <span class="n">slide</span> <span class="ow">in</span> <span class="n">IDs</span><span class="p">}</span>
<span class="bp">self</span><span class="o">.</span><span class="n">slides</span> <span class="o">=</span> <span class="p">{</span><span class="n">slide</span><span class="p">:</span><span class="n">da</span><span class="o">.</span><span class="n">from_zarr</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">input_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">.zarr'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">slide</span><span class="p">)))</span> <span class="k">for</span> <span class="n">slide</span> <span class="ow">in</span> <span class="n">IDs</span><span class="p">}</span>
<span class="c1">#print(self.slide_info)</span>
<span class="k">if</span> <span class="n">original_set</span> <span class="o">==</span><span class="s1">'pass'</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span><span class="o">=</span><span class="kc">False</span>
<span class="c1">#print(self.patch_info[self.targets].unique())</span>
<span class="k">if</span> <span class="n">oversampling_factor</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">]</span><span class="o">*</span><span class="nb">int</span><span class="p">(</span><span class="n">oversampling_factor</span><span class="p">),</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">reset_index</span><span class="p">(</span><span class="n">drop</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">oversampling_factor</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">.</span><span class="n">sample</span><span class="p">(</span><span class="n">frac</span><span class="o">=</span><span class="n">oversampling_factor</span><span class="p">)</span><span class="o">.</span><span class="n">reset_index</span><span class="p">(</span><span class="n">drop</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">length</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">n_segmentation_classes</span> <span class="o">=</span> <span class="n">n_segmentation_classes</span>
<span class="bp">self</span><span class="o">.</span><span class="n">gdl</span><span class="o">=</span><span class="n">gdl</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span> <span class="k">else</span> <span class="kc">False</span>
<span class="bp">self</span><span class="o">.</span><span class="n">binarized</span><span class="o">=</span><span class="kc">False</span>
<span class="bp">self</span><span class="o">.</span><span class="n">classify_annotations</span><span class="o">=</span><span class="n">classify_annotations</span>
<span class="nb">print</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">)</span>
<div class="viewcode-block" id="DynamicImageDataset.concat"><a class="viewcode-back" href="../../index.html#pathflowai.datasets.DynamicImageDataset.concat">[docs]</a> <span class="k">def</span> <span class="nf">concat</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other_dataset</span><span class="p">):</span>
<span class="sd">"""Concatenate this dataset with others. Updates its own internal attributes.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> other_dataset:DynamicImageDataset</span>
<span class="sd"> Other image dataset.</span>
<span class="sd"> """</span>
<span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">,</span> <span class="n">other_dataset</span><span class="o">.</span><span class="n">patch_info</span><span class="p">],</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">reset_index</span><span class="p">(</span><span class="n">drop</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">length</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">segmentation_maps</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">other_dataset</span><span class="o">.</span><span class="n">segmentation_maps</span><span class="p">)</span></div>
<span class="c1">#print(self.segmentation_maps.keys())</span>
<div class="viewcode-block" id="DynamicImageDataset.retain_ID"><a class="viewcode-back" href="../../index.html#pathflowai.datasets.DynamicImageDataset.retain_ID">[docs]</a> <span class="k">def</span> <span class="nf">retain_ID</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ID</span><span class="p">):</span>
<span class="sd">"""Reduce the sample set to just images from one ID.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> ID:str</span>
<span class="sd"> Basename/ID to predict on.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> self</span>
<span class="sd"> """</span>
<span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">]</span><span class="o">==</span><span class="n">ID</span><span class="p">]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">length</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">return</span> <span class="bp">self</span></div>
<div class="viewcode-block" id="DynamicImageDataset.split_by_ID"><a class="viewcode-back" href="../../index.html#pathflowai.datasets.DynamicImageDataset.split_by_ID">[docs]</a> <span class="k">def</span> <span class="nf">split_by_ID</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Generator similar to groupby, but splits up by ID, generates (ID,data) using retain_ID.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> generator</span>
<span class="sd"> ID, DynamicDataset</span>
<span class="sd"> """</span>
<span class="k">for</span> <span class="n">ID</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">]</span><span class="o">.</span><span class="n">unique</span><span class="p">():</span>
<span class="n">new_dataset</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="k">yield</span> <span class="n">ID</span><span class="p">,</span> <span class="n">new_dataset</span><span class="o">.</span><span class="n">retain_ID</span><span class="p">(</span><span class="n">ID</span><span class="p">)</span></div>
<div class="viewcode-block" id="DynamicImageDataset.get_class_weights"><a class="viewcode-back" href="../../index.html#pathflowai.datasets.DynamicImageDataset.get_class_weights">[docs]</a> <span class="k">def</span> <span class="nf">get_class_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">i</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span><span class="c1">#[0,1]</span>
<span class="sd">"""Weight loss function with weights inversely proportional to the class appearence.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> i:int</span>
<span class="sd"> If multi-target, class used for weighting.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> self</span>
<span class="sd"> Dataset.</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span><span class="p">:</span>
<span class="n">weights</span><span class="o">=</span><span class="mf">1.</span><span class="o">/</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="nb">str</span><span class="p">,</span><span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_segmentation_classes</span><span class="p">))))]</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">values</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">mt_bce</span><span class="p">:</span>
<span class="n">weights</span><span class="o">=</span><span class="mf">1.</span><span class="o">/</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">]</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">values</span><span class="p">)</span>
<span class="n">weights</span><span class="o">=</span><span class="n">weights</span><span class="o">/</span><span class="nb">sum</span><span class="p">(</span><span class="n">weights</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">binarized</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">)</span><span class="o">></span><span class="mi">1</span><span class="p">:</span>
<span class="n">y</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">,</span><span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="k">elif</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">)</span><span class="o">!=</span><span class="nb">type</span><span class="p">(</span><span class="s1">''</span><span class="p">)):</span>
<span class="n">y</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">y</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">[</span><span class="n">i</span><span class="p">]]</span>
<span class="n">y</span><span class="o">=</span><span class="n">y</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span><span class="o">.</span><span class="n">flatten</span><span class="p">()</span>
<span class="n">weights</span><span class="o">=</span><span class="n">compute_class_weight</span><span class="p">(</span><span class="n">class_weight</span><span class="o">=</span><span class="s1">'balanced'</span><span class="p">,</span><span class="n">classes</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">y</span><span class="p">),</span><span class="n">y</span><span class="o">=</span><span class="n">y</span><span class="p">)</span>
<span class="k">return</span> <span class="n">weights</span></div>
<div class="viewcode-block" id="DynamicImageDataset.binarize_annotations"><a class="viewcode-back" href="../../index.html#pathflowai.datasets.DynamicImageDataset.binarize_annotations">[docs]</a> <span class="k">def</span> <span class="nf">binarize_annotations</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">binarizer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_targets</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">binary_threshold</span><span class="o">=</span><span class="mf">0.</span><span class="p">):</span>
<span class="sd">"""Label binarize some annotations or threshold them if classifying slide annotations.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> binarizer:LabelBinarizer</span>
<span class="sd"> Binarizes the labels of a column(s)</span>
<span class="sd"> num_targets:int</span>
<span class="sd"> Number of desired targets to preidict on.</span>
<span class="sd"> binary_threshold:float</span>
<span class="sd"> Amount of annotation in patch before positive annotation.</span>
<span class="sd"> Returns</span>
<span class="sd"> -------</span>
<span class="sd"> binarizer</span>
<span class="sd"> """</span>
<span class="n">annotations</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">[</span><span class="s1">'annotation'</span><span class="p">]</span>
<span class="n">annots</span><span class="o">=</span><span class="p">[</span><span class="n">annot</span> <span class="k">for</span> <span class="n">annot</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span><span class="mi">6</span><span class="p">:])</span> <span class="k">if</span> <span class="n">annot</span> <span class="o">!=</span><span class="s1">'area'</span><span class="p">]</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">mt_bce</span> <span class="ow">and</span> <span class="n">num_targets</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
<span class="k">if</span> <span class="n">binarizer</span> <span class="o">==</span> <span class="kc">None</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">binarizer</span> <span class="o">=</span> <span class="n">LabelBinarizer</span><span class="p">()</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">annotations</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">binarizer</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">binarizer</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">targets</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">binarizer</span><span class="o">.</span><span class="n">classes_</span>
<span class="n">annotation_labels</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">binarizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">annotations</span><span class="p">),</span><span class="n">index</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">.</span><span class="n">index</span><span class="p">,</span><span class="n">columns</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">float</span><span class="p">)</span>
<span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="n">annotation_labels</span><span class="p">):</span>
<span class="k">if</span> <span class="n">col</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">.</span><span class="n">loc</span><span class="p">[:,</span><span class="n">col</span><span class="p">]</span><span class="o">=</span><span class="n">annotation_labels</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">=</span><span class="n">annotation_labels</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">binarizer</span><span class="o">=</span><span class="kc">None</span>
<span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="o">=</span><span class="n">annots</span>
<span class="k">if</span> <span class="n">num_targets</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">targets</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]]</span>
<span class="k">if</span> <span class="n">binary_threshold</span><span class="o">></span><span class="mf">0.</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">.</span><span class="n">loc</span><span class="p">[:,</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">]</span><span class="o">=</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">]</span><span class="o">>=</span><span class="n">binary_threshold</span><span class="p">)</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">)</span>
<span class="c1">#self.patch_info = pd.concat([self.patch_info,annotation_labels],axis=1)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">binarized</span><span class="o">=</span><span class="kc">True</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">binarizer</span></div>
<div class="viewcode-block" id="DynamicImageDataset.subsample"><a class="viewcode-back" href="../../index.html#pathflowai.datasets.DynamicImageDataset.subsample">[docs]</a> <span class="k">def</span> <span class="nf">subsample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">p</span><span class="p">):</span>
<span class="sd">"""Sample subset of dataset.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> p:float</span>
<span class="sd"> Fraction to subsample.</span>
<span class="sd"> """</span>
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">42</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">.</span><span class="n">sample</span><span class="p">(</span><span class="n">frac</span><span class="o">=</span><span class="n">p</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">length</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span></div>
<span class="c1">#@pysnooper.snoop('get_item.log')</span>
<span class="k">def</span> <span class="nf">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">i</span><span class="p">):</span>
<span class="n">patch_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">patch_info</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
<span class="n">ID</span> <span class="o">=</span> <span class="n">patch_info</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">]</span>
<span class="n">targets</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span>
<span class="n">use_long</span><span class="o">=</span><span class="kc">False</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span><span class="p">:</span>
<span class="n">y</span> <span class="o">=</span> <span class="n">patch_info</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">targets</span><span class="p">]</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">):</span>
<span class="n">y</span><span class="o">=</span><span class="n">y</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">float</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">binarized</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">mt_bce</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">y</span><span class="p">)</span><span class="o">></span><span class="mi">1</span><span class="p">:</span>
<span class="n">y</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">y</span><span class="o">.</span><span class="n">argmax</span><span class="p">())</span>
<span class="n">use_long</span><span class="o">=</span><span class="kc">True</span>
<span class="n">y</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">y</span><span class="o">.</span><span class="n">shape</span><span class="p">:</span>
<span class="n">y</span><span class="o">=</span><span class="n">y</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="n">xs</span> <span class="o">=</span> <span class="n">patch_info</span><span class="p">[</span><span class="s1">'x'</span><span class="p">]</span>
<span class="n">ys</span> <span class="o">=</span> <span class="n">patch_info</span><span class="p">[</span><span class="s1">'y'</span><span class="p">]</span>
<span class="n">patch_size</span> <span class="o">=</span> <span class="n">patch_info</span><span class="p">[</span><span class="s1">'patch_size'</span><span class="p">]</span>
<span class="n">y</span><span class="o">=</span><span class="p">(</span><span class="n">y</span> <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span> <span class="k">else</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">segmentation_maps</span><span class="p">[</span><span class="n">ID</span><span class="p">][</span><span class="n">xs</span><span class="p">:</span><span class="n">xs</span><span class="o">+</span><span class="n">patch_size</span><span class="p">,</span><span class="n">ys</span><span class="p">:</span><span class="n">ys</span><span class="o">+</span><span class="n">patch_size</span><span class="p">]))</span>
<span class="n">image</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">transform_fn</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">slides</span><span class="p">[</span><span class="n">ID</span><span class="p">][</span><span class="n">xs</span><span class="p">:</span><span class="n">xs</span><span class="o">+</span><span class="n">patch_size</span><span class="p">,</span><span class="n">ys</span><span class="p">:</span><span class="n">ys</span><span class="o">+</span><span class="n">patch_size</span><span class="p">,:</span><span class="mi">3</span><span class="p">]</span><span class="o">.</span><span class="n">compute</span><span class="p">()</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">),</span> <span class="n">y</span><span class="p">)</span><span class="c1">#.unsqueeze(0) # transpose .transpose([1,0,2])</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">segmentation</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">mt_bce</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">classify_annotations</span> <span class="ow">and</span> <span class="n">use_long</span><span class="p">:</span>
<span class="n">y</span><span class="o">=</span><span class="n">y</span><span class="o">.</span><span class="n">long</span><span class="p">()</span>
<span class="c1">#image_size=image.size()</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">gdl</span><span class="p">:</span>
<span class="n">y</span><span class="o">=</span><span class="n">class2one_hot</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_segmentation_classes</span><span class="p">)</span>
<span class="c1"># y=one_hot2dist(y)</span>
<span class="k">return</span> <span class="n">image</span><span class="p">,</span> <span class="n">y</span>
<span class="k">def</span> <span class="nf">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">length</span></div>
</pre></div>
</div>
</div>
<footer>
<hr/>
<div role="contentinfo">
<p>
© Copyright 2019, Joshua Levy
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>
Datasets
Models
