Datasets
Models
Downloads: 1
Diff of
/docs/_modules/pathflowai/utils.html
[000000]
..
[e9500f]
Switch to side-by-side view
--- a +++ b/docs/_modules/pathflowai/utils.html @@ -0,0 +1,1035 @@ + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>pathflowai.utils — PathFlowAI 0.1 documentation</title> + + + + + + + + + <script type="text/javascript" src="../../_static/js/modernizr.min.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../_static/jquery.js"></script> + <script type="text/javascript" src="../../_static/underscore.js"></script> + <script type="text/javascript" src="../../_static/doctools.js"></script> + <script type="text/javascript" src="../../_static/language_data.js"></script> + <script async="async" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script> + + <script type="text/javascript" src="../../_static/js/theme.js"></script> + + + + + <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" /> + <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" /> + <link rel="index" title="Index" href="../../genindex.html" /> + <link rel="search" title="Search" href="../../search.html" /> +</head> + +<body class="wy-body-for-nav"> + + + <div class="wy-grid-for-nav"> + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-scroll"> + <div class="wy-side-nav-search" > + + + + <a href="../../index.html" class="icon icon-home"> PathFlowAI + + + + </a> + + + + + + + +<div role="search"> + <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + + + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + + + + + <!-- Local TOC --> + <div class="local-toc"></div> + + + </div> + </div> + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" aria-label="top navigation"> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../../index.html">PathFlowAI</a> + + </nav> + + + <div class="wy-nav-content"> + + <div class="rst-content"> + + + + + + + + + + + + + + + + + +<div role="navigation" aria-label="breadcrumbs navigation"> + + <ul class="wy-breadcrumbs"> + + <li><a href="../../index.html">Docs</a> »</li> + + <li><a href="../index.html">Module code</a> »</li> + + <li>pathflowai.utils</li> + + + <li class="wy-breadcrumbs-aside"> + + </li> + + </ul> + + + <hr/> +</div> + <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> + <div itemprop="articleBody"> + + <h1>Source code for pathflowai.utils</h1><div class="highlight"><pre> +<span></span><span class="sd">"""</span> +<span class="sd">utils.py</span> +<span class="sd">=======================</span> +<span class="sd">General utilities that still need to be broken up into preprocessing, machine learning input preparation, and output submodules.</span> +<span class="sd">"""</span> + +<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span> +<span class="kn">from</span> <span class="nn">bs4</span> <span class="k">import</span> <span class="n">BeautifulSoup</span> +<span class="kn">from</span> <span class="nn">shapely.geometry</span> <span class="k">import</span> <span class="n">Point</span> +<span class="kn">from</span> <span class="nn">shapely.geometry.polygon</span> <span class="k">import</span> <span class="n">Polygon</span> +<span class="kn">import</span> <span class="nn">glob</span> +<span class="kn">from</span> <span class="nn">os.path</span> <span class="k">import</span> <span class="n">join</span> +<span class="kn">import</span> <span class="nn">plotly.graph_objs</span> <span class="k">as</span> <span class="nn">go</span> +<span class="kn">import</span> <span class="nn">plotly.offline</span> <span class="k">as</span> <span class="nn">py</span> +<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span><span class="o">,</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span> +<span class="kn">import</span> <span class="nn">scipy.sparse</span> <span class="k">as</span> <span class="nn">sps</span> +<span class="kn">from</span> <span class="nn">PIL</span> <span class="k">import</span> <span class="n">Image</span><span class="p">,</span> <span class="n">ImageDraw</span> +<span class="n">Image</span><span class="o">.</span><span class="n">MAX_IMAGE_PIXELS</span><span class="o">=</span><span class="mf">1e10</span> +<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span> +<span class="kn">import</span> <span class="nn">scipy.sparse</span> <span class="k">as</span> <span class="nn">sps</span> +<span class="kn">from</span> <span class="nn">os.path</span> <span class="k">import</span> <span class="n">join</span> +<span class="kn">import</span> <span class="nn">os</span><span class="o">,</span> <span class="nn">subprocess</span><span class="o">,</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span> +<span class="kn">import</span> <span class="nn">sqlite3</span> +<span class="kn">import</span> <span class="nn">torch</span> +<span class="kn">from</span> <span class="nn">torch.utils.data</span> <span class="k">import</span> <span class="n">Dataset</span><span class="c1">#, DataLoader</span> +<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="k">import</span> <span class="n">train_test_split</span> +<span class="kn">import</span> <span class="nn">pysnooper</span> + +<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span> +<span class="kn">import</span> <span class="nn">dask.array</span> <span class="k">as</span> <span class="nn">da</span> +<span class="kn">import</span> <span class="nn">dask</span> +<span class="kn">import</span> <span class="nn">openslide</span> +<span class="kn">from</span> <span class="nn">openslide</span> <span class="k">import</span> <span class="n">deepzoom</span> +<span class="c1">#import xarray as xr, sparse</span> +<span class="kn">import</span> <span class="nn">pickle</span> +<span class="kn">import</span> <span class="nn">copy</span> + +<span class="kn">import</span> <span class="nn">nonechucks</span> <span class="k">as</span> <span class="nn">nc</span> + +<span class="kn">from</span> <span class="nn">nonechucks</span> <span class="k">import</span> <span class="n">SafeDataLoader</span> <span class="k">as</span> <span class="n">DataLoader</span> + +<div class="viewcode-block" id="load_sql_df"><a class="viewcode-back" href="../../index.html#pathflowai.utils.load_sql_df">[docs]</a><span class="k">def</span> <span class="nf">load_sql_df</span><span class="p">(</span><span class="n">sql_file</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">):</span> + <span class="sd">"""Load pandas dataframe from SQL, accessing particular patch size within SQL.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> sql_file:str</span> +<span class="sd"> SQL db.</span> +<span class="sd"> patch_size:int</span> +<span class="sd"> Patch size.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> dataframe</span> +<span class="sd"> Patch level information.</span> + +<span class="sd"> """</span> + <span class="n">conn</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">sql_file</span><span class="p">)</span> + <span class="n">df</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">read_sql</span><span class="p">(</span><span class="s1">'select * from "</span><span class="si">{}</span><span class="s1">";'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span><span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">)</span> + <span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> + <span class="k">return</span> <span class="n">df</span></div> + +<div class="viewcode-block" id="df2sql"><a class="viewcode-back" href="../../index.html#pathflowai.utils.df2sql">[docs]</a><span class="k">def</span> <span class="nf">df2sql</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">sql_file</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">'replace'</span><span class="p">):</span> + <span class="sd">"""Write dataframe containing patch level information to SQL db.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> df:dataframe</span> +<span class="sd"> Dataframe containing patch information.</span> +<span class="sd"> sql_file:str</span> +<span class="sd"> SQL database.</span> +<span class="sd"> patch_size:int</span> +<span class="sd"> Size of patches.</span> +<span class="sd"> mode:str</span> +<span class="sd"> Replace or append.</span> + +<span class="sd"> """</span> + <span class="n">conn</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">sql_file</span><span class="p">)</span> + <span class="n">df</span><span class="o">.</span><span class="n">set_index</span><span class="p">(</span><span class="s1">'index'</span><span class="p">)</span><span class="o">.</span><span class="n">to_sql</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span> <span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">,</span> <span class="n">if_exists</span><span class="o">=</span><span class="n">mode</span><span class="p">)</span> + <span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div> + + +<span class="c1">#########</span> + +<span class="c1"># https://github.com/qupath/qupath/wiki/Supported-image-formats</span> +<div class="viewcode-block" id="svs2dask_array"><a class="viewcode-back" href="../../index.html#pathflowai.utils.svs2dask_array">[docs]</a><span class="k">def</span> <span class="nf">svs2dask_array</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">tile_size</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span> <span class="n">overlap</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">remove_last</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">allow_unknown_chunksizes</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="sd">"""Convert SVS, TIF or TIFF to dask array.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> svs_file:str</span> +<span class="sd"> Image file.</span> +<span class="sd"> tile_size:int</span> +<span class="sd"> Size of chunk to be read in.</span> +<span class="sd"> overlap:int</span> +<span class="sd"> Do not modify, overlap between neighboring tiles.</span> +<span class="sd"> remove_last:bool</span> +<span class="sd"> Remove last tile because it has a custom size.</span> +<span class="sd"> allow_unknown_chunksizes: bool</span> +<span class="sd"> Allow different chunk sizes, more flexible, but slowdown.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> dask.array</span> +<span class="sd"> Dask Array.</span> + +<span class="sd"> >>> arr=svs2dask_array(svs_file, tile_size=1000, overlap=0, remove_last=True, allow_unknown_chunksizes=False)</span> +<span class="sd"> >>> arr2=arr.compute()</span> +<span class="sd"> >>> arr3=to_pil(cv2.resize(arr2, dsize=(1440,700), interpolation=cv2.INTER_CUBIC))</span> +<span class="sd"> >>> arr3.save(test_image_name)"""</span> + <span class="n">img</span><span class="o">=</span><span class="n">openslide</span><span class="o">.</span><span class="n">open_slide</span><span class="p">(</span><span class="n">svs_file</span><span class="p">)</span> + <span class="n">gen</span><span class="o">=</span><span class="n">deepzoom</span><span class="o">.</span><span class="n">DeepZoomGenerator</span><span class="p">(</span><span class="n">img</span><span class="p">,</span> <span class="n">tile_size</span><span class="o">=</span><span class="n">tile_size</span><span class="p">,</span> <span class="n">overlap</span><span class="o">=</span><span class="n">overlap</span><span class="p">,</span> <span class="n">limit_bounds</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> + <span class="n">max_level</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">gen</span><span class="o">.</span><span class="n">level_dimensions</span><span class="p">)</span><span class="o">-</span><span class="mi">1</span> + <span class="n">n_tiles_x</span><span class="p">,</span> <span class="n">n_tiles_y</span> <span class="o">=</span> <span class="n">gen</span><span class="o">.</span><span class="n">level_tiles</span><span class="p">[</span><span class="n">max_level</span><span class="p">]</span> + <span class="n">get_tile</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">i</span><span class="p">,</span><span class="n">j</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">gen</span><span class="o">.</span><span class="n">get_tile</span><span class="p">(</span><span class="n">max_level</span><span class="p">,(</span><span class="n">i</span><span class="p">,</span><span class="n">j</span><span class="p">)))</span><span class="o">.</span><span class="n">transpose</span><span class="p">((</span><span class="mi">1</span><span class="p">,</span><span class="mi">0</span><span class="p">,</span><span class="mi">2</span><span class="p">))</span> + <span class="n">sample_tile</span> <span class="o">=</span> <span class="n">get_tile</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="mi">0</span><span class="p">)</span> + <span class="n">sample_tile_shape</span> <span class="o">=</span> <span class="n">sample_tile</span><span class="o">.</span><span class="n">shape</span> + <span class="n">dask_get_tile</span> <span class="o">=</span> <span class="n">dask</span><span class="o">.</span><span class="n">delayed</span><span class="p">(</span><span class="n">get_tile</span><span class="p">,</span> <span class="n">pure</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> + <span class="n">arr</span><span class="o">=</span><span class="n">da</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">da</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">da</span><span class="o">.</span><span class="n">from_delayed</span><span class="p">(</span><span class="n">dask_get_tile</span><span class="p">(</span><span class="n">i</span><span class="p">,</span><span class="n">j</span><span class="p">),</span><span class="n">sample_tile_shape</span><span class="p">,</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">)</span> <span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_tiles_y</span> <span class="o">-</span> <span class="p">(</span><span class="mi">0</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">remove_last</span> <span class="k">else</span> <span class="mi">1</span><span class="p">))],</span><span class="n">allow_unknown_chunksizes</span><span class="o">=</span><span class="n">allow_unknown_chunksizes</span><span class="p">,</span><span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_tiles_x</span> <span class="o">-</span> <span class="p">(</span><span class="mi">0</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">remove_last</span> <span class="k">else</span> <span class="mi">1</span><span class="p">))],</span><span class="n">allow_unknown_chunksizes</span><span class="o">=</span><span class="n">allow_unknown_chunksizes</span><span class="p">)</span><span class="c1">#.transpose([1,0,2])</span> + <span class="k">return</span> <span class="n">arr</span></div> + +<div class="viewcode-block" id="img2npy_"><a class="viewcode-back" href="../../index.html#pathflowai.utils.img2npy_">[docs]</a><span class="k">def</span> <span class="nf">img2npy_</span><span class="p">(</span><span class="n">input_dir</span><span class="p">,</span><span class="n">basename</span><span class="p">,</span> <span class="n">svs_file</span><span class="p">):</span> + <span class="sd">"""Convert SVS, TIF, TIFF to NPY.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> input_dir:str</span> +<span class="sd"> Output file dir.</span> +<span class="sd"> basename:str</span> +<span class="sd"> Basename of output file</span> +<span class="sd"> svs_file:str</span> +<span class="sd"> SVS, TIF, TIFF file input.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> str</span> +<span class="sd"> NPY output file.</span> +<span class="sd"> """</span> + <span class="n">npy_out_file</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">input_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">.npy'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">basename</span><span class="p">))</span> + <span class="n">arr</span> <span class="o">=</span> <span class="n">svs2dask_array</span><span class="p">(</span><span class="n">svs_file</span><span class="p">)</span> + <span class="n">np</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">npy_out_file</span><span class="p">,</span><span class="n">arr</span><span class="o">.</span><span class="n">compute</span><span class="p">())</span> + <span class="k">return</span> <span class="n">npy_out_file</span></div> + +<div class="viewcode-block" id="load_image"><a class="viewcode-back" href="../../index.html#pathflowai.utils.load_image">[docs]</a><span class="k">def</span> <span class="nf">load_image</span><span class="p">(</span><span class="n">svs_file</span><span class="p">):</span> + <span class="sd">"""Load SVS, TIF, TIFF</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> svs_file:type</span> +<span class="sd"> Description of parameter `svs_file`.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> type</span> +<span class="sd"> Description of returned object.</span> +<span class="sd"> """</span> + <span class="n">im</span> <span class="o">=</span> <span class="n">Image</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">svs_file</span><span class="p">)</span> + <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">im</span><span class="p">),(</span><span class="mi">1</span><span class="p">,</span><span class="mi">0</span><span class="p">)),</span> <span class="n">im</span><span class="o">.</span><span class="n">size</span></div> + +<div class="viewcode-block" id="create_purple_mask"><a class="viewcode-back" href="../../index.html#pathflowai.utils.create_purple_mask">[docs]</a><span class="k">def</span> <span class="nf">create_purple_mask</span><span class="p">(</span><span class="n">arr</span><span class="p">,</span> <span class="n">img_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sparse</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span> + <span class="sd">"""Create a gray scale intensity mask. This will be changed soon to support other thresholding QC methods.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> arr:dask.array</span> +<span class="sd"> Dask array containing image information.</span> +<span class="sd"> img_size:int</span> +<span class="sd"> Deprecated.</span> +<span class="sd"> sparse:bool</span> +<span class="sd"> Deprecated</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> dask.array</span> +<span class="sd"> Intensity, grayscale array over image.</span> + +<span class="sd"> """</span> + <span class="n">r</span><span class="p">,</span><span class="n">b</span><span class="p">,</span><span class="n">g</span><span class="o">=</span><span class="n">arr</span><span class="p">[:,:,</span><span class="mi">0</span><span class="p">],</span><span class="n">arr</span><span class="p">[:,:,</span><span class="mi">1</span><span class="p">],</span><span class="n">arr</span><span class="p">[:,:,</span><span class="mi">2</span><span class="p">]</span> + <span class="n">gray</span> <span class="o">=</span> <span class="mf">0.2989</span> <span class="o">*</span> <span class="n">r</span> <span class="o">+</span> <span class="mf">0.5870</span> <span class="o">*</span> <span class="n">g</span> <span class="o">+</span> <span class="mf">0.1140</span> <span class="o">*</span> <span class="n">b</span> + <span class="c1">#rb_avg = (r+b)/2</span> + <span class="n">mask</span><span class="o">=</span> <span class="p">((</span><span class="mf">255.</span><span class="o">-</span><span class="n">gray</span><span class="p">))</span><span class="c1"># >= threshold)#(r > g - 10) & (b > g - 10) & (rb_avg > g + 20)#np.vectorize(is_purple)(arr).astype(int)</span> + <span class="k">if</span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">sparse</span><span class="p">:</span> + <span class="n">mask</span> <span class="o">=</span> <span class="n">mask</span><span class="o">.</span><span class="n">nonzero</span><span class="p">()</span> + <span class="n">mask</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">mask</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">compute</span><span class="p">(),</span> <span class="n">mask</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">compute</span><span class="p">()])</span><span class="o">.</span><span class="n">T</span> + <span class="c1">#mask = (np.ones(len(mask[0])),mask)</span> + <span class="c1">#mask = sparse.COO.from_scipy_sparse(sps.coo_matrix(mask, img_size, dtype=np.uint8).tocsr())</span> + <span class="k">return</span> <span class="n">mask</span></div> + +<div class="viewcode-block" id="add_purple_mask"><a class="viewcode-back" href="../../index.html#pathflowai.utils.add_purple_mask">[docs]</a><span class="k">def</span> <span class="nf">add_purple_mask</span><span class="p">(</span><span class="n">arr</span><span class="p">):</span> + <span class="sd">"""Optional add intensity mask to the dask array.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> arr:dask.array</span> +<span class="sd"> Image data.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> array</span> +<span class="sd"> Image data with intensity added as forth channel.</span> + +<span class="sd"> """</span> + <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="n">arr</span><span class="p">,</span><span class="n">create_purple_mask</span><span class="p">(</span><span class="n">arr</span><span class="p">)),</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span></div> + +<div class="viewcode-block" id="create_sparse_annotation_arrays"><a class="viewcode-back" href="../../index.html#pathflowai.utils.create_sparse_annotation_arrays">[docs]</a><span class="k">def</span> <span class="nf">create_sparse_annotation_arrays</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">img_size</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[]):</span> + <span class="sd">"""Convert annotation xml to shapely objects and store in dictionary.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> xml_file:str</span> +<span class="sd"> XML file containing annotations.</span> +<span class="sd"> img_size:int</span> +<span class="sd"> Deprecated.</span> +<span class="sd"> annotations:list</span> +<span class="sd"> Annotations to look for in xml export.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> dict</span> +<span class="sd"> Dictionary with annotation-shapely object pairs.</span> + +<span class="sd"> """</span> + <span class="n">interior_points_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">annotation</span><span class="p">:</span><span class="n">parse_coord_return_boxes</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation_name</span> <span class="o">=</span> <span class="n">annotation</span><span class="p">,</span> <span class="n">return_coords</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="k">for</span> <span class="n">annotation</span> <span class="ow">in</span> <span class="n">annotations</span><span class="p">}</span><span class="c1">#grab_interior_points(xml_file, img_size, annotations=annotations) if annotations else {}</span> + <span class="k">return</span> <span class="p">{</span><span class="n">annotation</span><span class="p">:</span><span class="n">interior_points_dict</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span> <span class="k">for</span> <span class="n">annotation</span> <span class="ow">in</span> <span class="n">annotations</span><span class="p">}</span><span class="c1">#sparse.COO.from_scipy_sparse((sps.coo_matrix(interior_points_dict[annotation],img_size, dtype=np.uint8) if interior_points_dict[annotation] not None else sps.coo_matrix(img_size, dtype=np.uint8)).tocsr()) for annotation in annotations} # [sps.coo_matrix(img_size, dtype=np.uint8)]+</span></div> + +<div class="viewcode-block" id="load_process_image"><a class="viewcode-back" href="../../index.html#pathflowai.utils.load_process_image">[docs]</a><span class="k">def</span> <span class="nf">load_process_image</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">xml_file</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">npy_mask</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[]):</span> + <span class="sd">"""Load SVS-like image (including NPY), segmentation/classification annotations, generate dask array and dictionary of annotations.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> svs_file:str</span> +<span class="sd"> Image file</span> +<span class="sd"> xml_file:str</span> +<span class="sd"> Annotation file.</span> +<span class="sd"> npy_mask:array</span> +<span class="sd"> Numpy segmentation mask.</span> +<span class="sd"> annotations:list</span> +<span class="sd"> List of annotations in xml.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> array</span> +<span class="sd"> Dask array of image.</span> +<span class="sd"> dict</span> +<span class="sd"> Annotation masks.</span> + +<span class="sd"> """</span> + <span class="n">arr</span> <span class="o">=</span> <span class="n">npy2da</span><span class="p">(</span><span class="n">svs_file</span><span class="p">)</span> <span class="k">if</span> <span class="n">svs_file</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">'.npy'</span><span class="p">)</span> <span class="k">else</span> <span class="n">svs2dask_array</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">tile_size</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span> <span class="n">overlap</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span><span class="c1">#load_image(svs_file)</span> + <span class="n">img_size</span> <span class="o">=</span> <span class="n">arr</span><span class="o">.</span><span class="n">shape</span><span class="p">[:</span><span class="mi">2</span><span class="p">]</span> + <span class="n">masks</span> <span class="o">=</span> <span class="p">{}</span><span class="c1">#{'purple': create_purple_mask(arr,img_size,sparse=False)}</span> + <span class="k">if</span> <span class="n">xml_file</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">masks</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">create_sparse_annotation_arrays</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">img_size</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="n">annotations</span><span class="p">))</span> + <span class="k">if</span> <span class="n">npy_mask</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">masks</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">'annotations'</span><span class="p">:</span><span class="n">npy_mask</span><span class="p">})</span> + <span class="c1">#data = dict(image=(['x','y','rgb'],arr),**masks)</span> + <span class="c1">#data_arr = {'image':xr.Variable(['x','y','color'], arr)}</span> + <span class="c1">#purple_arr = {'mask':xr.Variable(['x','y'], masks['purple'])}</span> + <span class="c1">#mask_arr = {m:xr.Variable(['row','col'],masks[m]) for m in masks if m != 'purple'} if 'annotations' not in annotations else {'annotations':xr.Variable(['x','y'],masks['annotations'])}</span> + <span class="c1">#masks['purple'] = masks['purple'].reshape(*masks['purple'].shape,1)</span> + <span class="c1">#arr = da.concatenate([arr,masks.pop('purple')],axis=2)</span> + <span class="k">return</span> <span class="n">arr</span><span class="p">,</span> <span class="n">masks</span><span class="c1">#xr.Dataset.from_dict({k:v for k,v in list(data_arr.items())+list(purple_arr.items())+list(mask_arr.items())})#list(dict(image=data_arr,purple=purple_arr,annotations=mask_arr).items()))#arr, masks</span></div> + +<div class="viewcode-block" id="save_dataset"><a class="viewcode-back" href="../../index.html#pathflowai.utils.save_dataset">[docs]</a><span class="k">def</span> <span class="nf">save_dataset</span><span class="p">(</span><span class="n">arr</span><span class="p">,</span> <span class="n">masks</span><span class="p">,</span> <span class="n">out_zarr</span><span class="p">,</span> <span class="n">out_pkl</span><span class="p">):</span> + <span class="sd">"""Saves dask array image, dictionary of annotations to zarr and pickle respectively.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> arr:array</span> +<span class="sd"> Image.</span> +<span class="sd"> masks:dict</span> +<span class="sd"> Dictionary of annotation shapes.</span> +<span class="sd"> out_zarr:str</span> +<span class="sd"> Zarr output file for image.</span> +<span class="sd"> out_pkl:str</span> +<span class="sd"> Pickle output file.</span> +<span class="sd"> """</span> + <span class="n">arr</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">'uint8'</span><span class="p">)</span><span class="o">.</span><span class="n">to_zarr</span><span class="p">(</span><span class="n">out_zarr</span><span class="p">,</span> <span class="n">overwrite</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> + <span class="n">pickle</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">masks</span><span class="p">,</span><span class="nb">open</span><span class="p">(</span><span class="n">out_pkl</span><span class="p">,</span><span class="s1">'wb'</span><span class="p">))</span></div> + + <span class="c1">#dataset.to_netcdf(out_netcdf, compute=False)</span> + <span class="c1">#pickle.dump(dataset, open(out_pkl,'wb'), protocol=-1)</span> + +<div class="viewcode-block" id="run_preprocessing_pipeline"><a class="viewcode-back" href="../../index.html#pathflowai.utils.run_preprocessing_pipeline">[docs]</a><span class="k">def</span> <span class="nf">run_preprocessing_pipeline</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">xml_file</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">npy_mask</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">out_zarr</span><span class="o">=</span><span class="s1">'output_zarr.zarr'</span><span class="p">,</span> <span class="n">out_pkl</span><span class="o">=</span><span class="s1">'output.pkl'</span><span class="p">):</span> + <span class="sd">"""Run preprocessing pipeline. Store image into zarr format, segmentations maintain as npy, and xml annotations as pickle.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> svs_file:str</span> +<span class="sd"> Input image file.</span> +<span class="sd"> xml_file:str</span> +<span class="sd"> Input annotation file.</span> +<span class="sd"> npy_mask:str</span> +<span class="sd"> NPY segmentation mask.</span> +<span class="sd"> annotations:list</span> +<span class="sd"> List of annotations.</span> +<span class="sd"> out_zarr:str</span> +<span class="sd"> Output zarr for image.</span> +<span class="sd"> out_pkl:str</span> +<span class="sd"> Output pickle for annotations.</span> +<span class="sd"> """</span> + <span class="c1">#save_dataset(load_process_image(svs_file, xml_file, npy_mask, annotations), out_netcdf)</span> + <span class="n">arr</span><span class="p">,</span> <span class="n">masks</span> <span class="o">=</span> <span class="n">load_process_image</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">xml_file</span><span class="p">,</span> <span class="n">npy_mask</span><span class="p">,</span> <span class="n">annotations</span><span class="p">)</span> + <span class="n">save_dataset</span><span class="p">(</span><span class="n">arr</span><span class="p">,</span> <span class="n">masks</span><span class="p">,</span><span class="n">out_zarr</span><span class="p">,</span> <span class="n">out_pkl</span><span class="p">)</span></div> + +<span class="c1">###################</span> + +<div class="viewcode-block" id="adjust_mask"><a class="viewcode-back" href="../../index.html#pathflowai.utils.adjust_mask">[docs]</a><span class="k">def</span> <span class="nf">adjust_mask</span><span class="p">(</span><span class="n">mask_file</span><span class="p">,</span> <span class="n">dask_img_array_file</span><span class="p">,</span> <span class="n">out_npy</span><span class="p">,</span> <span class="n">n_neighbors</span><span class="p">):</span> + <span class="sd">"""Fixes segmentation masks to reduce coarse annotations over empty regions.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> mask_file:str</span> +<span class="sd"> NPY segmentation mask.</span> +<span class="sd"> dask_img_array_file:str</span> +<span class="sd"> Dask image file.</span> +<span class="sd"> out_npy:str</span> +<span class="sd"> Output numpy file.</span> +<span class="sd"> n_neighbors:int</span> +<span class="sd"> Number nearest neighbors for dilation and erosion of mask from background to not background.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> str</span> +<span class="sd"> Output numpy file.</span> + +<span class="sd"> """</span> + <span class="kn">from</span> <span class="nn">dask_image.ndmorph</span> <span class="k">import</span> <span class="n">binary_opening</span> + <span class="kn">from</span> <span class="nn">dask.distributed</span> <span class="k">import</span> <span class="n">Client</span> + <span class="c1">#c=Client()</span> + <span class="n">dask_img_array</span><span class="o">=</span><span class="n">da</span><span class="o">.</span><span class="n">from_zarr</span><span class="p">(</span><span class="n">dask_img_array_file</span><span class="p">)</span> + <span class="n">mask</span><span class="o">=</span><span class="n">npy2da</span><span class="p">(</span><span class="n">mask_file</span><span class="p">)</span> + <span class="n">is_tissue_mask</span> <span class="o">=</span> <span class="n">mask</span><span class="o">></span><span class="mf">0.</span> + <span class="n">is_tissue_mask_img</span><span class="o">=</span><span class="p">((</span><span class="n">dask_img_array</span><span class="p">[</span><span class="o">...</span><span class="p">,</span><span class="mi">0</span><span class="p">]</span><span class="o">></span><span class="mf">200.</span><span class="p">)</span> <span class="o">&</span> <span class="p">(</span><span class="n">dask_img_array</span><span class="p">[</span><span class="o">...</span><span class="p">,</span><span class="mi">1</span><span class="p">]</span><span class="o">></span><span class="mf">200.</span><span class="p">)</span><span class="o">&</span> <span class="p">(</span><span class="n">dask_img_array</span><span class="p">[</span><span class="o">...</span><span class="p">,</span><span class="mi">2</span><span class="p">]</span><span class="o">></span><span class="mf">200.</span><span class="p">))</span> <span class="o">==</span> <span class="mi">0</span> + <span class="n">opening</span><span class="o">=</span><span class="n">binary_opening</span><span class="p">(</span><span class="n">is_tissue_mask_img</span><span class="p">,</span><span class="n">structure</span><span class="o">=</span><span class="n">da</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">n_neighbors</span><span class="p">,</span><span class="n">n_neighbors</span><span class="p">)))</span><span class="c1">#,mask=is_tissue_mask)</span> + <span class="n">mask</span><span class="p">[(</span><span class="n">opening</span><span class="o">==</span><span class="mi">0</span><span class="p">)</span><span class="o">&</span><span class="p">(</span><span class="n">is_tissue_mask</span><span class="o">==</span><span class="mi">1</span><span class="p">)]</span><span class="o">=</span><span class="mi">0</span> + <span class="n">np</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">out_npy</span><span class="p">,</span><span class="n">mask</span><span class="o">.</span><span class="n">compute</span><span class="p">())</span> + <span class="c1">#c.close()</span> + <span class="k">return</span> <span class="n">out_npy</span></div> + +<span class="c1">###################</span> + +<div class="viewcode-block" id="process_svs"><a class="viewcode-back" href="../../index.html#pathflowai.utils.process_svs">[docs]</a><span class="k">def</span> <span class="nf">process_svs</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">xml_file</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">output_dir</span><span class="o">=</span><span class="s1">'./'</span><span class="p">):</span> + <span class="sd">"""Store images into npy format and store annotations into pickle dictionary.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> svs_file:str</span> +<span class="sd"> Image file.</span> +<span class="sd"> xml_file:str</span> +<span class="sd"> Annotations file.</span> +<span class="sd"> annotations:list</span> +<span class="sd"> List of annotations in image.</span> +<span class="sd"> output_dir:str</span> +<span class="sd"> Output directory.</span> +<span class="sd"> """</span> + <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span><span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> + <span class="n">basename</span> <span class="o">=</span> <span class="n">svs_file</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> + <span class="n">arr</span><span class="p">,</span> <span class="n">masks</span> <span class="o">=</span> <span class="n">load_process_image</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">xml_file</span><span class="p">)</span> + <span class="n">np</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">.npy'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">basename</span><span class="p">)),</span><span class="n">arr</span><span class="p">)</span> + <span class="n">pickle</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">masks</span><span class="p">,</span> <span class="nb">open</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">.pkl'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">basename</span><span class="p">)),</span><span class="s1">'wb'</span><span class="p">),</span> <span class="n">protocol</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span></div> + +<span class="c1">####################</span> + +<div class="viewcode-block" id="load_dataset"><a class="viewcode-back" href="../../index.html#pathflowai.utils.load_dataset">[docs]</a><span class="k">def</span> <span class="nf">load_dataset</span><span class="p">(</span><span class="n">in_zarr</span><span class="p">,</span> <span class="n">in_pkl</span><span class="p">):</span> + <span class="sd">"""Load ZARR image and annotations pickle.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> in_zarr:str</span> +<span class="sd"> Input image.</span> +<span class="sd"> in_pkl:str</span> +<span class="sd"> Input annotations.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> dask.array</span> +<span class="sd"> Image array.</span> +<span class="sd"> dict</span> +<span class="sd"> Annotations dictionary.</span> + +<span class="sd"> """</span> + <span class="k">return</span> <span class="n">da</span><span class="o">.</span><span class="n">from_zarr</span><span class="p">(</span><span class="n">in_zarr</span><span class="p">),</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">in_pkl</span><span class="p">,</span><span class="s1">'rb'</span><span class="p">))</span><span class="c1">#xr.open_dataset(in_netcdf)</span></div> + +<div class="viewcode-block" id="is_valid_patch"><a class="viewcode-back" href="../../index.html#pathflowai.utils.is_valid_patch">[docs]</a><span class="k">def</span> <span class="nf">is_valid_patch</span><span class="p">(</span><span class="n">xs</span><span class="p">,</span><span class="n">ys</span><span class="p">,</span><span class="n">patch_size</span><span class="p">,</span><span class="n">purple_mask</span><span class="p">,</span><span class="n">intensity_threshold</span><span class="p">,</span><span class="n">threshold</span><span class="o">=</span><span class="mf">0.5</span><span class="p">):</span> + <span class="sd">"""Deprecated, computes whether patch is valid."""</span> + <span class="nb">print</span><span class="p">(</span><span class="n">xs</span><span class="p">,</span><span class="n">ys</span><span class="p">)</span> + <span class="k">return</span> <span class="p">(</span><span class="n">purple_mask</span><span class="p">[</span><span class="n">xs</span><span class="p">:</span><span class="n">xs</span><span class="o">+</span><span class="n">patch_size</span><span class="p">,</span><span class="n">ys</span><span class="p">:</span><span class="n">ys</span><span class="o">+</span><span class="n">patch_size</span><span class="p">]</span><span class="o">>=</span><span class="n">intensity_threshold</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> <span class="o">></span> <span class="n">threshold</span></div> + +<span class="c1">#@pysnooper.snoop("extract_patch.log")</span> +<div class="viewcode-block" id="extract_patch_information"><a class="viewcode-back" href="../../index.html#pathflowai.utils.extract_patch_information">[docs]</a><span class="k">def</span> <span class="nf">extract_patch_information</span><span class="p">(</span><span class="n">basename</span><span class="p">,</span> <span class="n">input_dir</span><span class="o">=</span><span class="s1">'./'</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">threshold</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">patch_size</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">generate_finetune_segmentation</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">target_class</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">intensity_threshold</span><span class="o">=</span><span class="mf">100.</span><span class="p">,</span> <span class="n">target_threshold</span><span class="o">=</span><span class="mf">0.</span><span class="p">,</span> <span class="n">adj_mask</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">basic_preprocess</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">tries</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span> + <span class="sd">"""Final step of preprocessing pipeline. Break up image into patches, include if not background and of a certain intensity, find area of each annotation type in patch, spatial information, image ID and dump data to SQL table.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> basename:str</span> +<span class="sd"> Patient ID.</span> +<span class="sd"> input_dir:str</span> +<span class="sd"> Input directory.</span> +<span class="sd"> annotations:list</span> +<span class="sd"> List of annotations to record, these can be different tissue types, must correspond with XML labels.</span> +<span class="sd"> threshold:float</span> +<span class="sd"> Value between 0 and 1 that indicates the minimum amount of patch that musn't be background for inclusion.</span> +<span class="sd"> patch_size:int</span> +<span class="sd"> Patch size of patches; this will become one of the tables.</span> +<span class="sd"> generate_finetune_segmentation:bool</span> +<span class="sd"> Deprecated.</span> +<span class="sd"> target_class:int</span> +<span class="sd"> Number of segmentation classes desired, from 0th class to target_class-1 will be annotated in SQL.</span> +<span class="sd"> intensity_threshold:float</span> +<span class="sd"> Value between 0 and 255 that represents minimum intensity to not include as background. Will be modified with new transforms.</span> +<span class="sd"> target_threshold:float</span> +<span class="sd"> Deprecated.</span> +<span class="sd"> adj_mask:str</span> +<span class="sd"> Adjusted mask if performed binary opening operations in previous preprocessing step.</span> +<span class="sd"> basic_preprocess:bool</span> +<span class="sd"> Do not store patch level information.</span> +<span class="sd"> tries:int</span> +<span class="sd"> Number of tries in case there is a Dask timeout, run again.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> dataframe</span> +<span class="sd"> Patch information.</span> + +<span class="sd"> """</span> + <span class="c1">#from collections import OrderedDict</span> + <span class="c1">#annotations=OrderedDict(annotations)</span> + <span class="c1">#from dask.multiprocessing import get</span> + <span class="kn">import</span> <span class="nn">dask</span> + <span class="kn">import</span> <span class="nn">time</span> + <span class="kn">from</span> <span class="nn">dask</span> <span class="k">import</span> <span class="n">dataframe</span> <span class="k">as</span> <span class="n">dd</span> + <span class="kn">import</span> <span class="nn">dask.array</span> <span class="k">as</span> <span class="nn">da</span> + <span class="kn">import</span> <span class="nn">multiprocessing</span> + <span class="kn">from</span> <span class="nn">shapely.ops</span> <span class="k">import</span> <span class="n">unary_union</span> + <span class="kn">from</span> <span class="nn">shapely.geometry</span> <span class="k">import</span> <span class="n">MultiPolygon</span> + <span class="kn">from</span> <span class="nn">itertools</span> <span class="k">import</span> <span class="n">product</span> + <span class="c1">#from distributed import Client,LocalCluster</span> + <span class="n">max_tries</span><span class="o">=</span><span class="mi">4</span> + <span class="n">kargs</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">basename</span><span class="o">=</span><span class="n">basename</span><span class="p">,</span> <span class="n">input_dir</span><span class="o">=</span><span class="n">input_dir</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="n">annotations</span><span class="p">,</span> <span class="n">threshold</span><span class="o">=</span><span class="n">threshold</span><span class="p">,</span> <span class="n">patch_size</span><span class="o">=</span><span class="n">patch_size</span><span class="p">,</span> <span class="n">generate_finetune_segmentation</span><span class="o">=</span><span class="n">generate_finetune_segmentation</span><span class="p">,</span> <span class="n">target_class</span><span class="o">=</span><span class="n">target_class</span><span class="p">,</span> <span class="n">intensity_threshold</span><span class="o">=</span><span class="n">intensity_threshold</span><span class="p">,</span> <span class="n">target_threshold</span><span class="o">=</span><span class="n">target_threshold</span><span class="p">,</span> <span class="n">adj_mask</span><span class="o">=</span><span class="n">adj_mask</span><span class="p">,</span> <span class="n">basic_preprocess</span><span class="o">=</span><span class="n">basic_preprocess</span><span class="p">,</span> <span class="n">tries</span><span class="o">=</span><span class="n">tries</span><span class="p">)</span> + <span class="k">try</span><span class="p">:</span> + <span class="c1">#,</span> + <span class="c1"># 'distributed.scheduler.allowed-failures':20,</span> + <span class="c1"># 'num-workers':20}):</span> + <span class="c1">#cluster=LocalCluster()</span> + <span class="c1">#cluster.adapt(minimum=10, maximum=100)</span> + <span class="c1">#cluster = LocalCluster(threads_per_worker=1, n_workers=20, memory_limit="80G")</span> + <span class="c1">#client=Client()#Client(cluster)#processes=True)#cluster,</span> + + <span class="n">arr</span><span class="p">,</span> <span class="n">masks</span> <span class="o">=</span> <span class="n">load_dataset</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">input_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">.zarr'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">basename</span><span class="p">)),</span><span class="n">join</span><span class="p">(</span><span class="n">input_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">_mask.pkl'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">basename</span><span class="p">)))</span> + <span class="k">if</span> <span class="s1">'annotations'</span> <span class="ow">in</span> <span class="n">masks</span><span class="p">:</span> + <span class="n">segmentation</span> <span class="o">=</span> <span class="kc">True</span> + <span class="c1">#if generate_finetune_segmentation:</span> + <span class="n">segmentation_mask</span> <span class="o">=</span> <span class="n">npy2da</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">input_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">_mask.npy'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">basename</span><span class="p">))</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">adj_mask</span> <span class="k">else</span> <span class="n">adj_mask</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">segmentation</span> <span class="o">=</span> <span class="kc">False</span> + <span class="c1">#masks=np.load(masks['annotations'])</span> + <span class="c1">#npy_file = join(input_dir,'{}.npy'.format(basename))</span> + <span class="n">purple_mask</span> <span class="o">=</span> <span class="n">create_purple_mask</span><span class="p">(</span><span class="n">arr</span><span class="p">)</span> + <span class="n">x_max</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">arr</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> + <span class="n">y_max</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">arr</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> + <span class="n">x_steps</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">x_max</span><span class="o">-</span><span class="n">patch_size</span><span class="p">)</span> <span class="o">/</span> <span class="n">patch_size</span> <span class="p">)</span> + <span class="n">y_steps</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">y_max</span><span class="o">-</span><span class="n">patch_size</span><span class="p">)</span> <span class="o">/</span> <span class="n">patch_size</span> <span class="p">)</span> + <span class="k">for</span> <span class="n">annotation</span> <span class="ow">in</span> <span class="n">annotations</span><span class="p">:</span> + <span class="k">try</span><span class="p">:</span> + <span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span><span class="o">=</span><span class="p">[</span><span class="n">unary_union</span><span class="p">(</span><span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">])]</span> <span class="k">if</span> <span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span> <span class="k">else</span> <span class="p">[]</span> + <span class="k">except</span><span class="p">:</span> + <span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span><span class="o">=</span><span class="p">[</span><span class="n">MultiPolygon</span><span class="p">(</span><span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">])]</span> <span class="k">if</span> <span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span> <span class="k">else</span> <span class="p">[]</span> + + <span class="n">patch_info</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">([([</span><span class="n">basename</span><span class="p">,</span><span class="n">i</span><span class="o">*</span><span class="n">patch_size</span><span class="p">,</span><span class="n">j</span><span class="o">*</span><span class="n">patch_size</span><span class="p">,</span><span class="n">patch_size</span><span class="p">,</span><span class="s1">'NA'</span><span class="p">]</span><span class="o">+</span><span class="p">[</span><span class="mf">0.</span><span class="p">]</span><span class="o">*</span><span class="p">(</span><span class="n">target_class</span> <span class="k">if</span> <span class="n">segmentation</span> <span class="k">else</span> <span class="nb">len</span><span class="p">(</span><span class="n">annotations</span><span class="p">)))</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span><span class="n">j</span> <span class="ow">in</span> <span class="n">product</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">x_steps</span><span class="o">+</span><span class="mi">1</span><span class="p">),</span><span class="nb">range</span><span class="p">(</span><span class="n">y_steps</span><span class="o">+</span><span class="mi">1</span><span class="p">))],</span><span class="n">columns</span><span class="o">=</span><span class="p">([</span><span class="s1">'ID'</span><span class="p">,</span><span class="s1">'x'</span><span class="p">,</span><span class="s1">'y'</span><span class="p">,</span><span class="s1">'patch_size'</span><span class="p">,</span><span class="s1">'annotation'</span><span class="p">]</span><span class="o">+</span><span class="p">(</span><span class="n">annotations</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">segmentation</span> <span class="k">else</span> <span class="nb">list</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">target_class</span><span class="p">)]))))</span><span class="c1">#[dask.delayed(return_line_info)(i,j) for (i,j) in product(range(x_steps+1),range(y_steps+1))]</span> + <span class="k">if</span> <span class="n">basic_preprocess</span><span class="p">:</span> + <span class="n">patch_info</span><span class="o">=</span><span class="n">patch_info</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,:</span><span class="mi">4</span><span class="p">]</span> + <span class="n">valid_patches</span><span class="o">=</span><span class="p">[]</span> + <span class="k">for</span> <span class="n">xs</span><span class="p">,</span><span class="n">ys</span> <span class="ow">in</span> <span class="n">patch_info</span><span class="p">[[</span><span class="s1">'x'</span><span class="p">,</span><span class="s1">'y'</span><span class="p">]]</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">tolist</span><span class="p">():</span> + <span class="n">valid_patches</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">purple_mask</span><span class="p">[</span><span class="n">xs</span><span class="p">:</span><span class="n">xs</span><span class="o">+</span><span class="n">patch_size</span><span class="p">,</span><span class="n">ys</span><span class="p">:</span><span class="n">ys</span><span class="o">+</span><span class="n">patch_size</span><span class="p">]</span><span class="o">>=</span><span class="n">intensity_threshold</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> <span class="o">></span> <span class="n">threshold</span><span class="p">)</span> <span class="c1"># dask.delayed(is_valid_patch)(xs,ys,patch_size,purple_mask,intensity_threshold,threshold)</span> + <span class="n">valid_patches</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">da</span><span class="o">.</span><span class="n">compute</span><span class="p">(</span><span class="o">*</span><span class="n">valid_patches</span><span class="p">))</span> + <span class="nb">print</span><span class="p">(</span><span class="s1">'Valid Patches Complete'</span><span class="p">)</span> + <span class="c1">#print(valid_patches)</span> + <span class="n">patch_info</span><span class="o">=</span><span class="n">patch_info</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">valid_patches</span><span class="p">]</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">basic_preprocess</span><span class="p">:</span> + <span class="n">area_info</span><span class="o">=</span><span class="p">[]</span> + <span class="k">if</span> <span class="n">segmentation</span><span class="p">:</span> + <span class="n">patch_info</span><span class="o">.</span><span class="n">loc</span><span class="p">[:,</span><span class="s1">'annotation'</span><span class="p">]</span><span class="o">=</span><span class="s1">'segment'</span> + <span class="k">for</span> <span class="n">xs</span><span class="p">,</span><span class="n">ys</span> <span class="ow">in</span> <span class="n">patch_info</span><span class="p">[[</span><span class="s1">'x'</span><span class="p">,</span><span class="s1">'y'</span><span class="p">]]</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">tolist</span><span class="p">():</span> + <span class="n">xf</span><span class="o">=</span><span class="n">xs</span><span class="o">+</span><span class="n">patch_size</span> + <span class="n">yf</span><span class="o">=</span><span class="n">ys</span><span class="o">+</span><span class="n">patch_size</span> + <span class="c1">#print(xs,ys)</span> + <span class="n">area_info</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">da</span><span class="o">.</span><span class="n">histogram</span><span class="p">(</span><span class="n">segmentation_mask</span><span class="p">[</span><span class="n">xs</span><span class="p">:</span><span class="n">xf</span><span class="p">,</span><span class="n">ys</span><span class="p">:</span><span class="n">yf</span><span class="p">],</span><span class="nb">range</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="n">target_class</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span><span class="n">bins</span><span class="o">=</span><span class="n">target_class</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span> + <span class="c1">#area_info.append(dask.delayed(seg_line)(xs,ys,patch_size,segmentation_mask,target_class))</span> + <span class="k">else</span><span class="p">:</span> + <span class="k">for</span> <span class="n">xs</span><span class="p">,</span><span class="n">ys</span> <span class="ow">in</span> <span class="n">patch_info</span><span class="p">[[</span><span class="s1">'x'</span><span class="p">,</span><span class="s1">'y'</span><span class="p">]]</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">tolist</span><span class="p">():</span> + <span class="n">area_info</span><span class="o">.</span><span class="n">append</span><span class="p">([</span><span class="n">dask</span><span class="o">.</span><span class="n">delayed</span><span class="p">(</span><span class="n">is_coords_in_box</span><span class="p">)([</span><span class="n">xs</span><span class="p">,</span><span class="n">ys</span><span class="p">],</span><span class="n">patch_size</span><span class="p">,</span><span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">])</span> <span class="k">for</span> <span class="n">annotation</span> <span class="ow">in</span> <span class="n">annotations</span><span class="p">])</span> + <span class="c1">#area_info=da.concatenate(area_info,axis=0).compute()</span> + <span class="n">area_info</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">dask</span><span class="o">.</span><span class="n">compute</span><span class="p">(</span><span class="o">*</span><span class="n">area_info</span><span class="p">))</span><span class="c1">#da.concatenate(area_info,axis=0).compute(dtype=np.float16,scheduler='threaded')).astype(np.float16)</span> + <span class="nb">print</span><span class="p">(</span><span class="s1">'Area Info Complete'</span><span class="p">)</span> + <span class="k">if</span> <span class="n">segmentation</span><span class="p">:</span> + <span class="n">area_info</span> <span class="o">=</span> <span class="n">area_info</span><span class="o">/</span><span class="n">np</span><span class="o">.</span><span class="n">float16</span><span class="p">(</span><span class="n">patch_size</span><span class="o">*</span><span class="n">patch_size</span><span class="p">)</span> + <span class="c1">#print(area_info)</span> + <span class="n">patch_info</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span><span class="mi">5</span><span class="p">:]</span><span class="o">=</span><span class="n">area_info</span> + <span class="c1">#print(patch_info)</span> + <span class="c1">#print(patch_info.dtypes)</span> + <span class="n">annot</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="n">patch_info</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span><span class="mi">5</span><span class="p">:])</span> + <span class="n">patch_info</span><span class="o">.</span><span class="n">loc</span><span class="p">[:,</span><span class="s1">'annotation'</span><span class="p">]</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">vectorize</span><span class="p">(</span><span class="k">lambda</span> <span class="n">i</span><span class="p">:</span> <span class="n">annot</span><span class="p">[</span><span class="n">patch_info</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">,</span><span class="mi">5</span><span class="p">:]</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">argmax</span><span class="p">()])(</span><span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">patch_info</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span><span class="c1">#patch_info[np.arange(target_class).astype(str).tolist()].values.argmax(1).astype(str)</span> + <span class="c1">#client.close()</span> + <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> + <span class="nb">print</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> + <span class="n">kargs</span><span class="p">[</span><span class="s1">'tries'</span><span class="p">]</span><span class="o">+=</span><span class="mi">1</span> + <span class="k">if</span> <span class="n">kargs</span><span class="p">[</span><span class="s1">'tries'</span><span class="p">]</span><span class="o">==</span><span class="n">max_tries</span><span class="p">:</span> + <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">'Exceeded past maximum number of tries.'</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="nb">print</span><span class="p">(</span><span class="s1">'Restarting preprocessing again.'</span><span class="p">)</span> + <span class="n">extract_patch_information</span><span class="p">(</span><span class="o">**</span><span class="n">kargs</span><span class="p">)</span> + + <span class="k">return</span> <span class="n">patch_info</span></div> + +<div class="viewcode-block" id="generate_patch_pipeline"><a class="viewcode-back" href="../../index.html#pathflowai.utils.generate_patch_pipeline">[docs]</a><span class="k">def</span> <span class="nf">generate_patch_pipeline</span><span class="p">(</span><span class="n">basename</span><span class="p">,</span> <span class="n">input_dir</span><span class="o">=</span><span class="s1">'./'</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">threshold</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">patch_size</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">out_db</span><span class="o">=</span><span class="s1">'patch_info.db'</span><span class="p">,</span> <span class="n">generate_finetune_segmentation</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">target_class</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">intensity_threshold</span><span class="o">=</span><span class="mf">100.</span><span class="p">,</span> <span class="n">target_threshold</span><span class="o">=</span><span class="mf">0.</span><span class="p">,</span> <span class="n">adj_mask</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">basic_preprocess</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="sd">"""Short summary.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> basename:str</span> +<span class="sd"> Patient ID.</span> +<span class="sd"> input_dir:str</span> +<span class="sd"> Input directory.</span> +<span class="sd"> annotations:list</span> +<span class="sd"> List of annotations to record, these can be different tissue types, must correspond with XML labels.</span> +<span class="sd"> threshold:float</span> +<span class="sd"> Value between 0 and 1 that indicates the minimum amount of patch that musn't be background for inclusion.</span> +<span class="sd"> patch_size:int</span> +<span class="sd"> Patch size of patches; this will become one of the tables.</span> +<span class="sd"> out_db:str</span> +<span class="sd"> Output SQL database.</span> +<span class="sd"> generate_finetune_segmentation:bool</span> +<span class="sd"> Deprecated.</span> +<span class="sd"> target_class:int</span> +<span class="sd"> Number of segmentation classes desired, from 0th class to target_class-1 will be annotated in SQL.</span> +<span class="sd"> intensity_threshold:float</span> +<span class="sd"> Value between 0 and 255 that represents minimum intensity to not include as background. Will be modified with new transforms.</span> +<span class="sd"> target_threshold:float</span> +<span class="sd"> Deprecated.</span> +<span class="sd"> adj_mask:str</span> +<span class="sd"> Adjusted mask if performed binary opening operations in previous preprocessing step.</span> +<span class="sd"> basic_preprocess:bool</span> +<span class="sd"> Do not store patch level information.</span> +<span class="sd"> """</span> + <span class="n">patch_info</span> <span class="o">=</span> <span class="n">extract_patch_information</span><span class="p">(</span><span class="n">basename</span><span class="p">,</span> <span class="n">input_dir</span><span class="p">,</span> <span class="n">annotations</span><span class="p">,</span> <span class="n">threshold</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">,</span> <span class="n">generate_finetune_segmentation</span><span class="o">=</span><span class="n">generate_finetune_segmentation</span><span class="p">,</span> <span class="n">target_class</span><span class="o">=</span><span class="n">target_class</span><span class="p">,</span> <span class="n">intensity_threshold</span><span class="o">=</span><span class="n">intensity_threshold</span><span class="p">,</span> <span class="n">target_threshold</span><span class="o">=</span><span class="n">target_threshold</span><span class="p">,</span> <span class="n">adj_mask</span><span class="o">=</span><span class="n">adj_mask</span><span class="p">,</span> <span class="n">basic_preprocess</span><span class="o">=</span><span class="n">basic_preprocess</span><span class="p">)</span> + <span class="n">conn</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">out_db</span><span class="p">)</span> + <span class="n">patch_info</span><span class="o">.</span><span class="n">to_sql</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span> <span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">,</span> <span class="n">if_exists</span><span class="o">=</span><span class="s1">'append'</span><span class="p">)</span> + <span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div> + + +<span class="c1"># now output csv</span> +<div class="viewcode-block" id="save_all_patch_info"><a class="viewcode-back" href="../../index.html#pathflowai.utils.save_all_patch_info">[docs]</a><span class="k">def</span> <span class="nf">save_all_patch_info</span><span class="p">(</span><span class="n">basenames</span><span class="p">,</span> <span class="n">input_dir</span><span class="o">=</span><span class="s1">'./'</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">threshold</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">patch_size</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">output_pkl</span><span class="o">=</span><span class="s1">'patch_info.pkl'</span><span class="p">):</span> + <span class="sd">"""Deprecated."""</span> + <span class="n">df</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">extract_patch_information</span><span class="p">(</span><span class="n">basename</span><span class="p">,</span> <span class="n">input_dir</span><span class="p">,</span> <span class="n">annotations</span><span class="p">,</span> <span class="n">threshold</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">)</span> <span class="k">for</span> <span class="n">basename</span> <span class="ow">in</span> <span class="n">basenames</span><span class="p">])</span><span class="o">.</span><span class="n">reset_index</span><span class="p">(</span><span class="n">drop</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> + <span class="n">df</span><span class="o">.</span><span class="n">to_pickle</span><span class="p">(</span><span class="n">output_pkl</span><span class="p">)</span></div> + +<span class="c1">#########</span> + + +<div class="viewcode-block" id="create_train_val_test"><a class="viewcode-back" href="../../index.html#pathflowai.utils.create_train_val_test">[docs]</a><span class="k">def</span> <span class="nf">create_train_val_test</span><span class="p">(</span><span class="n">train_val_test_pkl</span><span class="p">,</span> <span class="n">input_info_db</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">):</span> + <span class="sd">"""Create dataframe that splits slides into training validation and test.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> train_val_test_pkl:str</span> +<span class="sd"> Pickle for training validation and test slides.</span> +<span class="sd"> input_info_db:str</span> +<span class="sd"> Patch information SQL database.</span> +<span class="sd"> patch_size:int</span> +<span class="sd"> Patch size looking to access.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> dataframe</span> +<span class="sd"> Train test validation splits.</span> + +<span class="sd"> """</span> + <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">train_val_test_pkl</span><span class="p">):</span> + <span class="n">IDs</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_pickle</span><span class="p">(</span><span class="n">train_val_test_pkl</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">conn</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">input_info_db</span><span class="p">)</span> + <span class="n">df</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">read_sql</span><span class="p">(</span><span class="s1">'select * from "</span><span class="si">{}</span><span class="s1">";'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span><span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">)</span> + <span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> + <span class="n">IDs</span><span class="o">=</span><span class="n">df</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">]</span><span class="o">.</span><span class="n">unique</span><span class="p">()</span> + <span class="n">IDs</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">IDs</span><span class="p">,</span><span class="n">columns</span><span class="o">=</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">])</span> + <span class="n">IDs_train</span><span class="p">,</span> <span class="n">IDs_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">IDs</span><span class="p">)</span> + <span class="n">IDs_train</span><span class="p">,</span> <span class="n">IDs_val</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">IDs_train</span><span class="p">)</span> + <span class="n">IDs_train</span><span class="p">[</span><span class="s1">'set'</span><span class="p">]</span><span class="o">=</span><span class="s1">'train'</span> + <span class="n">IDs_val</span><span class="p">[</span><span class="s1">'set'</span><span class="p">]</span><span class="o">=</span><span class="s1">'val'</span> + <span class="n">IDs_test</span><span class="p">[</span><span class="s1">'set'</span><span class="p">]</span><span class="o">=</span><span class="s1">'test'</span> + <span class="n">IDs</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">IDs_train</span><span class="p">,</span><span class="n">IDs_val</span><span class="p">,</span><span class="n">IDs_test</span><span class="p">])</span> + <span class="n">IDs</span><span class="o">.</span><span class="n">to_pickle</span><span class="p">(</span><span class="n">train_val_test_pkl</span><span class="p">)</span> + <span class="k">return</span> <span class="n">IDs</span></div> + +<span class="k">def</span> <span class="nf">modify_patch_info</span><span class="p">(</span><span class="n">input_info_db</span><span class="o">=</span><span class="s1">'patch_info.db'</span><span class="p">,</span> <span class="n">slide_labels</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(),</span> <span class="n">pos_annotation_class</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">patch_size</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">segmentation</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">other_annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">target_segmentation_class</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> <span class="n">target_threshold</span><span class="o">=</span><span class="mf">0.</span><span class="p">,</span> <span class="n">classify_annotations</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="sd">"""Modify the patch information to get ready for deep learning, incorporate whole slide labels if needed.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> input_info_db:str</span> +<span class="sd"> SQL DB file.</span> +<span class="sd"> slide_labels:dataframe</span> +<span class="sd"> Dataframe with whole slide labels.</span> +<span class="sd"> pos_annotation_class:str</span> +<span class="sd"> Tissue/annotation label to label with whole slide image label, if not supplied, any slide's patches receive the whole slide label.</span> +<span class="sd"> patch_size:int</span> +<span class="sd"> Patch size.</span> +<span class="sd"> segmentation:bool</span> +<span class="sd"> Segmentation?</span> +<span class="sd"> other_annotations:list</span> +<span class="sd"> Other annotations to access from patch information.</span> +<span class="sd"> target_segmentation_class:int</span> +<span class="sd"> Segmentation class to threshold.</span> +<span class="sd"> target_threshold:float</span> +<span class="sd"> Include patch if patch has target area greater than this.</span> +<span class="sd"> classify_annotations:bool</span> +<span class="sd"> Classifying annotations for pretraining, or final model?</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> dataframe</span> +<span class="sd"> Modified patch information.</span> + +<span class="sd"> """</span> + <span class="n">conn</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">input_info_db</span><span class="p">)</span> + <span class="n">df</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">read_sql</span><span class="p">(</span><span class="s1">'select * from "</span><span class="si">{}</span><span class="s1">";'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span><span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">)</span> + <span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> + <span class="c1">#print(df)</span> + <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">drop_duplicates</span><span class="p">()</span> + <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">],</span><span class="n">slide_labels</span><span class="o">.</span><span class="n">index</span><span class="p">)]</span> + <span class="c1">#print(classify_annotations)</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">segmentation</span><span class="p">:</span> + <span class="k">if</span> <span class="n">classify_annotations</span><span class="p">:</span> + <span class="n">targets</span><span class="o">=</span><span class="n">df</span><span class="p">[</span><span class="s1">'annotation'</span><span class="p">]</span><span class="o">.</span><span class="n">unique</span><span class="p">()</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> + <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">targets</span><span class="p">)</span><span class="o">==</span><span class="mi">1</span><span class="p">:</span> + <span class="n">targets</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span><span class="mi">5</span><span class="p">:])</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">targets</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">slide_labels</span><span class="p">)</span> + <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">pos_annotation_class</span><span class="p">)</span><span class="o">==</span><span class="nb">type</span><span class="p">(</span><span class="s1">''</span><span class="p">):</span> + <span class="n">included_annotations</span> <span class="o">=</span> <span class="p">[</span><span class="n">pos_annotation_class</span><span class="p">]</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">included_annotations</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">pos_annotation_class</span><span class="p">)</span> + <span class="n">included_annotations</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">other_annotations</span><span class="p">)</span> + <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">'annotation'</span><span class="p">],</span><span class="n">included_annotations</span><span class="p">)]</span> + <span class="k">for</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">targets</span><span class="p">:</span> + <span class="n">df</span><span class="p">[</span><span class="n">target</span><span class="p">]</span><span class="o">=</span><span class="mf">0.</span> + <span class="k">for</span> <span class="n">slide</span> <span class="ow">in</span> <span class="n">slide_labels</span><span class="o">.</span><span class="n">index</span><span class="p">:</span> + <span class="n">slide_bool</span><span class="o">=</span><span class="p">((</span><span class="n">df</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">]</span><span class="o">==</span><span class="n">slide</span><span class="p">)</span> <span class="o">&</span> <span class="n">df</span><span class="p">[</span><span class="n">pos_annotation_class</span><span class="p">]</span><span class="o">></span><span class="mf">0.</span><span class="p">)</span> <span class="k">if</span> <span class="n">pos_annotation_class</span> <span class="k">else</span> <span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">]</span><span class="o">==</span><span class="n">slide</span><span class="p">)</span> <span class="c1"># (df['annotation']==pos_annotation_class)</span> + <span class="k">if</span> <span class="n">slide_bool</span><span class="o">.</span><span class="n">sum</span><span class="p">():</span> + <span class="n">df</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">slide_bool</span><span class="p">,</span><span class="n">targets</span><span class="p">]</span> <span class="o">=</span> <span class="n">slide_labels</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">slide</span><span class="p">,</span><span class="n">targets</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="c1">#1.</span> + <span class="n">df</span><span class="p">[</span><span class="s1">'area'</span><span class="p">]</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">vectorize</span><span class="p">(</span><span class="k">lambda</span> <span class="n">i</span><span class="p">:</span> <span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s1">'annotation'</span><span class="p">]])(</span><span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span> + <span class="k">if</span> <span class="s1">'area'</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="ow">and</span> <span class="n">target_threshold</span><span class="o">></span><span class="mf">0.</span><span class="p">:</span> + <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">df</span><span class="p">[</span><span class="s1">'area'</span><span class="p">]</span><span class="o">>=</span><span class="n">target_threshold</span><span class="p">]</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">df</span><span class="p">[</span><span class="s1">'target'</span><span class="p">]</span><span class="o">=</span><span class="mf">0.</span> + <span class="k">if</span> <span class="n">target_segmentation_class</span> <span class="o">>=</span><span class="mi">0</span><span class="p">:</span> + <span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">df</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">target_segmentation_class</span><span class="p">)]</span><span class="o">>=</span><span class="n">target_threshold</span><span class="p">]</span> + <span class="k">return</span> <span class="n">df</span> + +<div class="viewcode-block" id="npy2da"><a class="viewcode-back" href="../../index.html#pathflowai.utils.npy2da">[docs]</a><span class="k">def</span> <span class="nf">npy2da</span><span class="p">(</span><span class="n">npy_file</span><span class="p">):</span> + <span class="sd">"""Numpy to dask array.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> npy_file:str</span> +<span class="sd"> Input npy file.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> dask.array</span> +<span class="sd"> Converted numpy array to dask.</span> + +<span class="sd"> """</span> + <span class="k">return</span> <span class="n">da</span><span class="o">.</span><span class="n">from_array</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">npy_file</span><span class="p">,</span> <span class="n">mmap_mode</span> <span class="o">=</span> <span class="s1">'r+'</span><span class="p">))</span></div> + +<div class="viewcode-block" id="grab_interior_points"><a class="viewcode-back" href="../../index.html#pathflowai.utils.grab_interior_points">[docs]</a><span class="k">def</span> <span class="nf">grab_interior_points</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">img_size</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[]):</span> + <span class="sd">"""Deprecated."""</span> + <span class="n">interior_point_dict</span> <span class="o">=</span> <span class="p">{}</span> + <span class="k">for</span> <span class="n">annotation</span> <span class="ow">in</span> <span class="n">annotations</span><span class="p">:</span> + <span class="k">try</span><span class="p">:</span> + <span class="n">interior_point_dict</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span> <span class="o">=</span> <span class="n">parse_coord_return_boxes</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation</span><span class="p">,</span> <span class="n">return_coords</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="c1"># boxes2interior(img_size,</span> + <span class="k">except</span><span class="p">:</span> + <span class="n">interior_point_dict</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span><span class="c1">#np.array([[],[]])</span> + <span class="k">return</span> <span class="n">interior_point_dict</span></div> + +<div class="viewcode-block" id="boxes2interior"><a class="viewcode-back" href="../../index.html#pathflowai.utils.boxes2interior">[docs]</a><span class="k">def</span> <span class="nf">boxes2interior</span><span class="p">(</span><span class="n">img_size</span><span class="p">,</span> <span class="n">polygons</span><span class="p">):</span> + <span class="sd">"""Deprecated."""</span> + <span class="n">img</span> <span class="o">=</span> <span class="n">Image</span><span class="o">.</span><span class="n">new</span><span class="p">(</span><span class="s1">'L'</span><span class="p">,</span> <span class="n">img_size</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span> + <span class="k">for</span> <span class="n">polygon</span> <span class="ow">in</span> <span class="n">polygons</span><span class="p">:</span> + <span class="n">ImageDraw</span><span class="o">.</span><span class="n">Draw</span><span class="p">(</span><span class="n">img</span><span class="p">)</span><span class="o">.</span><span class="n">polygon</span><span class="p">(</span><span class="n">polygon</span><span class="p">,</span> <span class="n">outline</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">fill</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> + <span class="n">mask</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">img</span><span class="p">)</span><span class="o">.</span><span class="n">nonzero</span><span class="p">()</span> + <span class="c1">#mask = (np.ones(len(mask[0])),mask)</span> + <span class="k">return</span> <span class="n">mask</span></div> + +<div class="viewcode-block" id="parse_coord_return_boxes"><a class="viewcode-back" href="../../index.html#pathflowai.utils.parse_coord_return_boxes">[docs]</a><span class="k">def</span> <span class="nf">parse_coord_return_boxes</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation_name</span> <span class="o">=</span> <span class="s1">''</span><span class="p">,</span> <span class="n">return_coords</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span> + <span class="sd">"""Get list of shapely objects for each annotation in the XML object.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> xml_file:str</span> +<span class="sd"> Annotation file.</span> +<span class="sd"> annotation_name:str</span> +<span class="sd"> Name of xml annotation.</span> +<span class="sd"> return_coords:bool</span> +<span class="sd"> Just return list of coords over shapes.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> list</span> +<span class="sd"> List of shapely objects.</span> + +<span class="sd"> """</span> + <span class="n">boxes</span> <span class="o">=</span> <span class="p">[]</span> + <span class="n">xml_data</span> <span class="o">=</span> <span class="n">BeautifulSoup</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">xml_file</span><span class="p">),</span><span class="s1">'html'</span><span class="p">)</span> + <span class="c1">#print(xml_data.findAll('annotation'))</span> + <span class="c1">#print(xml_data.findAll('Annotation'))</span> + <span class="k">for</span> <span class="n">annotation</span> <span class="ow">in</span> <span class="n">xml_data</span><span class="o">.</span><span class="n">findAll</span><span class="p">(</span><span class="s1">'annotation'</span><span class="p">):</span> + <span class="k">if</span> <span class="n">annotation</span><span class="p">[</span><span class="s1">'partofgroup'</span><span class="p">]</span> <span class="o">==</span> <span class="n">annotation_name</span><span class="p">:</span> + <span class="k">for</span> <span class="n">coordinates</span> <span class="ow">in</span> <span class="n">annotation</span><span class="o">.</span><span class="n">findAll</span><span class="p">(</span><span class="s1">'coordinates'</span><span class="p">):</span> + <span class="c1"># FIXME may need to change x and y coordinates</span> + <span class="n">coords</span> <span class="o">=</span> <span class="p">[(</span><span class="n">coordinate</span><span class="p">[</span><span class="s1">'x'</span><span class="p">],</span><span class="n">coordinate</span><span class="p">[</span><span class="s1">'y'</span><span class="p">])</span> <span class="k">for</span> <span class="n">coordinate</span> <span class="ow">in</span> <span class="n">coordinates</span><span class="o">.</span><span class="n">findAll</span><span class="p">(</span><span class="s1">'coordinate'</span><span class="p">)]</span> + <span class="k">if</span> <span class="n">return_coords</span><span class="p">:</span> + <span class="n">boxes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">coords</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">boxes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">Polygon</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">coords</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">float</span><span class="p">)))</span> + <span class="k">return</span> <span class="n">boxes</span></div> + +<div class="viewcode-block" id="is_coords_in_box"><a class="viewcode-back" href="../../index.html#pathflowai.utils.is_coords_in_box">[docs]</a><span class="k">def</span> <span class="nf">is_coords_in_box</span><span class="p">(</span><span class="n">coords</span><span class="p">,</span><span class="n">patch_size</span><span class="p">,</span><span class="n">boxes</span><span class="p">):</span> + <span class="sd">"""Get area of annotation in patch.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> coords:array</span> +<span class="sd"> X,Y coordinates of patch.</span> +<span class="sd"> patch_size:int</span> +<span class="sd"> Patch size.</span> +<span class="sd"> boxes:list</span> +<span class="sd"> Shapely objects for annotations.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> float</span> +<span class="sd"> Area of annotation type.</span> + +<span class="sd"> """</span> + <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">boxes</span><span class="p">):</span> + <span class="n">points</span><span class="o">=</span><span class="n">Polygon</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span><span class="mi">0</span><span class="p">],[</span><span class="mi">1</span><span class="p">,</span><span class="mi">0</span><span class="p">],[</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">],[</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">]])</span><span class="o">*</span><span class="n">patch_size</span><span class="o">+</span><span class="n">coords</span><span class="p">)</span> + <span class="n">area</span><span class="o">=</span><span class="n">points</span><span class="o">.</span><span class="n">intersection</span><span class="p">(</span><span class="n">boxes</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">area</span><span class="o">/</span><span class="nb">float</span><span class="p">(</span><span class="n">points</span><span class="o">.</span><span class="n">area</span><span class="p">)</span><span class="c1">#any(list(map(lambda x: x.intersects(points),boxes)))#return_image_coord(nx=nx,ny=ny,xi=xi,yi=yi, output_point=output_point)</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">area</span><span class="o">=</span><span class="mf">0.</span> + <span class="k">return</span> <span class="n">area</span></div> + +<div class="viewcode-block" id="is_image_in_boxes"><a class="viewcode-back" href="../../index.html#pathflowai.utils.is_image_in_boxes">[docs]</a><span class="k">def</span> <span class="nf">is_image_in_boxes</span><span class="p">(</span><span class="n">image_coord_dict</span><span class="p">,</span> <span class="n">boxes</span><span class="p">):</span> + <span class="sd">"""Find if image intersects with annotations.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> image_coord_dict:dict</span> +<span class="sd"> Dictionary of patches.</span> +<span class="sd"> boxes:list</span> +<span class="sd"> Shapely annotation shapes.</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> dict</span> +<span class="sd"> Dictionary of whether image intersects with any of the annotations.</span> + +<span class="sd"> """</span> + <span class="k">return</span> <span class="p">{</span><span class="n">image</span><span class="p">:</span> <span class="nb">any</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">intersects</span><span class="p">(</span><span class="n">image_coord_dict</span><span class="p">[</span><span class="n">image</span><span class="p">]),</span><span class="n">boxes</span><span class="p">)))</span> <span class="k">for</span> <span class="n">image</span> <span class="ow">in</span> <span class="n">image_coord_dict</span><span class="p">}</span></div> + +<div class="viewcode-block" id="images2coord_dict"><a class="viewcode-back" href="../../index.html#pathflowai.utils.images2coord_dict">[docs]</a><span class="k">def</span> <span class="nf">images2coord_dict</span><span class="p">(</span><span class="n">images</span><span class="p">,</span> <span class="n">output_point</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="sd">"""Deprecated"""</span> + <span class="k">return</span> <span class="p">{</span><span class="n">image</span><span class="p">:</span> <span class="n">image2coords</span><span class="p">(</span><span class="n">image</span><span class="p">,</span> <span class="n">output_point</span><span class="p">)</span> <span class="k">for</span> <span class="n">image</span> <span class="ow">in</span> <span class="n">images</span><span class="p">}</span></div> + +<div class="viewcode-block" id="dir2images"><a class="viewcode-back" href="../../index.html#pathflowai.utils.dir2images">[docs]</a><span class="k">def</span> <span class="nf">dir2images</span><span class="p">(</span><span class="n">image_dir</span><span class="p">):</span> + <span class="sd">"""Deprecated"""</span> + <span class="k">return</span> <span class="n">glob</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">image_dir</span><span class="p">,</span><span class="s1">'*.jpg'</span><span class="p">))</span></div> + +<div class="viewcode-block" id="return_image_in_boxes_dict"><a class="viewcode-back" href="../../index.html#pathflowai.utils.return_image_in_boxes_dict">[docs]</a><span class="k">def</span> <span class="nf">return_image_in_boxes_dict</span><span class="p">(</span><span class="n">image_dir</span><span class="p">,</span> <span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span> + <span class="sd">"""Deprecated"""</span> + <span class="n">boxes</span> <span class="o">=</span> <span class="n">parse_coord_return_boxes</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation</span><span class="p">)</span> + <span class="n">images</span> <span class="o">=</span> <span class="n">dir2images</span><span class="p">(</span><span class="n">image_dir</span><span class="p">)</span> + <span class="n">coord_dict</span> <span class="o">=</span> <span class="n">images2coord_dict</span><span class="p">(</span><span class="n">images</span><span class="p">)</span> + <span class="k">return</span> <span class="n">is_image_in_boxes</span><span class="p">(</span><span class="n">image_coord_dict</span><span class="o">=</span><span class="n">coord_dict</span><span class="p">,</span><span class="n">boxes</span><span class="o">=</span><span class="n">boxes</span><span class="p">)</span></div> + +<div class="viewcode-block" id="image2coords"><a class="viewcode-back" href="../../index.html#pathflowai.utils.image2coords">[docs]</a><span class="k">def</span> <span class="nf">image2coords</span><span class="p">(</span><span class="n">image_file</span><span class="p">,</span> <span class="n">output_point</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="sd">"""Deprecated."""</span> + <span class="n">nx</span><span class="p">,</span><span class="n">ny</span><span class="p">,</span><span class="n">yi</span><span class="p">,</span><span class="n">xi</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">image_file</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="mi">1</span><span class="p">:])</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> + <span class="k">return</span> <span class="n">return_image_coord</span><span class="p">(</span><span class="n">nx</span><span class="o">=</span><span class="n">nx</span><span class="p">,</span><span class="n">ny</span><span class="o">=</span><span class="n">ny</span><span class="p">,</span><span class="n">xi</span><span class="o">=</span><span class="n">xi</span><span class="p">,</span><span class="n">yi</span><span class="o">=</span><span class="n">yi</span><span class="p">,</span> <span class="n">output_point</span><span class="o">=</span><span class="n">output_point</span><span class="p">)</span></div> + +<div class="viewcode-block" id="retain_images"><a class="viewcode-back" href="../../index.html#pathflowai.utils.retain_images">[docs]</a><span class="k">def</span> <span class="nf">retain_images</span><span class="p">(</span><span class="n">image_dir</span><span class="p">,</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span> + <span class="sd">"""Deprecated"""</span> + <span class="n">image_in_boxes_dict</span><span class="o">=</span><span class="n">return_image_in_boxes_dict</span><span class="p">(</span><span class="n">image_dir</span><span class="p">,</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation</span><span class="p">)</span> + <span class="k">return</span> <span class="p">[</span><span class="n">img</span> <span class="k">for</span> <span class="n">img</span> <span class="ow">in</span> <span class="n">image_in_boxes_dict</span> <span class="k">if</span> <span class="n">image_in_boxes_dict</span><span class="p">[</span><span class="n">img</span><span class="p">]]</span></div> + +<div class="viewcode-block" id="return_image_coord"><a class="viewcode-back" href="../../index.html#pathflowai.utils.return_image_coord">[docs]</a><span class="k">def</span> <span class="nf">return_image_coord</span><span class="p">(</span><span class="n">nx</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span><span class="n">ny</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span><span class="n">xl</span><span class="o">=</span><span class="mi">3333</span><span class="p">,</span><span class="n">yl</span><span class="o">=</span><span class="mi">3333</span><span class="p">,</span><span class="n">xi</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span><span class="n">yi</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span><span class="n">xc</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span><span class="n">yc</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span><span class="n">dimx</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span><span class="n">dimy</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">output_point</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="sd">"""Deprecated"""</span> + <span class="k">if</span> <span class="n">output_point</span><span class="p">:</span> + <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">xc</span><span class="p">,</span><span class="n">yc</span><span class="p">])</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">nx</span><span class="o">*</span><span class="n">xl</span><span class="o">+</span><span class="n">xi</span><span class="o">+</span><span class="n">dimx</span><span class="o">/</span><span class="mi">2</span><span class="p">,</span><span class="n">ny</span><span class="o">*</span><span class="n">yl</span><span class="o">+</span><span class="n">yi</span><span class="o">+</span><span class="n">dimy</span><span class="o">/</span><span class="mi">2</span><span class="p">])</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">static_point</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">nx</span><span class="o">*</span><span class="n">xl</span><span class="o">+</span><span class="n">xi</span><span class="p">,</span><span class="n">ny</span><span class="o">*</span><span class="n">yl</span><span class="o">+</span><span class="n">yi</span><span class="p">])</span> + <span class="n">points</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">xc</span><span class="p">,</span><span class="n">yc</span><span class="p">])</span><span class="o">*</span><span class="p">(</span><span class="n">static_point</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">new_point</span><span class="p">)))</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> <span class="k">for</span> <span class="n">new_point</span> <span class="ow">in</span> <span class="p">[[</span><span class="mi">0</span><span class="p">,</span><span class="mi">0</span><span class="p">],[</span><span class="n">dimx</span><span class="p">,</span><span class="mi">0</span><span class="p">],[</span><span class="n">dimx</span><span class="p">,</span><span class="n">dimy</span><span class="p">],[</span><span class="mi">0</span><span class="p">,</span><span class="n">dimy</span><span class="p">]]])</span> + <span class="k">return</span> <span class="n">Polygon</span><span class="p">(</span><span class="n">points</span><span class="p">)</span><span class="c1">#Point(*((np.array([xc,yc])*np.array([nx*xl+xi+dimx/2,ny*yl+yi+dimy/2])).tolist())) # [::-1]</span></div> + +<div class="viewcode-block" id="fix_name"><a class="viewcode-back" href="../../index.html#pathflowai.utils.fix_name">[docs]</a><span class="k">def</span> <span class="nf">fix_name</span><span class="p">(</span><span class="n">basename</span><span class="p">):</span> + <span class="sd">"""Fixes illegitimate basename, deprecated."""</span> + <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">basename</span><span class="p">)</span> <span class="o"><</span> <span class="mi">3</span><span class="p">:</span> + <span class="k">return</span> <span class="s1">'</span><span class="si">{}</span><span class="s1">0</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">*</span><span class="n">basename</span><span class="p">)</span> + <span class="k">return</span> <span class="n">basename</span></div> + +<div class="viewcode-block" id="fix_names"><a class="viewcode-back" href="../../index.html#pathflowai.utils.fix_names">[docs]</a><span class="k">def</span> <span class="nf">fix_names</span><span class="p">(</span><span class="n">file_dir</span><span class="p">):</span> + <span class="sd">"""Fixes basenames, deprecated."""</span> + <span class="k">for</span> <span class="n">filename</span> <span class="ow">in</span> <span class="n">glob</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">file_dir</span><span class="p">,</span><span class="s1">'*'</span><span class="p">)):</span> + <span class="n">basename</span> <span class="o">=</span> <span class="n">filename</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> + <span class="n">basename</span><span class="p">,</span> <span class="n">suffix</span> <span class="o">=</span> <span class="n">basename</span><span class="p">[:</span><span class="n">basename</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)],</span> <span class="n">basename</span><span class="p">[</span><span class="n">basename</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">'.'</span><span class="p">):]</span> + <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">basename</span><span class="p">)</span> <span class="o"><</span> <span class="mi">3</span><span class="p">:</span> + <span class="n">new_filename</span><span class="o">=</span><span class="n">join</span><span class="p">(</span><span class="n">file_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">0</span><span class="si">{}{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">*</span><span class="n">basename</span><span class="p">,</span><span class="n">suffix</span><span class="p">))</span> + <span class="nb">print</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span><span class="n">new_filename</span><span class="p">)</span> + <span class="n">subprocess</span><span class="o">.</span><span class="n">call</span><span class="p">(</span><span class="s1">'mv </span><span class="si">{}</span><span class="s1"> </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span><span class="n">new_filename</span><span class="p">),</span><span class="n">shell</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div> + +<span class="c1">#######</span> + +<span class="c1">#@pysnooper.snoop('seg2npy.log')</span> +<div class="viewcode-block" id="segmentation_predictions2npy"><a class="viewcode-back" href="../../index.html#pathflowai.utils.segmentation_predictions2npy">[docs]</a><span class="k">def</span> <span class="nf">segmentation_predictions2npy</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">patch_info</span><span class="p">,</span> <span class="n">segmentation_map</span><span class="p">,</span> <span class="n">npy_output</span><span class="p">):</span> + <span class="sd">"""Convert segmentation predictions from model to numpy masks.</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> y_pred:list</span> +<span class="sd"> List of patch segmentation masks</span> +<span class="sd"> patch_info:dataframe</span> +<span class="sd"> Patch information from DB.</span> +<span class="sd"> segmentation_map:array</span> +<span class="sd"> Existing segmentation mask.</span> +<span class="sd"> npy_output:str</span> +<span class="sd"> Output npy file.</span> +<span class="sd"> """</span> + <span class="n">segmentation_map</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">segmentation_map</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="o">-</span><span class="mi">2</span><span class="p">:])</span> + <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">patch_info</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]):</span> + <span class="n">patch_info_i</span> <span class="o">=</span> <span class="n">patch_info</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> + <span class="n">ID</span> <span class="o">=</span> <span class="n">patch_info_i</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">]</span> + <span class="n">xs</span> <span class="o">=</span> <span class="n">patch_info_i</span><span class="p">[</span><span class="s1">'x'</span><span class="p">]</span> + <span class="n">ys</span> <span class="o">=</span> <span class="n">patch_info_i</span><span class="p">[</span><span class="s1">'y'</span><span class="p">]</span> + <span class="n">patch_size</span> <span class="o">=</span> <span class="n">patch_info_i</span><span class="p">[</span><span class="s1">'patch_size'</span><span class="p">]</span> + <span class="n">prediction</span><span class="o">=</span><span class="n">y_pred</span><span class="p">[</span><span class="n">i</span><span class="p">,</span><span class="o">...</span><span class="p">]</span> + <span class="n">pred_shape</span><span class="o">=</span><span class="n">prediction</span><span class="o">.</span><span class="n">shape</span> + <span class="n">segmentation_map</span><span class="p">[</span><span class="n">xs</span><span class="p">:</span><span class="n">xs</span><span class="o">+</span><span class="n">patch_size</span><span class="p">,</span><span class="n">ys</span><span class="p">:</span><span class="n">ys</span><span class="o">+</span><span class="n">patch_size</span><span class="p">]</span> <span class="o">=</span> <span class="n">prediction</span> + <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">npy_output</span><span class="p">[:</span><span class="n">npy_output</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)],</span><span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> + <span class="n">np</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">npy_output</span><span class="p">,</span><span class="n">segmentation_map</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">))</span></div> +</pre></div> + + </div> + + </div> + <footer> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2019, Joshua Levy + + </p> + </div> + Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. + +</footer> + + </div> + </div> + + </section> + + </div> + + + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.Navigation.enable(true); + }); + </script> + + + + + + +</body> +</html> \ No newline at end of file