|
a |
|
b/docs/_modules/pathflowai/utils.html |
|
|
1 |
|
|
|
2 |
|
|
|
3 |
<!DOCTYPE html> |
|
|
4 |
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> |
|
|
5 |
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> |
|
|
6 |
<head> |
|
|
7 |
<meta charset="utf-8"> |
|
|
8 |
|
|
|
9 |
<meta name="viewport" content="width=device-width, initial-scale=1.0"> |
|
|
10 |
|
|
|
11 |
<title>pathflowai.utils — PathFlowAI 0.1 documentation</title> |
|
|
12 |
|
|
|
13 |
|
|
|
14 |
|
|
|
15 |
|
|
|
16 |
|
|
|
17 |
|
|
|
18 |
|
|
|
19 |
|
|
|
20 |
<script type="text/javascript" src="../../_static/js/modernizr.min.js"></script> |
|
|
21 |
|
|
|
22 |
|
|
|
23 |
<script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script> |
|
|
24 |
<script type="text/javascript" src="../../_static/jquery.js"></script> |
|
|
25 |
<script type="text/javascript" src="../../_static/underscore.js"></script> |
|
|
26 |
<script type="text/javascript" src="../../_static/doctools.js"></script> |
|
|
27 |
<script type="text/javascript" src="../../_static/language_data.js"></script> |
|
|
28 |
<script async="async" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script> |
|
|
29 |
|
|
|
30 |
<script type="text/javascript" src="../../_static/js/theme.js"></script> |
|
|
31 |
|
|
|
32 |
|
|
|
33 |
|
|
|
34 |
|
|
|
35 |
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" /> |
|
|
36 |
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" /> |
|
|
37 |
<link rel="index" title="Index" href="../../genindex.html" /> |
|
|
38 |
<link rel="search" title="Search" href="../../search.html" /> |
|
|
39 |
</head> |
|
|
40 |
|
|
|
41 |
<body class="wy-body-for-nav"> |
|
|
42 |
|
|
|
43 |
|
|
|
44 |
<div class="wy-grid-for-nav"> |
|
|
45 |
|
|
|
46 |
<nav data-toggle="wy-nav-shift" class="wy-nav-side"> |
|
|
47 |
<div class="wy-side-scroll"> |
|
|
48 |
<div class="wy-side-nav-search" > |
|
|
49 |
|
|
|
50 |
|
|
|
51 |
|
|
|
52 |
<a href="../../index.html" class="icon icon-home"> PathFlowAI |
|
|
53 |
|
|
|
54 |
|
|
|
55 |
|
|
|
56 |
</a> |
|
|
57 |
|
|
|
58 |
|
|
|
59 |
|
|
|
60 |
|
|
|
61 |
|
|
|
62 |
|
|
|
63 |
|
|
|
64 |
<div role="search"> |
|
|
65 |
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get"> |
|
|
66 |
<input type="text" name="q" placeholder="Search docs" /> |
|
|
67 |
<input type="hidden" name="check_keywords" value="yes" /> |
|
|
68 |
<input type="hidden" name="area" value="default" /> |
|
|
69 |
</form> |
|
|
70 |
</div> |
|
|
71 |
|
|
|
72 |
|
|
|
73 |
</div> |
|
|
74 |
|
|
|
75 |
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> |
|
|
76 |
|
|
|
77 |
|
|
|
78 |
|
|
|
79 |
|
|
|
80 |
|
|
|
81 |
|
|
|
82 |
<!-- Local TOC --> |
|
|
83 |
<div class="local-toc"></div> |
|
|
84 |
|
|
|
85 |
|
|
|
86 |
</div> |
|
|
87 |
</div> |
|
|
88 |
</nav> |
|
|
89 |
|
|
|
90 |
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> |
|
|
91 |
|
|
|
92 |
|
|
|
93 |
<nav class="wy-nav-top" aria-label="top navigation"> |
|
|
94 |
|
|
|
95 |
<i data-toggle="wy-nav-top" class="fa fa-bars"></i> |
|
|
96 |
<a href="../../index.html">PathFlowAI</a> |
|
|
97 |
|
|
|
98 |
</nav> |
|
|
99 |
|
|
|
100 |
|
|
|
101 |
<div class="wy-nav-content"> |
|
|
102 |
|
|
|
103 |
<div class="rst-content"> |
|
|
104 |
|
|
|
105 |
|
|
|
106 |
|
|
|
107 |
|
|
|
108 |
|
|
|
109 |
|
|
|
110 |
|
|
|
111 |
|
|
|
112 |
|
|
|
113 |
|
|
|
114 |
|
|
|
115 |
|
|
|
116 |
|
|
|
117 |
|
|
|
118 |
|
|
|
119 |
|
|
|
120 |
|
|
|
121 |
<div role="navigation" aria-label="breadcrumbs navigation"> |
|
|
122 |
|
|
|
123 |
<ul class="wy-breadcrumbs"> |
|
|
124 |
|
|
|
125 |
<li><a href="../../index.html">Docs</a> »</li> |
|
|
126 |
|
|
|
127 |
<li><a href="../index.html">Module code</a> »</li> |
|
|
128 |
|
|
|
129 |
<li>pathflowai.utils</li> |
|
|
130 |
|
|
|
131 |
|
|
|
132 |
<li class="wy-breadcrumbs-aside"> |
|
|
133 |
|
|
|
134 |
</li> |
|
|
135 |
|
|
|
136 |
</ul> |
|
|
137 |
|
|
|
138 |
|
|
|
139 |
<hr/> |
|
|
140 |
</div> |
|
|
141 |
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> |
|
|
142 |
<div itemprop="articleBody"> |
|
|
143 |
|
|
|
144 |
<h1>Source code for pathflowai.utils</h1><div class="highlight"><pre> |
|
|
145 |
<span></span><span class="sd">"""</span> |
|
|
146 |
<span class="sd">utils.py</span> |
|
|
147 |
<span class="sd">=======================</span> |
|
|
148 |
<span class="sd">General utilities that still need to be broken up into preprocessing, machine learning input preparation, and output submodules.</span> |
|
|
149 |
<span class="sd">"""</span> |
|
|
150 |
|
|
|
151 |
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span> |
|
|
152 |
<span class="kn">from</span> <span class="nn">bs4</span> <span class="k">import</span> <span class="n">BeautifulSoup</span> |
|
|
153 |
<span class="kn">from</span> <span class="nn">shapely.geometry</span> <span class="k">import</span> <span class="n">Point</span> |
|
|
154 |
<span class="kn">from</span> <span class="nn">shapely.geometry.polygon</span> <span class="k">import</span> <span class="n">Polygon</span> |
|
|
155 |
<span class="kn">import</span> <span class="nn">glob</span> |
|
|
156 |
<span class="kn">from</span> <span class="nn">os.path</span> <span class="k">import</span> <span class="n">join</span> |
|
|
157 |
<span class="kn">import</span> <span class="nn">plotly.graph_objs</span> <span class="k">as</span> <span class="nn">go</span> |
|
|
158 |
<span class="kn">import</span> <span class="nn">plotly.offline</span> <span class="k">as</span> <span class="nn">py</span> |
|
|
159 |
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span><span class="o">,</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span> |
|
|
160 |
<span class="kn">import</span> <span class="nn">scipy.sparse</span> <span class="k">as</span> <span class="nn">sps</span> |
|
|
161 |
<span class="kn">from</span> <span class="nn">PIL</span> <span class="k">import</span> <span class="n">Image</span><span class="p">,</span> <span class="n">ImageDraw</span> |
|
|
162 |
<span class="n">Image</span><span class="o">.</span><span class="n">MAX_IMAGE_PIXELS</span><span class="o">=</span><span class="mf">1e10</span> |
|
|
163 |
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span> |
|
|
164 |
<span class="kn">import</span> <span class="nn">scipy.sparse</span> <span class="k">as</span> <span class="nn">sps</span> |
|
|
165 |
<span class="kn">from</span> <span class="nn">os.path</span> <span class="k">import</span> <span class="n">join</span> |
|
|
166 |
<span class="kn">import</span> <span class="nn">os</span><span class="o">,</span> <span class="nn">subprocess</span><span class="o">,</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span> |
|
|
167 |
<span class="kn">import</span> <span class="nn">sqlite3</span> |
|
|
168 |
<span class="kn">import</span> <span class="nn">torch</span> |
|
|
169 |
<span class="kn">from</span> <span class="nn">torch.utils.data</span> <span class="k">import</span> <span class="n">Dataset</span><span class="c1">#, DataLoader</span> |
|
|
170 |
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="k">import</span> <span class="n">train_test_split</span> |
|
|
171 |
<span class="kn">import</span> <span class="nn">pysnooper</span> |
|
|
172 |
|
|
|
173 |
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span> |
|
|
174 |
<span class="kn">import</span> <span class="nn">dask.array</span> <span class="k">as</span> <span class="nn">da</span> |
|
|
175 |
<span class="kn">import</span> <span class="nn">dask</span> |
|
|
176 |
<span class="kn">import</span> <span class="nn">openslide</span> |
|
|
177 |
<span class="kn">from</span> <span class="nn">openslide</span> <span class="k">import</span> <span class="n">deepzoom</span> |
|
|
178 |
<span class="c1">#import xarray as xr, sparse</span> |
|
|
179 |
<span class="kn">import</span> <span class="nn">pickle</span> |
|
|
180 |
<span class="kn">import</span> <span class="nn">copy</span> |
|
|
181 |
|
|
|
182 |
<span class="kn">import</span> <span class="nn">nonechucks</span> <span class="k">as</span> <span class="nn">nc</span> |
|
|
183 |
|
|
|
184 |
<span class="kn">from</span> <span class="nn">nonechucks</span> <span class="k">import</span> <span class="n">SafeDataLoader</span> <span class="k">as</span> <span class="n">DataLoader</span> |
|
|
185 |
|
|
|
186 |
<div class="viewcode-block" id="load_sql_df"><a class="viewcode-back" href="../../index.html#pathflowai.utils.load_sql_df">[docs]</a><span class="k">def</span> <span class="nf">load_sql_df</span><span class="p">(</span><span class="n">sql_file</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">):</span> |
|
|
187 |
<span class="sd">"""Load pandas dataframe from SQL, accessing particular patch size within SQL.</span> |
|
|
188 |
|
|
|
189 |
<span class="sd"> Parameters</span> |
|
|
190 |
<span class="sd"> ----------</span> |
|
|
191 |
<span class="sd"> sql_file:str</span> |
|
|
192 |
<span class="sd"> SQL db.</span> |
|
|
193 |
<span class="sd"> patch_size:int</span> |
|
|
194 |
<span class="sd"> Patch size.</span> |
|
|
195 |
|
|
|
196 |
<span class="sd"> Returns</span> |
|
|
197 |
<span class="sd"> -------</span> |
|
|
198 |
<span class="sd"> dataframe</span> |
|
|
199 |
<span class="sd"> Patch level information.</span> |
|
|
200 |
|
|
|
201 |
<span class="sd"> """</span> |
|
|
202 |
<span class="n">conn</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">sql_file</span><span class="p">)</span> |
|
|
203 |
<span class="n">df</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">read_sql</span><span class="p">(</span><span class="s1">'select * from "</span><span class="si">{}</span><span class="s1">";'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span><span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">)</span> |
|
|
204 |
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> |
|
|
205 |
<span class="k">return</span> <span class="n">df</span></div> |
|
|
206 |
|
|
|
207 |
<div class="viewcode-block" id="df2sql"><a class="viewcode-back" href="../../index.html#pathflowai.utils.df2sql">[docs]</a><span class="k">def</span> <span class="nf">df2sql</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">sql_file</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">'replace'</span><span class="p">):</span> |
|
|
208 |
<span class="sd">"""Write dataframe containing patch level information to SQL db.</span> |
|
|
209 |
|
|
|
210 |
<span class="sd"> Parameters</span> |
|
|
211 |
<span class="sd"> ----------</span> |
|
|
212 |
<span class="sd"> df:dataframe</span> |
|
|
213 |
<span class="sd"> Dataframe containing patch information.</span> |
|
|
214 |
<span class="sd"> sql_file:str</span> |
|
|
215 |
<span class="sd"> SQL database.</span> |
|
|
216 |
<span class="sd"> patch_size:int</span> |
|
|
217 |
<span class="sd"> Size of patches.</span> |
|
|
218 |
<span class="sd"> mode:str</span> |
|
|
219 |
<span class="sd"> Replace or append.</span> |
|
|
220 |
|
|
|
221 |
<span class="sd"> """</span> |
|
|
222 |
<span class="n">conn</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">sql_file</span><span class="p">)</span> |
|
|
223 |
<span class="n">df</span><span class="o">.</span><span class="n">set_index</span><span class="p">(</span><span class="s1">'index'</span><span class="p">)</span><span class="o">.</span><span class="n">to_sql</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span> <span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">,</span> <span class="n">if_exists</span><span class="o">=</span><span class="n">mode</span><span class="p">)</span> |
|
|
224 |
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div> |
|
|
225 |
|
|
|
226 |
|
|
|
227 |
<span class="c1">#########</span> |
|
|
228 |
|
|
|
229 |
<span class="c1"># https://github.com/qupath/qupath/wiki/Supported-image-formats</span> |
|
|
230 |
<div class="viewcode-block" id="svs2dask_array"><a class="viewcode-back" href="../../index.html#pathflowai.utils.svs2dask_array">[docs]</a><span class="k">def</span> <span class="nf">svs2dask_array</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">tile_size</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span> <span class="n">overlap</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">remove_last</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">allow_unknown_chunksizes</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> |
|
|
231 |
<span class="sd">"""Convert SVS, TIF or TIFF to dask array.</span> |
|
|
232 |
|
|
|
233 |
<span class="sd"> Parameters</span> |
|
|
234 |
<span class="sd"> ----------</span> |
|
|
235 |
<span class="sd"> svs_file:str</span> |
|
|
236 |
<span class="sd"> Image file.</span> |
|
|
237 |
<span class="sd"> tile_size:int</span> |
|
|
238 |
<span class="sd"> Size of chunk to be read in.</span> |
|
|
239 |
<span class="sd"> overlap:int</span> |
|
|
240 |
<span class="sd"> Do not modify, overlap between neighboring tiles.</span> |
|
|
241 |
<span class="sd"> remove_last:bool</span> |
|
|
242 |
<span class="sd"> Remove last tile because it has a custom size.</span> |
|
|
243 |
<span class="sd"> allow_unknown_chunksizes: bool</span> |
|
|
244 |
<span class="sd"> Allow different chunk sizes, more flexible, but slowdown.</span> |
|
|
245 |
|
|
|
246 |
<span class="sd"> Returns</span> |
|
|
247 |
<span class="sd"> -------</span> |
|
|
248 |
<span class="sd"> dask.array</span> |
|
|
249 |
<span class="sd"> Dask Array.</span> |
|
|
250 |
|
|
|
251 |
<span class="sd"> >>> arr=svs2dask_array(svs_file, tile_size=1000, overlap=0, remove_last=True, allow_unknown_chunksizes=False)</span> |
|
|
252 |
<span class="sd"> >>> arr2=arr.compute()</span> |
|
|
253 |
<span class="sd"> >>> arr3=to_pil(cv2.resize(arr2, dsize=(1440,700), interpolation=cv2.INTER_CUBIC))</span> |
|
|
254 |
<span class="sd"> >>> arr3.save(test_image_name)"""</span> |
|
|
255 |
<span class="n">img</span><span class="o">=</span><span class="n">openslide</span><span class="o">.</span><span class="n">open_slide</span><span class="p">(</span><span class="n">svs_file</span><span class="p">)</span> |
|
|
256 |
<span class="n">gen</span><span class="o">=</span><span class="n">deepzoom</span><span class="o">.</span><span class="n">DeepZoomGenerator</span><span class="p">(</span><span class="n">img</span><span class="p">,</span> <span class="n">tile_size</span><span class="o">=</span><span class="n">tile_size</span><span class="p">,</span> <span class="n">overlap</span><span class="o">=</span><span class="n">overlap</span><span class="p">,</span> <span class="n">limit_bounds</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
|
|
257 |
<span class="n">max_level</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">gen</span><span class="o">.</span><span class="n">level_dimensions</span><span class="p">)</span><span class="o">-</span><span class="mi">1</span> |
|
|
258 |
<span class="n">n_tiles_x</span><span class="p">,</span> <span class="n">n_tiles_y</span> <span class="o">=</span> <span class="n">gen</span><span class="o">.</span><span class="n">level_tiles</span><span class="p">[</span><span class="n">max_level</span><span class="p">]</span> |
|
|
259 |
<span class="n">get_tile</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">i</span><span class="p">,</span><span class="n">j</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">gen</span><span class="o">.</span><span class="n">get_tile</span><span class="p">(</span><span class="n">max_level</span><span class="p">,(</span><span class="n">i</span><span class="p">,</span><span class="n">j</span><span class="p">)))</span><span class="o">.</span><span class="n">transpose</span><span class="p">((</span><span class="mi">1</span><span class="p">,</span><span class="mi">0</span><span class="p">,</span><span class="mi">2</span><span class="p">))</span> |
|
|
260 |
<span class="n">sample_tile</span> <span class="o">=</span> <span class="n">get_tile</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="mi">0</span><span class="p">)</span> |
|
|
261 |
<span class="n">sample_tile_shape</span> <span class="o">=</span> <span class="n">sample_tile</span><span class="o">.</span><span class="n">shape</span> |
|
|
262 |
<span class="n">dask_get_tile</span> <span class="o">=</span> <span class="n">dask</span><span class="o">.</span><span class="n">delayed</span><span class="p">(</span><span class="n">get_tile</span><span class="p">,</span> <span class="n">pure</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
|
|
263 |
<span class="n">arr</span><span class="o">=</span><span class="n">da</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">da</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">da</span><span class="o">.</span><span class="n">from_delayed</span><span class="p">(</span><span class="n">dask_get_tile</span><span class="p">(</span><span class="n">i</span><span class="p">,</span><span class="n">j</span><span class="p">),</span><span class="n">sample_tile_shape</span><span class="p">,</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">)</span> <span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_tiles_y</span> <span class="o">-</span> <span class="p">(</span><span class="mi">0</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">remove_last</span> <span class="k">else</span> <span class="mi">1</span><span class="p">))],</span><span class="n">allow_unknown_chunksizes</span><span class="o">=</span><span class="n">allow_unknown_chunksizes</span><span class="p">,</span><span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_tiles_x</span> <span class="o">-</span> <span class="p">(</span><span class="mi">0</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">remove_last</span> <span class="k">else</span> <span class="mi">1</span><span class="p">))],</span><span class="n">allow_unknown_chunksizes</span><span class="o">=</span><span class="n">allow_unknown_chunksizes</span><span class="p">)</span><span class="c1">#.transpose([1,0,2])</span> |
|
|
264 |
<span class="k">return</span> <span class="n">arr</span></div> |
|
|
265 |
|
|
|
266 |
<div class="viewcode-block" id="img2npy_"><a class="viewcode-back" href="../../index.html#pathflowai.utils.img2npy_">[docs]</a><span class="k">def</span> <span class="nf">img2npy_</span><span class="p">(</span><span class="n">input_dir</span><span class="p">,</span><span class="n">basename</span><span class="p">,</span> <span class="n">svs_file</span><span class="p">):</span> |
|
|
267 |
<span class="sd">"""Convert SVS, TIF, TIFF to NPY.</span> |
|
|
268 |
|
|
|
269 |
<span class="sd"> Parameters</span> |
|
|
270 |
<span class="sd"> ----------</span> |
|
|
271 |
<span class="sd"> input_dir:str</span> |
|
|
272 |
<span class="sd"> Output file dir.</span> |
|
|
273 |
<span class="sd"> basename:str</span> |
|
|
274 |
<span class="sd"> Basename of output file</span> |
|
|
275 |
<span class="sd"> svs_file:str</span> |
|
|
276 |
<span class="sd"> SVS, TIF, TIFF file input.</span> |
|
|
277 |
|
|
|
278 |
<span class="sd"> Returns</span> |
|
|
279 |
<span class="sd"> -------</span> |
|
|
280 |
<span class="sd"> str</span> |
|
|
281 |
<span class="sd"> NPY output file.</span> |
|
|
282 |
<span class="sd"> """</span> |
|
|
283 |
<span class="n">npy_out_file</span> <span class="o">=</span> <span class="n">join</span><span class="p">(</span><span class="n">input_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">.npy'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">basename</span><span class="p">))</span> |
|
|
284 |
<span class="n">arr</span> <span class="o">=</span> <span class="n">svs2dask_array</span><span class="p">(</span><span class="n">svs_file</span><span class="p">)</span> |
|
|
285 |
<span class="n">np</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">npy_out_file</span><span class="p">,</span><span class="n">arr</span><span class="o">.</span><span class="n">compute</span><span class="p">())</span> |
|
|
286 |
<span class="k">return</span> <span class="n">npy_out_file</span></div> |
|
|
287 |
|
|
|
288 |
<div class="viewcode-block" id="load_image"><a class="viewcode-back" href="../../index.html#pathflowai.utils.load_image">[docs]</a><span class="k">def</span> <span class="nf">load_image</span><span class="p">(</span><span class="n">svs_file</span><span class="p">):</span> |
|
|
289 |
<span class="sd">"""Load SVS, TIF, TIFF</span> |
|
|
290 |
|
|
|
291 |
<span class="sd"> Parameters</span> |
|
|
292 |
<span class="sd"> ----------</span> |
|
|
293 |
<span class="sd"> svs_file:type</span> |
|
|
294 |
<span class="sd"> Description of parameter `svs_file`.</span> |
|
|
295 |
|
|
|
296 |
<span class="sd"> Returns</span> |
|
|
297 |
<span class="sd"> -------</span> |
|
|
298 |
<span class="sd"> type</span> |
|
|
299 |
<span class="sd"> Description of returned object.</span> |
|
|
300 |
<span class="sd"> """</span> |
|
|
301 |
<span class="n">im</span> <span class="o">=</span> <span class="n">Image</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">svs_file</span><span class="p">)</span> |
|
|
302 |
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">im</span><span class="p">),(</span><span class="mi">1</span><span class="p">,</span><span class="mi">0</span><span class="p">)),</span> <span class="n">im</span><span class="o">.</span><span class="n">size</span></div> |
|
|
303 |
|
|
|
304 |
<div class="viewcode-block" id="create_purple_mask"><a class="viewcode-back" href="../../index.html#pathflowai.utils.create_purple_mask">[docs]</a><span class="k">def</span> <span class="nf">create_purple_mask</span><span class="p">(</span><span class="n">arr</span><span class="p">,</span> <span class="n">img_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">sparse</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span> |
|
|
305 |
<span class="sd">"""Create a gray scale intensity mask. This will be changed soon to support other thresholding QC methods.</span> |
|
|
306 |
|
|
|
307 |
<span class="sd"> Parameters</span> |
|
|
308 |
<span class="sd"> ----------</span> |
|
|
309 |
<span class="sd"> arr:dask.array</span> |
|
|
310 |
<span class="sd"> Dask array containing image information.</span> |
|
|
311 |
<span class="sd"> img_size:int</span> |
|
|
312 |
<span class="sd"> Deprecated.</span> |
|
|
313 |
<span class="sd"> sparse:bool</span> |
|
|
314 |
<span class="sd"> Deprecated</span> |
|
|
315 |
|
|
|
316 |
<span class="sd"> Returns</span> |
|
|
317 |
<span class="sd"> -------</span> |
|
|
318 |
<span class="sd"> dask.array</span> |
|
|
319 |
<span class="sd"> Intensity, grayscale array over image.</span> |
|
|
320 |
|
|
|
321 |
<span class="sd"> """</span> |
|
|
322 |
<span class="n">r</span><span class="p">,</span><span class="n">b</span><span class="p">,</span><span class="n">g</span><span class="o">=</span><span class="n">arr</span><span class="p">[:,:,</span><span class="mi">0</span><span class="p">],</span><span class="n">arr</span><span class="p">[:,:,</span><span class="mi">1</span><span class="p">],</span><span class="n">arr</span><span class="p">[:,:,</span><span class="mi">2</span><span class="p">]</span> |
|
|
323 |
<span class="n">gray</span> <span class="o">=</span> <span class="mf">0.2989</span> <span class="o">*</span> <span class="n">r</span> <span class="o">+</span> <span class="mf">0.5870</span> <span class="o">*</span> <span class="n">g</span> <span class="o">+</span> <span class="mf">0.1140</span> <span class="o">*</span> <span class="n">b</span> |
|
|
324 |
<span class="c1">#rb_avg = (r+b)/2</span> |
|
|
325 |
<span class="n">mask</span><span class="o">=</span> <span class="p">((</span><span class="mf">255.</span><span class="o">-</span><span class="n">gray</span><span class="p">))</span><span class="c1"># >= threshold)#(r > g - 10) & (b > g - 10) & (rb_avg > g + 20)#np.vectorize(is_purple)(arr).astype(int)</span> |
|
|
326 |
<span class="k">if</span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">sparse</span><span class="p">:</span> |
|
|
327 |
<span class="n">mask</span> <span class="o">=</span> <span class="n">mask</span><span class="o">.</span><span class="n">nonzero</span><span class="p">()</span> |
|
|
328 |
<span class="n">mask</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">mask</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">compute</span><span class="p">(),</span> <span class="n">mask</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">compute</span><span class="p">()])</span><span class="o">.</span><span class="n">T</span> |
|
|
329 |
<span class="c1">#mask = (np.ones(len(mask[0])),mask)</span> |
|
|
330 |
<span class="c1">#mask = sparse.COO.from_scipy_sparse(sps.coo_matrix(mask, img_size, dtype=np.uint8).tocsr())</span> |
|
|
331 |
<span class="k">return</span> <span class="n">mask</span></div> |
|
|
332 |
|
|
|
333 |
<div class="viewcode-block" id="add_purple_mask"><a class="viewcode-back" href="../../index.html#pathflowai.utils.add_purple_mask">[docs]</a><span class="k">def</span> <span class="nf">add_purple_mask</span><span class="p">(</span><span class="n">arr</span><span class="p">):</span> |
|
|
334 |
<span class="sd">"""Optional add intensity mask to the dask array.</span> |
|
|
335 |
|
|
|
336 |
<span class="sd"> Parameters</span> |
|
|
337 |
<span class="sd"> ----------</span> |
|
|
338 |
<span class="sd"> arr:dask.array</span> |
|
|
339 |
<span class="sd"> Image data.</span> |
|
|
340 |
|
|
|
341 |
<span class="sd"> Returns</span> |
|
|
342 |
<span class="sd"> -------</span> |
|
|
343 |
<span class="sd"> array</span> |
|
|
344 |
<span class="sd"> Image data with intensity added as forth channel.</span> |
|
|
345 |
|
|
|
346 |
<span class="sd"> """</span> |
|
|
347 |
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="n">arr</span><span class="p">,</span><span class="n">create_purple_mask</span><span class="p">(</span><span class="n">arr</span><span class="p">)),</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span></div> |
|
|
348 |
|
|
|
349 |
<div class="viewcode-block" id="create_sparse_annotation_arrays"><a class="viewcode-back" href="../../index.html#pathflowai.utils.create_sparse_annotation_arrays">[docs]</a><span class="k">def</span> <span class="nf">create_sparse_annotation_arrays</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">img_size</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[]):</span> |
|
|
350 |
<span class="sd">"""Convert annotation xml to shapely objects and store in dictionary.</span> |
|
|
351 |
|
|
|
352 |
<span class="sd"> Parameters</span> |
|
|
353 |
<span class="sd"> ----------</span> |
|
|
354 |
<span class="sd"> xml_file:str</span> |
|
|
355 |
<span class="sd"> XML file containing annotations.</span> |
|
|
356 |
<span class="sd"> img_size:int</span> |
|
|
357 |
<span class="sd"> Deprecated.</span> |
|
|
358 |
<span class="sd"> annotations:list</span> |
|
|
359 |
<span class="sd"> Annotations to look for in xml export.</span> |
|
|
360 |
|
|
|
361 |
<span class="sd"> Returns</span> |
|
|
362 |
<span class="sd"> -------</span> |
|
|
363 |
<span class="sd"> dict</span> |
|
|
364 |
<span class="sd"> Dictionary with annotation-shapely object pairs.</span> |
|
|
365 |
|
|
|
366 |
<span class="sd"> """</span> |
|
|
367 |
<span class="n">interior_points_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">annotation</span><span class="p">:</span><span class="n">parse_coord_return_boxes</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation_name</span> <span class="o">=</span> <span class="n">annotation</span><span class="p">,</span> <span class="n">return_coords</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="k">for</span> <span class="n">annotation</span> <span class="ow">in</span> <span class="n">annotations</span><span class="p">}</span><span class="c1">#grab_interior_points(xml_file, img_size, annotations=annotations) if annotations else {}</span> |
|
|
368 |
<span class="k">return</span> <span class="p">{</span><span class="n">annotation</span><span class="p">:</span><span class="n">interior_points_dict</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span> <span class="k">for</span> <span class="n">annotation</span> <span class="ow">in</span> <span class="n">annotations</span><span class="p">}</span><span class="c1">#sparse.COO.from_scipy_sparse((sps.coo_matrix(interior_points_dict[annotation],img_size, dtype=np.uint8) if interior_points_dict[annotation] not None else sps.coo_matrix(img_size, dtype=np.uint8)).tocsr()) for annotation in annotations} # [sps.coo_matrix(img_size, dtype=np.uint8)]+</span></div> |
|
|
369 |
|
|
|
370 |
<div class="viewcode-block" id="load_process_image"><a class="viewcode-back" href="../../index.html#pathflowai.utils.load_process_image">[docs]</a><span class="k">def</span> <span class="nf">load_process_image</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">xml_file</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">npy_mask</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[]):</span> |
|
|
371 |
<span class="sd">"""Load SVS-like image (including NPY), segmentation/classification annotations, generate dask array and dictionary of annotations.</span> |
|
|
372 |
|
|
|
373 |
<span class="sd"> Parameters</span> |
|
|
374 |
<span class="sd"> ----------</span> |
|
|
375 |
<span class="sd"> svs_file:str</span> |
|
|
376 |
<span class="sd"> Image file</span> |
|
|
377 |
<span class="sd"> xml_file:str</span> |
|
|
378 |
<span class="sd"> Annotation file.</span> |
|
|
379 |
<span class="sd"> npy_mask:array</span> |
|
|
380 |
<span class="sd"> Numpy segmentation mask.</span> |
|
|
381 |
<span class="sd"> annotations:list</span> |
|
|
382 |
<span class="sd"> List of annotations in xml.</span> |
|
|
383 |
|
|
|
384 |
<span class="sd"> Returns</span> |
|
|
385 |
<span class="sd"> -------</span> |
|
|
386 |
<span class="sd"> array</span> |
|
|
387 |
<span class="sd"> Dask array of image.</span> |
|
|
388 |
<span class="sd"> dict</span> |
|
|
389 |
<span class="sd"> Annotation masks.</span> |
|
|
390 |
|
|
|
391 |
<span class="sd"> """</span> |
|
|
392 |
<span class="n">arr</span> <span class="o">=</span> <span class="n">npy2da</span><span class="p">(</span><span class="n">svs_file</span><span class="p">)</span> <span class="k">if</span> <span class="n">svs_file</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">'.npy'</span><span class="p">)</span> <span class="k">else</span> <span class="n">svs2dask_array</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">tile_size</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span> <span class="n">overlap</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span><span class="c1">#load_image(svs_file)</span> |
|
|
393 |
<span class="n">img_size</span> <span class="o">=</span> <span class="n">arr</span><span class="o">.</span><span class="n">shape</span><span class="p">[:</span><span class="mi">2</span><span class="p">]</span> |
|
|
394 |
<span class="n">masks</span> <span class="o">=</span> <span class="p">{}</span><span class="c1">#{'purple': create_purple_mask(arr,img_size,sparse=False)}</span> |
|
|
395 |
<span class="k">if</span> <span class="n">xml_file</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
|
|
396 |
<span class="n">masks</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">create_sparse_annotation_arrays</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">img_size</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="n">annotations</span><span class="p">))</span> |
|
|
397 |
<span class="k">if</span> <span class="n">npy_mask</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
|
|
398 |
<span class="n">masks</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">'annotations'</span><span class="p">:</span><span class="n">npy_mask</span><span class="p">})</span> |
|
|
399 |
<span class="c1">#data = dict(image=(['x','y','rgb'],arr),**masks)</span> |
|
|
400 |
<span class="c1">#data_arr = {'image':xr.Variable(['x','y','color'], arr)}</span> |
|
|
401 |
<span class="c1">#purple_arr = {'mask':xr.Variable(['x','y'], masks['purple'])}</span> |
|
|
402 |
<span class="c1">#mask_arr = {m:xr.Variable(['row','col'],masks[m]) for m in masks if m != 'purple'} if 'annotations' not in annotations else {'annotations':xr.Variable(['x','y'],masks['annotations'])}</span> |
|
|
403 |
<span class="c1">#masks['purple'] = masks['purple'].reshape(*masks['purple'].shape,1)</span> |
|
|
404 |
<span class="c1">#arr = da.concatenate([arr,masks.pop('purple')],axis=2)</span> |
|
|
405 |
<span class="k">return</span> <span class="n">arr</span><span class="p">,</span> <span class="n">masks</span><span class="c1">#xr.Dataset.from_dict({k:v for k,v in list(data_arr.items())+list(purple_arr.items())+list(mask_arr.items())})#list(dict(image=data_arr,purple=purple_arr,annotations=mask_arr).items()))#arr, masks</span></div> |
|
|
406 |
|
|
|
407 |
<div class="viewcode-block" id="save_dataset"><a class="viewcode-back" href="../../index.html#pathflowai.utils.save_dataset">[docs]</a><span class="k">def</span> <span class="nf">save_dataset</span><span class="p">(</span><span class="n">arr</span><span class="p">,</span> <span class="n">masks</span><span class="p">,</span> <span class="n">out_zarr</span><span class="p">,</span> <span class="n">out_pkl</span><span class="p">):</span> |
|
|
408 |
<span class="sd">"""Saves dask array image, dictionary of annotations to zarr and pickle respectively.</span> |
|
|
409 |
|
|
|
410 |
<span class="sd"> Parameters</span> |
|
|
411 |
<span class="sd"> ----------</span> |
|
|
412 |
<span class="sd"> arr:array</span> |
|
|
413 |
<span class="sd"> Image.</span> |
|
|
414 |
<span class="sd"> masks:dict</span> |
|
|
415 |
<span class="sd"> Dictionary of annotation shapes.</span> |
|
|
416 |
<span class="sd"> out_zarr:str</span> |
|
|
417 |
<span class="sd"> Zarr output file for image.</span> |
|
|
418 |
<span class="sd"> out_pkl:str</span> |
|
|
419 |
<span class="sd"> Pickle output file.</span> |
|
|
420 |
<span class="sd"> """</span> |
|
|
421 |
<span class="n">arr</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">'uint8'</span><span class="p">)</span><span class="o">.</span><span class="n">to_zarr</span><span class="p">(</span><span class="n">out_zarr</span><span class="p">,</span> <span class="n">overwrite</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
|
|
422 |
<span class="n">pickle</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">masks</span><span class="p">,</span><span class="nb">open</span><span class="p">(</span><span class="n">out_pkl</span><span class="p">,</span><span class="s1">'wb'</span><span class="p">))</span></div> |
|
|
423 |
|
|
|
424 |
<span class="c1">#dataset.to_netcdf(out_netcdf, compute=False)</span> |
|
|
425 |
<span class="c1">#pickle.dump(dataset, open(out_pkl,'wb'), protocol=-1)</span> |
|
|
426 |
|
|
|
427 |
<div class="viewcode-block" id="run_preprocessing_pipeline"><a class="viewcode-back" href="../../index.html#pathflowai.utils.run_preprocessing_pipeline">[docs]</a><span class="k">def</span> <span class="nf">run_preprocessing_pipeline</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">xml_file</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">npy_mask</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">out_zarr</span><span class="o">=</span><span class="s1">'output_zarr.zarr'</span><span class="p">,</span> <span class="n">out_pkl</span><span class="o">=</span><span class="s1">'output.pkl'</span><span class="p">):</span> |
|
|
428 |
<span class="sd">"""Run preprocessing pipeline. Store image into zarr format, segmentations maintain as npy, and xml annotations as pickle.</span> |
|
|
429 |
|
|
|
430 |
<span class="sd"> Parameters</span> |
|
|
431 |
<span class="sd"> ----------</span> |
|
|
432 |
<span class="sd"> svs_file:str</span> |
|
|
433 |
<span class="sd"> Input image file.</span> |
|
|
434 |
<span class="sd"> xml_file:str</span> |
|
|
435 |
<span class="sd"> Input annotation file.</span> |
|
|
436 |
<span class="sd"> npy_mask:str</span> |
|
|
437 |
<span class="sd"> NPY segmentation mask.</span> |
|
|
438 |
<span class="sd"> annotations:list</span> |
|
|
439 |
<span class="sd"> List of annotations.</span> |
|
|
440 |
<span class="sd"> out_zarr:str</span> |
|
|
441 |
<span class="sd"> Output zarr for image.</span> |
|
|
442 |
<span class="sd"> out_pkl:str</span> |
|
|
443 |
<span class="sd"> Output pickle for annotations.</span> |
|
|
444 |
<span class="sd"> """</span> |
|
|
445 |
<span class="c1">#save_dataset(load_process_image(svs_file, xml_file, npy_mask, annotations), out_netcdf)</span> |
|
|
446 |
<span class="n">arr</span><span class="p">,</span> <span class="n">masks</span> <span class="o">=</span> <span class="n">load_process_image</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">xml_file</span><span class="p">,</span> <span class="n">npy_mask</span><span class="p">,</span> <span class="n">annotations</span><span class="p">)</span> |
|
|
447 |
<span class="n">save_dataset</span><span class="p">(</span><span class="n">arr</span><span class="p">,</span> <span class="n">masks</span><span class="p">,</span><span class="n">out_zarr</span><span class="p">,</span> <span class="n">out_pkl</span><span class="p">)</span></div> |
|
|
448 |
|
|
|
449 |
<span class="c1">###################</span> |
|
|
450 |
|
|
|
451 |
<div class="viewcode-block" id="adjust_mask"><a class="viewcode-back" href="../../index.html#pathflowai.utils.adjust_mask">[docs]</a><span class="k">def</span> <span class="nf">adjust_mask</span><span class="p">(</span><span class="n">mask_file</span><span class="p">,</span> <span class="n">dask_img_array_file</span><span class="p">,</span> <span class="n">out_npy</span><span class="p">,</span> <span class="n">n_neighbors</span><span class="p">):</span> |
|
|
452 |
<span class="sd">"""Fixes segmentation masks to reduce coarse annotations over empty regions.</span> |
|
|
453 |
|
|
|
454 |
<span class="sd"> Parameters</span> |
|
|
455 |
<span class="sd"> ----------</span> |
|
|
456 |
<span class="sd"> mask_file:str</span> |
|
|
457 |
<span class="sd"> NPY segmentation mask.</span> |
|
|
458 |
<span class="sd"> dask_img_array_file:str</span> |
|
|
459 |
<span class="sd"> Dask image file.</span> |
|
|
460 |
<span class="sd"> out_npy:str</span> |
|
|
461 |
<span class="sd"> Output numpy file.</span> |
|
|
462 |
<span class="sd"> n_neighbors:int</span> |
|
|
463 |
<span class="sd"> Number nearest neighbors for dilation and erosion of mask from background to not background.</span> |
|
|
464 |
|
|
|
465 |
<span class="sd"> Returns</span> |
|
|
466 |
<span class="sd"> -------</span> |
|
|
467 |
<span class="sd"> str</span> |
|
|
468 |
<span class="sd"> Output numpy file.</span> |
|
|
469 |
|
|
|
470 |
<span class="sd"> """</span> |
|
|
471 |
<span class="kn">from</span> <span class="nn">dask_image.ndmorph</span> <span class="k">import</span> <span class="n">binary_opening</span> |
|
|
472 |
<span class="kn">from</span> <span class="nn">dask.distributed</span> <span class="k">import</span> <span class="n">Client</span> |
|
|
473 |
<span class="c1">#c=Client()</span> |
|
|
474 |
<span class="n">dask_img_array</span><span class="o">=</span><span class="n">da</span><span class="o">.</span><span class="n">from_zarr</span><span class="p">(</span><span class="n">dask_img_array_file</span><span class="p">)</span> |
|
|
475 |
<span class="n">mask</span><span class="o">=</span><span class="n">npy2da</span><span class="p">(</span><span class="n">mask_file</span><span class="p">)</span> |
|
|
476 |
<span class="n">is_tissue_mask</span> <span class="o">=</span> <span class="n">mask</span><span class="o">></span><span class="mf">0.</span> |
|
|
477 |
<span class="n">is_tissue_mask_img</span><span class="o">=</span><span class="p">((</span><span class="n">dask_img_array</span><span class="p">[</span><span class="o">...</span><span class="p">,</span><span class="mi">0</span><span class="p">]</span><span class="o">></span><span class="mf">200.</span><span class="p">)</span> <span class="o">&</span> <span class="p">(</span><span class="n">dask_img_array</span><span class="p">[</span><span class="o">...</span><span class="p">,</span><span class="mi">1</span><span class="p">]</span><span class="o">></span><span class="mf">200.</span><span class="p">)</span><span class="o">&</span> <span class="p">(</span><span class="n">dask_img_array</span><span class="p">[</span><span class="o">...</span><span class="p">,</span><span class="mi">2</span><span class="p">]</span><span class="o">></span><span class="mf">200.</span><span class="p">))</span> <span class="o">==</span> <span class="mi">0</span> |
|
|
478 |
<span class="n">opening</span><span class="o">=</span><span class="n">binary_opening</span><span class="p">(</span><span class="n">is_tissue_mask_img</span><span class="p">,</span><span class="n">structure</span><span class="o">=</span><span class="n">da</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">n_neighbors</span><span class="p">,</span><span class="n">n_neighbors</span><span class="p">)))</span><span class="c1">#,mask=is_tissue_mask)</span> |
|
|
479 |
<span class="n">mask</span><span class="p">[(</span><span class="n">opening</span><span class="o">==</span><span class="mi">0</span><span class="p">)</span><span class="o">&</span><span class="p">(</span><span class="n">is_tissue_mask</span><span class="o">==</span><span class="mi">1</span><span class="p">)]</span><span class="o">=</span><span class="mi">0</span> |
|
|
480 |
<span class="n">np</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">out_npy</span><span class="p">,</span><span class="n">mask</span><span class="o">.</span><span class="n">compute</span><span class="p">())</span> |
|
|
481 |
<span class="c1">#c.close()</span> |
|
|
482 |
<span class="k">return</span> <span class="n">out_npy</span></div> |
|
|
483 |
|
|
|
484 |
<span class="c1">###################</span> |
|
|
485 |
|
|
|
486 |
<div class="viewcode-block" id="process_svs"><a class="viewcode-back" href="../../index.html#pathflowai.utils.process_svs">[docs]</a><span class="k">def</span> <span class="nf">process_svs</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">xml_file</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">output_dir</span><span class="o">=</span><span class="s1">'./'</span><span class="p">):</span> |
|
|
487 |
<span class="sd">"""Store images into npy format and store annotations into pickle dictionary.</span> |
|
|
488 |
|
|
|
489 |
<span class="sd"> Parameters</span> |
|
|
490 |
<span class="sd"> ----------</span> |
|
|
491 |
<span class="sd"> svs_file:str</span> |
|
|
492 |
<span class="sd"> Image file.</span> |
|
|
493 |
<span class="sd"> xml_file:str</span> |
|
|
494 |
<span class="sd"> Annotations file.</span> |
|
|
495 |
<span class="sd"> annotations:list</span> |
|
|
496 |
<span class="sd"> List of annotations in image.</span> |
|
|
497 |
<span class="sd"> output_dir:str</span> |
|
|
498 |
<span class="sd"> Output directory.</span> |
|
|
499 |
<span class="sd"> """</span> |
|
|
500 |
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span><span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
|
|
501 |
<span class="n">basename</span> <span class="o">=</span> <span class="n">svs_file</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> |
|
|
502 |
<span class="n">arr</span><span class="p">,</span> <span class="n">masks</span> <span class="o">=</span> <span class="n">load_process_image</span><span class="p">(</span><span class="n">svs_file</span><span class="p">,</span> <span class="n">xml_file</span><span class="p">)</span> |
|
|
503 |
<span class="n">np</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">.npy'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">basename</span><span class="p">)),</span><span class="n">arr</span><span class="p">)</span> |
|
|
504 |
<span class="n">pickle</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">masks</span><span class="p">,</span> <span class="nb">open</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">output_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">.pkl'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">basename</span><span class="p">)),</span><span class="s1">'wb'</span><span class="p">),</span> <span class="n">protocol</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span></div> |
|
|
505 |
|
|
|
506 |
<span class="c1">####################</span> |
|
|
507 |
|
|
|
508 |
<div class="viewcode-block" id="load_dataset"><a class="viewcode-back" href="../../index.html#pathflowai.utils.load_dataset">[docs]</a><span class="k">def</span> <span class="nf">load_dataset</span><span class="p">(</span><span class="n">in_zarr</span><span class="p">,</span> <span class="n">in_pkl</span><span class="p">):</span> |
|
|
509 |
<span class="sd">"""Load ZARR image and annotations pickle.</span> |
|
|
510 |
|
|
|
511 |
<span class="sd"> Parameters</span> |
|
|
512 |
<span class="sd"> ----------</span> |
|
|
513 |
<span class="sd"> in_zarr:str</span> |
|
|
514 |
<span class="sd"> Input image.</span> |
|
|
515 |
<span class="sd"> in_pkl:str</span> |
|
|
516 |
<span class="sd"> Input annotations.</span> |
|
|
517 |
|
|
|
518 |
<span class="sd"> Returns</span> |
|
|
519 |
<span class="sd"> -------</span> |
|
|
520 |
<span class="sd"> dask.array</span> |
|
|
521 |
<span class="sd"> Image array.</span> |
|
|
522 |
<span class="sd"> dict</span> |
|
|
523 |
<span class="sd"> Annotations dictionary.</span> |
|
|
524 |
|
|
|
525 |
<span class="sd"> """</span> |
|
|
526 |
<span class="k">return</span> <span class="n">da</span><span class="o">.</span><span class="n">from_zarr</span><span class="p">(</span><span class="n">in_zarr</span><span class="p">),</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">in_pkl</span><span class="p">,</span><span class="s1">'rb'</span><span class="p">))</span><span class="c1">#xr.open_dataset(in_netcdf)</span></div> |
|
|
527 |
|
|
|
528 |
<div class="viewcode-block" id="is_valid_patch"><a class="viewcode-back" href="../../index.html#pathflowai.utils.is_valid_patch">[docs]</a><span class="k">def</span> <span class="nf">is_valid_patch</span><span class="p">(</span><span class="n">xs</span><span class="p">,</span><span class="n">ys</span><span class="p">,</span><span class="n">patch_size</span><span class="p">,</span><span class="n">purple_mask</span><span class="p">,</span><span class="n">intensity_threshold</span><span class="p">,</span><span class="n">threshold</span><span class="o">=</span><span class="mf">0.5</span><span class="p">):</span> |
|
|
529 |
<span class="sd">"""Deprecated, computes whether patch is valid."""</span> |
|
|
530 |
<span class="nb">print</span><span class="p">(</span><span class="n">xs</span><span class="p">,</span><span class="n">ys</span><span class="p">)</span> |
|
|
531 |
<span class="k">return</span> <span class="p">(</span><span class="n">purple_mask</span><span class="p">[</span><span class="n">xs</span><span class="p">:</span><span class="n">xs</span><span class="o">+</span><span class="n">patch_size</span><span class="p">,</span><span class="n">ys</span><span class="p">:</span><span class="n">ys</span><span class="o">+</span><span class="n">patch_size</span><span class="p">]</span><span class="o">>=</span><span class="n">intensity_threshold</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> <span class="o">></span> <span class="n">threshold</span></div> |
|
|
532 |
|
|
|
533 |
<span class="c1">#@pysnooper.snoop("extract_patch.log")</span> |
|
|
534 |
<div class="viewcode-block" id="extract_patch_information"><a class="viewcode-back" href="../../index.html#pathflowai.utils.extract_patch_information">[docs]</a><span class="k">def</span> <span class="nf">extract_patch_information</span><span class="p">(</span><span class="n">basename</span><span class="p">,</span> <span class="n">input_dir</span><span class="o">=</span><span class="s1">'./'</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">threshold</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">patch_size</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">generate_finetune_segmentation</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">target_class</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">intensity_threshold</span><span class="o">=</span><span class="mf">100.</span><span class="p">,</span> <span class="n">target_threshold</span><span class="o">=</span><span class="mf">0.</span><span class="p">,</span> <span class="n">adj_mask</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">basic_preprocess</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">tries</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span> |
|
|
535 |
<span class="sd">"""Final step of preprocessing pipeline. Break up image into patches, include if not background and of a certain intensity, find area of each annotation type in patch, spatial information, image ID and dump data to SQL table.</span> |
|
|
536 |
|
|
|
537 |
<span class="sd"> Parameters</span> |
|
|
538 |
<span class="sd"> ----------</span> |
|
|
539 |
<span class="sd"> basename:str</span> |
|
|
540 |
<span class="sd"> Patient ID.</span> |
|
|
541 |
<span class="sd"> input_dir:str</span> |
|
|
542 |
<span class="sd"> Input directory.</span> |
|
|
543 |
<span class="sd"> annotations:list</span> |
|
|
544 |
<span class="sd"> List of annotations to record, these can be different tissue types, must correspond with XML labels.</span> |
|
|
545 |
<span class="sd"> threshold:float</span> |
|
|
546 |
<span class="sd"> Value between 0 and 1 that indicates the minimum amount of patch that musn't be background for inclusion.</span> |
|
|
547 |
<span class="sd"> patch_size:int</span> |
|
|
548 |
<span class="sd"> Patch size of patches; this will become one of the tables.</span> |
|
|
549 |
<span class="sd"> generate_finetune_segmentation:bool</span> |
|
|
550 |
<span class="sd"> Deprecated.</span> |
|
|
551 |
<span class="sd"> target_class:int</span> |
|
|
552 |
<span class="sd"> Number of segmentation classes desired, from 0th class to target_class-1 will be annotated in SQL.</span> |
|
|
553 |
<span class="sd"> intensity_threshold:float</span> |
|
|
554 |
<span class="sd"> Value between 0 and 255 that represents minimum intensity to not include as background. Will be modified with new transforms.</span> |
|
|
555 |
<span class="sd"> target_threshold:float</span> |
|
|
556 |
<span class="sd"> Deprecated.</span> |
|
|
557 |
<span class="sd"> adj_mask:str</span> |
|
|
558 |
<span class="sd"> Adjusted mask if performed binary opening operations in previous preprocessing step.</span> |
|
|
559 |
<span class="sd"> basic_preprocess:bool</span> |
|
|
560 |
<span class="sd"> Do not store patch level information.</span> |
|
|
561 |
<span class="sd"> tries:int</span> |
|
|
562 |
<span class="sd"> Number of tries in case there is a Dask timeout, run again.</span> |
|
|
563 |
|
|
|
564 |
<span class="sd"> Returns</span> |
|
|
565 |
<span class="sd"> -------</span> |
|
|
566 |
<span class="sd"> dataframe</span> |
|
|
567 |
<span class="sd"> Patch information.</span> |
|
|
568 |
|
|
|
569 |
<span class="sd"> """</span> |
|
|
570 |
<span class="c1">#from collections import OrderedDict</span> |
|
|
571 |
<span class="c1">#annotations=OrderedDict(annotations)</span> |
|
|
572 |
<span class="c1">#from dask.multiprocessing import get</span> |
|
|
573 |
<span class="kn">import</span> <span class="nn">dask</span> |
|
|
574 |
<span class="kn">import</span> <span class="nn">time</span> |
|
|
575 |
<span class="kn">from</span> <span class="nn">dask</span> <span class="k">import</span> <span class="n">dataframe</span> <span class="k">as</span> <span class="n">dd</span> |
|
|
576 |
<span class="kn">import</span> <span class="nn">dask.array</span> <span class="k">as</span> <span class="nn">da</span> |
|
|
577 |
<span class="kn">import</span> <span class="nn">multiprocessing</span> |
|
|
578 |
<span class="kn">from</span> <span class="nn">shapely.ops</span> <span class="k">import</span> <span class="n">unary_union</span> |
|
|
579 |
<span class="kn">from</span> <span class="nn">shapely.geometry</span> <span class="k">import</span> <span class="n">MultiPolygon</span> |
|
|
580 |
<span class="kn">from</span> <span class="nn">itertools</span> <span class="k">import</span> <span class="n">product</span> |
|
|
581 |
<span class="c1">#from distributed import Client,LocalCluster</span> |
|
|
582 |
<span class="n">max_tries</span><span class="o">=</span><span class="mi">4</span> |
|
|
583 |
<span class="n">kargs</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">basename</span><span class="o">=</span><span class="n">basename</span><span class="p">,</span> <span class="n">input_dir</span><span class="o">=</span><span class="n">input_dir</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="n">annotations</span><span class="p">,</span> <span class="n">threshold</span><span class="o">=</span><span class="n">threshold</span><span class="p">,</span> <span class="n">patch_size</span><span class="o">=</span><span class="n">patch_size</span><span class="p">,</span> <span class="n">generate_finetune_segmentation</span><span class="o">=</span><span class="n">generate_finetune_segmentation</span><span class="p">,</span> <span class="n">target_class</span><span class="o">=</span><span class="n">target_class</span><span class="p">,</span> <span class="n">intensity_threshold</span><span class="o">=</span><span class="n">intensity_threshold</span><span class="p">,</span> <span class="n">target_threshold</span><span class="o">=</span><span class="n">target_threshold</span><span class="p">,</span> <span class="n">adj_mask</span><span class="o">=</span><span class="n">adj_mask</span><span class="p">,</span> <span class="n">basic_preprocess</span><span class="o">=</span><span class="n">basic_preprocess</span><span class="p">,</span> <span class="n">tries</span><span class="o">=</span><span class="n">tries</span><span class="p">)</span> |
|
|
584 |
<span class="k">try</span><span class="p">:</span> |
|
|
585 |
<span class="c1">#,</span> |
|
|
586 |
<span class="c1"># 'distributed.scheduler.allowed-failures':20,</span> |
|
|
587 |
<span class="c1"># 'num-workers':20}):</span> |
|
|
588 |
<span class="c1">#cluster=LocalCluster()</span> |
|
|
589 |
<span class="c1">#cluster.adapt(minimum=10, maximum=100)</span> |
|
|
590 |
<span class="c1">#cluster = LocalCluster(threads_per_worker=1, n_workers=20, memory_limit="80G")</span> |
|
|
591 |
<span class="c1">#client=Client()#Client(cluster)#processes=True)#cluster,</span> |
|
|
592 |
|
|
|
593 |
<span class="n">arr</span><span class="p">,</span> <span class="n">masks</span> <span class="o">=</span> <span class="n">load_dataset</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">input_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">.zarr'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">basename</span><span class="p">)),</span><span class="n">join</span><span class="p">(</span><span class="n">input_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">_mask.pkl'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">basename</span><span class="p">)))</span> |
|
|
594 |
<span class="k">if</span> <span class="s1">'annotations'</span> <span class="ow">in</span> <span class="n">masks</span><span class="p">:</span> |
|
|
595 |
<span class="n">segmentation</span> <span class="o">=</span> <span class="kc">True</span> |
|
|
596 |
<span class="c1">#if generate_finetune_segmentation:</span> |
|
|
597 |
<span class="n">segmentation_mask</span> <span class="o">=</span> <span class="n">npy2da</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">input_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">_mask.npy'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">basename</span><span class="p">))</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">adj_mask</span> <span class="k">else</span> <span class="n">adj_mask</span><span class="p">)</span> |
|
|
598 |
<span class="k">else</span><span class="p">:</span> |
|
|
599 |
<span class="n">segmentation</span> <span class="o">=</span> <span class="kc">False</span> |
|
|
600 |
<span class="c1">#masks=np.load(masks['annotations'])</span> |
|
|
601 |
<span class="c1">#npy_file = join(input_dir,'{}.npy'.format(basename))</span> |
|
|
602 |
<span class="n">purple_mask</span> <span class="o">=</span> <span class="n">create_purple_mask</span><span class="p">(</span><span class="n">arr</span><span class="p">)</span> |
|
|
603 |
<span class="n">x_max</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">arr</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> |
|
|
604 |
<span class="n">y_max</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">arr</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> |
|
|
605 |
<span class="n">x_steps</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">x_max</span><span class="o">-</span><span class="n">patch_size</span><span class="p">)</span> <span class="o">/</span> <span class="n">patch_size</span> <span class="p">)</span> |
|
|
606 |
<span class="n">y_steps</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">y_max</span><span class="o">-</span><span class="n">patch_size</span><span class="p">)</span> <span class="o">/</span> <span class="n">patch_size</span> <span class="p">)</span> |
|
|
607 |
<span class="k">for</span> <span class="n">annotation</span> <span class="ow">in</span> <span class="n">annotations</span><span class="p">:</span> |
|
|
608 |
<span class="k">try</span><span class="p">:</span> |
|
|
609 |
<span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span><span class="o">=</span><span class="p">[</span><span class="n">unary_union</span><span class="p">(</span><span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">])]</span> <span class="k">if</span> <span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span> <span class="k">else</span> <span class="p">[]</span> |
|
|
610 |
<span class="k">except</span><span class="p">:</span> |
|
|
611 |
<span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span><span class="o">=</span><span class="p">[</span><span class="n">MultiPolygon</span><span class="p">(</span><span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">])]</span> <span class="k">if</span> <span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span> <span class="k">else</span> <span class="p">[]</span> |
|
|
612 |
|
|
|
613 |
<span class="n">patch_info</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">([([</span><span class="n">basename</span><span class="p">,</span><span class="n">i</span><span class="o">*</span><span class="n">patch_size</span><span class="p">,</span><span class="n">j</span><span class="o">*</span><span class="n">patch_size</span><span class="p">,</span><span class="n">patch_size</span><span class="p">,</span><span class="s1">'NA'</span><span class="p">]</span><span class="o">+</span><span class="p">[</span><span class="mf">0.</span><span class="p">]</span><span class="o">*</span><span class="p">(</span><span class="n">target_class</span> <span class="k">if</span> <span class="n">segmentation</span> <span class="k">else</span> <span class="nb">len</span><span class="p">(</span><span class="n">annotations</span><span class="p">)))</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span><span class="n">j</span> <span class="ow">in</span> <span class="n">product</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">x_steps</span><span class="o">+</span><span class="mi">1</span><span class="p">),</span><span class="nb">range</span><span class="p">(</span><span class="n">y_steps</span><span class="o">+</span><span class="mi">1</span><span class="p">))],</span><span class="n">columns</span><span class="o">=</span><span class="p">([</span><span class="s1">'ID'</span><span class="p">,</span><span class="s1">'x'</span><span class="p">,</span><span class="s1">'y'</span><span class="p">,</span><span class="s1">'patch_size'</span><span class="p">,</span><span class="s1">'annotation'</span><span class="p">]</span><span class="o">+</span><span class="p">(</span><span class="n">annotations</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">segmentation</span> <span class="k">else</span> <span class="nb">list</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">target_class</span><span class="p">)]))))</span><span class="c1">#[dask.delayed(return_line_info)(i,j) for (i,j) in product(range(x_steps+1),range(y_steps+1))]</span> |
|
|
614 |
<span class="k">if</span> <span class="n">basic_preprocess</span><span class="p">:</span> |
|
|
615 |
<span class="n">patch_info</span><span class="o">=</span><span class="n">patch_info</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,:</span><span class="mi">4</span><span class="p">]</span> |
|
|
616 |
<span class="n">valid_patches</span><span class="o">=</span><span class="p">[]</span> |
|
|
617 |
<span class="k">for</span> <span class="n">xs</span><span class="p">,</span><span class="n">ys</span> <span class="ow">in</span> <span class="n">patch_info</span><span class="p">[[</span><span class="s1">'x'</span><span class="p">,</span><span class="s1">'y'</span><span class="p">]]</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">tolist</span><span class="p">():</span> |
|
|
618 |
<span class="n">valid_patches</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">purple_mask</span><span class="p">[</span><span class="n">xs</span><span class="p">:</span><span class="n">xs</span><span class="o">+</span><span class="n">patch_size</span><span class="p">,</span><span class="n">ys</span><span class="p">:</span><span class="n">ys</span><span class="o">+</span><span class="n">patch_size</span><span class="p">]</span><span class="o">>=</span><span class="n">intensity_threshold</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> <span class="o">></span> <span class="n">threshold</span><span class="p">)</span> <span class="c1"># dask.delayed(is_valid_patch)(xs,ys,patch_size,purple_mask,intensity_threshold,threshold)</span> |
|
|
619 |
<span class="n">valid_patches</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">da</span><span class="o">.</span><span class="n">compute</span><span class="p">(</span><span class="o">*</span><span class="n">valid_patches</span><span class="p">))</span> |
|
|
620 |
<span class="nb">print</span><span class="p">(</span><span class="s1">'Valid Patches Complete'</span><span class="p">)</span> |
|
|
621 |
<span class="c1">#print(valid_patches)</span> |
|
|
622 |
<span class="n">patch_info</span><span class="o">=</span><span class="n">patch_info</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">valid_patches</span><span class="p">]</span> |
|
|
623 |
<span class="k">if</span> <span class="ow">not</span> <span class="n">basic_preprocess</span><span class="p">:</span> |
|
|
624 |
<span class="n">area_info</span><span class="o">=</span><span class="p">[]</span> |
|
|
625 |
<span class="k">if</span> <span class="n">segmentation</span><span class="p">:</span> |
|
|
626 |
<span class="n">patch_info</span><span class="o">.</span><span class="n">loc</span><span class="p">[:,</span><span class="s1">'annotation'</span><span class="p">]</span><span class="o">=</span><span class="s1">'segment'</span> |
|
|
627 |
<span class="k">for</span> <span class="n">xs</span><span class="p">,</span><span class="n">ys</span> <span class="ow">in</span> <span class="n">patch_info</span><span class="p">[[</span><span class="s1">'x'</span><span class="p">,</span><span class="s1">'y'</span><span class="p">]]</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">tolist</span><span class="p">():</span> |
|
|
628 |
<span class="n">xf</span><span class="o">=</span><span class="n">xs</span><span class="o">+</span><span class="n">patch_size</span> |
|
|
629 |
<span class="n">yf</span><span class="o">=</span><span class="n">ys</span><span class="o">+</span><span class="n">patch_size</span> |
|
|
630 |
<span class="c1">#print(xs,ys)</span> |
|
|
631 |
<span class="n">area_info</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">da</span><span class="o">.</span><span class="n">histogram</span><span class="p">(</span><span class="n">segmentation_mask</span><span class="p">[</span><span class="n">xs</span><span class="p">:</span><span class="n">xf</span><span class="p">,</span><span class="n">ys</span><span class="p">:</span><span class="n">yf</span><span class="p">],</span><span class="nb">range</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="n">target_class</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span><span class="n">bins</span><span class="o">=</span><span class="n">target_class</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span> |
|
|
632 |
<span class="c1">#area_info.append(dask.delayed(seg_line)(xs,ys,patch_size,segmentation_mask,target_class))</span> |
|
|
633 |
<span class="k">else</span><span class="p">:</span> |
|
|
634 |
<span class="k">for</span> <span class="n">xs</span><span class="p">,</span><span class="n">ys</span> <span class="ow">in</span> <span class="n">patch_info</span><span class="p">[[</span><span class="s1">'x'</span><span class="p">,</span><span class="s1">'y'</span><span class="p">]]</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">tolist</span><span class="p">():</span> |
|
|
635 |
<span class="n">area_info</span><span class="o">.</span><span class="n">append</span><span class="p">([</span><span class="n">dask</span><span class="o">.</span><span class="n">delayed</span><span class="p">(</span><span class="n">is_coords_in_box</span><span class="p">)([</span><span class="n">xs</span><span class="p">,</span><span class="n">ys</span><span class="p">],</span><span class="n">patch_size</span><span class="p">,</span><span class="n">masks</span><span class="p">[</span><span class="n">annotation</span><span class="p">])</span> <span class="k">for</span> <span class="n">annotation</span> <span class="ow">in</span> <span class="n">annotations</span><span class="p">])</span> |
|
|
636 |
<span class="c1">#area_info=da.concatenate(area_info,axis=0).compute()</span> |
|
|
637 |
<span class="n">area_info</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">dask</span><span class="o">.</span><span class="n">compute</span><span class="p">(</span><span class="o">*</span><span class="n">area_info</span><span class="p">))</span><span class="c1">#da.concatenate(area_info,axis=0).compute(dtype=np.float16,scheduler='threaded')).astype(np.float16)</span> |
|
|
638 |
<span class="nb">print</span><span class="p">(</span><span class="s1">'Area Info Complete'</span><span class="p">)</span> |
|
|
639 |
<span class="k">if</span> <span class="n">segmentation</span><span class="p">:</span> |
|
|
640 |
<span class="n">area_info</span> <span class="o">=</span> <span class="n">area_info</span><span class="o">/</span><span class="n">np</span><span class="o">.</span><span class="n">float16</span><span class="p">(</span><span class="n">patch_size</span><span class="o">*</span><span class="n">patch_size</span><span class="p">)</span> |
|
|
641 |
<span class="c1">#print(area_info)</span> |
|
|
642 |
<span class="n">patch_info</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span><span class="mi">5</span><span class="p">:]</span><span class="o">=</span><span class="n">area_info</span> |
|
|
643 |
<span class="c1">#print(patch_info)</span> |
|
|
644 |
<span class="c1">#print(patch_info.dtypes)</span> |
|
|
645 |
<span class="n">annot</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="n">patch_info</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span><span class="mi">5</span><span class="p">:])</span> |
|
|
646 |
<span class="n">patch_info</span><span class="o">.</span><span class="n">loc</span><span class="p">[:,</span><span class="s1">'annotation'</span><span class="p">]</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">vectorize</span><span class="p">(</span><span class="k">lambda</span> <span class="n">i</span><span class="p">:</span> <span class="n">annot</span><span class="p">[</span><span class="n">patch_info</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">,</span><span class="mi">5</span><span class="p">:]</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">argmax</span><span class="p">()])(</span><span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">patch_info</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span><span class="c1">#patch_info[np.arange(target_class).astype(str).tolist()].values.argmax(1).astype(str)</span> |
|
|
647 |
<span class="c1">#client.close()</span> |
|
|
648 |
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
|
|
649 |
<span class="nb">print</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> |
|
|
650 |
<span class="n">kargs</span><span class="p">[</span><span class="s1">'tries'</span><span class="p">]</span><span class="o">+=</span><span class="mi">1</span> |
|
|
651 |
<span class="k">if</span> <span class="n">kargs</span><span class="p">[</span><span class="s1">'tries'</span><span class="p">]</span><span class="o">==</span><span class="n">max_tries</span><span class="p">:</span> |
|
|
652 |
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">'Exceeded past maximum number of tries.'</span><span class="p">)</span> |
|
|
653 |
<span class="k">else</span><span class="p">:</span> |
|
|
654 |
<span class="nb">print</span><span class="p">(</span><span class="s1">'Restarting preprocessing again.'</span><span class="p">)</span> |
|
|
655 |
<span class="n">extract_patch_information</span><span class="p">(</span><span class="o">**</span><span class="n">kargs</span><span class="p">)</span> |
|
|
656 |
|
|
|
657 |
<span class="k">return</span> <span class="n">patch_info</span></div> |
|
|
658 |
|
|
|
659 |
<div class="viewcode-block" id="generate_patch_pipeline"><a class="viewcode-back" href="../../index.html#pathflowai.utils.generate_patch_pipeline">[docs]</a><span class="k">def</span> <span class="nf">generate_patch_pipeline</span><span class="p">(</span><span class="n">basename</span><span class="p">,</span> <span class="n">input_dir</span><span class="o">=</span><span class="s1">'./'</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">threshold</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">patch_size</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">out_db</span><span class="o">=</span><span class="s1">'patch_info.db'</span><span class="p">,</span> <span class="n">generate_finetune_segmentation</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">target_class</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">intensity_threshold</span><span class="o">=</span><span class="mf">100.</span><span class="p">,</span> <span class="n">target_threshold</span><span class="o">=</span><span class="mf">0.</span><span class="p">,</span> <span class="n">adj_mask</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">basic_preprocess</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> |
|
|
660 |
<span class="sd">"""Short summary.</span> |
|
|
661 |
|
|
|
662 |
<span class="sd"> Parameters</span> |
|
|
663 |
<span class="sd"> ----------</span> |
|
|
664 |
<span class="sd"> basename:str</span> |
|
|
665 |
<span class="sd"> Patient ID.</span> |
|
|
666 |
<span class="sd"> input_dir:str</span> |
|
|
667 |
<span class="sd"> Input directory.</span> |
|
|
668 |
<span class="sd"> annotations:list</span> |
|
|
669 |
<span class="sd"> List of annotations to record, these can be different tissue types, must correspond with XML labels.</span> |
|
|
670 |
<span class="sd"> threshold:float</span> |
|
|
671 |
<span class="sd"> Value between 0 and 1 that indicates the minimum amount of patch that musn't be background for inclusion.</span> |
|
|
672 |
<span class="sd"> patch_size:int</span> |
|
|
673 |
<span class="sd"> Patch size of patches; this will become one of the tables.</span> |
|
|
674 |
<span class="sd"> out_db:str</span> |
|
|
675 |
<span class="sd"> Output SQL database.</span> |
|
|
676 |
<span class="sd"> generate_finetune_segmentation:bool</span> |
|
|
677 |
<span class="sd"> Deprecated.</span> |
|
|
678 |
<span class="sd"> target_class:int</span> |
|
|
679 |
<span class="sd"> Number of segmentation classes desired, from 0th class to target_class-1 will be annotated in SQL.</span> |
|
|
680 |
<span class="sd"> intensity_threshold:float</span> |
|
|
681 |
<span class="sd"> Value between 0 and 255 that represents minimum intensity to not include as background. Will be modified with new transforms.</span> |
|
|
682 |
<span class="sd"> target_threshold:float</span> |
|
|
683 |
<span class="sd"> Deprecated.</span> |
|
|
684 |
<span class="sd"> adj_mask:str</span> |
|
|
685 |
<span class="sd"> Adjusted mask if performed binary opening operations in previous preprocessing step.</span> |
|
|
686 |
<span class="sd"> basic_preprocess:bool</span> |
|
|
687 |
<span class="sd"> Do not store patch level information.</span> |
|
|
688 |
<span class="sd"> """</span> |
|
|
689 |
<span class="n">patch_info</span> <span class="o">=</span> <span class="n">extract_patch_information</span><span class="p">(</span><span class="n">basename</span><span class="p">,</span> <span class="n">input_dir</span><span class="p">,</span> <span class="n">annotations</span><span class="p">,</span> <span class="n">threshold</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">,</span> <span class="n">generate_finetune_segmentation</span><span class="o">=</span><span class="n">generate_finetune_segmentation</span><span class="p">,</span> <span class="n">target_class</span><span class="o">=</span><span class="n">target_class</span><span class="p">,</span> <span class="n">intensity_threshold</span><span class="o">=</span><span class="n">intensity_threshold</span><span class="p">,</span> <span class="n">target_threshold</span><span class="o">=</span><span class="n">target_threshold</span><span class="p">,</span> <span class="n">adj_mask</span><span class="o">=</span><span class="n">adj_mask</span><span class="p">,</span> <span class="n">basic_preprocess</span><span class="o">=</span><span class="n">basic_preprocess</span><span class="p">)</span> |
|
|
690 |
<span class="n">conn</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">out_db</span><span class="p">)</span> |
|
|
691 |
<span class="n">patch_info</span><span class="o">.</span><span class="n">to_sql</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span> <span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">,</span> <span class="n">if_exists</span><span class="o">=</span><span class="s1">'append'</span><span class="p">)</span> |
|
|
692 |
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div> |
|
|
693 |
|
|
|
694 |
|
|
|
695 |
<span class="c1"># now output csv</span> |
|
|
696 |
<div class="viewcode-block" id="save_all_patch_info"><a class="viewcode-back" href="../../index.html#pathflowai.utils.save_all_patch_info">[docs]</a><span class="k">def</span> <span class="nf">save_all_patch_info</span><span class="p">(</span><span class="n">basenames</span><span class="p">,</span> <span class="n">input_dir</span><span class="o">=</span><span class="s1">'./'</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">threshold</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">patch_size</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">output_pkl</span><span class="o">=</span><span class="s1">'patch_info.pkl'</span><span class="p">):</span> |
|
|
697 |
<span class="sd">"""Deprecated."""</span> |
|
|
698 |
<span class="n">df</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">extract_patch_information</span><span class="p">(</span><span class="n">basename</span><span class="p">,</span> <span class="n">input_dir</span><span class="p">,</span> <span class="n">annotations</span><span class="p">,</span> <span class="n">threshold</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">)</span> <span class="k">for</span> <span class="n">basename</span> <span class="ow">in</span> <span class="n">basenames</span><span class="p">])</span><span class="o">.</span><span class="n">reset_index</span><span class="p">(</span><span class="n">drop</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
|
|
699 |
<span class="n">df</span><span class="o">.</span><span class="n">to_pickle</span><span class="p">(</span><span class="n">output_pkl</span><span class="p">)</span></div> |
|
|
700 |
|
|
|
701 |
<span class="c1">#########</span> |
|
|
702 |
|
|
|
703 |
|
|
|
704 |
<div class="viewcode-block" id="create_train_val_test"><a class="viewcode-back" href="../../index.html#pathflowai.utils.create_train_val_test">[docs]</a><span class="k">def</span> <span class="nf">create_train_val_test</span><span class="p">(</span><span class="n">train_val_test_pkl</span><span class="p">,</span> <span class="n">input_info_db</span><span class="p">,</span> <span class="n">patch_size</span><span class="p">):</span> |
|
|
705 |
<span class="sd">"""Create dataframe that splits slides into training validation and test.</span> |
|
|
706 |
|
|
|
707 |
<span class="sd"> Parameters</span> |
|
|
708 |
<span class="sd"> ----------</span> |
|
|
709 |
<span class="sd"> train_val_test_pkl:str</span> |
|
|
710 |
<span class="sd"> Pickle for training validation and test slides.</span> |
|
|
711 |
<span class="sd"> input_info_db:str</span> |
|
|
712 |
<span class="sd"> Patch information SQL database.</span> |
|
|
713 |
<span class="sd"> patch_size:int</span> |
|
|
714 |
<span class="sd"> Patch size looking to access.</span> |
|
|
715 |
|
|
|
716 |
<span class="sd"> Returns</span> |
|
|
717 |
<span class="sd"> -------</span> |
|
|
718 |
<span class="sd"> dataframe</span> |
|
|
719 |
<span class="sd"> Train test validation splits.</span> |
|
|
720 |
|
|
|
721 |
<span class="sd"> """</span> |
|
|
722 |
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">train_val_test_pkl</span><span class="p">):</span> |
|
|
723 |
<span class="n">IDs</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_pickle</span><span class="p">(</span><span class="n">train_val_test_pkl</span><span class="p">)</span> |
|
|
724 |
<span class="k">else</span><span class="p">:</span> |
|
|
725 |
<span class="n">conn</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">input_info_db</span><span class="p">)</span> |
|
|
726 |
<span class="n">df</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">read_sql</span><span class="p">(</span><span class="s1">'select * from "</span><span class="si">{}</span><span class="s1">";'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span><span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">)</span> |
|
|
727 |
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> |
|
|
728 |
<span class="n">IDs</span><span class="o">=</span><span class="n">df</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">]</span><span class="o">.</span><span class="n">unique</span><span class="p">()</span> |
|
|
729 |
<span class="n">IDs</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">IDs</span><span class="p">,</span><span class="n">columns</span><span class="o">=</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">])</span> |
|
|
730 |
<span class="n">IDs_train</span><span class="p">,</span> <span class="n">IDs_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">IDs</span><span class="p">)</span> |
|
|
731 |
<span class="n">IDs_train</span><span class="p">,</span> <span class="n">IDs_val</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">IDs_train</span><span class="p">)</span> |
|
|
732 |
<span class="n">IDs_train</span><span class="p">[</span><span class="s1">'set'</span><span class="p">]</span><span class="o">=</span><span class="s1">'train'</span> |
|
|
733 |
<span class="n">IDs_val</span><span class="p">[</span><span class="s1">'set'</span><span class="p">]</span><span class="o">=</span><span class="s1">'val'</span> |
|
|
734 |
<span class="n">IDs_test</span><span class="p">[</span><span class="s1">'set'</span><span class="p">]</span><span class="o">=</span><span class="s1">'test'</span> |
|
|
735 |
<span class="n">IDs</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">IDs_train</span><span class="p">,</span><span class="n">IDs_val</span><span class="p">,</span><span class="n">IDs_test</span><span class="p">])</span> |
|
|
736 |
<span class="n">IDs</span><span class="o">.</span><span class="n">to_pickle</span><span class="p">(</span><span class="n">train_val_test_pkl</span><span class="p">)</span> |
|
|
737 |
<span class="k">return</span> <span class="n">IDs</span></div> |
|
|
738 |
|
|
|
739 |
<span class="k">def</span> <span class="nf">modify_patch_info</span><span class="p">(</span><span class="n">input_info_db</span><span class="o">=</span><span class="s1">'patch_info.db'</span><span class="p">,</span> <span class="n">slide_labels</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(),</span> <span class="n">pos_annotation_class</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span> <span class="n">patch_size</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">segmentation</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">other_annotations</span><span class="o">=</span><span class="p">[],</span> <span class="n">target_segmentation_class</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> <span class="n">target_threshold</span><span class="o">=</span><span class="mf">0.</span><span class="p">,</span> <span class="n">classify_annotations</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> |
|
|
740 |
<span class="sd">"""Modify the patch information to get ready for deep learning, incorporate whole slide labels if needed.</span> |
|
|
741 |
|
|
|
742 |
<span class="sd"> Parameters</span> |
|
|
743 |
<span class="sd"> ----------</span> |
|
|
744 |
<span class="sd"> input_info_db:str</span> |
|
|
745 |
<span class="sd"> SQL DB file.</span> |
|
|
746 |
<span class="sd"> slide_labels:dataframe</span> |
|
|
747 |
<span class="sd"> Dataframe with whole slide labels.</span> |
|
|
748 |
<span class="sd"> pos_annotation_class:str</span> |
|
|
749 |
<span class="sd"> Tissue/annotation label to label with whole slide image label, if not supplied, any slide's patches receive the whole slide label.</span> |
|
|
750 |
<span class="sd"> patch_size:int</span> |
|
|
751 |
<span class="sd"> Patch size.</span> |
|
|
752 |
<span class="sd"> segmentation:bool</span> |
|
|
753 |
<span class="sd"> Segmentation?</span> |
|
|
754 |
<span class="sd"> other_annotations:list</span> |
|
|
755 |
<span class="sd"> Other annotations to access from patch information.</span> |
|
|
756 |
<span class="sd"> target_segmentation_class:int</span> |
|
|
757 |
<span class="sd"> Segmentation class to threshold.</span> |
|
|
758 |
<span class="sd"> target_threshold:float</span> |
|
|
759 |
<span class="sd"> Include patch if patch has target area greater than this.</span> |
|
|
760 |
<span class="sd"> classify_annotations:bool</span> |
|
|
761 |
<span class="sd"> Classifying annotations for pretraining, or final model?</span> |
|
|
762 |
|
|
|
763 |
<span class="sd"> Returns</span> |
|
|
764 |
<span class="sd"> -------</span> |
|
|
765 |
<span class="sd"> dataframe</span> |
|
|
766 |
<span class="sd"> Modified patch information.</span> |
|
|
767 |
|
|
|
768 |
<span class="sd"> """</span> |
|
|
769 |
<span class="n">conn</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">input_info_db</span><span class="p">)</span> |
|
|
770 |
<span class="n">df</span><span class="o">=</span><span class="n">pd</span><span class="o">.</span><span class="n">read_sql</span><span class="p">(</span><span class="s1">'select * from "</span><span class="si">{}</span><span class="s1">";'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">patch_size</span><span class="p">),</span><span class="n">con</span><span class="o">=</span><span class="n">conn</span><span class="p">)</span> |
|
|
771 |
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> |
|
|
772 |
<span class="c1">#print(df)</span> |
|
|
773 |
<span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">drop_duplicates</span><span class="p">()</span> |
|
|
774 |
<span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">],</span><span class="n">slide_labels</span><span class="o">.</span><span class="n">index</span><span class="p">)]</span> |
|
|
775 |
<span class="c1">#print(classify_annotations)</span> |
|
|
776 |
<span class="k">if</span> <span class="ow">not</span> <span class="n">segmentation</span><span class="p">:</span> |
|
|
777 |
<span class="k">if</span> <span class="n">classify_annotations</span><span class="p">:</span> |
|
|
778 |
<span class="n">targets</span><span class="o">=</span><span class="n">df</span><span class="p">[</span><span class="s1">'annotation'</span><span class="p">]</span><span class="o">.</span><span class="n">unique</span><span class="p">()</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> |
|
|
779 |
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">targets</span><span class="p">)</span><span class="o">==</span><span class="mi">1</span><span class="p">:</span> |
|
|
780 |
<span class="n">targets</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[:,</span><span class="mi">5</span><span class="p">:])</span> |
|
|
781 |
<span class="k">else</span><span class="p">:</span> |
|
|
782 |
<span class="n">targets</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">slide_labels</span><span class="p">)</span> |
|
|
783 |
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">pos_annotation_class</span><span class="p">)</span><span class="o">==</span><span class="nb">type</span><span class="p">(</span><span class="s1">''</span><span class="p">):</span> |
|
|
784 |
<span class="n">included_annotations</span> <span class="o">=</span> <span class="p">[</span><span class="n">pos_annotation_class</span><span class="p">]</span> |
|
|
785 |
<span class="k">else</span><span class="p">:</span> |
|
|
786 |
<span class="n">included_annotations</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">pos_annotation_class</span><span class="p">)</span> |
|
|
787 |
<span class="n">included_annotations</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">other_annotations</span><span class="p">)</span> |
|
|
788 |
<span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">isin</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">'annotation'</span><span class="p">],</span><span class="n">included_annotations</span><span class="p">)]</span> |
|
|
789 |
<span class="k">for</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">targets</span><span class="p">:</span> |
|
|
790 |
<span class="n">df</span><span class="p">[</span><span class="n">target</span><span class="p">]</span><span class="o">=</span><span class="mf">0.</span> |
|
|
791 |
<span class="k">for</span> <span class="n">slide</span> <span class="ow">in</span> <span class="n">slide_labels</span><span class="o">.</span><span class="n">index</span><span class="p">:</span> |
|
|
792 |
<span class="n">slide_bool</span><span class="o">=</span><span class="p">((</span><span class="n">df</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">]</span><span class="o">==</span><span class="n">slide</span><span class="p">)</span> <span class="o">&</span> <span class="n">df</span><span class="p">[</span><span class="n">pos_annotation_class</span><span class="p">]</span><span class="o">></span><span class="mf">0.</span><span class="p">)</span> <span class="k">if</span> <span class="n">pos_annotation_class</span> <span class="k">else</span> <span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">]</span><span class="o">==</span><span class="n">slide</span><span class="p">)</span> <span class="c1"># (df['annotation']==pos_annotation_class)</span> |
|
|
793 |
<span class="k">if</span> <span class="n">slide_bool</span><span class="o">.</span><span class="n">sum</span><span class="p">():</span> |
|
|
794 |
<span class="n">df</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">slide_bool</span><span class="p">,</span><span class="n">targets</span><span class="p">]</span> <span class="o">=</span> <span class="n">slide_labels</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">slide</span><span class="p">,</span><span class="n">targets</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="c1">#1.</span> |
|
|
795 |
<span class="n">df</span><span class="p">[</span><span class="s1">'area'</span><span class="p">]</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">vectorize</span><span class="p">(</span><span class="k">lambda</span> <span class="n">i</span><span class="p">:</span> <span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="s1">'annotation'</span><span class="p">]])(</span><span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span> |
|
|
796 |
<span class="k">if</span> <span class="s1">'area'</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="ow">and</span> <span class="n">target_threshold</span><span class="o">></span><span class="mf">0.</span><span class="p">:</span> |
|
|
797 |
<span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">df</span><span class="p">[</span><span class="s1">'area'</span><span class="p">]</span><span class="o">>=</span><span class="n">target_threshold</span><span class="p">]</span> |
|
|
798 |
<span class="k">else</span><span class="p">:</span> |
|
|
799 |
<span class="n">df</span><span class="p">[</span><span class="s1">'target'</span><span class="p">]</span><span class="o">=</span><span class="mf">0.</span> |
|
|
800 |
<span class="k">if</span> <span class="n">target_segmentation_class</span> <span class="o">>=</span><span class="mi">0</span><span class="p">:</span> |
|
|
801 |
<span class="n">df</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">df</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">target_segmentation_class</span><span class="p">)]</span><span class="o">>=</span><span class="n">target_threshold</span><span class="p">]</span> |
|
|
802 |
<span class="k">return</span> <span class="n">df</span> |
|
|
803 |
|
|
|
804 |
<div class="viewcode-block" id="npy2da"><a class="viewcode-back" href="../../index.html#pathflowai.utils.npy2da">[docs]</a><span class="k">def</span> <span class="nf">npy2da</span><span class="p">(</span><span class="n">npy_file</span><span class="p">):</span> |
|
|
805 |
<span class="sd">"""Numpy to dask array.</span> |
|
|
806 |
|
|
|
807 |
<span class="sd"> Parameters</span> |
|
|
808 |
<span class="sd"> ----------</span> |
|
|
809 |
<span class="sd"> npy_file:str</span> |
|
|
810 |
<span class="sd"> Input npy file.</span> |
|
|
811 |
|
|
|
812 |
<span class="sd"> Returns</span> |
|
|
813 |
<span class="sd"> -------</span> |
|
|
814 |
<span class="sd"> dask.array</span> |
|
|
815 |
<span class="sd"> Converted numpy array to dask.</span> |
|
|
816 |
|
|
|
817 |
<span class="sd"> """</span> |
|
|
818 |
<span class="k">return</span> <span class="n">da</span><span class="o">.</span><span class="n">from_array</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">npy_file</span><span class="p">,</span> <span class="n">mmap_mode</span> <span class="o">=</span> <span class="s1">'r+'</span><span class="p">))</span></div> |
|
|
819 |
|
|
|
820 |
<div class="viewcode-block" id="grab_interior_points"><a class="viewcode-back" href="../../index.html#pathflowai.utils.grab_interior_points">[docs]</a><span class="k">def</span> <span class="nf">grab_interior_points</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">img_size</span><span class="p">,</span> <span class="n">annotations</span><span class="o">=</span><span class="p">[]):</span> |
|
|
821 |
<span class="sd">"""Deprecated."""</span> |
|
|
822 |
<span class="n">interior_point_dict</span> <span class="o">=</span> <span class="p">{}</span> |
|
|
823 |
<span class="k">for</span> <span class="n">annotation</span> <span class="ow">in</span> <span class="n">annotations</span><span class="p">:</span> |
|
|
824 |
<span class="k">try</span><span class="p">:</span> |
|
|
825 |
<span class="n">interior_point_dict</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span> <span class="o">=</span> <span class="n">parse_coord_return_boxes</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation</span><span class="p">,</span> <span class="n">return_coords</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="c1"># boxes2interior(img_size,</span> |
|
|
826 |
<span class="k">except</span><span class="p">:</span> |
|
|
827 |
<span class="n">interior_point_dict</span><span class="p">[</span><span class="n">annotation</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span><span class="c1">#np.array([[],[]])</span> |
|
|
828 |
<span class="k">return</span> <span class="n">interior_point_dict</span></div> |
|
|
829 |
|
|
|
830 |
<div class="viewcode-block" id="boxes2interior"><a class="viewcode-back" href="../../index.html#pathflowai.utils.boxes2interior">[docs]</a><span class="k">def</span> <span class="nf">boxes2interior</span><span class="p">(</span><span class="n">img_size</span><span class="p">,</span> <span class="n">polygons</span><span class="p">):</span> |
|
|
831 |
<span class="sd">"""Deprecated."""</span> |
|
|
832 |
<span class="n">img</span> <span class="o">=</span> <span class="n">Image</span><span class="o">.</span><span class="n">new</span><span class="p">(</span><span class="s1">'L'</span><span class="p">,</span> <span class="n">img_size</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span> |
|
|
833 |
<span class="k">for</span> <span class="n">polygon</span> <span class="ow">in</span> <span class="n">polygons</span><span class="p">:</span> |
|
|
834 |
<span class="n">ImageDraw</span><span class="o">.</span><span class="n">Draw</span><span class="p">(</span><span class="n">img</span><span class="p">)</span><span class="o">.</span><span class="n">polygon</span><span class="p">(</span><span class="n">polygon</span><span class="p">,</span> <span class="n">outline</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">fill</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> |
|
|
835 |
<span class="n">mask</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">img</span><span class="p">)</span><span class="o">.</span><span class="n">nonzero</span><span class="p">()</span> |
|
|
836 |
<span class="c1">#mask = (np.ones(len(mask[0])),mask)</span> |
|
|
837 |
<span class="k">return</span> <span class="n">mask</span></div> |
|
|
838 |
|
|
|
839 |
<div class="viewcode-block" id="parse_coord_return_boxes"><a class="viewcode-back" href="../../index.html#pathflowai.utils.parse_coord_return_boxes">[docs]</a><span class="k">def</span> <span class="nf">parse_coord_return_boxes</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation_name</span> <span class="o">=</span> <span class="s1">''</span><span class="p">,</span> <span class="n">return_coords</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span> |
|
|
840 |
<span class="sd">"""Get list of shapely objects for each annotation in the XML object.</span> |
|
|
841 |
|
|
|
842 |
<span class="sd"> Parameters</span> |
|
|
843 |
<span class="sd"> ----------</span> |
|
|
844 |
<span class="sd"> xml_file:str</span> |
|
|
845 |
<span class="sd"> Annotation file.</span> |
|
|
846 |
<span class="sd"> annotation_name:str</span> |
|
|
847 |
<span class="sd"> Name of xml annotation.</span> |
|
|
848 |
<span class="sd"> return_coords:bool</span> |
|
|
849 |
<span class="sd"> Just return list of coords over shapes.</span> |
|
|
850 |
|
|
|
851 |
<span class="sd"> Returns</span> |
|
|
852 |
<span class="sd"> -------</span> |
|
|
853 |
<span class="sd"> list</span> |
|
|
854 |
<span class="sd"> List of shapely objects.</span> |
|
|
855 |
|
|
|
856 |
<span class="sd"> """</span> |
|
|
857 |
<span class="n">boxes</span> <span class="o">=</span> <span class="p">[]</span> |
|
|
858 |
<span class="n">xml_data</span> <span class="o">=</span> <span class="n">BeautifulSoup</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">xml_file</span><span class="p">),</span><span class="s1">'html'</span><span class="p">)</span> |
|
|
859 |
<span class="c1">#print(xml_data.findAll('annotation'))</span> |
|
|
860 |
<span class="c1">#print(xml_data.findAll('Annotation'))</span> |
|
|
861 |
<span class="k">for</span> <span class="n">annotation</span> <span class="ow">in</span> <span class="n">xml_data</span><span class="o">.</span><span class="n">findAll</span><span class="p">(</span><span class="s1">'annotation'</span><span class="p">):</span> |
|
|
862 |
<span class="k">if</span> <span class="n">annotation</span><span class="p">[</span><span class="s1">'partofgroup'</span><span class="p">]</span> <span class="o">==</span> <span class="n">annotation_name</span><span class="p">:</span> |
|
|
863 |
<span class="k">for</span> <span class="n">coordinates</span> <span class="ow">in</span> <span class="n">annotation</span><span class="o">.</span><span class="n">findAll</span><span class="p">(</span><span class="s1">'coordinates'</span><span class="p">):</span> |
|
|
864 |
<span class="c1"># FIXME may need to change x and y coordinates</span> |
|
|
865 |
<span class="n">coords</span> <span class="o">=</span> <span class="p">[(</span><span class="n">coordinate</span><span class="p">[</span><span class="s1">'x'</span><span class="p">],</span><span class="n">coordinate</span><span class="p">[</span><span class="s1">'y'</span><span class="p">])</span> <span class="k">for</span> <span class="n">coordinate</span> <span class="ow">in</span> <span class="n">coordinates</span><span class="o">.</span><span class="n">findAll</span><span class="p">(</span><span class="s1">'coordinate'</span><span class="p">)]</span> |
|
|
866 |
<span class="k">if</span> <span class="n">return_coords</span><span class="p">:</span> |
|
|
867 |
<span class="n">boxes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">coords</span><span class="p">)</span> |
|
|
868 |
<span class="k">else</span><span class="p">:</span> |
|
|
869 |
<span class="n">boxes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">Polygon</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">coords</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">float</span><span class="p">)))</span> |
|
|
870 |
<span class="k">return</span> <span class="n">boxes</span></div> |
|
|
871 |
|
|
|
872 |
<div class="viewcode-block" id="is_coords_in_box"><a class="viewcode-back" href="../../index.html#pathflowai.utils.is_coords_in_box">[docs]</a><span class="k">def</span> <span class="nf">is_coords_in_box</span><span class="p">(</span><span class="n">coords</span><span class="p">,</span><span class="n">patch_size</span><span class="p">,</span><span class="n">boxes</span><span class="p">):</span> |
|
|
873 |
<span class="sd">"""Get area of annotation in patch.</span> |
|
|
874 |
|
|
|
875 |
<span class="sd"> Parameters</span> |
|
|
876 |
<span class="sd"> ----------</span> |
|
|
877 |
<span class="sd"> coords:array</span> |
|
|
878 |
<span class="sd"> X,Y coordinates of patch.</span> |
|
|
879 |
<span class="sd"> patch_size:int</span> |
|
|
880 |
<span class="sd"> Patch size.</span> |
|
|
881 |
<span class="sd"> boxes:list</span> |
|
|
882 |
<span class="sd"> Shapely objects for annotations.</span> |
|
|
883 |
|
|
|
884 |
<span class="sd"> Returns</span> |
|
|
885 |
<span class="sd"> -------</span> |
|
|
886 |
<span class="sd"> float</span> |
|
|
887 |
<span class="sd"> Area of annotation type.</span> |
|
|
888 |
|
|
|
889 |
<span class="sd"> """</span> |
|
|
890 |
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">boxes</span><span class="p">):</span> |
|
|
891 |
<span class="n">points</span><span class="o">=</span><span class="n">Polygon</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span><span class="mi">0</span><span class="p">],[</span><span class="mi">1</span><span class="p">,</span><span class="mi">0</span><span class="p">],[</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">],[</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">]])</span><span class="o">*</span><span class="n">patch_size</span><span class="o">+</span><span class="n">coords</span><span class="p">)</span> |
|
|
892 |
<span class="n">area</span><span class="o">=</span><span class="n">points</span><span class="o">.</span><span class="n">intersection</span><span class="p">(</span><span class="n">boxes</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">area</span><span class="o">/</span><span class="nb">float</span><span class="p">(</span><span class="n">points</span><span class="o">.</span><span class="n">area</span><span class="p">)</span><span class="c1">#any(list(map(lambda x: x.intersects(points),boxes)))#return_image_coord(nx=nx,ny=ny,xi=xi,yi=yi, output_point=output_point)</span> |
|
|
893 |
<span class="k">else</span><span class="p">:</span> |
|
|
894 |
<span class="n">area</span><span class="o">=</span><span class="mf">0.</span> |
|
|
895 |
<span class="k">return</span> <span class="n">area</span></div> |
|
|
896 |
|
|
|
897 |
<div class="viewcode-block" id="is_image_in_boxes"><a class="viewcode-back" href="../../index.html#pathflowai.utils.is_image_in_boxes">[docs]</a><span class="k">def</span> <span class="nf">is_image_in_boxes</span><span class="p">(</span><span class="n">image_coord_dict</span><span class="p">,</span> <span class="n">boxes</span><span class="p">):</span> |
|
|
898 |
<span class="sd">"""Find if image intersects with annotations.</span> |
|
|
899 |
|
|
|
900 |
<span class="sd"> Parameters</span> |
|
|
901 |
<span class="sd"> ----------</span> |
|
|
902 |
<span class="sd"> image_coord_dict:dict</span> |
|
|
903 |
<span class="sd"> Dictionary of patches.</span> |
|
|
904 |
<span class="sd"> boxes:list</span> |
|
|
905 |
<span class="sd"> Shapely annotation shapes.</span> |
|
|
906 |
|
|
|
907 |
<span class="sd"> Returns</span> |
|
|
908 |
<span class="sd"> -------</span> |
|
|
909 |
<span class="sd"> dict</span> |
|
|
910 |
<span class="sd"> Dictionary of whether image intersects with any of the annotations.</span> |
|
|
911 |
|
|
|
912 |
<span class="sd"> """</span> |
|
|
913 |
<span class="k">return</span> <span class="p">{</span><span class="n">image</span><span class="p">:</span> <span class="nb">any</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">intersects</span><span class="p">(</span><span class="n">image_coord_dict</span><span class="p">[</span><span class="n">image</span><span class="p">]),</span><span class="n">boxes</span><span class="p">)))</span> <span class="k">for</span> <span class="n">image</span> <span class="ow">in</span> <span class="n">image_coord_dict</span><span class="p">}</span></div> |
|
|
914 |
|
|
|
915 |
<div class="viewcode-block" id="images2coord_dict"><a class="viewcode-back" href="../../index.html#pathflowai.utils.images2coord_dict">[docs]</a><span class="k">def</span> <span class="nf">images2coord_dict</span><span class="p">(</span><span class="n">images</span><span class="p">,</span> <span class="n">output_point</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> |
|
|
916 |
<span class="sd">"""Deprecated"""</span> |
|
|
917 |
<span class="k">return</span> <span class="p">{</span><span class="n">image</span><span class="p">:</span> <span class="n">image2coords</span><span class="p">(</span><span class="n">image</span><span class="p">,</span> <span class="n">output_point</span><span class="p">)</span> <span class="k">for</span> <span class="n">image</span> <span class="ow">in</span> <span class="n">images</span><span class="p">}</span></div> |
|
|
918 |
|
|
|
919 |
<div class="viewcode-block" id="dir2images"><a class="viewcode-back" href="../../index.html#pathflowai.utils.dir2images">[docs]</a><span class="k">def</span> <span class="nf">dir2images</span><span class="p">(</span><span class="n">image_dir</span><span class="p">):</span> |
|
|
920 |
<span class="sd">"""Deprecated"""</span> |
|
|
921 |
<span class="k">return</span> <span class="n">glob</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">image_dir</span><span class="p">,</span><span class="s1">'*.jpg'</span><span class="p">))</span></div> |
|
|
922 |
|
|
|
923 |
<div class="viewcode-block" id="return_image_in_boxes_dict"><a class="viewcode-back" href="../../index.html#pathflowai.utils.return_image_in_boxes_dict">[docs]</a><span class="k">def</span> <span class="nf">return_image_in_boxes_dict</span><span class="p">(</span><span class="n">image_dir</span><span class="p">,</span> <span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span> |
|
|
924 |
<span class="sd">"""Deprecated"""</span> |
|
|
925 |
<span class="n">boxes</span> <span class="o">=</span> <span class="n">parse_coord_return_boxes</span><span class="p">(</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation</span><span class="p">)</span> |
|
|
926 |
<span class="n">images</span> <span class="o">=</span> <span class="n">dir2images</span><span class="p">(</span><span class="n">image_dir</span><span class="p">)</span> |
|
|
927 |
<span class="n">coord_dict</span> <span class="o">=</span> <span class="n">images2coord_dict</span><span class="p">(</span><span class="n">images</span><span class="p">)</span> |
|
|
928 |
<span class="k">return</span> <span class="n">is_image_in_boxes</span><span class="p">(</span><span class="n">image_coord_dict</span><span class="o">=</span><span class="n">coord_dict</span><span class="p">,</span><span class="n">boxes</span><span class="o">=</span><span class="n">boxes</span><span class="p">)</span></div> |
|
|
929 |
|
|
|
930 |
<div class="viewcode-block" id="image2coords"><a class="viewcode-back" href="../../index.html#pathflowai.utils.image2coords">[docs]</a><span class="k">def</span> <span class="nf">image2coords</span><span class="p">(</span><span class="n">image_file</span><span class="p">,</span> <span class="n">output_point</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> |
|
|
931 |
<span class="sd">"""Deprecated."""</span> |
|
|
932 |
<span class="n">nx</span><span class="p">,</span><span class="n">ny</span><span class="p">,</span><span class="n">yi</span><span class="p">,</span><span class="n">xi</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">image_file</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="mi">1</span><span class="p">:])</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> |
|
|
933 |
<span class="k">return</span> <span class="n">return_image_coord</span><span class="p">(</span><span class="n">nx</span><span class="o">=</span><span class="n">nx</span><span class="p">,</span><span class="n">ny</span><span class="o">=</span><span class="n">ny</span><span class="p">,</span><span class="n">xi</span><span class="o">=</span><span class="n">xi</span><span class="p">,</span><span class="n">yi</span><span class="o">=</span><span class="n">yi</span><span class="p">,</span> <span class="n">output_point</span><span class="o">=</span><span class="n">output_point</span><span class="p">)</span></div> |
|
|
934 |
|
|
|
935 |
<div class="viewcode-block" id="retain_images"><a class="viewcode-back" href="../../index.html#pathflowai.utils.retain_images">[docs]</a><span class="k">def</span> <span class="nf">retain_images</span><span class="p">(</span><span class="n">image_dir</span><span class="p">,</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation</span><span class="o">=</span><span class="s1">''</span><span class="p">):</span> |
|
|
936 |
<span class="sd">"""Deprecated"""</span> |
|
|
937 |
<span class="n">image_in_boxes_dict</span><span class="o">=</span><span class="n">return_image_in_boxes_dict</span><span class="p">(</span><span class="n">image_dir</span><span class="p">,</span><span class="n">xml_file</span><span class="p">,</span> <span class="n">annotation</span><span class="p">)</span> |
|
|
938 |
<span class="k">return</span> <span class="p">[</span><span class="n">img</span> <span class="k">for</span> <span class="n">img</span> <span class="ow">in</span> <span class="n">image_in_boxes_dict</span> <span class="k">if</span> <span class="n">image_in_boxes_dict</span><span class="p">[</span><span class="n">img</span><span class="p">]]</span></div> |
|
|
939 |
|
|
|
940 |
<div class="viewcode-block" id="return_image_coord"><a class="viewcode-back" href="../../index.html#pathflowai.utils.return_image_coord">[docs]</a><span class="k">def</span> <span class="nf">return_image_coord</span><span class="p">(</span><span class="n">nx</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span><span class="n">ny</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span><span class="n">xl</span><span class="o">=</span><span class="mi">3333</span><span class="p">,</span><span class="n">yl</span><span class="o">=</span><span class="mi">3333</span><span class="p">,</span><span class="n">xi</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span><span class="n">yi</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span><span class="n">xc</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span><span class="n">yc</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span><span class="n">dimx</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span><span class="n">dimy</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">output_point</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> |
|
|
941 |
<span class="sd">"""Deprecated"""</span> |
|
|
942 |
<span class="k">if</span> <span class="n">output_point</span><span class="p">:</span> |
|
|
943 |
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">xc</span><span class="p">,</span><span class="n">yc</span><span class="p">])</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">nx</span><span class="o">*</span><span class="n">xl</span><span class="o">+</span><span class="n">xi</span><span class="o">+</span><span class="n">dimx</span><span class="o">/</span><span class="mi">2</span><span class="p">,</span><span class="n">ny</span><span class="o">*</span><span class="n">yl</span><span class="o">+</span><span class="n">yi</span><span class="o">+</span><span class="n">dimy</span><span class="o">/</span><span class="mi">2</span><span class="p">])</span> |
|
|
944 |
<span class="k">else</span><span class="p">:</span> |
|
|
945 |
<span class="n">static_point</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">nx</span><span class="o">*</span><span class="n">xl</span><span class="o">+</span><span class="n">xi</span><span class="p">,</span><span class="n">ny</span><span class="o">*</span><span class="n">yl</span><span class="o">+</span><span class="n">yi</span><span class="p">])</span> |
|
|
946 |
<span class="n">points</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">xc</span><span class="p">,</span><span class="n">yc</span><span class="p">])</span><span class="o">*</span><span class="p">(</span><span class="n">static_point</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">new_point</span><span class="p">)))</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> <span class="k">for</span> <span class="n">new_point</span> <span class="ow">in</span> <span class="p">[[</span><span class="mi">0</span><span class="p">,</span><span class="mi">0</span><span class="p">],[</span><span class="n">dimx</span><span class="p">,</span><span class="mi">0</span><span class="p">],[</span><span class="n">dimx</span><span class="p">,</span><span class="n">dimy</span><span class="p">],[</span><span class="mi">0</span><span class="p">,</span><span class="n">dimy</span><span class="p">]]])</span> |
|
|
947 |
<span class="k">return</span> <span class="n">Polygon</span><span class="p">(</span><span class="n">points</span><span class="p">)</span><span class="c1">#Point(*((np.array([xc,yc])*np.array([nx*xl+xi+dimx/2,ny*yl+yi+dimy/2])).tolist())) # [::-1]</span></div> |
|
|
948 |
|
|
|
949 |
<div class="viewcode-block" id="fix_name"><a class="viewcode-back" href="../../index.html#pathflowai.utils.fix_name">[docs]</a><span class="k">def</span> <span class="nf">fix_name</span><span class="p">(</span><span class="n">basename</span><span class="p">):</span> |
|
|
950 |
<span class="sd">"""Fixes illegitimate basename, deprecated."""</span> |
|
|
951 |
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">basename</span><span class="p">)</span> <span class="o"><</span> <span class="mi">3</span><span class="p">:</span> |
|
|
952 |
<span class="k">return</span> <span class="s1">'</span><span class="si">{}</span><span class="s1">0</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">*</span><span class="n">basename</span><span class="p">)</span> |
|
|
953 |
<span class="k">return</span> <span class="n">basename</span></div> |
|
|
954 |
|
|
|
955 |
<div class="viewcode-block" id="fix_names"><a class="viewcode-back" href="../../index.html#pathflowai.utils.fix_names">[docs]</a><span class="k">def</span> <span class="nf">fix_names</span><span class="p">(</span><span class="n">file_dir</span><span class="p">):</span> |
|
|
956 |
<span class="sd">"""Fixes basenames, deprecated."""</span> |
|
|
957 |
<span class="k">for</span> <span class="n">filename</span> <span class="ow">in</span> <span class="n">glob</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span><span class="n">join</span><span class="p">(</span><span class="n">file_dir</span><span class="p">,</span><span class="s1">'*'</span><span class="p">)):</span> |
|
|
958 |
<span class="n">basename</span> <span class="o">=</span> <span class="n">filename</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> |
|
|
959 |
<span class="n">basename</span><span class="p">,</span> <span class="n">suffix</span> <span class="o">=</span> <span class="n">basename</span><span class="p">[:</span><span class="n">basename</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)],</span> <span class="n">basename</span><span class="p">[</span><span class="n">basename</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">'.'</span><span class="p">):]</span> |
|
|
960 |
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">basename</span><span class="p">)</span> <span class="o"><</span> <span class="mi">3</span><span class="p">:</span> |
|
|
961 |
<span class="n">new_filename</span><span class="o">=</span><span class="n">join</span><span class="p">(</span><span class="n">file_dir</span><span class="p">,</span><span class="s1">'</span><span class="si">{}</span><span class="s1">0</span><span class="si">{}{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">*</span><span class="n">basename</span><span class="p">,</span><span class="n">suffix</span><span class="p">))</span> |
|
|
962 |
<span class="nb">print</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span><span class="n">new_filename</span><span class="p">)</span> |
|
|
963 |
<span class="n">subprocess</span><span class="o">.</span><span class="n">call</span><span class="p">(</span><span class="s1">'mv </span><span class="si">{}</span><span class="s1"> </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span><span class="n">new_filename</span><span class="p">),</span><span class="n">shell</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div> |
|
|
964 |
|
|
|
965 |
<span class="c1">#######</span> |
|
|
966 |
|
|
|
967 |
<span class="c1">#@pysnooper.snoop('seg2npy.log')</span> |
|
|
968 |
<div class="viewcode-block" id="segmentation_predictions2npy"><a class="viewcode-back" href="../../index.html#pathflowai.utils.segmentation_predictions2npy">[docs]</a><span class="k">def</span> <span class="nf">segmentation_predictions2npy</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">patch_info</span><span class="p">,</span> <span class="n">segmentation_map</span><span class="p">,</span> <span class="n">npy_output</span><span class="p">):</span> |
|
|
969 |
<span class="sd">"""Convert segmentation predictions from model to numpy masks.</span> |
|
|
970 |
|
|
|
971 |
<span class="sd"> Parameters</span> |
|
|
972 |
<span class="sd"> ----------</span> |
|
|
973 |
<span class="sd"> y_pred:list</span> |
|
|
974 |
<span class="sd"> List of patch segmentation masks</span> |
|
|
975 |
<span class="sd"> patch_info:dataframe</span> |
|
|
976 |
<span class="sd"> Patch information from DB.</span> |
|
|
977 |
<span class="sd"> segmentation_map:array</span> |
|
|
978 |
<span class="sd"> Existing segmentation mask.</span> |
|
|
979 |
<span class="sd"> npy_output:str</span> |
|
|
980 |
<span class="sd"> Output npy file.</span> |
|
|
981 |
<span class="sd"> """</span> |
|
|
982 |
<span class="n">segmentation_map</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">segmentation_map</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="o">-</span><span class="mi">2</span><span class="p">:])</span> |
|
|
983 |
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">patch_info</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]):</span> |
|
|
984 |
<span class="n">patch_info_i</span> <span class="o">=</span> <span class="n">patch_info</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> |
|
|
985 |
<span class="n">ID</span> <span class="o">=</span> <span class="n">patch_info_i</span><span class="p">[</span><span class="s1">'ID'</span><span class="p">]</span> |
|
|
986 |
<span class="n">xs</span> <span class="o">=</span> <span class="n">patch_info_i</span><span class="p">[</span><span class="s1">'x'</span><span class="p">]</span> |
|
|
987 |
<span class="n">ys</span> <span class="o">=</span> <span class="n">patch_info_i</span><span class="p">[</span><span class="s1">'y'</span><span class="p">]</span> |
|
|
988 |
<span class="n">patch_size</span> <span class="o">=</span> <span class="n">patch_info_i</span><span class="p">[</span><span class="s1">'patch_size'</span><span class="p">]</span> |
|
|
989 |
<span class="n">prediction</span><span class="o">=</span><span class="n">y_pred</span><span class="p">[</span><span class="n">i</span><span class="p">,</span><span class="o">...</span><span class="p">]</span> |
|
|
990 |
<span class="n">pred_shape</span><span class="o">=</span><span class="n">prediction</span><span class="o">.</span><span class="n">shape</span> |
|
|
991 |
<span class="n">segmentation_map</span><span class="p">[</span><span class="n">xs</span><span class="p">:</span><span class="n">xs</span><span class="o">+</span><span class="n">patch_size</span><span class="p">,</span><span class="n">ys</span><span class="p">:</span><span class="n">ys</span><span class="o">+</span><span class="n">patch_size</span><span class="p">]</span> <span class="o">=</span> <span class="n">prediction</span> |
|
|
992 |
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">npy_output</span><span class="p">[:</span><span class="n">npy_output</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)],</span><span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
|
|
993 |
<span class="n">np</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">npy_output</span><span class="p">,</span><span class="n">segmentation_map</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">))</span></div> |
|
|
994 |
</pre></div> |
|
|
995 |
|
|
|
996 |
</div> |
|
|
997 |
|
|
|
998 |
</div> |
|
|
999 |
<footer> |
|
|
1000 |
|
|
|
1001 |
|
|
|
1002 |
<hr/> |
|
|
1003 |
|
|
|
1004 |
<div role="contentinfo"> |
|
|
1005 |
<p> |
|
|
1006 |
© Copyright 2019, Joshua Levy |
|
|
1007 |
|
|
|
1008 |
</p> |
|
|
1009 |
</div> |
|
|
1010 |
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. |
|
|
1011 |
|
|
|
1012 |
</footer> |
|
|
1013 |
|
|
|
1014 |
</div> |
|
|
1015 |
</div> |
|
|
1016 |
|
|
|
1017 |
</section> |
|
|
1018 |
|
|
|
1019 |
</div> |
|
|
1020 |
|
|
|
1021 |
|
|
|
1022 |
|
|
|
1023 |
<script type="text/javascript"> |
|
|
1024 |
jQuery(function () { |
|
|
1025 |
SphinxRtdTheme.Navigation.enable(true); |
|
|
1026 |
}); |
|
|
1027 |
</script> |
|
|
1028 |
|
|
|
1029 |
|
|
|
1030 |
|
|
|
1031 |
|
|
|
1032 |
|
|
|
1033 |
|
|
|
1034 |
</body> |
|
|
1035 |
</html> |