<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1" />
<meta name="generator" content="pdoc 0.8.1" />
<title>VITAE.preprocess API documentation</title>
<meta name="description" content="" />
<link href='https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.0/normalize.min.css' rel='stylesheet'>
<link href='https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/8.0.0/sanitize.min.css' rel='stylesheet'>
<link href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css" rel="stylesheet">
<style>.flex{display:flex !important}body{line-height:1.5em}#content{padding:20px}#sidebar{padding:30px;overflow:hidden}#sidebar > *:last-child{margin-bottom:2cm}.http-server-breadcrumbs{font-size:130%;margin:0 0 15px 0}#footer{font-size:.75em;padding:5px 30px;border-top:1px solid #ddd;text-align:right}#footer p{margin:0 0 0 1em;display:inline-block}#footer p:last-child{margin-right:30px}h1,h2,h3,h4,h5{font-weight:300}h1{font-size:2.5em;line-height:1.1em}h2{font-size:1.75em;margin:1em 0 .50em 0}h3{font-size:1.4em;margin:25px 0 10px 0}h4{margin:0;font-size:105%}a{color:#058;text-decoration:none;transition:color .3s ease-in-out}a:hover{color:#e82}.title code{font-weight:bold}h2[id^="header-"]{margin-top:2em}.ident{color:#900}pre code{background:#f8f8f8;font-size:.8em;line-height:1.4em}code{background:#f2f2f1;padding:1px 4px;overflow-wrap:break-word}h1 code{background:transparent}pre{background:#f8f8f8;border:0;border-top:1px solid #ccc;border-bottom:1px solid #ccc;margin:1em 0;padding:1ex}#http-server-module-list{display:flex;flex-flow:column}#http-server-module-list div{display:flex}#http-server-module-list dt{min-width:10%}#http-server-module-list p{margin-top:0}.toc ul,#index{list-style-type:none;margin:0;padding:0}#index code{background:transparent}#index h3{border-bottom:1px solid #ddd}#index ul{padding:0}#index h4{margin-top:.6em;font-weight:bold}@media (min-width:200ex){#index .two-column{column-count:2}}@media (min-width:300ex){#index .two-column{column-count:3}}dl{margin-bottom:2em}dl dl:last-child{margin-bottom:4em}dd{margin:0 0 1em 3em}#header-classes + dl > dd{margin-bottom:3em}dd dd{margin-left:2em}dd p{margin:10px 0}.name{background:#eee;font-weight:bold;font-size:.85em;padding:5px 10px;display:inline-block;min-width:40%}.name:hover{background:#e0e0e0}.name > span:first-child{white-space:nowrap}.name.class > span:nth-child(2){margin-left:.4em}.inherited{color:#999;border-left:5px solid #eee;padding-left:1em}.inheritance em{font-style:normal;font-weight:bold}.desc h2{font-weight:400;font-size:1.25em}.desc h3{font-size:1em}.desc dt code{background:inherit}.source summary,.git-link-div{color:#666;text-align:right;font-weight:400;font-size:.8em;text-transform:uppercase}.source summary > *{white-space:nowrap;cursor:pointer}.git-link{color:inherit;margin-left:1em}.source pre{max-height:500px;overflow:auto;margin:0}.source pre code{font-size:12px;overflow:visible}.hlist{list-style:none}.hlist li{display:inline}.hlist li:after{content:',\2002'}.hlist li:last-child:after{content:none}.hlist .hlist{display:inline;padding-left:1em}img{max-width:100%}.admonition{padding:.1em .5em;margin-bottom:1em}.admonition-title{font-weight:bold}.admonition.note,.admonition.info,.admonition.important{background:#aef}.admonition.todo,.admonition.versionadded,.admonition.tip,.admonition.hint{background:#dfd}.admonition.warning,.admonition.versionchanged,.admonition.deprecated{background:#fd4}.admonition.error,.admonition.danger,.admonition.caution{background:lightpink}</style>
<style media="screen and (min-width: 700px)">@media screen and (min-width:700px){#sidebar{width:30%;height:100vh;overflow:auto;position:sticky;top:0}#content{width:70%;max-width:100ch;padding:3em 4em;border-left:1px solid #ddd}pre code{font-size:1em}.item .name{font-size:1em}main{display:flex;flex-direction:row-reverse;justify-content:flex-end}.toc ul ul,#index ul{padding-left:1.5em}.toc > ul > li{margin-top:.5em}}</style>
<style media="print">@media print{#sidebar h1{page-break-before:always}.source{display:none}}@media print{*{background:transparent !important;color:#000 !important;box-shadow:none !important;text-shadow:none !important}a[href]:after{content:" (" attr(href) ")";font-size:90%}a[href][title]:after{content:none}abbr[title]:after{content:" (" attr(title) ")"}.ir a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}@page{margin:0.5cm}p,h2,h3{orphans:3;widows:3}h1,h2,h3,h4,h5,h6{page-break-after:avoid}}</style>
<script async src='https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS_CHTML'></script>
</head>
<body>
<main>
<article id="content">
<header>
<h1 class="title">Module <code>VITAE.preprocess</code></h1>
</header>
<section id="section-intro">
<details class="source">
<summary>
<span>Expand source code</span>
</summary>
<pre><code class="python"># -*- coding: utf-8 -*-
from typing import Optional
import numpy as np
import pandas as pd
from skmisc import loess
from sklearn import preprocessing
import warnings
from sklearn.decomposition import PCA
from VITAE.utils import _check_expression, _check_variability
def normalize_gene_expression(x, K : float = 1e4, transform_fn : str = 'log'):
'''Normalize the gene expression counts for each cell by the total expression counts,
divide this by a size scale factor, which is determined by total counts and a constant K
then log-transforms the result.
Parameters
----------
x : np.array
\([N, G^{raw}]\) The raw count data.
K : float, optional
The normalizing constant.
transform_fn : str, optional
Either 'log' or 'sqrt'.
Returns
----------
x_normalized : np.array
\([N, G^{raw}]\) The log-normalized data.
scale_factor : np.array
\([N, ]\) The scale factors.
'''
scale_factor = np.sum(x,axis=1, keepdims=True)/K
if transform_fn=='log':
x_normalized = np.log(x/scale_factor + 1)
else:
x_normalized = np.where(x>0, np.sqrt(x/scale_factor), 0)
print('min normalized value: ' + str(np.min(x_normalized)))
print('max normalized value: ' + str(np.max(x_normalized)))
return x_normalized, scale_factor
def feature_select(x, gene_num : int = 2000):
'''Select highly variable genes (HVGs)
(See [Stuart *et al*, (2019)](https://www.nature.com/articles/nbt.4096) and its early version [preprint](https://www.biorxiv.org/content/10.1101/460147v1.full.pdf)
Page 12-13: Data preprocessing - Feature selection for individual datasets).
Parameters
----------
x : np.array
\([N, G^{raw}]\) The raw count data.
gene_num : int, optional
The number of genes to retain.
Returns
----------
x : np.array
\([N, G]\) The count data after gene selection.
index : np.array
\([G, ]\) The selected index of genes.
'''
n, p = x.shape
# mean and variance of each gene of the unnormalized data
mean, var = np.mean(x, axis=0), np.var(x, axis=0, ddof=1)
# model log10(var)~log10(mean) by local fitting of polynomials of degree 2
loess_model = loess.loess(np.log10(mean), np.log10(var),
span = 0.3, degree = 2, family='gaussian'
)
loess_model.fit()
fitted = loess_model.outputs.fitted_values
# standardized feature
z = (x - mean)/np.sqrt(10**fitted)
# clipped the standardized features to remove outliers
z = np.clip(z, -np.inf, np.sqrt(n))
# the variance of standardized features across all cells represents a measure of
# single cell dispersion after controlling for mean expression
feature_score = np.sum(z**2, axis=0)/(n-1)
# feature selection
index = feature_score.argsort()[::-1][0:gene_num]
return x[:, index], index
def preprocess(adata = None, processed: bool = False, dimred: bool = False,
x = None, c = None, label_names = None, raw_cell_names = None, raw_gene_names = None,
K: float = 1e4, transform_fn: str = 'log', gene_num: int = 2000, data_type: str = 'UMI',
npc: int = 64, random_state=0):
'''Preprocess count data.
Parameters
----------
adata : AnnData, optional
The scanpy object.
processed : boolean
Whether adata has been processed.
dimred : boolean
Whether the processed adata is after dimension reduction.
x : np.array, optional
\([N^{raw}, G^{raw}]\) The raw count matrix.
c : np.array
\([N^{raw}, s]\) The covariate matrix.
label_names : np.array
\([N^{raw}, ]\) The true or estimated cell types.
raw_cell_names : np.array
\([N^{raw}, ]\) The names of cells.
raw_gene_names : np.array
\([G^{raw}, ]\) The names of genes.
K : int, optional
The normalizing constant.
transform_fn : str
The transform function used to normalize the gene expression after scaling. Either 'log' or 'sqrt'.
gene_num : int, optional
The number of genes to retain.
data_type : str, optional
'UMI', 'non-UMI', or 'Gaussian'.
npc : int, optional
The number of PCs to retain, only used if `data_type='Gaussian'`.
random_state : int, optional
The random state for PCA. With different random states, the resulted PCA scores are slightly different.
Returns
----------
x_normalized : np.array
\([N, G]\) The preprocessed matrix.
expression : np.array
\([N, G^{raw}]\) The expression matrix after log-normalization and before scaling.
x : np.array
\([N, G]\) The raw count matrix after gene selections.
c : np.array
\([N, s]\) The covariates.
cell_names : np.array
\([N, ]\) The cell names.
gene_names : np.array
\([G^{raw}, ]\) The gene names.
selected_gene_names :
\([G, ]\) The selected gene names.
scale_factor :
\([N, ]\) The scale factors.
labels : np.array
\([N, ]\) The encoded labels.
label_names : np.array
\([N, ]\) The label names.
le : sklearn.preprocessing.LabelEncoder
The label encoder.
gene_scalar : sklearn.preprocessing.StandardScaler
The gene scaler.
'''
# if input is a scanpy data
if adata is not None:
import scanpy as sc
# if the input scanpy is processed
if processed:
x_normalized = x = adata.X
gene_names = adata.var_names.values
expression = None
scale_factor = np.ones(x.shape[0])
# if the input scanpy is not processed
else:
dimred = False
x = adata.X.copy()
adata, expression, gene_names, cell_mask, gene_mask, gene_mask2 = _recipe_seurat(adata, gene_num)
x_normalized = adata.X.copy()
scale_factor = adata.obs.counts_per_cell.values / 1e4
x = x[cell_mask,:][:,gene_mask][:,gene_mask2]
if label_names is None:
try:
label_names = adata.obs.cell_types
except:
if label_names is not None and processed is False:
label_names = label_names[cell_mask]
cell_names = adata.obs_names.values
selected_gene_names = adata.var_names.values
gene_scalar = None
# if input is a count matrix
else:
# remove cells that have no expression
expressed = _check_expression(x)
print('Removing %d cells without expression.'%(np.sum(expressed==0)))
x = x[expressed==1,:]
if c is not None:
c = c[expressed==1,:]
if label_names is not None:
label_names = label_names[expressed==1]
# remove genes without variability
variable = _check_variability(x)
print('Removing %d genes without variability.'%(np.sum(variable==0)))
x = x[:, variable==1]
gene_names = raw_gene_names[variable==1]
# log-normalization
expression, scale_factor = normalize_gene_expression(x, K, transform_fn)
# feature selection
x, index = feature_select(x, gene_num)
selected_expression = expression[:, index]
# per-gene standardization
gene_scalar = preprocessing.StandardScaler()
x_normalized = gene_scalar.fit_transform(selected_expression)
cell_names = raw_cell_names[expressed==1]
selected_gene_names = gene_names[index]
if (data_type=='Gaussian') and (dimred is False):
# use arpack solver and extend precision to get deterministic result
pca = PCA(n_components = npc, random_state=random_state, svd_solver='arpack')
x_normalized = x = pca.fit_transform(x_normalized.astype(np.float64)).astype(np.float32)
if c is not None:
c_scalar = preprocessing.StandardScaler()
c = c_scalar.fit_transform(c)
if label_names is None:
warnings.warn('No labels for cells!')
labels = None
le = None
else:
le = preprocessing.LabelEncoder()
labels = le.fit_transform(label_names)
print('Number of cells in each class: ')
table = pd.value_counts(label_names)
table.index = pd.Series(le.transform(table.index).astype(str)) \
+ ' <---> ' + table.index
table = table.sort_index()
print(table)
return (x_normalized, expression, x, c,
cell_names, gene_names, selected_gene_names,
scale_factor, labels, label_names, le, gene_scalar)
def _recipe_seurat(adata, gene_num):
"""
Normalization and filtering as of Seurat [Satija15]_.
This uses a particular preprocessing
"""
import scanpy as sc
cell_mask = sc.pp.filter_cells(adata, min_genes=200, inplace=False)[0]
adata = adata[cell_mask,:]
gene_mask = sc.pp.filter_genes(adata, min_cells=3, inplace=False)[0]
adata = adata[:,gene_mask]
gene_names = adata.var_names.values
sc.pp.normalize_total(adata, target_sum=1e4, key_added='counts_per_cell')
filter_result = sc.pp.filter_genes_dispersion(
adata.X, min_mean=0.0125, max_mean=3, min_disp=0.5, log=False, n_top_genes=gene_num)
sc.pp.log1p(adata)
expression = adata.X.copy()
adata._inplace_subset_var(filter_result.gene_subset) # filter genes
sc.pp.scale(adata, max_value=10)
return adata, expression, gene_names, cell_mask, gene_mask, filter_result.gene_subset</code></pre>
</details>
</section>
<section>
</section>
<section>
</section>
<section>
<h2 class="section-title" id="header-functions">Functions</h2>
<dl>
<dt id="VITAE.preprocess.normalize_gene_expression"><code class="name flex">
<span>def <span class="ident">normalize_gene_expression</span></span>(<span>x, K: float = 10000.0, transform_fn: str = 'log')</span>
</code></dt>
<dd>
<div class="desc"><p>Normalize the gene expression counts for each cell by the total expression counts,
divide this by a size scale factor, which is determined by total counts and a constant K
then log-transforms the result.</p>
<h2 id="parameters">Parameters</h2>
<dl>
<dt><strong><code>x</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N, G^{raw}]</span><script type="math/tex">[N, G^{raw}]</script></span> The raw count data.</dd>
<dt><strong><code>K</code></strong> : <code>float</code>, optional</dt>
<dd>The normalizing constant.</dd>
<dt><strong><code>transform_fn</code></strong> : <code>str</code>, optional</dt>
<dd>Either 'log' or 'sqrt'.</dd>
</dl>
<h2 id="returns">Returns</h2>
<dl>
<dt><strong><code>x_normalized</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N, G^{raw}]</span><script type="math/tex">[N, G^{raw}]</script></span> The log-normalized data.</dd>
<dt><strong><code>scale_factor</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The scale factors.</dd>
</dl></div>
<details class="source">
<summary>
<span>Expand source code</span>
</summary>
<pre><code class="python">def normalize_gene_expression(x, K : float = 1e4, transform_fn : str = 'log'):
'''Normalize the gene expression counts for each cell by the total expression counts,
divide this by a size scale factor, which is determined by total counts and a constant K
then log-transforms the result.
Parameters
----------
x : np.array
\([N, G^{raw}]\) The raw count data.
K : float, optional
The normalizing constant.
transform_fn : str, optional
Either 'log' or 'sqrt'.
Returns
----------
x_normalized : np.array
\([N, G^{raw}]\) The log-normalized data.
scale_factor : np.array
\([N, ]\) The scale factors.
'''
scale_factor = np.sum(x,axis=1, keepdims=True)/K
if transform_fn=='log':
x_normalized = np.log(x/scale_factor + 1)
else:
x_normalized = np.where(x>0, np.sqrt(x/scale_factor), 0)
print('min normalized value: ' + str(np.min(x_normalized)))
print('max normalized value: ' + str(np.max(x_normalized)))
return x_normalized, scale_factor</code></pre>
</details>
</dd>
<dt id="VITAE.preprocess.feature_select"><code class="name flex">
<span>def <span class="ident">feature_select</span></span>(<span>x, gene_num: int = 2000)</span>
</code></dt>
<dd>
<div class="desc"><p>Select highly variable genes (HVGs)
(See <a href="https://www.nature.com/articles/nbt.4096">Stuart <em>et al</em>, (2019)</a> and its early version <a href="https://www.biorxiv.org/content/10.1101/460147v1.full.pdf">preprint</a>
Page 12-13: Data preprocessing - Feature selection for individual datasets).</p>
<h2 id="parameters">Parameters</h2>
<dl>
<dt><strong><code>x</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N, G^{raw}]</span><script type="math/tex">[N, G^{raw}]</script></span> The raw count data.</dd>
<dt><strong><code>gene_num</code></strong> : <code>int</code>, optional</dt>
<dd>The number of genes to retain.</dd>
</dl>
<h2 id="returns">Returns</h2>
<dl>
<dt><strong><code>x</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N, G]</span><script type="math/tex">[N, G]</script></span> The count data after gene selection.</dd>
<dt><strong><code>index</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[G, ]</span><script type="math/tex">[G, ]</script></span> The selected index of genes.</dd>
</dl></div>
<details class="source">
<summary>
<span>Expand source code</span>
</summary>
<pre><code class="python">def feature_select(x, gene_num : int = 2000):
'''Select highly variable genes (HVGs)
(See [Stuart *et al*, (2019)](https://www.nature.com/articles/nbt.4096) and its early version [preprint](https://www.biorxiv.org/content/10.1101/460147v1.full.pdf)
Page 12-13: Data preprocessing - Feature selection for individual datasets).
Parameters
----------
x : np.array
\([N, G^{raw}]\) The raw count data.
gene_num : int, optional
The number of genes to retain.
Returns
----------
x : np.array
\([N, G]\) The count data after gene selection.
index : np.array
\([G, ]\) The selected index of genes.
'''
n, p = x.shape
# mean and variance of each gene of the unnormalized data
mean, var = np.mean(x, axis=0), np.var(x, axis=0, ddof=1)
# model log10(var)~log10(mean) by local fitting of polynomials of degree 2
loess_model = loess.loess(np.log10(mean), np.log10(var),
span = 0.3, degree = 2, family='gaussian'
)
loess_model.fit()
fitted = loess_model.outputs.fitted_values
# standardized feature
z = (x - mean)/np.sqrt(10**fitted)
# clipped the standardized features to remove outliers
z = np.clip(z, -np.inf, np.sqrt(n))
# the variance of standardized features across all cells represents a measure of
# single cell dispersion after controlling for mean expression
feature_score = np.sum(z**2, axis=0)/(n-1)
# feature selection
index = feature_score.argsort()[::-1][0:gene_num]
return x[:, index], index</code></pre>
</details>
</dd>
<dt id="VITAE.preprocess.preprocess"><code class="name flex">
<span>def <span class="ident">preprocess</span></span>(<span>adata=None, processed: bool = False, dimred: bool = False, x=None, c=None, label_names=None, raw_cell_names=None, raw_gene_names=None, K: float = 10000.0, transform_fn: str = 'log', gene_num: int = 2000, data_type: str = 'UMI', npc: int = 64, random_state=0)</span>
</code></dt>
<dd>
<div class="desc"><p>Preprocess count data.</p>
<h2 id="parameters">Parameters</h2>
<dl>
<dt><strong><code>adata</code></strong> : <code>AnnData</code>, optional</dt>
<dd>The scanpy object.</dd>
<dt><strong><code>processed</code></strong> : <code>boolean</code></dt>
<dd>Whether adata has been processed.</dd>
<dt><strong><code>dimred</code></strong> : <code>boolean</code></dt>
<dd>Whether the processed adata is after dimension reduction.</dd>
<dt><strong><code>x</code></strong> : <code>np.array</code>, optional</dt>
<dd><span><span class="MathJax_Preview">[N^{raw}, G^{raw}]</span><script type="math/tex">[N^{raw}, G^{raw}]</script></span> The raw count matrix.</dd>
<dt><strong><code>c</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N^{raw}, s]</span><script type="math/tex">[N^{raw}, s]</script></span> The covariate matrix.</dd>
<dt><strong><code>label_names</code></strong> : <code>np.array </code></dt>
<dd><span><span class="MathJax_Preview">[N^{raw}, ]</span><script type="math/tex">[N^{raw}, ]</script></span> The true or estimated cell types.</dd>
<dt><strong><code>raw_cell_names</code></strong> : <code>np.array
</code></dt>
<dd><span><span class="MathJax_Preview">[N^{raw}, ]</span><script type="math/tex">[N^{raw}, ]</script></span> The names of cells.</dd>
<dt><strong><code>raw_gene_names</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[G^{raw}, ]</span><script type="math/tex">[G^{raw}, ]</script></span> The names of genes.</dd>
<dt><strong><code>K</code></strong> : <code>int</code>, optional</dt>
<dd>The normalizing constant.</dd>
<dt><strong><code>transform_fn</code></strong> : <code>str</code></dt>
<dd>The transform function used to normalize the gene expression after scaling. Either 'log' or 'sqrt'.</dd>
<dt><strong><code>gene_num</code></strong> : <code>int</code>, optional</dt>
<dd>The number of genes to retain.</dd>
<dt><strong><code>data_type</code></strong> : <code>str</code>, optional</dt>
<dd>'UMI', 'non-UMI', or 'Gaussian'.</dd>
<dt><strong><code>npc</code></strong> : <code>int</code>, optional</dt>
<dd>The number of PCs to retain, only used if <code>data_type='Gaussian'</code>.</dd>
<dt><strong><code>random_state</code></strong> : <code>int</code>, optional</dt>
<dd>The random state for PCA. With different random states, the resulted PCA scores are slightly different.</dd>
</dl>
<h2 id="returns">Returns</h2>
<dl>
<dt><strong><code>x_normalized</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N, G]</span><script type="math/tex">[N, G]</script></span> The preprocessed matrix.</dd>
<dt><strong><code>expression</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N, G^{raw}]</span><script type="math/tex">[N, G^{raw}]</script></span> The expression matrix after log-normalization and before scaling.</dd>
<dt><strong><code>x</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N, G]</span><script type="math/tex">[N, G]</script></span> The raw count matrix after gene selections.</dd>
<dt><strong><code>c</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N, s]</span><script type="math/tex">[N, s]</script></span> The covariates.</dd>
<dt><strong><code>cell_names</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The cell names.</dd>
<dt><strong><code>gene_names</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[G^{raw}, ]</span><script type="math/tex">[G^{raw}, ]</script></span> The gene names.</dd>
<dt><strong><code>selected_gene_names</code></strong></dt>
<dd><span><span class="MathJax_Preview">[G, ]</span><script type="math/tex">[G, ]</script></span> The selected gene names.</dd>
<dt><strong><code>scale_factor</code></strong></dt>
<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The scale factors.</dd>
<dt><strong><code>labels</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The encoded labels.</dd>
<dt><strong><code>label_names</code></strong> : <code>np.array</code></dt>
<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The label names.</dd>
<dt><strong><code>le</code></strong> : <code>sklearn.preprocessing.LabelEncoder</code></dt>
<dd>The label encoder.</dd>
<dt><strong><code>gene_scalar</code></strong> : <code>sklearn.preprocessing.StandardScaler</code></dt>
<dd>The gene scaler.</dd>
</dl></div>
<details class="source">
<summary>
<span>Expand source code</span>
</summary>
<pre><code class="python">def preprocess(adata = None, processed: bool = False, dimred: bool = False,
x = None, c = None, label_names = None, raw_cell_names = None, raw_gene_names = None,
K: float = 1e4, transform_fn: str = 'log', gene_num: int = 2000, data_type: str = 'UMI',
npc: int = 64, random_state=0):
'''Preprocess count data.
Parameters
----------
adata : AnnData, optional
The scanpy object.
processed : boolean
Whether adata has been processed.
dimred : boolean
Whether the processed adata is after dimension reduction.
x : np.array, optional
\([N^{raw}, G^{raw}]\) The raw count matrix.
c : np.array
\([N^{raw}, s]\) The covariate matrix.
label_names : np.array
\([N^{raw}, ]\) The true or estimated cell types.
raw_cell_names : np.array
\([N^{raw}, ]\) The names of cells.
raw_gene_names : np.array
\([G^{raw}, ]\) The names of genes.
K : int, optional
The normalizing constant.
transform_fn : str
The transform function used to normalize the gene expression after scaling. Either 'log' or 'sqrt'.
gene_num : int, optional
The number of genes to retain.
data_type : str, optional
'UMI', 'non-UMI', or 'Gaussian'.
npc : int, optional
The number of PCs to retain, only used if `data_type='Gaussian'`.
random_state : int, optional
The random state for PCA. With different random states, the resulted PCA scores are slightly different.
Returns
----------
x_normalized : np.array
\([N, G]\) The preprocessed matrix.
expression : np.array
\([N, G^{raw}]\) The expression matrix after log-normalization and before scaling.
x : np.array
\([N, G]\) The raw count matrix after gene selections.
c : np.array
\([N, s]\) The covariates.
cell_names : np.array
\([N, ]\) The cell names.
gene_names : np.array
\([G^{raw}, ]\) The gene names.
selected_gene_names :
\([G, ]\) The selected gene names.
scale_factor :
\([N, ]\) The scale factors.
labels : np.array
\([N, ]\) The encoded labels.
label_names : np.array
\([N, ]\) The label names.
le : sklearn.preprocessing.LabelEncoder
The label encoder.
gene_scalar : sklearn.preprocessing.StandardScaler
The gene scaler.
'''
# if input is a scanpy data
if adata is not None:
import scanpy as sc
# if the input scanpy is processed
if processed:
x_normalized = x = adata.X
gene_names = adata.var_names.values
expression = None
scale_factor = np.ones(x.shape[0])
# if the input scanpy is not processed
else:
dimred = False
x = adata.X.copy()
adata, expression, gene_names, cell_mask, gene_mask, gene_mask2 = _recipe_seurat(adata, gene_num)
x_normalized = adata.X.copy()
scale_factor = adata.obs.counts_per_cell.values / 1e4
x = x[cell_mask,:][:,gene_mask][:,gene_mask2]
if label_names is None:
try:
label_names = adata.obs.cell_types
except:
if label_names is not None and processed is False:
label_names = label_names[cell_mask]
cell_names = adata.obs_names.values
selected_gene_names = adata.var_names.values
gene_scalar = None
# if input is a count matrix
else:
# remove cells that have no expression
expressed = _check_expression(x)
print('Removing %d cells without expression.'%(np.sum(expressed==0)))
x = x[expressed==1,:]
if c is not None:
c = c[expressed==1,:]
if label_names is not None:
label_names = label_names[expressed==1]
# remove genes without variability
variable = _check_variability(x)
print('Removing %d genes without variability.'%(np.sum(variable==0)))
x = x[:, variable==1]
gene_names = raw_gene_names[variable==1]
# log-normalization
expression, scale_factor = normalize_gene_expression(x, K, transform_fn)
# feature selection
x, index = feature_select(x, gene_num)
selected_expression = expression[:, index]
# per-gene standardization
gene_scalar = preprocessing.StandardScaler()
x_normalized = gene_scalar.fit_transform(selected_expression)
cell_names = raw_cell_names[expressed==1]
selected_gene_names = gene_names[index]
if (data_type=='Gaussian') and (dimred is False):
# use arpack solver and extend precision to get deterministic result
pca = PCA(n_components = npc, random_state=random_state, svd_solver='arpack')
x_normalized = x = pca.fit_transform(x_normalized.astype(np.float64)).astype(np.float32)
if c is not None:
c_scalar = preprocessing.StandardScaler()
c = c_scalar.fit_transform(c)
if label_names is None:
warnings.warn('No labels for cells!')
labels = None
le = None
else:
le = preprocessing.LabelEncoder()
labels = le.fit_transform(label_names)
print('Number of cells in each class: ')
table = pd.value_counts(label_names)
table.index = pd.Series(le.transform(table.index).astype(str)) \
+ ' <---> ' + table.index
table = table.sort_index()
print(table)
return (x_normalized, expression, x, c,
cell_names, gene_names, selected_gene_names,
scale_factor, labels, label_names, le, gene_scalar)</code></pre>
</details>
</dd>
</dl>
</section>
<section>
</section>
</article>
<nav id="sidebar">
<h1>Index</h1>
<div class="toc">
<ul></ul>
</div>
<ul id="index">
<li><h3>Super-module</h3>
<ul>
<li><code><a title="VITAE" href="index.html">VITAE</a></code></li>
</ul>
</li>
<li><h3><a href="#header-functions">Functions</a></h3>
<ul class="">
<li><code><a title="VITAE.preprocess.normalize_gene_expression" href="#VITAE.preprocess.normalize_gene_expression">normalize_gene_expression</a></code></li>
<li><code><a title="VITAE.preprocess.feature_select" href="#VITAE.preprocess.feature_select">feature_select</a></code></li>
<li><code><a title="VITAE.preprocess.preprocess" href="#VITAE.preprocess.preprocess">preprocess</a></code></li>
</ul>
</li>
</ul>
</nav>
</main>
<footer id="footer">
<p>Generated by <a href="https://pdoc3.github.io/pdoc"><cite>pdoc</cite> 0.8.1</a>.</p>
</footer>
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
<script>hljs.initHighlightingOnLoad()</script>
</body>
</html>
Datasets
Models
