VITAE / Git / Diff of /docs/preprocess.html

Models:

MarcoTheBlack/

VITAE

Downloads: 1

Diff of /docs/preprocess.html [000000] .. [2c6b19]

Switch to unified view

 b/docs/preprocess.html
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1" />
+<meta name="generator" content="pdoc 0.8.1" />
+<title>VITAE.preprocess API documentation</title>
+<meta name="description" content="" />
+<link href='https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.0/normalize.min.css' rel='stylesheet'>
+<link href='https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/8.0.0/sanitize.min.css' rel='stylesheet'>
+<link href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css" rel="stylesheet">
+<style>.flex{display:flex !important}body{line-height:1.5em}#content{padding:20px}#sidebar{padding:30px;overflow:hidden}#sidebar > *:last-child{margin-bottom:2cm}.http-server-breadcrumbs{font-size:130%;margin:0 0 15px 0}#footer{font-size:.75em;padding:5px 30px;border-top:1px solid #ddd;text-align:right}#footer p{margin:0 0 0 1em;display:inline-block}#footer p:last-child{margin-right:30px}h1,h2,h3,h4,h5{font-weight:300}h1{font-size:2.5em;line-height:1.1em}h2{font-size:1.75em;margin:1em 0 .50em 0}h3{font-size:1.4em;margin:25px 0 10px 0}h4{margin:0;font-size:105%}a{color:#058;text-decoration:none;transition:color .3s ease-in-out}a:hover{color:#e82}.title code{font-weight:bold}h2[id^="header-"]{margin-top:2em}.ident{color:#900}pre code{background:#f8f8f8;font-size:.8em;line-height:1.4em}code{background:#f2f2f1;padding:1px 4px;overflow-wrap:break-word}h1 code{background:transparent}pre{background:#f8f8f8;border:0;border-top:1px solid #ccc;border-bottom:1px solid #ccc;margin:1em 0;padding:1ex}#http-server-module-list{display:flex;flex-flow:column}#http-server-module-list div{display:flex}#http-server-module-list dt{min-width:10%}#http-server-module-list p{margin-top:0}.toc ul,#index{list-style-type:none;margin:0;padding:0}#index code{background:transparent}#index h3{border-bottom:1px solid #ddd}#index ul{padding:0}#index h4{margin-top:.6em;font-weight:bold}@media (min-width:200ex){#index .two-column{column-count:2}}@media (min-width:300ex){#index .two-column{column-count:3}}dl{margin-bottom:2em}dl dl:last-child{margin-bottom:4em}dd{margin:0 0 1em 3em}#header-classes + dl > dd{margin-bottom:3em}dd dd{margin-left:2em}dd p{margin:10px 0}.name{background:#eee;font-weight:bold;font-size:.85em;padding:5px 10px;display:inline-block;min-width:40%}.name:hover{background:#e0e0e0}.name > span:first-child{white-space:nowrap}.name.class > span:nth-child(2){margin-left:.4em}.inherited{color:#999;border-left:5px solid #eee;padding-left:1em}.inheritance em{font-style:normal;font-weight:bold}.desc h2{font-weight:400;font-size:1.25em}.desc h3{font-size:1em}.desc dt code{background:inherit}.source summary,.git-link-div{color:#666;text-align:right;font-weight:400;font-size:.8em;text-transform:uppercase}.source summary > *{white-space:nowrap;cursor:pointer}.git-link{color:inherit;margin-left:1em}.source pre{max-height:500px;overflow:auto;margin:0}.source pre code{font-size:12px;overflow:visible}.hlist{list-style:none}.hlist li{display:inline}.hlist li:after{content:',\2002'}.hlist li:last-child:after{content:none}.hlist .hlist{display:inline;padding-left:1em}img{max-width:100%}.admonition{padding:.1em .5em;margin-bottom:1em}.admonition-title{font-weight:bold}.admonition.note,.admonition.info,.admonition.important{background:#aef}.admonition.todo,.admonition.versionadded,.admonition.tip,.admonition.hint{background:#dfd}.admonition.warning,.admonition.versionchanged,.admonition.deprecated{background:#fd4}.admonition.error,.admonition.danger,.admonition.caution{background:lightpink}</style>
+<style media="screen and (min-width: 700px)">@media screen and (min-width:700px){#sidebar{width:30%;height:100vh;overflow:auto;position:sticky;top:0}#content{width:70%;max-width:100ch;padding:3em 4em;border-left:1px solid #ddd}pre code{font-size:1em}.item .name{font-size:1em}main{display:flex;flex-direction:row-reverse;justify-content:flex-end}.toc ul ul,#index ul{padding-left:1.5em}.toc > ul > li{margin-top:.5em}}</style>
+<style media="print">@media print{#sidebar h1{page-break-before:always}.source{display:none}}@media print{*{background:transparent !important;color:#000 !important;box-shadow:none !important;text-shadow:none !important}a[href]:after{content:" (" attr(href) ")";font-size:90%}a[href][title]:after{content:none}abbr[title]:after{content:" (" attr(title) ")"}.ir a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}@page{margin:0.5cm}p,h2,h3{orphans:3;widows:3}h1,h2,h3,h4,h5,h6{page-break-after:avoid}}</style>
+<script async src='https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS_CHTML'></script>
+</head>
+<body>
+<main>
+<article id="content">
+<header>
+<h1 class="title">Module <code>VITAE.preprocess</code></h1>
+</header>
+<section id="section-intro">
+<details class="source">
+<summary>
+<span>Expand source code</span>
+</summary>
+<pre><code class="python"># -*- coding: utf-8 -*-
+from typing import Optional
+import numpy as np
+import pandas as pd
+from skmisc import loess
+from sklearn import preprocessing
+import warnings
+from sklearn.decomposition import PCA
+from VITAE.utils import _check_expression, _check_variability
+def normalize_gene_expression(x, K : float = 1e4, transform_fn : str = &#39;log&#39;):
+    &#39;&#39;&#39;Normalize the gene expression counts for each cell by the total expression counts,
+    divide this by a size scale factor, which is determined by total counts and a constant K
+    then log-transforms the result.
+    Parameters
+    ----------
+    x : np.array
+        \([N, G^{raw}]\) The raw count data.
+    K : float, optional
+        The normalizing constant.
+    transform_fn : str, optional
+        Either &#39;log&#39; or &#39;sqrt&#39;.
+    Returns
+    ----------
+    x_normalized : np.array
+        \([N, G^{raw}]\) The log-normalized data.
+    scale_factor : np.array
+        \([N, ]\) The scale factors.
+    &#39;&#39;&#39;
+    scale_factor = np.sum(x,axis=1, keepdims=True)/K
+    if transform_fn==&#39;log&#39;:
+        x_normalized = np.log(x/scale_factor + 1)
+    else:
+        x_normalized = np.where(x&gt;0, np.sqrt(x/scale_factor), 0)
+    print(&#39;min normalized value: &#39; + str(np.min(x_normalized)))
+    print(&#39;max normalized value: &#39; + str(np.max(x_normalized)))
+    return x_normalized, scale_factor
+def feature_select(x, gene_num : int = 2000):
+    &#39;&#39;&#39;Select highly variable genes (HVGs)
+    (See [Stuart *et al*, (2019)](https://www.nature.com/articles/nbt.4096) and its early version [preprint](https://www.biorxiv.org/content/10.1101/460147v1.full.pdf)
+    Page 12-13: Data preprocessing - Feature selection for individual datasets).
+    Parameters
+    ----------
+    x : np.array
+        \([N, G^{raw}]\) The raw count data.
+    gene_num : int, optional
+        The number of genes to retain.
+    Returns
+    ----------
+    x : np.array
+        \([N, G]\) The count data after gene selection.
+    index : np.array
+        \([G, ]\) The selected index of genes.
+    &#39;&#39;&#39;
+    n, p = x.shape
+    # mean and variance of each gene of the unnormalized data
+    mean, var = np.mean(x, axis=0), np.var(x, axis=0, ddof=1)
+    # model log10(var)~log10(mean) by local fitting of polynomials of degree 2
+    loess_model = loess.loess(np.log10(mean), np.log10(var),
+                    span = 0.3, degree = 2, family=&#39;gaussian&#39;
+                    )
+    loess_model.fit()
+    fitted = loess_model.outputs.fitted_values
+    # standardized feature
+    z = (x - mean)/np.sqrt(10**fitted)
+    # clipped the standardized features to remove outliers
+    z = np.clip(z, -np.inf, np.sqrt(n))
+    # the variance of standardized features across all cells represents a measure of
+    # single cell dispersion after controlling for mean expression
+    feature_score = np.sum(z**2, axis=0)/(n-1)
+    # feature selection
+    index = feature_score.argsort()[::-1][0:gene_num]
+    return x[:, index], index
+def preprocess(adata = None, processed: bool = False, dimred: bool = False,
+            x = None, c = None, label_names = None, raw_cell_names = None, raw_gene_names = None,
+            K: float = 1e4, transform_fn: str = &#39;log&#39;, gene_num: int = 2000, data_type: str = &#39;UMI&#39;,
+            npc: int = 64, random_state=0):
+    &#39;&#39;&#39;Preprocess count data.
+    Parameters
+    ----------
+    adata : AnnData, optional
+        The scanpy object.
+    processed : boolean
+        Whether adata has been processed.
+    dimred : boolean
+        Whether the processed adata is after dimension reduction.
+    x : np.array, optional
+        \([N^{raw}, G^{raw}]\) The raw count matrix.
+    c : np.array
+        \([N^{raw}, s]\) The covariate matrix.
+    label_names : np.array
+        \([N^{raw}, ]\) The true or estimated cell types.
+    raw_cell_names : np.array
+        \([N^{raw}, ]\) The names of cells.
+    raw_gene_names : np.array
+        \([G^{raw}, ]\) The names of genes.
+    K : int, optional
+        The normalizing constant.
+    transform_fn : str
+        The transform function used to normalize the gene expression after scaling. Either &#39;log&#39; or &#39;sqrt&#39;.
+    gene_num : int, optional
+        The number of genes to retain.
+    data_type : str, optional
+        &#39;UMI&#39;, &#39;non-UMI&#39;, or &#39;Gaussian&#39;.
+    npc : int, optional
+        The number of PCs to retain, only used if `data_type=&#39;Gaussian&#39;`.
+    random_state : int, optional
+        The random state for PCA. With different random states, the resulted PCA scores are slightly different.
+    Returns
+    ----------
+    x_normalized : np.array
+        \([N, G]\) The preprocessed matrix.
+    expression : np.array
+        \([N, G^{raw}]\) The expression matrix after log-normalization and before scaling.
+    x : np.array
+        \([N, G]\) The raw count matrix after gene selections.
+    c : np.array
+        \([N, s]\) The covariates.
+    cell_names : np.array
+        \([N, ]\) The cell names.
+    gene_names : np.array
+        \([G^{raw}, ]\) The gene names.
+    selected_gene_names :
+        \([G, ]\) The selected gene names.
+    scale_factor :
+        \([N, ]\) The scale factors.
+    labels : np.array
+        \([N, ]\) The encoded labels.
+    label_names : np.array
+        \([N, ]\) The label names.
+    le : sklearn.preprocessing.LabelEncoder
+        The label encoder.
+    gene_scalar : sklearn.preprocessing.StandardScaler
+        The gene scaler.
+    &#39;&#39;&#39;
+    # if input is a scanpy data
+    if adata is not None:
+        import scanpy as sc
+        # if the input scanpy is processed
+        if processed:
+            x_normalized = x = adata.X
+            gene_names = adata.var_names.values
+            expression = None
+            scale_factor = np.ones(x.shape[0])
+        # if the input scanpy is not processed
+        else:
+            dimred = False
+            x = adata.X.copy()
+            adata, expression, gene_names, cell_mask, gene_mask, gene_mask2 = _recipe_seurat(adata, gene_num)
+            x_normalized = adata.X.copy()
+            scale_factor = adata.obs.counts_per_cell.values / 1e4
+            x = x[cell_mask,:][:,gene_mask][:,gene_mask2]
+        if label_names is None:
+            try:
+                label_names = adata.obs.cell_types
+            except:
+                if label_names is not None and processed is False:
+                    label_names = label_names[cell_mask]
+        cell_names = adata.obs_names.values
+        selected_gene_names = adata.var_names.values
+        gene_scalar = None
+    # if input is a count matrix
+    else:
+        # remove cells that have no expression
+        expressed = _check_expression(x)
+        print(&#39;Removing %d cells without expression.&#39;%(np.sum(expressed==0)))
+        x = x[expressed==1,:]
+        if c is not None:
+            c = c[expressed==1,:]
+        if label_names is not None:
+            label_names = label_names[expressed==1]
+        # remove genes without variability
+        variable = _check_variability(x)
+        print(&#39;Removing %d genes without variability.&#39;%(np.sum(variable==0)))
+        x = x[:, variable==1]
+        gene_names = raw_gene_names[variable==1]
+        # log-normalization
+        expression, scale_factor = normalize_gene_expression(x, K, transform_fn)
+        # feature selection
+        x, index = feature_select(x, gene_num)
+        selected_expression = expression[:, index]
+        # per-gene standardization
+        gene_scalar = preprocessing.StandardScaler()
+        x_normalized = gene_scalar.fit_transform(selected_expression)
+        cell_names = raw_cell_names[expressed==1]
+        selected_gene_names = gene_names[index]
+    if (data_type==&#39;Gaussian&#39;) and (dimred is False):
+        # use arpack solver and extend precision to get deterministic result
+        pca = PCA(n_components = npc, random_state=random_state, svd_solver=&#39;arpack&#39;)
+        x_normalized = x = pca.fit_transform(x_normalized.astype(np.float64)).astype(np.float32)
+    if c is not None:
+        c_scalar = preprocessing.StandardScaler()
+        c = c_scalar.fit_transform(c)
+    if label_names is None:
+        warnings.warn(&#39;No labels for cells!&#39;)
+        labels = None
+        le = None
+    else:
+        le = preprocessing.LabelEncoder()
+        labels = le.fit_transform(label_names)
+        print(&#39;Number of cells in each class: &#39;)
+        table = pd.value_counts(label_names)
+        table.index = pd.Series(le.transform(table.index).astype(str)) \
+            + &#39; &lt;---&gt; &#39; + table.index
+        table = table.sort_index()
+        print(table)
+    return (x_normalized, expression, x, c,
+        cell_names, gene_names, selected_gene_names,
+        scale_factor, labels, label_names, le, gene_scalar)
+def _recipe_seurat(adata, gene_num):
+    &#34;&#34;&#34;
+    Normalization and filtering as of Seurat [Satija15]_.
+    This uses a particular preprocessing
+    &#34;&#34;&#34;
+    import scanpy as sc
+    cell_mask = sc.pp.filter_cells(adata, min_genes=200, inplace=False)[0]
+    adata = adata[cell_mask,:]
+    gene_mask = sc.pp.filter_genes(adata, min_cells=3, inplace=False)[0]
+    adata = adata[:,gene_mask]
+    gene_names = adata.var_names.values
+    sc.pp.normalize_total(adata, target_sum=1e4, key_added=&#39;counts_per_cell&#39;)
+    filter_result = sc.pp.filter_genes_dispersion(
+        adata.X, min_mean=0.0125, max_mean=3, min_disp=0.5, log=False, n_top_genes=gene_num)
+    sc.pp.log1p(adata)
+    expression = adata.X.copy()
+    adata._inplace_subset_var(filter_result.gene_subset)  # filter genes
+    sc.pp.scale(adata, max_value=10)
+    return adata, expression, gene_names, cell_mask, gene_mask, filter_result.gene_subset</code></pre>
+</details>
+</section>
+<section>
+</section>
+<section>
+</section>
+<section>
+<h2 class="section-title" id="header-functions">Functions</h2>
+<dl>
+<dt id="VITAE.preprocess.normalize_gene_expression"><code class="name flex">
+<span>def <span class="ident">normalize_gene_expression</span></span>(<span>x, K: float = 10000.0, transform_fn: str = 'log')</span>
+</code></dt>
+<dd>
+<div class="desc"><p>Normalize the gene expression counts for each cell by the total expression counts,
+divide this by a size scale factor, which is determined by total counts and a constant K
+then log-transforms the result.</p>
+<h2 id="parameters">Parameters</h2>
+<dl>
+<dt><strong><code>x</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N, G^{raw}]</span><script type="math/tex">[N, G^{raw}]</script></span> The raw count data.</dd>
+<dt><strong><code>K</code></strong> :&ensp;<code>float</code>, optional</dt>
+<dd>The normalizing constant.</dd>
+<dt><strong><code>transform_fn</code></strong> :&ensp;<code>str</code>, optional</dt>
+<dd>Either 'log' or 'sqrt'.</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>x_normalized</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N, G^{raw}]</span><script type="math/tex">[N, G^{raw}]</script></span> The log-normalized data.</dd>
+<dt><strong><code>scale_factor</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The scale factors.</dd>
+</dl></div>
+<details class="source">
+<summary>
+<span>Expand source code</span>
+</summary>
+<pre><code class="python">def normalize_gene_expression(x, K : float = 1e4, transform_fn : str = &#39;log&#39;):
+    &#39;&#39;&#39;Normalize the gene expression counts for each cell by the total expression counts,
+    divide this by a size scale factor, which is determined by total counts and a constant K
+    then log-transforms the result.
+    Parameters
+    ----------
+    x : np.array
+        \([N, G^{raw}]\) The raw count data.
+    K : float, optional
+        The normalizing constant.
+    transform_fn : str, optional
+        Either &#39;log&#39; or &#39;sqrt&#39;.
+    Returns
+    ----------
+    x_normalized : np.array
+        \([N, G^{raw}]\) The log-normalized data.
+    scale_factor : np.array
+        \([N, ]\) The scale factors.
+    &#39;&#39;&#39;
+    scale_factor = np.sum(x,axis=1, keepdims=True)/K
+    if transform_fn==&#39;log&#39;:
+        x_normalized = np.log(x/scale_factor + 1)
+    else:
+        x_normalized = np.where(x&gt;0, np.sqrt(x/scale_factor), 0)
+    print(&#39;min normalized value: &#39; + str(np.min(x_normalized)))
+    print(&#39;max normalized value: &#39; + str(np.max(x_normalized)))
+    return x_normalized, scale_factor</code></pre>
+</details>
+</dd>
+<dt id="VITAE.preprocess.feature_select"><code class="name flex">
+<span>def <span class="ident">feature_select</span></span>(<span>x, gene_num: int = 2000)</span>
+</code></dt>
+<dd>
+<div class="desc"><p>Select highly variable genes (HVGs)
+(See <a href="https://www.nature.com/articles/nbt.4096">Stuart <em>et al</em>, (2019)</a> and its early version <a href="https://www.biorxiv.org/content/10.1101/460147v1.full.pdf">preprint</a>
+Page 12-13: Data preprocessing - Feature selection for individual datasets).</p>
+<h2 id="parameters">Parameters</h2>
+<dl>
+<dt><strong><code>x</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N, G^{raw}]</span><script type="math/tex">[N, G^{raw}]</script></span> The raw count data.</dd>
+<dt><strong><code>gene_num</code></strong> :&ensp;<code>int</code>, optional</dt>
+<dd>The number of genes to retain.</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>x</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N, G]</span><script type="math/tex">[N, G]</script></span> The count data after gene selection.</dd>
+<dt><strong><code>index</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[G, ]</span><script type="math/tex">[G, ]</script></span> The selected index of genes.</dd>
+</dl></div>
+<details class="source">
+<summary>
+<span>Expand source code</span>
+</summary>
+<pre><code class="python">def feature_select(x, gene_num : int = 2000):
+    &#39;&#39;&#39;Select highly variable genes (HVGs)
+    (See [Stuart *et al*, (2019)](https://www.nature.com/articles/nbt.4096) and its early version [preprint](https://www.biorxiv.org/content/10.1101/460147v1.full.pdf)
+    Page 12-13: Data preprocessing - Feature selection for individual datasets).
+    Parameters
+    ----------
+    x : np.array
+        \([N, G^{raw}]\) The raw count data.
+    gene_num : int, optional
+        The number of genes to retain.
+    Returns
+    ----------
+    x : np.array
+        \([N, G]\) The count data after gene selection.
+    index : np.array
+        \([G, ]\) The selected index of genes.
+    &#39;&#39;&#39;
+    n, p = x.shape
+    # mean and variance of each gene of the unnormalized data
+    mean, var = np.mean(x, axis=0), np.var(x, axis=0, ddof=1)
+    # model log10(var)~log10(mean) by local fitting of polynomials of degree 2
+    loess_model = loess.loess(np.log10(mean), np.log10(var),
+                    span = 0.3, degree = 2, family=&#39;gaussian&#39;
+                    )
+    loess_model.fit()
+    fitted = loess_model.outputs.fitted_values
+    # standardized feature
+    z = (x - mean)/np.sqrt(10**fitted)
+    # clipped the standardized features to remove outliers
+    z = np.clip(z, -np.inf, np.sqrt(n))
+    # the variance of standardized features across all cells represents a measure of
+    # single cell dispersion after controlling for mean expression
+    feature_score = np.sum(z**2, axis=0)/(n-1)
+    # feature selection
+    index = feature_score.argsort()[::-1][0:gene_num]
+    return x[:, index], index</code></pre>
+</details>
+</dd>
+<dt id="VITAE.preprocess.preprocess"><code class="name flex">
+<span>def <span class="ident">preprocess</span></span>(<span>adata=None, processed: bool = False, dimred: bool = False, x=None, c=None, label_names=None, raw_cell_names=None, raw_gene_names=None, K: float = 10000.0, transform_fn: str = 'log', gene_num: int = 2000, data_type: str = 'UMI', npc: int = 64, random_state=0)</span>
+</code></dt>
+<dd>
+<div class="desc"><p>Preprocess count data.</p>
+<h2 id="parameters">Parameters</h2>
+<dl>
+<dt><strong><code>adata</code></strong> :&ensp;<code>AnnData</code>, optional</dt>
+<dd>The scanpy object.</dd>
+<dt><strong><code>processed</code></strong> :&ensp;<code>boolean</code></dt>
+<dd>Whether adata has been processed.</dd>
+<dt><strong><code>dimred</code></strong> :&ensp;<code>boolean</code></dt>
+<dd>Whether the processed adata is after dimension reduction.</dd>
+<dt><strong><code>x</code></strong> :&ensp;<code>np.array</code>, optional</dt>
+<dd><span><span class="MathJax_Preview">[N^{raw}, G^{raw}]</span><script type="math/tex">[N^{raw}, G^{raw}]</script></span> The raw count matrix.</dd>
+<dt><strong><code>c</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N^{raw}, s]</span><script type="math/tex">[N^{raw}, s]</script></span> The covariate matrix.</dd>
+<dt><strong><code>label_names</code></strong> :&ensp;<code>np.array </code></dt>
+<dd><span><span class="MathJax_Preview">[N^{raw}, ]</span><script type="math/tex">[N^{raw}, ]</script></span> The true or estimated cell types.</dd>
+<dt><strong><code>raw_cell_names</code></strong> :&ensp;<code>np.array
+</code></dt>
+<dd><span><span class="MathJax_Preview">[N^{raw}, ]</span><script type="math/tex">[N^{raw}, ]</script></span> The names of cells.</dd>
+<dt><strong><code>raw_gene_names</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[G^{raw}, ]</span><script type="math/tex">[G^{raw}, ]</script></span> The names of genes.</dd>
+<dt><strong><code>K</code></strong> :&ensp;<code>int</code>, optional</dt>
+<dd>The normalizing constant.</dd>
+<dt><strong><code>transform_fn</code></strong> :&ensp;<code>str</code></dt>
+<dd>The transform function used to normalize the gene expression after scaling. Either 'log' or 'sqrt'.</dd>
+<dt><strong><code>gene_num</code></strong> :&ensp;<code>int</code>, optional</dt>
+<dd>The number of genes to retain.</dd>
+<dt><strong><code>data_type</code></strong> :&ensp;<code>str</code>, optional</dt>
+<dd>'UMI', 'non-UMI', or 'Gaussian'.</dd>
+<dt><strong><code>npc</code></strong> :&ensp;<code>int</code>, optional</dt>
+<dd>The number of PCs to retain, only used if <code>data_type='Gaussian'</code>.</dd>
+<dt><strong><code>random_state</code></strong> :&ensp;<code>int</code>, optional</dt>
+<dd>The random state for PCA. With different random states, the resulted PCA scores are slightly different.</dd>
+</dl>
+<h2 id="returns">Returns</h2>
+<dl>
+<dt><strong><code>x_normalized</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N, G]</span><script type="math/tex">[N, G]</script></span> The preprocessed matrix.</dd>
+<dt><strong><code>expression</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N, G^{raw}]</span><script type="math/tex">[N, G^{raw}]</script></span> The expression matrix after log-normalization and before scaling.</dd>
+<dt><strong><code>x</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N, G]</span><script type="math/tex">[N, G]</script></span> The raw count matrix after gene selections.</dd>
+<dt><strong><code>c</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N, s]</span><script type="math/tex">[N, s]</script></span> The covariates.</dd>
+<dt><strong><code>cell_names</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The cell names.</dd>
+<dt><strong><code>gene_names</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[G^{raw}, ]</span><script type="math/tex">[G^{raw}, ]</script></span> The gene names.</dd>
+<dt><strong><code>selected_gene_names</code></strong></dt>
+<dd><span><span class="MathJax_Preview">[G, ]</span><script type="math/tex">[G, ]</script></span> The selected gene names.</dd>
+<dt><strong><code>scale_factor</code></strong></dt>
+<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The scale factors.</dd>
+<dt><strong><code>labels</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The encoded labels.</dd>
+<dt><strong><code>label_names</code></strong> :&ensp;<code>np.array</code></dt>
+<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The label names.</dd>
+<dt><strong><code>le</code></strong> :&ensp;<code>sklearn.preprocessing.LabelEncoder</code></dt>
+<dd>The label encoder.</dd>
+<dt><strong><code>gene_scalar</code></strong> :&ensp;<code>sklearn.preprocessing.StandardScaler</code></dt>
+<dd>The gene scaler.</dd>
+</dl></div>
+<details class="source">
+<summary>
+<span>Expand source code</span>
+</summary>
+<pre><code class="python">def preprocess(adata = None, processed: bool = False, dimred: bool = False,
+            x = None, c = None, label_names = None, raw_cell_names = None, raw_gene_names = None,
+            K: float = 1e4, transform_fn: str = &#39;log&#39;, gene_num: int = 2000, data_type: str = &#39;UMI&#39;,
+            npc: int = 64, random_state=0):
+    &#39;&#39;&#39;Preprocess count data.
+    Parameters
+    ----------
+    adata : AnnData, optional
+        The scanpy object.
+    processed : boolean
+        Whether adata has been processed.
+    dimred : boolean
+        Whether the processed adata is after dimension reduction.
+    x : np.array, optional
+        \([N^{raw}, G^{raw}]\) The raw count matrix.
+    c : np.array
+        \([N^{raw}, s]\) The covariate matrix.
+    label_names : np.array
+        \([N^{raw}, ]\) The true or estimated cell types.
+    raw_cell_names : np.array
+        \([N^{raw}, ]\) The names of cells.
+    raw_gene_names : np.array
+        \([G^{raw}, ]\) The names of genes.
+    K : int, optional
+        The normalizing constant.
+    transform_fn : str
+        The transform function used to normalize the gene expression after scaling. Either &#39;log&#39; or &#39;sqrt&#39;.
+    gene_num : int, optional
+        The number of genes to retain.
+    data_type : str, optional
+        &#39;UMI&#39;, &#39;non-UMI&#39;, or &#39;Gaussian&#39;.
+    npc : int, optional
+        The number of PCs to retain, only used if `data_type=&#39;Gaussian&#39;`.
+    random_state : int, optional
+        The random state for PCA. With different random states, the resulted PCA scores are slightly different.
+    Returns
+    ----------
+    x_normalized : np.array
+        \([N, G]\) The preprocessed matrix.
+    expression : np.array
+        \([N, G^{raw}]\) The expression matrix after log-normalization and before scaling.
+    x : np.array
+        \([N, G]\) The raw count matrix after gene selections.
+    c : np.array
+        \([N, s]\) The covariates.
+    cell_names : np.array
+        \([N, ]\) The cell names.
+    gene_names : np.array
+        \([G^{raw}, ]\) The gene names.
+    selected_gene_names :
+        \([G, ]\) The selected gene names.
+    scale_factor :
+        \([N, ]\) The scale factors.
+    labels : np.array
+        \([N, ]\) The encoded labels.
+    label_names : np.array
+        \([N, ]\) The label names.
+    le : sklearn.preprocessing.LabelEncoder
+        The label encoder.
+    gene_scalar : sklearn.preprocessing.StandardScaler
+        The gene scaler.
+    &#39;&#39;&#39;
+    # if input is a scanpy data
+    if adata is not None:
+        import scanpy as sc
+        # if the input scanpy is processed
+        if processed:
+            x_normalized = x = adata.X
+            gene_names = adata.var_names.values
+            expression = None
+            scale_factor = np.ones(x.shape[0])
+        # if the input scanpy is not processed
+        else:
+            dimred = False
+            x = adata.X.copy()
+            adata, expression, gene_names, cell_mask, gene_mask, gene_mask2 = _recipe_seurat(adata, gene_num)
+            x_normalized = adata.X.copy()
+            scale_factor = adata.obs.counts_per_cell.values / 1e4
+            x = x[cell_mask,:][:,gene_mask][:,gene_mask2]
+        if label_names is None:
+            try:
+                label_names = adata.obs.cell_types
+            except:
+                if label_names is not None and processed is False:
+                    label_names = label_names[cell_mask]
+        cell_names = adata.obs_names.values
+        selected_gene_names = adata.var_names.values
+        gene_scalar = None
+    # if input is a count matrix
+    else:
+        # remove cells that have no expression
+        expressed = _check_expression(x)
+        print(&#39;Removing %d cells without expression.&#39;%(np.sum(expressed==0)))
+        x = x[expressed==1,:]
+        if c is not None:
+            c = c[expressed==1,:]
+        if label_names is not None:
+            label_names = label_names[expressed==1]
+        # remove genes without variability
+        variable = _check_variability(x)
+        print(&#39;Removing %d genes without variability.&#39;%(np.sum(variable==0)))
+        x = x[:, variable==1]
+        gene_names = raw_gene_names[variable==1]
+        # log-normalization
+        expression, scale_factor = normalize_gene_expression(x, K, transform_fn)
+        # feature selection
+        x, index = feature_select(x, gene_num)
+        selected_expression = expression[:, index]
+        # per-gene standardization
+        gene_scalar = preprocessing.StandardScaler()
+        x_normalized = gene_scalar.fit_transform(selected_expression)
+        cell_names = raw_cell_names[expressed==1]
+        selected_gene_names = gene_names[index]
+    if (data_type==&#39;Gaussian&#39;) and (dimred is False):
+        # use arpack solver and extend precision to get deterministic result
+        pca = PCA(n_components = npc, random_state=random_state, svd_solver=&#39;arpack&#39;)
+        x_normalized = x = pca.fit_transform(x_normalized.astype(np.float64)).astype(np.float32)
+    if c is not None:
+        c_scalar = preprocessing.StandardScaler()
+        c = c_scalar.fit_transform(c)
+    if label_names is None:
+        warnings.warn(&#39;No labels for cells!&#39;)
+        labels = None
+        le = None
+    else:
+        le = preprocessing.LabelEncoder()
+        labels = le.fit_transform(label_names)
+        print(&#39;Number of cells in each class: &#39;)
+        table = pd.value_counts(label_names)
+        table.index = pd.Series(le.transform(table.index).astype(str)) \
+            + &#39; &lt;---&gt; &#39; + table.index
+        table = table.sort_index()
+        print(table)
+    return (x_normalized, expression, x, c,
+        cell_names, gene_names, selected_gene_names,
+        scale_factor, labels, label_names, le, gene_scalar)</code></pre>
+</details>
+</dd>
+</dl>
+</section>
+<section>
+</section>
+</article>
+<nav id="sidebar">
+<h1>Index</h1>
+<div class="toc">
+<ul></ul>
+</div>
+<ul id="index">
+<li><h3>Super-module</h3>
+<ul>
+<li><code><a title="VITAE" href="index.html">VITAE</a></code></li>
+</ul>
+</li>
+<li><h3><a href="#header-functions">Functions</a></h3>
+<ul class="">
+<li><code><a title="VITAE.preprocess.normalize_gene_expression" href="#VITAE.preprocess.normalize_gene_expression">normalize_gene_expression</a></code></li>
+<li><code><a title="VITAE.preprocess.feature_select" href="#VITAE.preprocess.feature_select">feature_select</a></code></li>
+<li><code><a title="VITAE.preprocess.preprocess" href="#VITAE.preprocess.preprocess">preprocess</a></code></li>
+</ul>
+</li>
+</ul>
+</nav>
+</main>
+<footer id="footer">
+<p>Generated by <a href="https://pdoc3.github.io/pdoc"><cite>pdoc</cite> 0.8.1</a>.</p>
+</footer>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+<script>hljs.initHighlightingOnLoad()</script>
+</body>
+</html>