Datasets
Models
Downloads: 1
Diff of
/docs/preprocess.html
[000000]
..
[2c6b19]
Switch to side-by-side view
--- a +++ b/docs/preprocess.html @@ -0,0 +1,689 @@ +<!doctype html> +<html lang="en"> +<head> +<meta charset="utf-8"> +<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1" /> +<meta name="generator" content="pdoc 0.8.1" /> +<title>VITAE.preprocess API documentation</title> +<meta name="description" content="" /> +<link href='https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.0/normalize.min.css' rel='stylesheet'> +<link href='https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/8.0.0/sanitize.min.css' rel='stylesheet'> +<link href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css" rel="stylesheet"> +<style>.flex{display:flex !important}body{line-height:1.5em}#content{padding:20px}#sidebar{padding:30px;overflow:hidden}#sidebar > *:last-child{margin-bottom:2cm}.http-server-breadcrumbs{font-size:130%;margin:0 0 15px 0}#footer{font-size:.75em;padding:5px 30px;border-top:1px solid #ddd;text-align:right}#footer p{margin:0 0 0 1em;display:inline-block}#footer p:last-child{margin-right:30px}h1,h2,h3,h4,h5{font-weight:300}h1{font-size:2.5em;line-height:1.1em}h2{font-size:1.75em;margin:1em 0 .50em 0}h3{font-size:1.4em;margin:25px 0 10px 0}h4{margin:0;font-size:105%}a{color:#058;text-decoration:none;transition:color .3s ease-in-out}a:hover{color:#e82}.title code{font-weight:bold}h2[id^="header-"]{margin-top:2em}.ident{color:#900}pre code{background:#f8f8f8;font-size:.8em;line-height:1.4em}code{background:#f2f2f1;padding:1px 4px;overflow-wrap:break-word}h1 code{background:transparent}pre{background:#f8f8f8;border:0;border-top:1px solid #ccc;border-bottom:1px solid #ccc;margin:1em 0;padding:1ex}#http-server-module-list{display:flex;flex-flow:column}#http-server-module-list div{display:flex}#http-server-module-list dt{min-width:10%}#http-server-module-list p{margin-top:0}.toc ul,#index{list-style-type:none;margin:0;padding:0}#index code{background:transparent}#index h3{border-bottom:1px solid #ddd}#index ul{padding:0}#index h4{margin-top:.6em;font-weight:bold}@media (min-width:200ex){#index .two-column{column-count:2}}@media (min-width:300ex){#index .two-column{column-count:3}}dl{margin-bottom:2em}dl dl:last-child{margin-bottom:4em}dd{margin:0 0 1em 3em}#header-classes + dl > dd{margin-bottom:3em}dd dd{margin-left:2em}dd p{margin:10px 0}.name{background:#eee;font-weight:bold;font-size:.85em;padding:5px 10px;display:inline-block;min-width:40%}.name:hover{background:#e0e0e0}.name > span:first-child{white-space:nowrap}.name.class > span:nth-child(2){margin-left:.4em}.inherited{color:#999;border-left:5px solid #eee;padding-left:1em}.inheritance em{font-style:normal;font-weight:bold}.desc h2{font-weight:400;font-size:1.25em}.desc h3{font-size:1em}.desc dt code{background:inherit}.source summary,.git-link-div{color:#666;text-align:right;font-weight:400;font-size:.8em;text-transform:uppercase}.source summary > *{white-space:nowrap;cursor:pointer}.git-link{color:inherit;margin-left:1em}.source pre{max-height:500px;overflow:auto;margin:0}.source pre code{font-size:12px;overflow:visible}.hlist{list-style:none}.hlist li{display:inline}.hlist li:after{content:',\2002'}.hlist li:last-child:after{content:none}.hlist .hlist{display:inline;padding-left:1em}img{max-width:100%}.admonition{padding:.1em .5em;margin-bottom:1em}.admonition-title{font-weight:bold}.admonition.note,.admonition.info,.admonition.important{background:#aef}.admonition.todo,.admonition.versionadded,.admonition.tip,.admonition.hint{background:#dfd}.admonition.warning,.admonition.versionchanged,.admonition.deprecated{background:#fd4}.admonition.error,.admonition.danger,.admonition.caution{background:lightpink}</style> +<style media="screen and (min-width: 700px)">@media screen and (min-width:700px){#sidebar{width:30%;height:100vh;overflow:auto;position:sticky;top:0}#content{width:70%;max-width:100ch;padding:3em 4em;border-left:1px solid #ddd}pre code{font-size:1em}.item .name{font-size:1em}main{display:flex;flex-direction:row-reverse;justify-content:flex-end}.toc ul ul,#index ul{padding-left:1.5em}.toc > ul > li{margin-top:.5em}}</style> +<style media="print">@media print{#sidebar h1{page-break-before:always}.source{display:none}}@media print{*{background:transparent !important;color:#000 !important;box-shadow:none !important;text-shadow:none !important}a[href]:after{content:" (" attr(href) ")";font-size:90%}a[href][title]:after{content:none}abbr[title]:after{content:" (" attr(title) ")"}.ir a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}@page{margin:0.5cm}p,h2,h3{orphans:3;widows:3}h1,h2,h3,h4,h5,h6{page-break-after:avoid}}</style> +<script async src='https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS_CHTML'></script> +</head> +<body> +<main> +<article id="content"> +<header> +<h1 class="title">Module <code>VITAE.preprocess</code></h1> +</header> +<section id="section-intro"> +<details class="source"> +<summary> +<span>Expand source code</span> +</summary> +<pre><code class="python"># -*- coding: utf-8 -*- +from typing import Optional +import numpy as np +import pandas as pd +from skmisc import loess +from sklearn import preprocessing +import warnings +from sklearn.decomposition import PCA +from VITAE.utils import _check_expression, _check_variability + + +def normalize_gene_expression(x, K : float = 1e4, transform_fn : str = 'log'): + '''Normalize the gene expression counts for each cell by the total expression counts, + divide this by a size scale factor, which is determined by total counts and a constant K + then log-transforms the result. + + Parameters + ---------- + x : np.array + \([N, G^{raw}]\) The raw count data. + K : float, optional + The normalizing constant. + transform_fn : str, optional + Either 'log' or 'sqrt'. + + Returns + ---------- + x_normalized : np.array + \([N, G^{raw}]\) The log-normalized data. + scale_factor : np.array + \([N, ]\) The scale factors. + ''' + scale_factor = np.sum(x,axis=1, keepdims=True)/K + if transform_fn=='log': + x_normalized = np.log(x/scale_factor + 1) + else: + x_normalized = np.where(x>0, np.sqrt(x/scale_factor), 0) + + print('min normalized value: ' + str(np.min(x_normalized))) + print('max normalized value: ' + str(np.max(x_normalized))) + return x_normalized, scale_factor + + +def feature_select(x, gene_num : int = 2000): + '''Select highly variable genes (HVGs) + (See [Stuart *et al*, (2019)](https://www.nature.com/articles/nbt.4096) and its early version [preprint](https://www.biorxiv.org/content/10.1101/460147v1.full.pdf) + Page 12-13: Data preprocessing - Feature selection for individual datasets). + + Parameters + ---------- + x : np.array + \([N, G^{raw}]\) The raw count data. + gene_num : int, optional + The number of genes to retain. + + Returns + ---------- + x : np.array + \([N, G]\) The count data after gene selection. + index : np.array + \([G, ]\) The selected index of genes. + ''' + + + n, p = x.shape + + # mean and variance of each gene of the unnormalized data + mean, var = np.mean(x, axis=0), np.var(x, axis=0, ddof=1) + + # model log10(var)~log10(mean) by local fitting of polynomials of degree 2 + loess_model = loess.loess(np.log10(mean), np.log10(var), + span = 0.3, degree = 2, family='gaussian' + ) + loess_model.fit() + fitted = loess_model.outputs.fitted_values + + # standardized feature + z = (x - mean)/np.sqrt(10**fitted) + + # clipped the standardized features to remove outliers + z = np.clip(z, -np.inf, np.sqrt(n)) + + # the variance of standardized features across all cells represents a measure of + # single cell dispersion after controlling for mean expression + feature_score = np.sum(z**2, axis=0)/(n-1) + + # feature selection + index = feature_score.argsort()[::-1][0:gene_num] + + return x[:, index], index + + +def preprocess(adata = None, processed: bool = False, dimred: bool = False, + x = None, c = None, label_names = None, raw_cell_names = None, raw_gene_names = None, + K: float = 1e4, transform_fn: str = 'log', gene_num: int = 2000, data_type: str = 'UMI', + npc: int = 64, random_state=0): + '''Preprocess count data. + + Parameters + ---------- + adata : AnnData, optional + The scanpy object. + processed : boolean + Whether adata has been processed. + dimred : boolean + Whether the processed adata is after dimension reduction. + x : np.array, optional + \([N^{raw}, G^{raw}]\) The raw count matrix. + c : np.array + \([N^{raw}, s]\) The covariate matrix. + label_names : np.array + \([N^{raw}, ]\) The true or estimated cell types. + raw_cell_names : np.array + \([N^{raw}, ]\) The names of cells. + raw_gene_names : np.array + \([G^{raw}, ]\) The names of genes. + K : int, optional + The normalizing constant. + transform_fn : str + The transform function used to normalize the gene expression after scaling. Either 'log' or 'sqrt'. + gene_num : int, optional + The number of genes to retain. + data_type : str, optional + 'UMI', 'non-UMI', or 'Gaussian'. + npc : int, optional + The number of PCs to retain, only used if `data_type='Gaussian'`. + random_state : int, optional + The random state for PCA. With different random states, the resulted PCA scores are slightly different. + + Returns + ---------- + x_normalized : np.array + \([N, G]\) The preprocessed matrix. + expression : np.array + \([N, G^{raw}]\) The expression matrix after log-normalization and before scaling. + x : np.array + \([N, G]\) The raw count matrix after gene selections. + c : np.array + \([N, s]\) The covariates. + cell_names : np.array + \([N, ]\) The cell names. + gene_names : np.array + \([G^{raw}, ]\) The gene names. + selected_gene_names : + \([G, ]\) The selected gene names. + scale_factor : + \([N, ]\) The scale factors. + labels : np.array + \([N, ]\) The encoded labels. + label_names : np.array + \([N, ]\) The label names. + le : sklearn.preprocessing.LabelEncoder + The label encoder. + gene_scalar : sklearn.preprocessing.StandardScaler + The gene scaler. + ''' + # if input is a scanpy data + if adata is not None: + import scanpy as sc + + # if the input scanpy is processed + if processed: + x_normalized = x = adata.X + gene_names = adata.var_names.values + expression = None + scale_factor = np.ones(x.shape[0]) + # if the input scanpy is not processed + else: + dimred = False + x = adata.X.copy() + adata, expression, gene_names, cell_mask, gene_mask, gene_mask2 = _recipe_seurat(adata, gene_num) + x_normalized = adata.X.copy() + scale_factor = adata.obs.counts_per_cell.values / 1e4 + x = x[cell_mask,:][:,gene_mask][:,gene_mask2] + + if label_names is None: + try: + label_names = adata.obs.cell_types + except: + if label_names is not None and processed is False: + label_names = label_names[cell_mask] + + cell_names = adata.obs_names.values + selected_gene_names = adata.var_names.values + gene_scalar = None + + # if input is a count matrix + else: + # remove cells that have no expression + expressed = _check_expression(x) + print('Removing %d cells without expression.'%(np.sum(expressed==0))) + x = x[expressed==1,:] + if c is not None: + c = c[expressed==1,:] + if label_names is not None: + label_names = label_names[expressed==1] + + # remove genes without variability + variable = _check_variability(x) + print('Removing %d genes without variability.'%(np.sum(variable==0))) + x = x[:, variable==1] + gene_names = raw_gene_names[variable==1] + + # log-normalization + expression, scale_factor = normalize_gene_expression(x, K, transform_fn) + + # feature selection + x, index = feature_select(x, gene_num) + selected_expression = expression[:, index] + + # per-gene standardization + gene_scalar = preprocessing.StandardScaler() + x_normalized = gene_scalar.fit_transform(selected_expression) + + cell_names = raw_cell_names[expressed==1] + selected_gene_names = gene_names[index] + + if (data_type=='Gaussian') and (dimred is False): + # use arpack solver and extend precision to get deterministic result + pca = PCA(n_components = npc, random_state=random_state, svd_solver='arpack') + x_normalized = x = pca.fit_transform(x_normalized.astype(np.float64)).astype(np.float32) + + if c is not None: + c_scalar = preprocessing.StandardScaler() + c = c_scalar.fit_transform(c) + + if label_names is None: + warnings.warn('No labels for cells!') + labels = None + le = None + else: + le = preprocessing.LabelEncoder() + labels = le.fit_transform(label_names) + print('Number of cells in each class: ') + table = pd.value_counts(label_names) + table.index = pd.Series(le.transform(table.index).astype(str)) \ + + ' <---> ' + table.index + table = table.sort_index() + print(table) + + return (x_normalized, expression, x, c, + cell_names, gene_names, selected_gene_names, + scale_factor, labels, label_names, le, gene_scalar) + + +def _recipe_seurat(adata, gene_num): + """ + Normalization and filtering as of Seurat [Satija15]_. + This uses a particular preprocessing + """ + import scanpy as sc + cell_mask = sc.pp.filter_cells(adata, min_genes=200, inplace=False)[0] + adata = adata[cell_mask,:] + gene_mask = sc.pp.filter_genes(adata, min_cells=3, inplace=False)[0] + adata = adata[:,gene_mask] + gene_names = adata.var_names.values + + sc.pp.normalize_total(adata, target_sum=1e4, key_added='counts_per_cell') + filter_result = sc.pp.filter_genes_dispersion( + adata.X, min_mean=0.0125, max_mean=3, min_disp=0.5, log=False, n_top_genes=gene_num) + + sc.pp.log1p(adata) + expression = adata.X.copy() + adata._inplace_subset_var(filter_result.gene_subset) # filter genes + sc.pp.scale(adata, max_value=10) + return adata, expression, gene_names, cell_mask, gene_mask, filter_result.gene_subset</code></pre> +</details> +</section> +<section> +</section> +<section> +</section> +<section> +<h2 class="section-title" id="header-functions">Functions</h2> +<dl> +<dt id="VITAE.preprocess.normalize_gene_expression"><code class="name flex"> +<span>def <span class="ident">normalize_gene_expression</span></span>(<span>x, K: float = 10000.0, transform_fn: str = 'log')</span> +</code></dt> +<dd> +<div class="desc"><p>Normalize the gene expression counts for each cell by the total expression counts, +divide this by a size scale factor, which is determined by total counts and a constant K +then log-transforms the result.</p> +<h2 id="parameters">Parameters</h2> +<dl> +<dt><strong><code>x</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N, G^{raw}]</span><script type="math/tex">[N, G^{raw}]</script></span> The raw count data.</dd> +<dt><strong><code>K</code></strong> : <code>float</code>, optional</dt> +<dd>The normalizing constant.</dd> +<dt><strong><code>transform_fn</code></strong> : <code>str</code>, optional</dt> +<dd>Either 'log' or 'sqrt'.</dd> +</dl> +<h2 id="returns">Returns</h2> +<dl> +<dt><strong><code>x_normalized</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N, G^{raw}]</span><script type="math/tex">[N, G^{raw}]</script></span> The log-normalized data.</dd> +<dt><strong><code>scale_factor</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The scale factors.</dd> +</dl></div> +<details class="source"> +<summary> +<span>Expand source code</span> +</summary> +<pre><code class="python">def normalize_gene_expression(x, K : float = 1e4, transform_fn : str = 'log'): + '''Normalize the gene expression counts for each cell by the total expression counts, + divide this by a size scale factor, which is determined by total counts and a constant K + then log-transforms the result. + + Parameters + ---------- + x : np.array + \([N, G^{raw}]\) The raw count data. + K : float, optional + The normalizing constant. + transform_fn : str, optional + Either 'log' or 'sqrt'. + + Returns + ---------- + x_normalized : np.array + \([N, G^{raw}]\) The log-normalized data. + scale_factor : np.array + \([N, ]\) The scale factors. + ''' + scale_factor = np.sum(x,axis=1, keepdims=True)/K + if transform_fn=='log': + x_normalized = np.log(x/scale_factor + 1) + else: + x_normalized = np.where(x>0, np.sqrt(x/scale_factor), 0) + + print('min normalized value: ' + str(np.min(x_normalized))) + print('max normalized value: ' + str(np.max(x_normalized))) + return x_normalized, scale_factor</code></pre> +</details> +</dd> +<dt id="VITAE.preprocess.feature_select"><code class="name flex"> +<span>def <span class="ident">feature_select</span></span>(<span>x, gene_num: int = 2000)</span> +</code></dt> +<dd> +<div class="desc"><p>Select highly variable genes (HVGs) +(See <a href="https://www.nature.com/articles/nbt.4096">Stuart <em>et al</em>, (2019)</a> and its early version <a href="https://www.biorxiv.org/content/10.1101/460147v1.full.pdf">preprint</a> +Page 12-13: Data preprocessing - Feature selection for individual datasets).</p> +<h2 id="parameters">Parameters</h2> +<dl> +<dt><strong><code>x</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N, G^{raw}]</span><script type="math/tex">[N, G^{raw}]</script></span> The raw count data.</dd> +<dt><strong><code>gene_num</code></strong> : <code>int</code>, optional</dt> +<dd>The number of genes to retain.</dd> +</dl> +<h2 id="returns">Returns</h2> +<dl> +<dt><strong><code>x</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N, G]</span><script type="math/tex">[N, G]</script></span> The count data after gene selection.</dd> +<dt><strong><code>index</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[G, ]</span><script type="math/tex">[G, ]</script></span> The selected index of genes.</dd> +</dl></div> +<details class="source"> +<summary> +<span>Expand source code</span> +</summary> +<pre><code class="python">def feature_select(x, gene_num : int = 2000): + '''Select highly variable genes (HVGs) + (See [Stuart *et al*, (2019)](https://www.nature.com/articles/nbt.4096) and its early version [preprint](https://www.biorxiv.org/content/10.1101/460147v1.full.pdf) + Page 12-13: Data preprocessing - Feature selection for individual datasets). + + Parameters + ---------- + x : np.array + \([N, G^{raw}]\) The raw count data. + gene_num : int, optional + The number of genes to retain. + + Returns + ---------- + x : np.array + \([N, G]\) The count data after gene selection. + index : np.array + \([G, ]\) The selected index of genes. + ''' + + + n, p = x.shape + + # mean and variance of each gene of the unnormalized data + mean, var = np.mean(x, axis=0), np.var(x, axis=0, ddof=1) + + # model log10(var)~log10(mean) by local fitting of polynomials of degree 2 + loess_model = loess.loess(np.log10(mean), np.log10(var), + span = 0.3, degree = 2, family='gaussian' + ) + loess_model.fit() + fitted = loess_model.outputs.fitted_values + + # standardized feature + z = (x - mean)/np.sqrt(10**fitted) + + # clipped the standardized features to remove outliers + z = np.clip(z, -np.inf, np.sqrt(n)) + + # the variance of standardized features across all cells represents a measure of + # single cell dispersion after controlling for mean expression + feature_score = np.sum(z**2, axis=0)/(n-1) + + # feature selection + index = feature_score.argsort()[::-1][0:gene_num] + + return x[:, index], index</code></pre> +</details> +</dd> +<dt id="VITAE.preprocess.preprocess"><code class="name flex"> +<span>def <span class="ident">preprocess</span></span>(<span>adata=None, processed: bool = False, dimred: bool = False, x=None, c=None, label_names=None, raw_cell_names=None, raw_gene_names=None, K: float = 10000.0, transform_fn: str = 'log', gene_num: int = 2000, data_type: str = 'UMI', npc: int = 64, random_state=0)</span> +</code></dt> +<dd> +<div class="desc"><p>Preprocess count data.</p> +<h2 id="parameters">Parameters</h2> +<dl> +<dt><strong><code>adata</code></strong> : <code>AnnData</code>, optional</dt> +<dd>The scanpy object.</dd> +<dt><strong><code>processed</code></strong> : <code>boolean</code></dt> +<dd>Whether adata has been processed.</dd> +<dt><strong><code>dimred</code></strong> : <code>boolean</code></dt> +<dd>Whether the processed adata is after dimension reduction.</dd> +<dt><strong><code>x</code></strong> : <code>np.array</code>, optional</dt> +<dd><span><span class="MathJax_Preview">[N^{raw}, G^{raw}]</span><script type="math/tex">[N^{raw}, G^{raw}]</script></span> The raw count matrix.</dd> +<dt><strong><code>c</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N^{raw}, s]</span><script type="math/tex">[N^{raw}, s]</script></span> The covariate matrix.</dd> +<dt><strong><code>label_names</code></strong> : <code>np.array </code></dt> +<dd><span><span class="MathJax_Preview">[N^{raw}, ]</span><script type="math/tex">[N^{raw}, ]</script></span> The true or estimated cell types.</dd> +<dt><strong><code>raw_cell_names</code></strong> : <code>np.array +</code></dt> +<dd><span><span class="MathJax_Preview">[N^{raw}, ]</span><script type="math/tex">[N^{raw}, ]</script></span> The names of cells.</dd> +<dt><strong><code>raw_gene_names</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[G^{raw}, ]</span><script type="math/tex">[G^{raw}, ]</script></span> The names of genes.</dd> +<dt><strong><code>K</code></strong> : <code>int</code>, optional</dt> +<dd>The normalizing constant.</dd> +<dt><strong><code>transform_fn</code></strong> : <code>str</code></dt> +<dd>The transform function used to normalize the gene expression after scaling. Either 'log' or 'sqrt'.</dd> +<dt><strong><code>gene_num</code></strong> : <code>int</code>, optional</dt> +<dd>The number of genes to retain.</dd> +<dt><strong><code>data_type</code></strong> : <code>str</code>, optional</dt> +<dd>'UMI', 'non-UMI', or 'Gaussian'.</dd> +<dt><strong><code>npc</code></strong> : <code>int</code>, optional</dt> +<dd>The number of PCs to retain, only used if <code>data_type='Gaussian'</code>.</dd> +<dt><strong><code>random_state</code></strong> : <code>int</code>, optional</dt> +<dd>The random state for PCA. With different random states, the resulted PCA scores are slightly different.</dd> +</dl> +<h2 id="returns">Returns</h2> +<dl> +<dt><strong><code>x_normalized</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N, G]</span><script type="math/tex">[N, G]</script></span> The preprocessed matrix.</dd> +<dt><strong><code>expression</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N, G^{raw}]</span><script type="math/tex">[N, G^{raw}]</script></span> The expression matrix after log-normalization and before scaling.</dd> +<dt><strong><code>x</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N, G]</span><script type="math/tex">[N, G]</script></span> The raw count matrix after gene selections.</dd> +<dt><strong><code>c</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N, s]</span><script type="math/tex">[N, s]</script></span> The covariates.</dd> +<dt><strong><code>cell_names</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The cell names.</dd> +<dt><strong><code>gene_names</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[G^{raw}, ]</span><script type="math/tex">[G^{raw}, ]</script></span> The gene names.</dd> +<dt><strong><code>selected_gene_names</code></strong></dt> +<dd><span><span class="MathJax_Preview">[G, ]</span><script type="math/tex">[G, ]</script></span> The selected gene names.</dd> +<dt><strong><code>scale_factor</code></strong></dt> +<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The scale factors.</dd> +<dt><strong><code>labels</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The encoded labels.</dd> +<dt><strong><code>label_names</code></strong> : <code>np.array</code></dt> +<dd><span><span class="MathJax_Preview">[N, ]</span><script type="math/tex">[N, ]</script></span> The label names.</dd> +<dt><strong><code>le</code></strong> : <code>sklearn.preprocessing.LabelEncoder</code></dt> +<dd>The label encoder.</dd> +<dt><strong><code>gene_scalar</code></strong> : <code>sklearn.preprocessing.StandardScaler</code></dt> +<dd>The gene scaler.</dd> +</dl></div> +<details class="source"> +<summary> +<span>Expand source code</span> +</summary> +<pre><code class="python">def preprocess(adata = None, processed: bool = False, dimred: bool = False, + x = None, c = None, label_names = None, raw_cell_names = None, raw_gene_names = None, + K: float = 1e4, transform_fn: str = 'log', gene_num: int = 2000, data_type: str = 'UMI', + npc: int = 64, random_state=0): + '''Preprocess count data. + + Parameters + ---------- + adata : AnnData, optional + The scanpy object. + processed : boolean + Whether adata has been processed. + dimred : boolean + Whether the processed adata is after dimension reduction. + x : np.array, optional + \([N^{raw}, G^{raw}]\) The raw count matrix. + c : np.array + \([N^{raw}, s]\) The covariate matrix. + label_names : np.array + \([N^{raw}, ]\) The true or estimated cell types. + raw_cell_names : np.array + \([N^{raw}, ]\) The names of cells. + raw_gene_names : np.array + \([G^{raw}, ]\) The names of genes. + K : int, optional + The normalizing constant. + transform_fn : str + The transform function used to normalize the gene expression after scaling. Either 'log' or 'sqrt'. + gene_num : int, optional + The number of genes to retain. + data_type : str, optional + 'UMI', 'non-UMI', or 'Gaussian'. + npc : int, optional + The number of PCs to retain, only used if `data_type='Gaussian'`. + random_state : int, optional + The random state for PCA. With different random states, the resulted PCA scores are slightly different. + + Returns + ---------- + x_normalized : np.array + \([N, G]\) The preprocessed matrix. + expression : np.array + \([N, G^{raw}]\) The expression matrix after log-normalization and before scaling. + x : np.array + \([N, G]\) The raw count matrix after gene selections. + c : np.array + \([N, s]\) The covariates. + cell_names : np.array + \([N, ]\) The cell names. + gene_names : np.array + \([G^{raw}, ]\) The gene names. + selected_gene_names : + \([G, ]\) The selected gene names. + scale_factor : + \([N, ]\) The scale factors. + labels : np.array + \([N, ]\) The encoded labels. + label_names : np.array + \([N, ]\) The label names. + le : sklearn.preprocessing.LabelEncoder + The label encoder. + gene_scalar : sklearn.preprocessing.StandardScaler + The gene scaler. + ''' + # if input is a scanpy data + if adata is not None: + import scanpy as sc + + # if the input scanpy is processed + if processed: + x_normalized = x = adata.X + gene_names = adata.var_names.values + expression = None + scale_factor = np.ones(x.shape[0]) + # if the input scanpy is not processed + else: + dimred = False + x = adata.X.copy() + adata, expression, gene_names, cell_mask, gene_mask, gene_mask2 = _recipe_seurat(adata, gene_num) + x_normalized = adata.X.copy() + scale_factor = adata.obs.counts_per_cell.values / 1e4 + x = x[cell_mask,:][:,gene_mask][:,gene_mask2] + + if label_names is None: + try: + label_names = adata.obs.cell_types + except: + if label_names is not None and processed is False: + label_names = label_names[cell_mask] + + cell_names = adata.obs_names.values + selected_gene_names = adata.var_names.values + gene_scalar = None + + # if input is a count matrix + else: + # remove cells that have no expression + expressed = _check_expression(x) + print('Removing %d cells without expression.'%(np.sum(expressed==0))) + x = x[expressed==1,:] + if c is not None: + c = c[expressed==1,:] + if label_names is not None: + label_names = label_names[expressed==1] + + # remove genes without variability + variable = _check_variability(x) + print('Removing %d genes without variability.'%(np.sum(variable==0))) + x = x[:, variable==1] + gene_names = raw_gene_names[variable==1] + + # log-normalization + expression, scale_factor = normalize_gene_expression(x, K, transform_fn) + + # feature selection + x, index = feature_select(x, gene_num) + selected_expression = expression[:, index] + + # per-gene standardization + gene_scalar = preprocessing.StandardScaler() + x_normalized = gene_scalar.fit_transform(selected_expression) + + cell_names = raw_cell_names[expressed==1] + selected_gene_names = gene_names[index] + + if (data_type=='Gaussian') and (dimred is False): + # use arpack solver and extend precision to get deterministic result + pca = PCA(n_components = npc, random_state=random_state, svd_solver='arpack') + x_normalized = x = pca.fit_transform(x_normalized.astype(np.float64)).astype(np.float32) + + if c is not None: + c_scalar = preprocessing.StandardScaler() + c = c_scalar.fit_transform(c) + + if label_names is None: + warnings.warn('No labels for cells!') + labels = None + le = None + else: + le = preprocessing.LabelEncoder() + labels = le.fit_transform(label_names) + print('Number of cells in each class: ') + table = pd.value_counts(label_names) + table.index = pd.Series(le.transform(table.index).astype(str)) \ + + ' <---> ' + table.index + table = table.sort_index() + print(table) + + return (x_normalized, expression, x, c, + cell_names, gene_names, selected_gene_names, + scale_factor, labels, label_names, le, gene_scalar)</code></pre> +</details> +</dd> +</dl> +</section> +<section> +</section> +</article> +<nav id="sidebar"> +<h1>Index</h1> +<div class="toc"> +<ul></ul> +</div> +<ul id="index"> +<li><h3>Super-module</h3> +<ul> +<li><code><a title="VITAE" href="index.html">VITAE</a></code></li> +</ul> +</li> +<li><h3><a href="#header-functions">Functions</a></h3> +<ul class=""> +<li><code><a title="VITAE.preprocess.normalize_gene_expression" href="#VITAE.preprocess.normalize_gene_expression">normalize_gene_expression</a></code></li> +<li><code><a title="VITAE.preprocess.feature_select" href="#VITAE.preprocess.feature_select">feature_select</a></code></li> +<li><code><a title="VITAE.preprocess.preprocess" href="#VITAE.preprocess.preprocess">preprocess</a></code></li> +</ul> +</li> +</ul> +</nav> +</main> +<footer id="footer"> +<p>Generated by <a href="https://pdoc3.github.io/pdoc"><cite>pdoc</cite> 0.8.1</a>.</p> +</footer> +<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script> +<script>hljs.initHighlightingOnLoad()</script> +</body> +</html> \ No newline at end of file