deepG / Git / [1c0e03] /docs/tutorial_html/data

Models:
MarcoTheBlack/
deepG
Downloads: 2
[1c0e03]: / docs / tutorial_html / data_generator.html
History
Download this file
1815 lines (1781 with data), 189.9 kB

<!DOCTYPE html>
<!-- Generated by pkgdown: do not edit by hand --><html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="description" content="deepG">
<title>Data Generator • deepG</title>
<!-- favicons --><link rel="icon" type="image/png" sizes="16x16" href="../favicon-16x16.png">
<link rel="icon" type="image/png" sizes="32x32" href="../favicon-32x32.png">
<link rel="apple-touch-icon" type="image/png" sizes="180x180" href="../apple-touch-icon.png">
<link rel="apple-touch-icon" type="image/png" sizes="120x120" href="../apple-touch-icon-120x120.png">
<link rel="apple-touch-icon" type="image/png" sizes="76x76" href="../apple-touch-icon-76x76.png">
<link rel="apple-touch-icon" type="image/png" sizes="60x60" href="../apple-touch-icon-60x60.png">
<script src="../deps/jquery-3.6.0/jquery-3.6.0.min.js"></script><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<link href="../deps/bootstrap-5.3.1/bootstrap.min.css" rel="stylesheet">
<script src="../deps/bootstrap-5.3.1/bootstrap.bundle.min.js"></script><!-- Font Awesome icons --><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous">
<!-- bootstrap-toc --><script src="https://cdn.jsdelivr.net/gh/afeld/bootstrap-toc@v1.0.1/dist/bootstrap-toc.min.js" integrity="sha256-4veVQbu7//Lk5TSmc7YV48MxtMy98e26cf5MrgZYnwo=" crossorigin="anonymous"></script><!-- headroom.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script><!-- clipboard.js --><script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.11/clipboard.min.js" integrity="sha512-7O5pXpc0oCRrxk8RUfDYFgn0nO1t+jLuIOQdOMRp4APB7uZ4vSjspzp5y6YDtDs4VzUSTbWzBFZ/LKJhnyFOKw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script><!-- search --><script src="https://cdnjs.cloudflare.com/ajax/libs/fuse.js/6.4.6/fuse.js" integrity="sha512-zv6Ywkjyktsohkbp9bb45V6tEMoWhzFzXis+LrMehmJZZSys19Yxf1dopHx7WzIKxr5tK2dVcYmaCk2uqdjF4A==" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/autocomplete.js/0.38.0/autocomplete.jquery.min.js" integrity="sha512-GU9ayf+66Xx2TmpxqJpliWbT5PiGYxpaG8rfnBEk1LL8l1KGkRShhngwdXK1UgqhAzWpZHSiYPc09/NwDQIGyg==" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mark.js/8.11.1/mark.min.js" integrity="sha512-5CYOlHXGh6QpOFA/TeTylKLWfB3ftPsde7AnmhuitiTX4K5SqCLBeKro6sPS8ilsz1Q4NRx3v8Ko2IBiszzdww==" crossorigin="anonymous"></script><!-- pkgdown --><script src="../pkgdown.js"></script><meta property="og:title" content="Data Generator">
<meta property="og:description" content="deepG">
<meta property="og:image" content="https://genomenet.github.io/deepG/logo.png">
<!-- mathjax --><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script><!--[if lt IE 9]>
<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
<![endif]-->
</head>
<body>
    <a href="#main" class="visually-hidden-focusable">Skip to contents</a>
    

    <nav class="navbar fixed-top navbar-light navbar-expand-lg bg-light" data-bs-theme="light"><div class="container">
    
    <a class="navbar-brand me-2" href="../index.html">deepG</a>

    <small class="nav-text text-default me-auto" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Released version">0.3.0</small>

    
    <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbar" aria-controls="navbar" aria-expanded="false" aria-label="Toggle navigation">
      <span class="navbar-toggler-icon"></span>
    </button>

    <div id="navbar" class="collapse navbar-collapse ms-3">
      <ul class="navbar-nav me-auto">
<li class="nav-item">
  <a class="nav-link" href="../reference/index.html">
    <span class="fa fa fa fa-file-alt"></span>
     
    Reference
  </a>
</li>
<li class="nav-item dropdown">
  <a href="#" class="nav-link dropdown-toggle" data-bs-toggle="dropdown" role="button" aria-expanded="false" aria-haspopup="true" id="dropdown-notebooks">Notebooks</a>
  <div class="dropdown-menu" aria-labelledby="dropdown-notebooks">
    <a class="external-link dropdown-item" href="https://colab.research.google.com/drive/175jIdXcDcgPUvaBo2rH2Lupbpjnp5O7G?usp=sharing">deepG tutorial</a>
    <a class="external-link dropdown-item" href="https://colab.research.google.com/drive/1Eolc0koMNM1zkuO4XyVM58ImeF1BpRiH?usp=sharing">Read-length level: Human contamination</a>
    <a class="external-link dropdown-item" href="https://colab.research.google.com/drive/1yiXSwFafXpMLHaov9iBTQLIDZ6bK1zYX?usp=sharing">Locus level: CRISPR detection</a>
    <a class="external-link dropdown-item" href="https://colab.research.google.com/drive/1G7bOFEX87cZNrM2tdRtTdkrZn5fM__g0?usp=sharing">Gene level: 16S rRNA detection</a>
    <a class="external-link dropdown-item" href="https://colab.research.google.com/drive/1BCggL-tfQF136YeJ8cKKi-zoBEDMgkNh?usp=sharing">Genome level: Bacterial morphology (Sporulation)</a>
    <a class="external-link dropdown-item" href="https://colab.research.google.com/drive/10xpRzGd3JeBAbqQYSCxzQUMctt01sx9D?usp=sharing">Full metagenome level: Colorectal cancer prediction</a>
    <a class="external-link dropdown-item" href="https://colab.research.google.com/drive/1kyYK7IU7GSfdpDzO_a8U3_qD4i3zTu6w?usp=sharing">BERT with deepG</a>
  </div>
</li>
<li class="active nav-item dropdown">
  <a href="#" class="nav-link dropdown-toggle" data-bs-toggle="dropdown" role="button" aria-expanded="false" aria-haspopup="true" id="dropdown-tutorials">Tutorials</a>
  <div class="dropdown-menu" aria-labelledby="dropdown-tutorials">
    <a class="dropdown-item" href="../articles/getting_started.html">Getting Started</a>
    <a class="dropdown-item" href="../articles/training_types.html">Training types</a>
    <a class="dropdown-item" href="../articles/data_generator.html">Data generator</a>
    <a class="dropdown-item" href="../articles/using_tb.html">Using tensorboard</a>
    <a class="dropdown-item" href="../articles/integrated_gradient.html">Integrated Gradient</a>
  </div>
</li>
      </ul>
<form class="form-inline my-2 my-lg-0" role="search">
        <input type="search" class="form-control me-sm-2" aria-label="Toggle navigation" name="search-input" data-search-index="../search.json" id="search-input" placeholder="Search for" autocomplete="off">
</form>

      <ul class="navbar-nav">
<li class="nav-item">
  <a class="external-link nav-link" href="https://github.com/GenomeNet/deepG/" aria-label="github">
    <span class="fab fa fab fa-github fa-lg"></span>
     
  </a>
</li>
      </ul>
</div>

    
  </div>
</nav><div class="container template-article">




<div class="row">
  <main id="main" class="col-md-9"><div class="page-header">
      <img src="../logo.png" class="logo" alt=""><h1>Data Generator</h1>
            
      
      <small class="dont-index">Source: <a href="https://github.com/GenomeNet/deepG/blob/HEAD/vignettes/data_generator.Rmd" class="external-link"><code>vignettes/data_generator.Rmd</code></a></small>
      <div class="d-none name"><code>data_generator.Rmd</code></div>
    </div>

    
    
<div class="sourceCode" id="cb1"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co">#devtools::install_github("GenomeNet/deepG")</span></span>
<span><span class="co">#library(deepG)</span></span>
<span><span class="co">#library(magrittr)</span></span></code></pre></div>
<style type="text/css">
mark.in {
  background-color: CornflowerBlue;
}

mark.out {
  background-color: IndianRed;
}

</style>
<div class="section level2">
<h2 id="introduction">Introduction<a class="anchor" aria-label="anchor" href="#introduction"></a>
</h2>
<p>The most common use case for the deepG data generator is to extract
samples from a collection of fasta (or fastq) files. The generator will
always return a list of length 2. The first element is the input <span class="math inline">\(X\)</span> and the second the target <span class="math inline">\(Y\)</span>. We can differentiate between 2
approaches</p>
<ul>
<li>
<strong>Language model</strong>: Part of a sequence is the input and
other part the target.
<ul>
<li>Example: Predict the next nucleotide given the previous 100
nucleotides.</li>
</ul>
</li>
<li>
<strong>Label classification</strong>: Assign a label to a sequence.
<ul>
<li>Example: Assign a label “virus” or “bacteria” to a sequence of
length 100.</li>
</ul>
</li>
</ul>
<p>Suppose we are given 2 fasta files called “a.fasta” and “b.fasta”
that look as follows:</p>
<div style="float: left;margin-right:10px">
<table class="table"><tr>
<td>
<strong>a.fasta</strong> <br><tt> &gt;header_a1 <br> AACCAAGG <br>
&gt;header_a2 <br> TTTGGG <br> &gt;header_a3 <br> ACGTACGT <br></tt>
</td>
</tr></table>
</div>
<div style="float: left">
<table class="table"><tr>
<td>
<strong>b.fasta</strong> <br><tt> &gt;header_b1 <br> GTGTGT <br>
&gt;header_b2 <br> AAGG <br></tt>
</td>
</tr></table>
</div>
<p><br><br><br><br><br><br><br><br><br></p>
<p>If we want to extract sequences of length 4 from these files, there
would be 17 possible samples (5 from <tt>AACCAAGG</tt>, 3 from
<tt>TTTGGG</tt>, …). A naive approach would be to extract the samples in
a sequential manner:</p>
<p><em>1. sample</em>:</p>
<div style="float: left;margin-right:10px">
<table class="table"><tr>
<td>
<strong>a.fasta</strong> <br><tt> &gt;header_a1 <br><mark class="in">AACC</mark>AAGG <br> &gt;header_a2 <br> TTTGGG <br>
&gt;header_a3 <br> ACGTACGT <br></tt>
</td>
</tr></table>
</div>
<div style="float: left">
<table class="table"><tr>
<td>
<strong>b.fasta</strong> <br><tt> &gt;header_b1 <br> GTGTGT <br>
&gt;header_b2 <br> AAGG <br></tt>
</td>
</tr></table>
</div>
<p><br><br><br><br><br><br><br><br><br></p>
<p><em>2. sample</em>:</p>
<div style="float: left;margin-right:10px">
<table class="table"><tr>
<td>
<strong>a.fasta</strong> <br><tt> &gt;header_a1 <br>
A<mark class="in">ACCA</mark>AGG <br> &gt;header_a2 <br> TTTGGG <br>
&gt;header_a3 <br> ACGTACGT <br></tt>
</td>
</tr></table>
</div>
<div style="float: left">
<table class="table"><tr>
<td>
<strong>b.fasta</strong> <br><tt> &gt;header_b1 <br> GTGTGT <br>
&gt;header_b2 <br> AAGG <br></tt>
</td>
</tr></table>
</div>
<p><br><br><br><br><br><br><br><br><br></p>
<p>…</p>
<p><br></p>
<p><em>17. sample</em>:</p>
<div style="float: left;margin-right:10px">
<table class="table"><tr>
<td>
<strong>a.fasta</strong> <br><tt> &gt;header_a1 <br> AACCAAGG <br>
&gt;header_a2 <br> TTTGGG <br> &gt;header_a3 <br> ACGTACGT <br></tt>
</td>
</tr></table>
</div>
<div style="float: left">
<table class="table"><tr>
<td>
<strong>b.fasta</strong> <br><tt> &gt;header_b1 <br> GTGTGT <br>
&gt;header_b2 <br><mark class="in">AAGG</mark><br></tt>
</td>
</tr></table>
</div>
<p><br><br><br><br><br><br><br><br><br></p>
<p><em>18. sample</em>:</p>
<div style="float: left;margin-right:10px">
<table class="table"><tr>
<td>
<strong>a.fasta</strong> <br><tt> &gt;header_a1 <br><mark class="in">AACC</mark>AAGG <br> &gt;header_a2 <br> TTTGGG <br>
&gt;header_a3 <br> ACGTACGT <br></tt>
</td>
</tr></table>
</div>
<div style="float: left">
<table class="table"><tr>
<td>
<strong>b.fasta</strong> <br><tt> &gt;header_b1 <br> GTGTGT <br>
&gt;header_b2 <br> AAGG <br></tt>
</td>
</tr></table>
</div>
<p><br><br><br><br><br><br><br><br><br></p>
<p>… <br><br></p>
<p>For longer sequences this is not a desirable strategy since the data
is very redundant (often just one nucleotide difference) and the model
would often see long stretches of data from the same source. Choosing
the samples completely at random can also be problematic since we would
constantly have to open new files. The deepG generators offers several
option to navigate the data sampling strategy to achieve a good balance
between the two approaches.</p>
</div>
<div class="section level2">
<h2 id="data-generator-options">Data generator options<a class="anchor" aria-label="anchor" href="#data-generator-options"></a>
</h2>
<p>In the following code examples, we will mostly use the sequence <tt>
<strong>abcdefghiiii</strong> </tt> to demonstrate some of the deepG
data generator options. (In real world application you would usually
have sequences from the <tt>ACGT</tt> vocabulary.)</p>
<div class="sourceCode" id="cb2"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">sequence</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste0</a></span><span class="op">(</span><span class="st">"a"</span>, <span class="st">"b"</span>, <span class="st">"c"</span>, <span class="st">"d"</span>, <span class="st">"e"</span>, <span class="st">"f"</span>, <span class="st">"g"</span>, <span class="st">"h"</span>, <span class="st">"i"</span>, <span class="st">"i"</span>, <span class="st">"i"</span>, <span class="st">"i"</span><span class="op">)</span></span>
<span><span class="va">vocabulary</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"a"</span>, <span class="st">"b"</span>, <span class="st">"c"</span>, <span class="st">"d"</span>, <span class="st">"e"</span>, <span class="st">"f"</span>, <span class="st">"g"</span>, <span class="st">"h"</span>, <span class="st">"i"</span><span class="op">)</span>  </span></code></pre></div>
<p>We may store this sequence in a fasta file</p>
<div class="sourceCode" id="cb3"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">temp_dir</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span><span class="op">)</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/files2.html" class="external-link">dir.create</a></span><span class="op">(</span><span class="va">temp_dir</span><span class="op">)</span></span>
<span><span class="va">dir_path</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste0</a></span><span class="op">(</span><span class="va">temp_dir</span>, <span class="st">"/dummy_data"</span><span class="op">)</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/files2.html" class="external-link">dir.create</a></span><span class="op">(</span><span class="va">dir_path</span><span class="op">)</span></span>
<span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>Sequence <span class="op">=</span> <span class="va">sequence</span>, Header <span class="op">=</span> <span class="st">"label_1"</span>, stringsAsFactors <span class="op">=</span> <span class="cn">FALSE</span><span class="op">)</span></span>
<span><span class="va">file_path</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="va">dir_path</span>, <span class="st">"a.fasta"</span><span class="op">)</span></span>
<span><span class="co"># sequence as fasta file</span></span>
<span><span class="fu">microseq</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/microseq/man/readFasta.html" class="external-link">writeFasta</a></span><span class="op">(</span>fdta <span class="op">=</span> <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://tibble.tidyverse.org/reference/as_tibble.html" class="external-link">as_tibble</a></span><span class="op">(</span><span class="va">df</span><span class="op">)</span>, out.file <span class="op">=</span> <span class="va">file_path</span><span class="op">)</span></span></code></pre></div>
<p>Since neural networks can only work with numeric data, we have to
encode sequences of characters with numeric data. Usually this is
achieved by one-hot-encoding; there are some other approaches
implemented: see <code>use_coverage</code>,
<code>use_quality_score</code> and <code>ambiguous_nuc</code>
sections.</p>
<div class="sourceCode" id="cb4"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># one-hot encoding example</span></span>
<span><span class="va">s</span> <span class="op">&lt;-</span>  <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"a"</span>, <span class="st">"c"</span>, <span class="st">"a"</span>, <span class="st">"f"</span>, <span class="st">"i"</span>, <span class="st">"b"</span><span class="op">)</span></span>
<span><span class="va">s_as_int_seq</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/vector.html" class="external-link">vector</a></span><span class="op">(</span><span class="st">"integer"</span>, <span class="fu"><a href="https://rdrr.io/r/base/length.html" class="external-link">length</a></span><span class="op">(</span><span class="va">s</span><span class="op">)</span><span class="op">)</span></span>
<span><span class="kw">for</span> <span class="op">(</span><span class="va">i</span> <span class="kw">in</span> <span class="fl">1</span><span class="op">:</span><span class="fu"><a href="https://rdrr.io/r/base/length.html" class="external-link">length</a></span><span class="op">(</span><span class="va">s</span><span class="op">)</span><span class="op">)</span> <span class="op">{</span></span>
<span>  <span class="va">s_as_int_seq</span><span class="op">[</span><span class="va">i</span><span class="op">]</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/which.html" class="external-link">which</a></span><span class="op">(</span><span class="va">s</span><span class="op">[</span><span class="va">i</span><span class="op">]</span> <span class="op">==</span> <span class="va">vocabulary</span><span class="op">)</span> <span class="op">-</span> <span class="fl">1</span></span>
<span><span class="op">}</span></span>
<span><span class="va">one_hot_sample</span> <span class="op">&lt;-</span> <span class="fu">keras</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/keras/man/to_categorical.html" class="external-link">to_categorical</a></span><span class="op">(</span><span class="va">s_as_int_seq</span><span class="op">)</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">one_hot_sample</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">one_hot_sample</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 0 0 1 0 0 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 0 0 0 0 1</span></span>
<span><span class="co">## [6,] 0 1 0 0 0 0 0 0 0</span></span></code></pre>
<div class="section level3">
<h3 id="maxlen">maxlen<a class="anchor" aria-label="anchor" href="#maxlen"></a>
</h3>
<p>The length of the input sequence.</p>
</div>
<div class="section level3">
<h3 id="vocabulary">vocabulary<a class="anchor" aria-label="anchor" href="#vocabulary"></a>
</h3>
<p>The set of allowed characters in a sequence. What happens to
characters outside the vocabulary can be controlled with the
<code>ambiguous_nuc</code> argument.</p>
</div>
<div class="section level3">
<h3 id="train_type">train_type<a class="anchor" aria-label="anchor" href="#train_type"></a>
</h3>
<p>The generator will always return a list of length 2. The first
element is the input <span class="math inline">\(X\)</span> and the
second the target <span class="math inline">\(Y\)</span>. The
<code>train_type</code> argument determines how <span class="math inline">\(X\)</span> and <span class="math inline">\(Y\)</span> get extracted. Possible arguments for
<u> <em>language models</em> </u> are:</p>
<ul>
<li>
<strong>“lm”</strong> or <strong>“lm_rds”</strong>: Given some
sequence <span class="math inline">\(s\)</span>, we take some subset of
that sequence as input and the rest as target. How to split <span class="math inline">\(s\)</span> can be specified in
<code>output_format</code> argument.</li>
</ul>
<p>Besides the language model approach, we can use <u> <em>label
classification</em> </u>. This means we map some label to a sequence.
For example, the target for some nucleotide sequence could be one of the
labels “bacteria” or “virus”. We have to specify how to extract a label
corresponding to a sequence. Possible arguments are:</p>
<ul>
<li><p><strong>“label_header”</strong>: get label from fasta
headers.</p></li>
<li><p><strong>“label_folder”</strong>: get label from folder, i.e. all
files in one folder must belong to the same class.</p></li>
<li>
<p><strong>“label_csv”</strong>: get label from csv file. Csv file
should have one column named “file”. The targets then correspond to
entries in that row (except “file” column). Example: if we are currently
working with a file called “a.fasta”, there should be a row in our csv
file with some target information for that file <br></p>
<table class="table">
<thead><tr class="header">
<th>file</th>
<th>label_1</th>
<th>label_2</th>
</tr></thead>
<tbody><tr class="odd">
<td>“a.fasta”</td>
<td>1</td>
<td>0</td>
</tr></tbody>
</table>
</li>
<li><p><strong>“label_rds”</strong>: rds file contains preprocessed list
of input and target tensors.</p></li>
</ul>
<p>Another option is <strong>“dummy_gen”</strong>: generator creates
random data once and repeatedly returns them.</p>
<p>Extract target from fasta header (fasta header is “label_1” in
example file):</p>
<div class="sourceCode" id="cb6"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># get target from header</span></span>
<span><span class="va">vocabulary_label</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste0</a></span><span class="op">(</span><span class="st">"label_"</span>, <span class="fl">1</span><span class="op">:</span><span class="fl">5</span><span class="op">)</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"label_header"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     vocabulary_label <span class="op">=</span> <span class="va">vocabulary_label</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span> </span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span> </span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary_label</span> </span>
<span><span class="va">x</span> <span class="co"># abcdef</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [6,] 0 0 0 0 0 1 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb8"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="co"># label_1 </span></span></code></pre></div>
<pre><code><span><span class="co">##      label_1 label_2 label_3 label_4 label_5</span></span>
<span><span class="co">## [1,]       1       0       0       0       0</span></span></code></pre>
<p>Extract target from fasta folder:</p>
<div class="sourceCode" id="cb10"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># create data for second class</span></span>
<span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>Sequence <span class="op">=</span> <span class="st">"AABAACAADAAE"</span>, Header <span class="op">=</span> <span class="st">"header_1"</span><span class="op">)</span></span>
<span><span class="va">file_path_2</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>fileext <span class="op">=</span> <span class="st">".fasta"</span><span class="op">)</span></span>
<span><span class="va">fasta_file</span> <span class="op">&lt;-</span> <span class="fu">microseq</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/microseq/man/readFasta.html" class="external-link">writeFasta</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">file_path_2</span><span class="op">)</span></span>
<span></span>
<span><span class="co"># get target from folder</span></span>
<span><span class="va">vocabulary_label</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste0</a></span><span class="op">(</span><span class="st">"label_"</span>, <span class="fl">1</span><span class="op">:</span><span class="fl">2</span><span class="op">)</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="va">file_path</span>, <span class="va">file_path_2</span><span class="op">)</span>, <span class="co"># one entry for each class</span></span>
<span>                     train_type <span class="op">=</span> <span class="st">"label_folder"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">8</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     vocabulary_label <span class="op">=</span> <span class="va">vocabulary_label</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span> </span>
<span><span class="va">x_1_1</span> <span class="op">&lt;-</span> <span class="va">x</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x_1_1</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x_1_1</span> <span class="co"># first sample from first class</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [6,] 0 0 0 0 0 1 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb12"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x_2_1</span> <span class="op">&lt;-</span> <span class="va">x</span><span class="op">[</span><span class="fl">5</span>, , <span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x_2_1</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x_2_1</span> <span class="co"># first sample from second class</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [4,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [5,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [6,] 0 0 1 0 0 0 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb14"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary_label</span> </span>
<span><span class="va">y</span> <span class="co"># 4 samples from each class  </span></span></code></pre></div>
<pre><code><span><span class="co">##      label_1 label_2</span></span>
<span><span class="co">## [1,]       1       0</span></span>
<span><span class="co">## [2,]       1       0</span></span>
<span><span class="co">## [3,]       1       0</span></span>
<span><span class="co">## [4,]       1       0</span></span>
<span><span class="co">## [5,]       0       1</span></span>
<span><span class="co">## [6,]       0       1</span></span>
<span><span class="co">## [7,]       0       1</span></span>
<span><span class="co">## [8,]       0       1</span></span></code></pre>
<p>Extract target from csv file:</p>
<div class="sourceCode" id="cb16"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># get target from csv</span></span>
<span><span class="va">file</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/basename.html" class="external-link">basename</a></span><span class="op">(</span><span class="va">file_path</span><span class="op">)</span>, <span class="st">"xyz.fasta"</span>, <span class="st">"abc.fasta"</span>, <span class="st">"x_123.fasta"</span><span class="op">)</span></span>
<span><span class="va">vocabulary_label</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste0</a></span><span class="op">(</span><span class="st">"label"</span>, <span class="fl">1</span><span class="op">:</span><span class="fl">4</span><span class="op">)</span></span>
<span><span class="va">label_1</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">1</span>, <span class="fl">0</span>, <span class="fl">0</span>, <span class="fl">0</span><span class="op">)</span></span>
<span><span class="va">label_2</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0</span>, <span class="fl">1</span>, <span class="fl">0</span>, <span class="fl">0</span><span class="op">)</span></span>
<span><span class="va">label_3</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0</span>, <span class="fl">0</span>, <span class="fl">1</span>, <span class="fl">0</span><span class="op">)</span></span>
<span><span class="va">label_4</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0</span>, <span class="fl">0</span>, <span class="fl">0</span>, <span class="fl">1</span><span class="op">)</span></span>
<span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span><span class="va">file</span>, <span class="va">label_1</span>, <span class="va">label_2</span>, <span class="va">label_3</span>, <span class="va">label_4</span><span class="op">)</span></span>
<span><span class="va">df</span></span></code></pre></div>
<pre><code><span><span class="co">##          file label_1 label_2 label_3 label_4</span></span>
<span><span class="co">## 1     a.fasta       1       0       0       0</span></span>
<span><span class="co">## 2   xyz.fasta       0       1       0       0</span></span>
<span><span class="co">## 3   abc.fasta       0       0       1       0</span></span>
<span><span class="co">## 4 x_123.fasta       0       0       0       1</span></span></code></pre>
<div class="sourceCode" id="cb18"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">csv_file</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>fileext <span class="op">=</span> <span class="st">".csv"</span><span class="op">)</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/utils/write.table.html" class="external-link">write.csv</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">csv_file</span>, row.names <span class="op">=</span> <span class="cn">FALSE</span><span class="op">)</span></span>
<span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"label_csv"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                     target_from_csv <span class="op">=</span> <span class="va">csv_file</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     vocabulary_label <span class="op">=</span> <span class="va">vocabulary_label</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span> </span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span> </span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary_label</span> </span>
<span><span class="va">x</span> <span class="co"># abcdef</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [6,] 0 0 0 0 0 1 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb20"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="co"># label_1 </span></span></code></pre></div>
<pre><code><span><span class="co">##      label1 label2 label3 label4</span></span>
<span><span class="co">## [1,]      1      0      0      0</span></span></code></pre>
<p>Examples for language models follow in the next section.</p>
</div>
<div class="section level3">
<h3 id="output_format">output_format<a class="anchor" aria-label="anchor" href="#output_format"></a>
</h3>
<p>The <code>output_format</code> determines the shape of the output for
a language model, i.e. part of a sequence is the input <span class="math inline">\(X\)</span> and another the target <span class="math inline">\(Y\)</span>. Assume a sequence <tt>abcdefg</tt> and
<code>maxlen = 6</code>. Output correspond as follows</p>
<p><strong>“target_right”</strong>: <span class="math inline">\(X=\)</span> <tt>abcdef</tt>, <span class="math inline">\(Y=\)</span> <tt>g</tt></p>
<p><strong>“target_middle_lstm”</strong>: <span class="math inline">\(X
=\)</span> (<span class="math inline">\(X_1 =\)</span> <tt>abc</tt>,
<span class="math inline">\(X_2 =\)</span> <tt>gfe</tt>), <span class="math inline">\(Y=\)</span> <tt>d</tt> (note reversed order of
<span class="math inline">\(X_2\)</span>)</p>
<p><strong>“target_middle_cnn”</strong>: <span class="math inline">\(X
=\)</span> <tt>abcefg</tt>, <span class="math inline">\(Y =\)</span>
<tt>d</tt></p>
<p><strong>“wavenet”</strong>: <span class="math inline">\(X =\)</span>
<tt>abcdef</tt>, <span class="math inline">\(Y =\)</span>
<tt>bcdefg</tt></p>
<div class="sourceCode" id="cb22"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># target_right</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span> </span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span> </span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x</span> <span class="co"># abcdef</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [6,] 0 0 0 0 0 1 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb24"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="co"># g </span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 0 0 0 1 0 0</span></span></code></pre>
<div class="sourceCode" id="cb26"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># target_middle_lstm</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_middle_lstm"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x_1</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span> </span>
<span><span class="va">x_2</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span> </span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span> </span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x_1</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x_2</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x_1</span> <span class="co"># abc</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 1 0 0 0 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb28"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x_2</span> <span class="co"># gfe</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 0 0 0 1 0 0</span></span>
<span><span class="co">## [2,] 0 0 0 0 0 1 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 0 0 1 0 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb30"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="co"># d </span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 1 0 0 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb32"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># target_middle_cnn</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_middle_cnn"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x</span> <span class="co"># abcefg</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 0 1 0 0 0</span></span>
<span><span class="co">## [6,] 0 0 0 0 0 0 1 0 0</span></span></code></pre>
<div class="sourceCode" id="cb34"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="co"># d</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 1 0 0 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb36"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># wavenet</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"wavenet"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span> </span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x</span> <span class="co"># abcdef</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [6,] 0 0 0 0 0 1 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb38"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="co"># bcdefg</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 0 1 0 0 0</span></span>
<span><span class="co">## [6,] 0 0 0 0 0 0 1 0 0</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="batch_size">batch_size<a class="anchor" aria-label="anchor" href="#batch_size"></a>
</h3>
<p>Number of samples in one batch.</p>
<div class="sourceCode" id="cb40"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># target_right</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">7</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span> </span>
<span><span class="fu"><a href="https://rdrr.io/r/base/dim.html" class="external-link">dim</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span></span></code></pre></div>
<pre><code><span><span class="co">## [1] 7 6 9</span></span></code></pre>
<div class="sourceCode" id="cb42"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/dim.html" class="external-link">dim</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span></span></code></pre></div>
<pre><code><span><span class="co">## [1] 7 9</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="step">step<a class="anchor" aria-label="anchor" href="#step"></a>
</h3>
<p>We may determine how frequently we want to take a sample. If
<code>step = 1</code> we take a sample at every possible step. Let’s
assume we want to predict the next character, i.e. part of the sequence
is the <mark class="in">input</mark> and next character the
<mark class="out">target</mark>. If
<code>maxlen = 3, step = 1</code>:</p>
<ol style="list-style-type: decimal">
<li><p>sample:
<tt><mark class="in">abc</mark><mark class="out">d</mark>efghiiii</tt></p></li>
<li><p>sample:
<tt>a<mark class="in">bcd</mark><mark class="out">e</mark>fghiiii</tt></p></li>
<li><p>sample:
<tt>ab<mark class="in">cde</mark><mark class="out">f</mark>ghiiii</tt></p></li>
</ol>
<p>if <code>step = 3</code></p>
<ol style="list-style-type: decimal">
<li><p>sample:
<tt><mark class="in">abc</mark><mark class="out">d</mark>efghiiii</tt></p></li>
<li><p>sample:
<tt>abc<mark class="in">def</mark><mark class="out">g</mark>hiiii</tt></p></li>
<li><p>sample:
<tt>abcdef<mark class="in">ghi</mark><mark class="out">i</mark>ii</tt></p></li>
</ol>
<div class="sourceCode" id="cb44"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">3</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     step <span class="op">=</span> <span class="fl">3</span>, </span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span> <span class="co">#encodes abc</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span> <span class="co"># encodes d</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 1 0 0 0 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb46"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 1 0 0 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb48"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># go 3 steps forward</span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span> <span class="co">#encodes def</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span> <span class="co"># encodes g</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 0 0 0 1 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb50"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 0 0 0 1 0 0</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="padding">padding<a class="anchor" aria-label="anchor" href="#padding"></a>
</h3>
<p>If the sequence is too short to create a single sample, we can pad
the sequence with zero-vectors. If <code>padding = FALSE</code> the
generator will go to next file/ fasta entry until it finds a sequence
long enough for a sample.</p>
<div class="sourceCode" id="cb52"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">15</span>, <span class="co"># maxlen is longer than sequence</span></span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     step <span class="op">=</span> <span class="fl">3</span>,</span>
<span>                     padding <span class="op">=</span> <span class="cn">TRUE</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span> </span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span> </span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x</span> <span class="co"># first 4 entries are zero-vectors</span></span></code></pre></div>
<pre><code><span><span class="co">##       a b c d e f g h i</span></span>
<span><span class="co">##  [1,] 0 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [2,] 0 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [3,] 0 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [4,] 0 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [5,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [6,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [7,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">##  [8,] 0 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">##  [9,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [10,] 0 0 0 0 0 1 0 0 0</span></span>
<span><span class="co">## [11,] 0 0 0 0 0 0 1 0 0</span></span>
<span><span class="co">## [12,] 0 0 0 0 0 0 0 1 0</span></span>
<span><span class="co">## [13,] 0 0 0 0 0 0 0 0 1</span></span>
<span><span class="co">## [14,] 0 0 0 0 0 0 0 0 1</span></span>
<span><span class="co">## [15,] 0 0 0 0 0 0 0 0 1</span></span></code></pre>
<div class="sourceCode" id="cb54"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 0 0 0 0 0 1</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="ambiguous_nuc">ambiguous_nuc<a class="anchor" aria-label="anchor" href="#ambiguous_nuc"></a>
</h3>
<p>A sequence might contain a character that does not lie inside our
vocabulary. For example, let’s assume we discard <tt>e</tt> from our
vocabulary. We have 4 options to handle this situation</p>
<ol style="list-style-type: decimal">
<li>encode as zero vector</li>
</ol>
<div class="sourceCode" id="cb56"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">vocabulary_2</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"a"</span>, <span class="st">"b"</span>, <span class="st">"c"</span>, <span class="st">"d"</span>, <span class="st">"f"</span>, <span class="st">"g"</span>, <span class="st">"h"</span>, <span class="st">"i"</span><span class="op">)</span> <span class="co"># exclude "e" from vocabulary</span></span>
<span></span>
<span><span class="co"># zero</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary_2</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span>,</span>
<span>                     ambiguous_nuc <span class="op">=</span> <span class="st">"zeros"</span><span class="op">)</span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span> </span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary_2</span></span>
<span><span class="va">x</span> <span class="co"># fifth row is zero vector </span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [6,] 0 0 0 0 1 0 0 0</span></span></code></pre>
<ol start="2" style="list-style-type: decimal">
<li>equal probability</li>
</ol>
<div class="sourceCode" id="cb58"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># equal</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                    train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                    batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                    maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                    vocabulary <span class="op">=</span> <span class="va">vocabulary_2</span>,</span>
<span>                    output_format <span class="op">=</span> <span class="st">"target_right"</span>,</span>
<span>                    ambiguous_nuc <span class="op">=</span> <span class="st">"equal"</span><span class="op">)</span> </span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary_2</span></span>
<span><span class="va">x</span> <span class="co"># fifth row is 1/8 for every entry </span></span></code></pre></div>
<pre><code><span><span class="co">##          a     b     c     d     f     g     h     i</span></span>
<span><span class="co">## [1,] 1.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000</span></span>
<span><span class="co">## [2,] 0.000 1.000 0.000 0.000 0.000 0.000 0.000 0.000</span></span>
<span><span class="co">## [3,] 0.000 0.000 1.000 0.000 0.000 0.000 0.000 0.000</span></span>
<span><span class="co">## [4,] 0.000 0.000 0.000 1.000 0.000 0.000 0.000 0.000</span></span>
<span><span class="co">## [5,] 0.125 0.125 0.125 0.125 0.125 0.125 0.125 0.125</span></span>
<span><span class="co">## [6,] 0.000 0.000 0.000 0.000 1.000 0.000 0.000 0.000</span></span></code></pre>
<ol start="3" style="list-style-type: decimal">
<li>use distribution of current file</li>
</ol>
<div class="sourceCode" id="cb60"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># empirical</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary_2</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span>,</span>
<span>                     ambiguous_nuc <span class="op">=</span> <span class="st">"empirical"</span><span class="op">)</span> </span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span> </span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary_2</span></span>
<span><span class="va">x</span> <span class="co"># fifth row is distribuation of file</span></span></code></pre></div>
<pre><code><span><span class="co">##               a          b          c          d          f          g</span></span>
<span><span class="co">## [1,] 1.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000</span></span>
<span><span class="co">## [2,] 0.00000000 1.00000000 0.00000000 0.00000000 0.00000000 0.00000000</span></span>
<span><span class="co">## [3,] 0.00000000 0.00000000 1.00000000 0.00000000 0.00000000 0.00000000</span></span>
<span><span class="co">## [4,] 0.00000000 0.00000000 0.00000000 1.00000000 0.00000000 0.00000000</span></span>
<span><span class="co">## [5,] 0.09090909 0.09090909 0.09090909 0.09090909 0.09090909 0.09090909</span></span>
<span><span class="co">## [6,] 0.00000000 0.00000000 0.00000000 0.00000000 1.00000000 0.00000000</span></span>
<span><span class="co">##               h         i</span></span>
<span><span class="co">## [1,] 0.00000000 0.0000000</span></span>
<span><span class="co">## [2,] 0.00000000 0.0000000</span></span>
<span><span class="co">## [3,] 0.00000000 0.0000000</span></span>
<span><span class="co">## [4,] 0.00000000 0.0000000</span></span>
<span><span class="co">## [5,] 0.09090909 0.3636364</span></span>
<span><span class="co">## [6,] 0.00000000 0.0000000</span></span></code></pre>
<ol start="4" style="list-style-type: decimal">
<li>discard</li>
</ol>
<div class="sourceCode" id="cb62"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># discard</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary_2</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span>,</span>
<span>                     ambiguous_nuc <span class="op">=</span> <span class="st">"discard"</span><span class="op">)</span> </span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary_2</span></span>
<span><span class="va">x</span> <span class="co"># first sample with only characters from vocabulary is fghiii|i</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 0 1 0 0 0</span></span>
<span><span class="co">## [2,] 0 0 0 0 0 1 0 0</span></span>
<span><span class="co">## [3,] 0 0 0 0 0 0 1 0</span></span>
<span><span class="co">## [4,] 0 0 0 0 0 0 0 1</span></span>
<span><span class="co">## [5,] 0 0 0 0 0 0 0 1</span></span>
<span><span class="co">## [6,] 0 0 0 0 0 0 0 1</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="proportion_per_seq">proportion_per_seq<a class="anchor" aria-label="anchor" href="#proportion_per_seq"></a>
</h3>
<p>The <code>proportion_per_seq</code> argument gives the option to use
a random subset instead of the full sequence.</p>
<div class="sourceCode" id="cb64"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/cat.html" class="external-link">cat</a></span><span class="op">(</span><span class="st">"sequence is "</span>, <span class="fu"><a href="https://rdrr.io/r/base/nchar.html" class="external-link">nchar</a></span><span class="op">(</span><span class="va">sequence</span><span class="op">)</span>, <span class="st">"characters long \n"</span><span class="op">)</span></span></code></pre></div>
<pre><code><span><span class="co">## sequence is  12 characters long</span></span></code></pre>
<div class="sourceCode" id="cb66"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">5</span>,</span>
<span>                     seed <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span>,</span>
<span>                     <span class="co"># take random subsequence using 50% of sequence </span></span>
<span>                     proportion_per_seq <span class="op">=</span> <span class="fl">0.5</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x</span> <span class="co"># defgh</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 1 0 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb68"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="co"># i</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 0 0 1 0 0 0</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="file_limit">file_limit<a class="anchor" aria-label="anchor" href="#file_limit"></a>
</h3>
<p>Integer or NULL. If integer, use only specified number of randomly
sampled files for training.</p>
</div>
<div class="section level3">
<h3 id="delete_used_files">delete_used_files<a class="anchor" aria-label="anchor" href="#delete_used_files"></a>
</h3>
<p>If true, delete file once used. Only applies for rds files.</p>
<div class="sourceCode" id="cb70"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/array.html" class="external-link">array</a></span><span class="op">(</span><span class="fl">0</span>, dim <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">1</span>,<span class="fl">5</span>,<span class="fl">4</span><span class="op">)</span><span class="op">)</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/matrix.html" class="external-link">matrix</a></span><span class="op">(</span><span class="fl">0</span>, ncol <span class="op">=</span> <span class="fl">1</span><span class="op">)</span></span>
<span><span class="va">rds_path</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>fileext <span class="op">=</span> <span class="st">".rds"</span><span class="op">)</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/readRDS.html" class="external-link">saveRDS</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span><span class="va">x</span>, <span class="va">y</span><span class="op">)</span>, <span class="va">rds_path</span><span class="op">)</span></span>
<span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">rds_path</span>,</span>
<span>                     delete_used_files <span class="op">=</span> <span class="cn">TRUE</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"label_rds"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">5</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/files.html" class="external-link">file.exists</a></span><span class="op">(</span><span class="va">rds_path</span><span class="op">)</span></span></code></pre></div>
<pre><code><span><span class="co">## [1] FALSE</span></span></code></pre>
<div class="sourceCode" id="cb72"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># z &lt;- gen()</span></span>
<span><span class="co"># When calling the generator again, it will wait until it finds a file again from the files listed in </span></span>
<span><span class="co"># the initial `path` argument. Can be used if another process(es) create rds files.</span></span></code></pre></div>
</div>
<div class="section level3">
<h3 id="max_samples">max_samples<a class="anchor" aria-label="anchor" href="#max_samples"></a>
</h3>
<p>Only use fixed number of samples per file. Randomly choose which
samples to use. (If <code>random_sampling = FALSE</code>, samples are
consecutive.)</p>
<div class="sourceCode" id="cb73"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">2</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">5</span>,</span>
<span>                     step <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     seed <span class="op">=</span> <span class="fl">3</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span>,</span>
<span>                     max_samples <span class="op">=</span> <span class="fl">2</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x1</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="va">x2</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">2</span>, , <span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x1</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x2</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x1</span> <span class="co"># bcdef</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 0 0 0 0 1 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 0 0 0 0 1 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 0 0 0 0 1 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 0 0 0 0 1</span></span></code></pre>
<div class="sourceCode" id="cb75"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x2</span> <span class="co"># cdefg</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 0 0 1 0 0 0</span></span>
<span><span class="co">## [2,] 0 0 0 0 0 0 1 0 0</span></span>
<span><span class="co">## [3,] 0 0 0 0 0 0 0 1 0</span></span>
<span><span class="co">## [4,] 0 0 0 0 0 0 0 0 1</span></span>
<span><span class="co">## [5,] 0 0 0 0 0 0 0 0 1</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="random_sampling">random_sampling<a class="anchor" aria-label="anchor" href="#random_sampling"></a>
</h3>
<p>If you use <code>max_samples</code>, generator will randomly choose
subset from all possible samples, but those samples are consecutive.
With <code>random_sampling = TRUE</code>, samples are completely
random.</p>
<div class="sourceCode" id="cb77"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">2</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">5</span>,</span>
<span>                     seed <span class="op">=</span> <span class="fl">66</span>,</span>
<span>                     random_sampling <span class="op">=</span> <span class="cn">TRUE</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span>,</span>
<span>                     max_samples <span class="op">=</span> <span class="fl">2</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x1</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="va">x2</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">2</span>, , <span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x1</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x2</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x1</span> <span class="co"># efghi</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 0 0 0 1 0 0</span></span>
<span><span class="co">## [2,] 0 0 0 0 0 0 0 1 0</span></span>
<span><span class="co">## [3,] 0 0 0 0 0 0 0 0 1</span></span>
<span><span class="co">## [4,] 0 0 0 0 0 0 0 0 1</span></span>
<span><span class="co">## [5,] 0 0 0 0 0 0 0 0 1</span></span></code></pre>
<div class="sourceCode" id="cb79"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x2</span> <span class="co"># defgh</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 0 0 0 0 1 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 0 0 0 0 1 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 0 0 0 0 1 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 0 0 0 0 1</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="target_len">target_len<a class="anchor" aria-label="anchor" href="#target_len"></a>
</h3>
<p>Target length for language model.</p>
<div class="sourceCode" id="cb81"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     target_len <span class="op">=</span> <span class="fl">3</span>, </span>
<span>                     maxlen <span class="op">=</span> <span class="fl">5</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="va">y1</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span><span class="op">[</span> , <span class="fl">1</span>, <span class="op">]</span></span>
<span><span class="va">y2</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span><span class="op">[</span> , <span class="fl">2</span>, <span class="op">]</span></span>
<span><span class="va">y3</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span><span class="op">[</span> , <span class="fl">3</span>, <span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/names.html" class="external-link">names</a></span><span class="op">(</span><span class="va">y1</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/names.html" class="external-link">names</a></span><span class="op">(</span><span class="va">y2</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/names.html" class="external-link">names</a></span><span class="op">(</span><span class="va">y3</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x</span> <span class="co"># abcde</span></span></code></pre></div>
<pre><code><span><span class="co">##      a b c d e f g h i</span></span>
<span><span class="co">## [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [2,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [3,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">## [4,] 0 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">## [5,] 0 0 0 0 1 0 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb83"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y1</span> <span class="co"># f</span></span></code></pre></div>
<pre><code><span><span class="co">## a b c d e f g h i </span></span>
<span><span class="co">## 0 0 0 0 0 1 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb85"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y2</span> <span class="co"># g</span></span></code></pre></div>
<pre><code><span><span class="co">## a b c d e f g h i </span></span>
<span><span class="co">## 0 0 0 0 0 0 1 0 0</span></span></code></pre>
<div class="sourceCode" id="cb87"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y3</span> <span class="co"># h</span></span></code></pre></div>
<pre><code><span><span class="co">## a b c d e f g h i </span></span>
<span><span class="co">## 0 0 0 0 0 0 0 1 0</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="n_gram-n_gram_stride">n_gram / n_gram_stride<a class="anchor" aria-label="anchor" href="#n_gram-n_gram_stride"></a>
</h3>
<p>Encode target in language model not character wise but combine n
characters to one target. <code>n_gram_stride</code> determines the
frequency of the n-gram encoding.</p>
<div class="sourceCode" id="cb89"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     target_len <span class="op">=</span> <span class="fl">6</span>, </span>
<span>                     n_gram <span class="op">=</span> <span class="fl">3</span>,</span>
<span>                     n_gram_stride <span class="op">=</span> <span class="fl">3</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">3</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">y1</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span><span class="op">[</span> , <span class="fl">1</span>, <span class="op">]</span></span>
<span><span class="va">y2</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span><span class="op">[</span> , <span class="fl">2</span>, <span class="op">]</span></span>
<span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/dim.html" class="external-link">dim</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span><span class="op">[</span><span class="fl">3</span><span class="op">]</span> <span class="op">==</span> <span class="fu"><a href="https://rdrr.io/r/base/length.html" class="external-link">length</a></span><span class="op">(</span><span class="va">vocabulary</span><span class="op">)</span><span class="op">^</span><span class="fl">3</span></span></code></pre></div>
<pre><code><span><span class="co">## [1] TRUE</span></span></code></pre>
<div class="sourceCode" id="cb91"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># x = abc as 3-gram</span></span>
<span><span class="co"># y1 = def as 3-gram</span></span>
<span><span class="co"># y2 = ghi as 3-gram</span></span></code></pre></div>
</div>
<div class="section level3">
<h3 id="add_noise">add_noise<a class="anchor" aria-label="anchor" href="#add_noise"></a>
</h3>
<p>Add noise to input. Must be a list that specifies noise distribution
or NULL (no noise). List contains arguments <code>noise_type</code>:
either <code>"normal"</code> or <code>"uniform"</code>. Optional
arguments <code>sd</code> or <code>mean</code> if
<code>noise_type</code> is <code>"normal"</code> (default is
<code>sd=1</code> and <code>mean=0</code>) or <code>min</code>,
<code>max</code> if <code>noise_type</code> is <code>"uniform"</code>
(default is <code>min=0</code>, <code>max=1</code>).</p>
<div class="sourceCode" id="cb92"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">file_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     add_noise <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>noise_type <span class="op">=</span> <span class="st">"normal"</span>, mean <span class="op">=</span> <span class="fl">0</span>, sd <span class="op">=</span> <span class="fl">0.01</span><span class="op">)</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">5</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/Round.html" class="external-link">round</a></span><span class="op">(</span><span class="va">x</span>, <span class="fl">3</span><span class="op">)</span> <span class="co"># abcde + noise</span></span></code></pre></div>
<pre><code><span><span class="co">##           a      b      c     d      e      f      g      h      i</span></span>
<span><span class="co">## [1,]  0.994  0.005 -0.006 0.008  0.006  0.014 -0.004  0.007 -0.001</span></span>
<span><span class="co">## [2,]  0.002  1.007 -0.022 0.006 -0.001 -0.001 -0.001  0.006  0.009</span></span>
<span><span class="co">## [3,] -0.008  0.006  1.011 0.009 -0.002  0.004  0.011 -0.007  0.004</span></span>
<span><span class="co">## [4,]  0.016 -0.003  0.000 1.008 -0.015 -0.001  0.008 -0.007 -0.006</span></span>
<span><span class="co">## [5,]  0.003  0.015  0.000 0.001  0.995 -0.014 -0.002  0.004  0.003</span></span></code></pre>
<div class="sourceCode" id="cb94"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="co"># f</span></span></code></pre></div>
<pre><code><span><span class="co">##                 a           b           c           d           e         f</span></span>
<span><span class="co">## [1,] -0.008204684 0.003898432 0.009438362 -0.01989352 0.004179416 0.9958501</span></span>
<span><span class="co">##                 g           h           i</span></span>
<span><span class="co">## [1,] -0.002533617 0.007685329 -0.01129363</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="proportion_entries">proportion_entries<a class="anchor" aria-label="anchor" href="#proportion_entries"></a>
</h3>
<p>If a fasta file has multiple entries, you can randomly choose a
subset. For example, if the file has 6 entries and
<code>proportion_entries = 0.5</code> the generator will randomly choose
only 3 of the entries.</p>
</div>
<div class="section level3">
<h3 id="shuffle_file_order">shuffle_file_order<a class="anchor" aria-label="anchor" href="#shuffle_file_order"></a>
</h3>
<p>Shuffle file order before iterating through files. Order gets
reshuffled after every iteration.</p>
</div>
<div class="section level3">
<h3 id="shuffle_input">shuffle_input<a class="anchor" aria-label="anchor" href="#shuffle_input"></a>
</h3>
<p>Whether to shuffle fasta entries if fasta file has multiple
entries.</p>
</div>
<div class="section level3">
<h3 id="reverse_complement">reverse_complement<a class="anchor" aria-label="anchor" href="#reverse_complement"></a>
</h3>
<p>If <code>TRUE</code>, randomly decide for every batch to use original
sequence or its reverse complement. Only implemented for <tt>ACGT</tt>
vocabulary.</p>
</div>
<div class="section level3">
<h3 id="sample_by_file_size">sample_by_file_size<a class="anchor" aria-label="anchor" href="#sample_by_file_size"></a>
</h3>
<p>Randomly choose new file by sampling according to file size (bigger
files more likely).</p>
</div>
<div class="section level3">
<h3 id="concat_seq">concat_seq<a class="anchor" aria-label="anchor" href="#concat_seq"></a>
</h3>
<p>Character string or <code>NULL</code>. If not <code>NULL</code> all
entries from file get concatenated to one sequence with
<code>concat_seq</code> string between them. Use
<code>concat_seq = ""</code> if you don’t want to add a new token.</p>
<div class="sourceCode" id="cb96"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>Sequence <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"AC"</span>, <span class="st">"AG"</span>, <span class="st">"AT"</span><span class="op">)</span>, Header <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste0</a></span><span class="op">(</span><span class="st">"header"</span>, <span class="fl">1</span><span class="op">:</span><span class="fl">3</span><span class="op">)</span><span class="op">)</span></span>
<span><span class="va">fasta_path</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>fileext <span class="op">=</span> <span class="st">".fasta"</span><span class="op">)</span></span>
<span><span class="va">fasta_file</span> <span class="op">&lt;-</span> <span class="fu">microseq</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/microseq/man/readFasta.html" class="external-link">writeFasta</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">fasta_path</span><span class="op">)</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">fasta_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">9</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span>, <span class="st">"Z"</span><span class="op">)</span>,</span>
<span>                     concat_seq <span class="op">=</span> <span class="st">"ZZ"</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span>, <span class="st">"Z"</span><span class="op">)</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span>, <span class="st">"Z"</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="co"># ACZZAGZZA</span></span></code></pre></div>
<pre><code><span><span class="co">##       A C G T Z</span></span>
<span><span class="co">##  [1,] 1 0 0 0 0</span></span>
<span><span class="co">##  [2,] 0 1 0 0 0</span></span>
<span><span class="co">##  [3,] 0 0 0 0 1</span></span>
<span><span class="co">##  [4,] 0 0 0 0 1</span></span>
<span><span class="co">##  [5,] 1 0 0 0 0</span></span>
<span><span class="co">##  [6,] 0 0 1 0 0</span></span>
<span><span class="co">##  [7,] 0 0 0 0 1</span></span>
<span><span class="co">##  [8,] 0 0 0 0 1</span></span>
<span><span class="co">##  [9,] 1 0 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb98"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="co"># T</span></span></code></pre></div>
<pre><code><span><span class="co">##      A C G T Z</span></span>
<span><span class="co">## [1,] 0 0 0 1 0</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="set_learning">set_learning<a class="anchor" aria-label="anchor" href="#set_learning"></a>
</h3>
<p>When you want to assign one label to set of samples. Only implemented
for <code>train_type = "label_folder"</code>. Input is a list with the
following parameters</p>
<ul>
<li>
<code>samples_per_target</code> how many samples to use for one
target</li>
<li>
<code>maxlen</code> length of one sample</li>
<li>
<code>reshape_mode</code>: <code>"time_dist", "multi_input"</code>
or <code>"concat"</code>.
<ul>
<li>If <code>reshape_mode = "multi_input"</code>, generator will produce
<code>samples_per_target</code> separate inputs, each of length
<code>maxlen</code>.</li>
<li>If <code>reshape_mode = "time_dist"</code>, generator will produce a
4D input array. The dimensions correspond to
<code>(batch_size, samples_per_target, maxlen, length(vocabulary))</code>.<br>
</li>
<li>If <code>reshape_mode</code> is <code>"concat"</code>, generator
will concatenate <code>samples_per_target</code> sequences of length
<code>maxlen</code> to one long sequence.</li>
</ul>
</li>
<li>If <code>reshape_mode = "concat"</code>, there is an additional
<code>buffer_len</code> argument: add new token between concatenated
samples
<ul>
<li>If <code>buffer_len</code> is an integer, the sub-sequences are
inter spaced with <code>buffer_len</code> rows. The input length is
(<code>maxlen</code> * <code>samples_per_target</code>) +
<code>buffer_len</code> * (<code>samples_per_target</code> - 1)</li>
</ul>
</li>
</ul>
<div class="sourceCode" id="cb100"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># create data for second label</span></span>
<span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>Sequence <span class="op">=</span> <span class="st">"AABAACAADAAE"</span>, Header <span class="op">=</span> <span class="st">"header_1"</span><span class="op">)</span></span>
<span><span class="va">file_path_2</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>fileext <span class="op">=</span> <span class="st">".fasta"</span><span class="op">)</span></span>
<span><span class="va">fasta_file</span> <span class="op">&lt;-</span> <span class="fu">microseq</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/microseq/man/readFasta.html" class="external-link">writeFasta</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">file_path_2</span><span class="op">)</span></span>
<span></span>
<span><span class="co"># multi_input </span></span>
<span><span class="va">set_learning</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>reshape_mode <span class="op">=</span> <span class="st">"multi_input"</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">4</span>,</span>
<span>                     samples_per_target <span class="op">=</span> <span class="fl">3</span><span class="op">)</span></span>
<span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="va">file_path</span>, <span class="va">file_path_2</span><span class="op">)</span>, <span class="co"># path has length 2 =&gt; 2 classes</span></span>
<span>                     train_type <span class="op">=</span> <span class="st">"label_folder"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">2</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">4</span>,</span>
<span>                     step <span class="op">=</span> <span class="fl">1</span>, </span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     set_learning <span class="op">=</span> <span class="va">set_learning</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/length.html" class="external-link">length</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="co"># 3 samples per target</span></span></code></pre></div>
<pre><code><span><span class="co">## [1] 3</span></span></code></pre>
<div class="sourceCode" id="cb102"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x_1_1</span> <span class="op">&lt;-</span> <span class="va">x</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="va">x_1_1</span> <span class="co"># abcd</span></span></code></pre></div>
<pre><code><span><span class="co">##      [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9]</span></span>
<span><span class="co">## [1,]    1    0    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [2,]    0    1    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [3,]    0    0    1    0    0    0    0    0    0</span></span>
<span><span class="co">## [4,]    0    0    0    1    0    0    0    0    0</span></span></code></pre>
<div class="sourceCode" id="cb104"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x_1_2</span> <span class="op">&lt;-</span> <span class="va">x</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="va">x_1_2</span> <span class="co"># bcde</span></span></code></pre></div>
<pre><code><span><span class="co">##      [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9]</span></span>
<span><span class="co">## [1,]    0    1    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [2,]    0    0    1    0    0    0    0    0    0</span></span>
<span><span class="co">## [3,]    0    0    0    1    0    0    0    0    0</span></span>
<span><span class="co">## [4,]    0    0    0    0    1    0    0    0    0</span></span></code></pre>
<div class="sourceCode" id="cb106"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x_1_3</span> <span class="op">&lt;-</span> <span class="va">x</span><span class="op">[[</span><span class="fl">3</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="va">x_1_3</span> <span class="co"># cdef</span></span></code></pre></div>
<pre><code><span><span class="co">##      [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9]</span></span>
<span><span class="co">## [1,]    0    0    1    0    0    0    0    0    0</span></span>
<span><span class="co">## [2,]    0    0    0    1    0    0    0    0    0</span></span>
<span><span class="co">## [3,]    0    0    0    0    1    0    0    0    0</span></span>
<span><span class="co">## [4,]    0    0    0    0    0    1    0    0    0</span></span></code></pre>
<div class="sourceCode" id="cb108"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x_2_1</span> <span class="op">&lt;-</span> <span class="va">x</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">2</span>, , <span class="op">]</span></span>
<span><span class="va">x_2_1</span> <span class="co"># aaba</span></span></code></pre></div>
<pre><code><span><span class="co">##      [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9]</span></span>
<span><span class="co">## [1,]    1    0    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [2,]    1    0    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [3,]    0    1    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [4,]    1    0    0    0    0    0    0    0    0</span></span></code></pre>
<div class="sourceCode" id="cb110"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x_2_2</span> <span class="op">&lt;-</span> <span class="va">x</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">2</span>, , <span class="op">]</span></span>
<span><span class="va">x_2_2</span> <span class="co"># abaa</span></span></code></pre></div>
<pre><code><span><span class="co">##      [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9]</span></span>
<span><span class="co">## [1,]    1    0    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [2,]    0    1    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [3,]    1    0    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [4,]    1    0    0    0    0    0    0    0    0</span></span></code></pre>
<div class="sourceCode" id="cb112"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x_2_3</span> <span class="op">&lt;-</span> <span class="va">x</span><span class="op">[[</span><span class="fl">3</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">2</span>, , <span class="op">]</span></span>
<span><span class="va">x_2_3</span> <span class="co"># baac</span></span></code></pre></div>
<pre><code><span><span class="co">##      [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9]</span></span>
<span><span class="co">## [1,]    0    1    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [2,]    1    0    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [3,]    1    0    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [4,]    0    0    1    0    0    0    0    0    0</span></span></code></pre>
<div class="sourceCode" id="cb114"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"label_1"</span>, <span class="st">"label_2"</span><span class="op">)</span></span>
<span><span class="va">y</span> </span></code></pre></div>
<pre><code><span><span class="co">##      label_1 label_2</span></span>
<span><span class="co">## [1,]       1       0</span></span>
<span><span class="co">## [2,]       0       1</span></span></code></pre>
<div class="sourceCode" id="cb116"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># concat </span></span>
<span><span class="va">set_learning</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>reshape_mode <span class="op">=</span> <span class="st">"concat"</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">4</span>,</span>
<span>                     samples_per_target <span class="op">=</span> <span class="fl">3</span><span class="op">)</span></span>
<span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="va">file_path</span>, <span class="va">file_path_2</span><span class="op">)</span>, <span class="co"># path has length 2 =&gt; 2 classes</span></span>
<span>                     train_type <span class="op">=</span> <span class="st">"label_folder"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">2</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">4</span>,</span>
<span>                     step <span class="op">=</span> <span class="fl">2</span>, </span>
<span>                     vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                     set_learning <span class="op">=</span> <span class="va">set_learning</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/dim.html" class="external-link">dim</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> </span></code></pre></div>
<pre><code><span><span class="co">## [1]  2 12  9</span></span></code></pre>
<div class="sourceCode" id="cb118"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x_1</span> <span class="op">&lt;-</span> <span class="va">x</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x_1</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x_1</span> <span class="co"># abcd | cdef | efgh</span></span></code></pre></div>
<pre><code><span><span class="co">##       a b c d e f g h i</span></span>
<span><span class="co">##  [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [2,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [3,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">##  [4,] 0 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">##  [5,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">##  [6,] 0 0 0 1 0 0 0 0 0</span></span>
<span><span class="co">##  [7,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">##  [8,] 0 0 0 0 0 1 0 0 0</span></span>
<span><span class="co">##  [9,] 0 0 0 0 1 0 0 0 0</span></span>
<span><span class="co">## [10,] 0 0 0 0 0 1 0 0 0</span></span>
<span><span class="co">## [11,] 0 0 0 0 0 0 1 0 0</span></span>
<span><span class="co">## [12,] 0 0 0 0 0 0 0 1 0</span></span></code></pre>
<div class="sourceCode" id="cb120"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x_2</span> <span class="op">&lt;-</span> <span class="va">x</span><span class="op">[</span><span class="fl">2</span>, , <span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x_2</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="va">vocabulary</span></span>
<span><span class="va">x_2</span> <span class="co"># aaba | baac | acaa</span></span></code></pre></div>
<pre><code><span><span class="co">##       a b c d e f g h i</span></span>
<span><span class="co">##  [1,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [2,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [3,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [4,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [5,] 0 1 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [6,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [7,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">##  [8,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">##  [9,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [10,] 0 0 1 0 0 0 0 0 0</span></span>
<span><span class="co">## [11,] 1 0 0 0 0 0 0 0 0</span></span>
<span><span class="co">## [12,] 1 0 0 0 0 0 0 0 0</span></span></code></pre>
<div class="sourceCode" id="cb122"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"label_1"</span>, <span class="st">"label_2"</span><span class="op">)</span></span>
<span><span class="va">y</span> </span></code></pre></div>
<pre><code><span><span class="co">##      label_1 label_2</span></span>
<span><span class="co">## [1,]       1       0</span></span>
<span><span class="co">## [2,]       0       1</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="use_quality_score">use_quality_score<a class="anchor" aria-label="anchor" href="#use_quality_score"></a>
</h3>
<p>If <code>TRUE</code>, instead of one-hot encoding, use quality score
of fastq file.</p>
<div class="sourceCode" id="cb124"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>Sequence <span class="op">=</span> <span class="st">"ACAGAT"</span>, Header <span class="op">=</span> <span class="st">"header_1"</span>, Quality <span class="op">=</span> <span class="st">"!#*=?I"</span><span class="op">)</span></span>
<span><span class="va">fastq_path</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>fileext <span class="op">=</span> <span class="st">".fastq"</span><span class="op">)</span></span>
<span><span class="va">fastq_file</span> <span class="op">&lt;-</span> <span class="fu">microseq</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/microseq/man/readFastq.html" class="external-link">writeFastq</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">fastq_path</span><span class="op">)</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">fastq_path</span>,</span>
<span>                     train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                     batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                     maxlen <span class="op">=</span> <span class="fl">5</span>,</span>
<span>                     format <span class="op">=</span> <span class="st">"fastq"</span>,</span>
<span>                     vocabulary <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span><span class="op">)</span>,</span>
<span>                     use_quality_score <span class="op">=</span> <span class="cn">TRUE</span>,</span>
<span>                     output_format <span class="op">=</span> <span class="st">"target_right"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span><span class="op">)</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="co"># ACAGA</span></span></code></pre></div>
<pre><code><span><span class="co">##                 A            C            G            T</span></span>
<span><span class="co">## [1,] 0.0000000000 0.3333333333 0.3333333333 0.3333333333</span></span>
<span><span class="co">## [2,] 0.2103191148 0.3690426555 0.2103191148 0.2103191148</span></span>
<span><span class="co">## [3,] 0.8741074588 0.0419641804 0.0419641804 0.0419641804</span></span>
<span><span class="co">## [4,] 0.0005282977 0.0005282977 0.9984151068 0.0005282977</span></span>
<span><span class="co">## [5,] 0.9990000000 0.0003333333 0.0003333333 0.0003333333</span></span></code></pre>
<div class="sourceCode" id="cb126"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="co"># T</span></span></code></pre></div>
<pre><code><span><span class="co">##                 A            C            G      T</span></span>
<span><span class="co">## [1,] 3.333333e-05 3.333333e-05 3.333333e-05 0.9999</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="use_coverage">use_coverage<a class="anchor" aria-label="anchor" href="#use_coverage"></a>
</h3>
<p>Integer or <code>NULL</code>. If not <code>NULL</code>, use coverage
as encoding rather than one-hot encoding. Coverage information must be
contained in fasta header: there must be a string “cov_n” in the header,
where n is some integer.</p>
<div class="sourceCode" id="cb128"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>Sequence <span class="op">=</span> <span class="st">"ACAGAT"</span>, Header <span class="op">=</span> <span class="st">"header_1_cov_8"</span><span class="op">)</span></span>
<span><span class="va">fasta_path</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>fileext <span class="op">=</span> <span class="st">".fasta"</span><span class="op">)</span></span>
<span><span class="va">fasta_file</span> <span class="op">&lt;-</span> <span class="fu">microseq</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/microseq/man/readFasta.html" class="external-link">writeFasta</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">fasta_path</span><span class="op">)</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span>  <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">fasta_path</span>,</span>
<span>                      train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                      batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                      maxlen <span class="op">=</span> <span class="fl">5</span>,</span>
<span>                      vocabulary <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span><span class="op">)</span>,</span>
<span>                      use_coverage <span class="op">=</span> <span class="fl">25</span>,</span>
<span>                      output_format <span class="op">=</span> <span class="st">"target_right"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span><span class="op">)</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="co"># ACAGA; 0.32 = 8/25</span></span></code></pre></div>
<pre><code><span><span class="co">##         A    C    G T</span></span>
<span><span class="co">## [1,] 0.32 0.00 0.00 0</span></span>
<span><span class="co">## [2,] 0.00 0.32 0.00 0</span></span>
<span><span class="co">## [3,] 0.32 0.00 0.00 0</span></span>
<span><span class="co">## [4,] 0.00 0.00 0.32 0</span></span>
<span><span class="co">## [5,] 0.32 0.00 0.00 0</span></span></code></pre>
<div class="sourceCode" id="cb130"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="co"># T</span></span></code></pre></div>
<pre><code><span><span class="co">##      A C G    T</span></span>
<span><span class="co">## [1,] 0 0 0 0.32</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="added_label_path">added_label_path<a class="anchor" aria-label="anchor" href="#added_label_path"></a>
</h3>
<p>It is possible to feed a network additional information associated to
a sequence. This information needs to be in a csv file. If all sequences
in one file share the same label, the csv file should have one column
named “file”.</p>
<p>We may add some additional input to our dummy data</p>
<div class="sourceCode" id="cb132"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">file</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/basename.html" class="external-link">basename</a></span><span class="op">(</span><span class="va">file_path</span><span class="op">)</span>, <span class="st">"some_file_name.fasta"</span><span class="op">)</span></span>
<span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>file <span class="op">=</span> <span class="va">file</span>,</span>
<span>                 label_1 <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">0</span>, <span class="fl">1</span><span class="op">)</span>, label_2 <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">1</span>, <span class="fl">0</span><span class="op">)</span>, label_3 <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">1</span>, <span class="fl">0</span><span class="op">)</span><span class="op">)</span></span>
<span><span class="va">df</span></span></code></pre></div>
<pre><code><span><span class="co">##                   file label_1 label_2 label_3</span></span>
<span><span class="co">## 1              a.fasta       0       1       1</span></span>
<span><span class="co">## 2 some_file_name.fasta       1       0       0</span></span></code></pre>
<div class="sourceCode" id="cb134"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/utils/write.table.html" class="external-link">write.csv</a></span><span class="op">(</span>x <span class="op">=</span> <span class="va">df</span>, file <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="va">dir_path</span>, <span class="st">"add_input.csv"</span><span class="op">)</span>, row.names <span class="op">=</span> <span class="cn">FALSE</span><span class="op">)</span></span></code></pre></div>
<p>If we add the path to the csv file, the generator will map additional
input to sequences:</p>
<div class="sourceCode" id="cb135"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">gen</span> <span class="op">&lt;-</span>  <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">dir_path</span>,</span>
<span>                      train_type <span class="op">=</span> <span class="st">"lm"</span>, </span>
<span>                      batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                      maxlen <span class="op">=</span> <span class="fl">5</span>,</span>
<span>                      output_format <span class="op">=</span> <span class="st">"target_right"</span>,</span>
<span>                      vocabulary <span class="op">=</span> <span class="va">vocabulary</span>,</span>
<span>                      added_label_path <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/file.path.html" class="external-link">file.path</a></span><span class="op">(</span><span class="va">dir_path</span>, <span class="st">"add_input.csv"</span><span class="op">)</span>,</span>
<span>                      add_input_as_seq <span class="op">=</span> <span class="cn">FALSE</span><span class="op">)</span>  <span class="co"># don't treat added input as sequence</span></span>
<span>                      </span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">added_label_input</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">added_label_input</span></span></code></pre></div>
<pre><code><span><span class="co">##      [,1] [,2] [,3]</span></span>
<span><span class="co">## [1,]    0    1    1</span></span></code></pre>
<div class="sourceCode" id="cb137"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">x</span><span class="op">[</span><span class="fl">1</span>, , <span class="op">]</span></span></code></pre></div>
<pre><code><span><span class="co">##      [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9]</span></span>
<span><span class="co">## [1,]    1    0    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [2,]    0    1    0    0    0    0    0    0    0</span></span>
<span><span class="co">## [3,]    0    0    1    0    0    0    0    0    0</span></span>
<span><span class="co">## [4,]    0    0    0    1    0    0    0    0    0</span></span>
<span><span class="co">## [5,]    0    0    0    0    1    0    0    0    0</span></span></code></pre>
<div class="sourceCode" id="cb139"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span> </span>
<span><span class="va">y</span></span></code></pre></div>
<pre><code><span><span class="co">##      [,1] [,2] [,3] [,4] [,5] [,6] [,7] [,8] [,9]</span></span>
<span><span class="co">## [1,]    0    0    0    0    0    1    0    0    0</span></span></code></pre>
<p>If we want to train a network with additional labels, we have to add
an additional input layer.</p>
<div class="sourceCode" id="cb141"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">model</span> <span class="op">&lt;-</span> <span class="fu"><a href="../reference/create_model_lstm_cnn.html">create_model_lstm_cnn</a></span><span class="op">(</span></span>
<span>  maxlen <span class="op">=</span> <span class="fl">5</span>,</span>
<span>  layer_lstm <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">8</span>, <span class="fl">8</span><span class="op">)</span>,</span>
<span>  layer_dense <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="fl">4</span><span class="op">)</span>,</span>
<span>  label_input <span class="op">=</span> <span class="fl">3</span> <span class="co"># additional input vector has length 3</span></span>
<span><span class="op">)</span></span></code></pre></div>
<pre><code><span><span class="co">## Model: "model"</span></span>
<span><span class="co">## __________________________________________________________________________________________________</span></span>
<span><span class="co">##  Layer (type)                Output Shape                 Param #   Connected to                  </span></span>
<span><span class="co">## ==================================================================================================</span></span>
<span><span class="co">##  input_1 (InputLayer)        [(None, 5, 4)]               0         []                            </span></span>
<span><span class="co">##                                                                                                   </span></span>
<span><span class="co">##  lstm (LSTM)                 (None, 5, 8)                 416       ['input_1[0][0]']             </span></span>
<span><span class="co">##                                                                                                   </span></span>
<span><span class="co">##  input_2 (InputLayer)        [(None, 3)]                  0         []                            </span></span>
<span><span class="co">##                                                                                                   </span></span>
<span><span class="co">##  lstm_1 (LSTM)               (None, 8)                    544       ['lstm[0][0]']                </span></span>
<span><span class="co">##                                                                                                   </span></span>
<span><span class="co">##  concatenate (Concatenate)   (None, 11)                   0         ['input_2[0][0]',             </span></span>
<span><span class="co">##                                                                      'lstm_1[0][0]']              </span></span>
<span><span class="co">##                                                                                                   </span></span>
<span><span class="co">##  dense (Dense)               (None, 4)                    48        ['concatenate[0][0]']         </span></span>
<span><span class="co">##                                                                                                   </span></span>
<span><span class="co">## ==================================================================================================</span></span>
<span><span class="co">## Total params: 1008 (3.94 KB)</span></span>
<span><span class="co">## Trainable params: 1008 (3.94 KB)</span></span>
<span><span class="co">## Non-trainable params: 0 (0.00 Byte)</span></span>
<span><span class="co">## __________________________________________________________________________________________________</span></span></code></pre>
<div class="sourceCode" id="cb143"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="co"># train_model(train_type = "lm", </span></span>
<span><span class="co">#             model = model,</span></span>
<span><span class="co">#             path = file.path(dir_path, "train_files_1"),</span></span>
<span><span class="co">#             path_val = file.path(dir_path, "validation_files_1"),</span></span>
<span><span class="co">#             added_label_path = file.path(dir_path, "add_input.csv"),</span></span>
<span><span class="co">#             steps_per_epoch = 5,</span></span>
<span><span class="co">#             batch_size = 8,</span></span>
<span><span class="co">#             epochs = 2)</span></span></code></pre></div>
</div>
<div class="section level3">
<h3 id="return_int">return_int<a class="anchor" aria-label="anchor" href="#return_int"></a>
</h3>
<p>Whether to return integer encoding rather than one-hot encoding.</p>
<div class="sourceCode" id="cb144"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>Sequence <span class="op">=</span> <span class="st">"ATCGC"</span>, Header <span class="op">=</span> <span class="st">"seq_1"</span><span class="op">)</span></span>
<span><span class="va">fasta_path</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>fileext <span class="op">=</span> <span class="st">".fasta"</span><span class="op">)</span></span>
<span><span class="va">fasta_file</span> <span class="op">&lt;-</span> <span class="fu">microseq</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/microseq/man/readFasta.html" class="external-link">writeFasta</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">fasta_path</span><span class="op">)</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span>  <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">fasta_path</span>,</span>
<span>                      train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                      batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                      return_int <span class="op">=</span> <span class="cn">TRUE</span>,</span>
<span>                      padding <span class="op">=</span> <span class="cn">TRUE</span>,</span>
<span>                      maxlen <span class="op">=</span> <span class="fl">8</span>,</span>
<span>                      vocabulary <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span><span class="op">)</span>,</span>
<span>                      output_format <span class="op">=</span> <span class="st">"target_right"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"pad"</span>, <span class="st">"pad"</span>, <span class="st">"pad"</span>, <span class="st">"pad"</span>, <span class="st">"A"</span>, <span class="st">"T"</span>, <span class="st">"C"</span>, <span class="st">"G"</span><span class="op">)</span></span>
<span><span class="va">x</span></span></code></pre></div>
<pre><code><span><span class="co">##      pad pad pad pad A T C G</span></span>
<span><span class="co">## [1,]   0   0   0   0 1 4 2 3</span></span></code></pre>
<div class="sourceCode" id="cb146"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"C"</span></span>
<span><span class="va">y</span></span></code></pre></div>
<pre><code><span><span class="co">##      C</span></span>
<span><span class="co">## [1,] 2</span></span></code></pre>
<p>Can also be combined with n-gram encoding:</p>
<div class="sourceCode" id="cb148"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>Sequence <span class="op">=</span> <span class="st">"AAACCCTTT"</span>, Header <span class="op">=</span> <span class="st">"seq_1"</span><span class="op">)</span></span>
<span><span class="va">fasta_path</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>fileext <span class="op">=</span> <span class="st">".fasta"</span><span class="op">)</span></span>
<span><span class="va">fasta_file</span> <span class="op">&lt;-</span> <span class="fu">microseq</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/microseq/man/readFasta.html" class="external-link">writeFasta</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">fasta_path</span><span class="op">)</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span>  <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">fasta_path</span>,</span>
<span>                      train_type <span class="op">=</span> <span class="st">"lm"</span>,</span>
<span>                      batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                      n_gram <span class="op">=</span> <span class="fl">3</span>,</span>
<span>                      n_gram_stride <span class="op">=</span> <span class="fl">3</span>,</span>
<span>                      return_int <span class="op">=</span> <span class="cn">TRUE</span>,</span>
<span>                      maxlen <span class="op">=</span> <span class="fl">6</span>,</span>
<span>                      target_len <span class="op">=</span> <span class="fl">3</span>,</span>
<span>                      vocabulary <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span><span class="op">)</span>,</span>
<span>                      output_format <span class="op">=</span> <span class="st">"target_right"</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"AAA"</span>, <span class="st">"CCC"</span><span class="op">)</span></span>
<span><span class="va">x</span></span></code></pre></div>
<pre><code><span><span class="co">##      AAA CCC</span></span>
<span><span class="co">## [1,]   1  22</span></span></code></pre>
<div class="sourceCode" id="cb150"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/colnames.html" class="external-link">colnames</a></span><span class="op">(</span><span class="va">y</span><span class="op">)</span> <span class="op">&lt;-</span> <span class="st">"TTT"</span></span>
<span><span class="va">y</span></span></code></pre></div>
<pre><code><span><span class="co">##      TTT</span></span>
<span><span class="co">## [1,]  64</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="reshape_xy">reshape_xy<a class="anchor" aria-label="anchor" href="#reshape_xy"></a>
</h3>
<p>Apply some function to the output of a generator call.</p>
<div class="sourceCode" id="cb152"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>Sequence <span class="op">=</span> <span class="st">"AAAATTTT"</span>, Header <span class="op">=</span> <span class="st">"header_1"</span><span class="op">)</span></span>
<span><span class="va">fasta_path</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>fileext <span class="op">=</span> <span class="st">".fasta"</span><span class="op">)</span></span>
<span><span class="va">fasta_file</span> <span class="op">&lt;-</span> <span class="fu">microseq</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/microseq/man/readFasta.html" class="external-link">writeFasta</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">fasta_path</span><span class="op">)</span></span>
<span><span class="va">fx</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="cn">NULL</span>, <span class="va">y</span> <span class="op">=</span> <span class="cn">NULL</span><span class="op">)</span> <span class="op">{</span></span>
<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="va">x</span> <span class="op">-</span> <span class="fl">1</span><span class="op">)</span></span>
<span><span class="op">}</span></span>
<span><span class="va">fy</span> <span class="op">&lt;-</span> <span class="kw">function</span><span class="op">(</span><span class="va">x</span> <span class="op">=</span> <span class="cn">NULL</span>, <span class="va">y</span> <span class="op">=</span> <span class="cn">NULL</span><span class="op">)</span> <span class="op">{</span></span>
<span>  <span class="kw"><a href="https://rdrr.io/r/base/function.html" class="external-link">return</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/Log.html" class="external-link">exp</a></span><span class="op">(</span><span class="va">y</span> <span class="op">*</span> <span class="fl">5</span><span class="op">)</span><span class="op">)</span></span>
<span><span class="op">}</span></span>
<span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span>  <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">fasta_path</span>,</span>
<span>                      reshape_xy <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>x <span class="op">=</span> <span class="va">fx</span>, y <span class="op">=</span> <span class="va">fy</span><span class="op">)</span>,</span>
<span>                      train_type <span class="op">=</span> <span class="st">"label_folder"</span>,</span>
<span>                      batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                      maxlen <span class="op">=</span> <span class="fl">8</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">x</span><span class="op">[</span><span class="fl">1</span>,,<span class="op">]</span></span></code></pre></div>
<pre><code><span><span class="co">##      [,1] [,2] [,3] [,4]</span></span>
<span><span class="co">## [1,]    0   -1   -1   -1</span></span>
<span><span class="co">## [2,]    0   -1   -1   -1</span></span>
<span><span class="co">## [3,]    0   -1   -1   -1</span></span>
<span><span class="co">## [4,]    0   -1   -1   -1</span></span>
<span><span class="co">## [5,]   -1   -1   -1    0</span></span>
<span><span class="co">## [6,]   -1   -1   -1    0</span></span>
<span><span class="co">## [7,]   -1   -1   -1    0</span></span>
<span><span class="co">## [8,]   -1   -1   -1    0</span></span></code></pre>
<div class="sourceCode" id="cb154"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">y</span></span></code></pre></div>
<pre><code><span><span class="co">##          [,1]</span></span>
<span><span class="co">## [1,] 148.4132</span></span></code></pre>
</div>
<div class="section level3">
<h3 id="masked_lm">masked_lm<a class="anchor" aria-label="anchor" href="#masked_lm"></a>
</h3>
<p>Masks some parts of input sequence. Can be used for training
BERT-like models.</p>
<div class="sourceCode" id="cb156"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">nt_seq</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/rep.html" class="external-link">rep</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span><span class="op">)</span>, each <span class="op">=</span> <span class="fl">25</span><span class="op">)</span> <span class="op"><a href="../reference/pipe.html">%&gt;%</a></span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste</a></span><span class="op">(</span>collapse <span class="op">=</span> <span class="st">""</span><span class="op">)</span></span>
<span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>Sequence <span class="op">=</span> <span class="va">nt_seq</span>, Header <span class="op">=</span> <span class="st">"seq_1"</span><span class="op">)</span></span>
<span><span class="va">fasta_path</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>fileext <span class="op">=</span> <span class="st">".fasta"</span><span class="op">)</span></span>
<span><span class="va">fasta_file</span> <span class="op">&lt;-</span> <span class="fu">microseq</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/microseq/man/readFasta.html" class="external-link">writeFasta</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">fasta_path</span><span class="op">)</span></span>
<span><span class="va">masked_lm</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>mask_rate <span class="op">=</span> <span class="fl">0.10</span>, <span class="co"># replace 10% of input with special mask token</span></span>
<span>                  random_rate <span class="op">=</span> <span class="fl">0.025</span>, <span class="co"># set 2.5% of input to random value</span></span>
<span>                  identity_rate <span class="op">=</span> <span class="fl">0.05</span>, <span class="co"># leave 5% unchanged</span></span>
<span>                  include_sw <span class="op">=</span> <span class="cn">TRUE</span><span class="op">)</span> <span class="co"># 0,1 matrix showing where masking was applied</span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span>  <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">fasta_path</span>,</span>
<span>                      train_type <span class="op">=</span> <span class="st">"masked_lm"</span>,</span>
<span>                      masked_lm <span class="op">=</span> <span class="va">masked_lm</span>,</span>
<span>                      batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                      n_gram <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                      n_gram_stride <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                      return_int <span class="op">=</span> <span class="cn">TRUE</span>,</span>
<span>                      maxlen <span class="op">=</span> <span class="fl">100</span>,</span>
<span>                      vocabulary <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span><span class="op">)</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">sw</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">3</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>x <span class="op">=</span> <span class="va">x</span><span class="op">[</span><span class="fl">1</span>, <span class="op">]</span>, y <span class="op">=</span> <span class="va">y</span><span class="op">[</span><span class="fl">1</span>, <span class="op">]</span>, sw <span class="op">=</span> <span class="va">sw</span><span class="op">[</span><span class="fl">1</span>, <span class="op">]</span><span class="op">)</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/utils/head.html" class="external-link">head</a></span><span class="op">(</span><span class="va">df</span><span class="op">)</span></span></code></pre></div>
<pre><code><span><span class="co">##   x y sw</span></span>
<span><span class="co">## 1 5 1  1</span></span>
<span><span class="co">## 2 1 1  0</span></span>
<span><span class="co">## 3 1 1  0</span></span>
<span><span class="co">## 4 1 1  0</span></span>
<span><span class="co">## 5 1 1  0</span></span>
<span><span class="co">## 6 1 1  0</span></span></code></pre>
<p>Whenever sw (sample weight) column is 0, x and y columns are
identical. Let’s look at rows where sw is 1:</p>
<div class="sourceCode" id="cb158"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">df</span> <span class="op"><a href="../reference/pipe.html">%&gt;%</a></span> <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/filter.html" class="external-link">filter</a></span><span class="op">(</span><span class="va">sw</span> <span class="op">==</span> <span class="fl">1</span><span class="op">)</span></span></code></pre></div>
<pre><code><span><span class="co">##    x y sw</span></span>
<span><span class="co">## 1  5 1  1</span></span>
<span><span class="co">## 2  1 1  1</span></span>
<span><span class="co">## 3  1 1  1</span></span>
<span><span class="co">## 4  5 1  1</span></span>
<span><span class="co">## 5  5 2  1</span></span>
<span><span class="co">## 6  5 2  1</span></span>
<span><span class="co">## 7  5 2  1</span></span>
<span><span class="co">## 8  3 3  1</span></span>
<span><span class="co">## 9  2 3  1</span></span>
<span><span class="co">## 10 3 3  1</span></span>
<span><span class="co">## 11 5 3  1</span></span>
<span><span class="co">## 12 5 3  1</span></span>
<span><span class="co">## 13 4 4  1</span></span>
<span><span class="co">## 14 5 4  1</span></span>
<span><span class="co">## 15 4 4  1</span></span>
<span><span class="co">## 16 5 4  1</span></span>
<span><span class="co">## 17 5 4  1</span></span>
<span><span class="co">## 18 4 4  1</span></span></code></pre>
<p>Here 5 is the mask token, this is always the size of the vocabulary +
1.</p>
<div class="sourceCode" id="cb160"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">df</span> <span class="op"><a href="../reference/pipe.html">%&gt;%</a></span> <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/filter.html" class="external-link">filter</a></span><span class="op">(</span><span class="va">sw</span> <span class="op">==</span> <span class="fl">1</span> <span class="op">&amp;</span> <span class="va">x</span> <span class="op">==</span> <span class="fl">5</span><span class="op">)</span> <span class="co"># 10% masked part</span></span></code></pre></div>
<pre><code><span><span class="co">##    x y sw</span></span>
<span><span class="co">## 1  5 1  1</span></span>
<span><span class="co">## 2  5 1  1</span></span>
<span><span class="co">## 3  5 2  1</span></span>
<span><span class="co">## 4  5 2  1</span></span>
<span><span class="co">## 5  5 2  1</span></span>
<span><span class="co">## 6  5 3  1</span></span>
<span><span class="co">## 7  5 3  1</span></span>
<span><span class="co">## 8  5 4  1</span></span>
<span><span class="co">## 9  5 4  1</span></span>
<span><span class="co">## 10 5 4  1</span></span></code></pre>
<div class="sourceCode" id="cb162"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">df</span> <span class="op"><a href="../reference/pipe.html">%&gt;%</a></span> <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/filter.html" class="external-link">filter</a></span><span class="op">(</span><span class="va">sw</span> <span class="op">==</span> <span class="fl">1</span> <span class="op">&amp;</span> <span class="va">x</span> <span class="op">!=</span> <span class="fl">5</span><span class="op">)</span> <span class="co"># 5% identity part and 2.5% random part (can randomly be the true value)</span></span></code></pre></div>
<pre><code><span><span class="co">##   x y sw</span></span>
<span><span class="co">## 1 1 1  1</span></span>
<span><span class="co">## 2 1 1  1</span></span>
<span><span class="co">## 3 3 3  1</span></span>
<span><span class="co">## 4 2 3  1</span></span>
<span><span class="co">## 5 3 3  1</span></span>
<span><span class="co">## 6 4 4  1</span></span>
<span><span class="co">## 7 4 4  1</span></span>
<span><span class="co">## 8 4 4  1</span></span></code></pre>
<p>Can be combined with n-gram encoding and masking of fixed block
size:</p>
<div class="sourceCode" id="cb164"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">nt_seq</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/rep.html" class="external-link">rep</a></span><span class="op">(</span><span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span><span class="op">)</span>, each <span class="op">=</span> <span class="fl">25</span><span class="op">)</span> <span class="op"><a href="../reference/pipe.html">%&gt;%</a></span> <span class="fu"><a href="https://rdrr.io/r/base/paste.html" class="external-link">paste</a></span><span class="op">(</span>collapse <span class="op">=</span> <span class="st">""</span><span class="op">)</span></span>
<span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>Sequence <span class="op">=</span> <span class="va">nt_seq</span>, Header <span class="op">=</span> <span class="st">"seq_1"</span><span class="op">)</span></span>
<span><span class="va">fasta_path</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/tempfile.html" class="external-link">tempfile</a></span><span class="op">(</span>fileext <span class="op">=</span> <span class="st">".fasta"</span><span class="op">)</span></span>
<span><span class="va">fasta_file</span> <span class="op">&lt;-</span> <span class="fu">microseq</span><span class="fu">::</span><span class="fu"><a href="https://rdrr.io/pkg/microseq/man/readFasta.html" class="external-link">writeFasta</a></span><span class="op">(</span><span class="va">df</span>, <span class="va">fasta_path</span><span class="op">)</span></span>
<span><span class="va">masked_lm</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/list.html" class="external-link">list</a></span><span class="op">(</span>mask_rate <span class="op">=</span> <span class="fl">0.10</span>, <span class="co"># replace 10% of input with special mask token</span></span>
<span>                  random_rate <span class="op">=</span> <span class="fl">0.05</span>, <span class="co"># set 5% of input to random value</span></span>
<span>                  identity_rate <span class="op">=</span> <span class="fl">0.05</span>, <span class="co"># leave 5% unchanged</span></span>
<span>                  include_sw <span class="op">=</span> <span class="cn">TRUE</span>, <span class="co"># 0,1 matrix showing where masking was applied</span></span>
<span>                  block_len <span class="op">=</span> <span class="fl">3</span><span class="op">)</span> <span class="co"># always mask at least 3 tokens in a row </span></span>
<span><span class="va">gen</span> <span class="op">&lt;-</span>  <span class="fu"><a href="../reference/get_generator.html">get_generator</a></span><span class="op">(</span>path <span class="op">=</span> <span class="va">fasta_path</span>,</span>
<span>                      train_type <span class="op">=</span> <span class="st">"masked_lm"</span>,</span>
<span>                      masked_lm <span class="op">=</span> <span class="va">masked_lm</span>,</span>
<span>                      batch_size <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                      n_gram <span class="op">=</span> <span class="fl">3</span>,</span>
<span>                      seed <span class="op">=</span> <span class="fl">12</span>,</span>
<span>                      n_gram_stride <span class="op">=</span> <span class="fl">1</span>,</span>
<span>                      return_int <span class="op">=</span> <span class="cn">TRUE</span>,</span>
<span>                      maxlen <span class="op">=</span> <span class="fl">100</span>,</span>
<span>                      vocabulary <span class="op">=</span> <span class="fu"><a href="https://rdrr.io/r/base/c.html" class="external-link">c</a></span><span class="op">(</span><span class="st">"A"</span>, <span class="st">"C"</span>, <span class="st">"G"</span>, <span class="st">"T"</span><span class="op">)</span><span class="op">)</span></span>
<span></span>
<span><span class="va">z</span> <span class="op">&lt;-</span> <span class="fu">gen</span><span class="op">(</span><span class="op">)</span></span>
<span><span class="va">x</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">1</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">y</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">2</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">sw</span> <span class="op">&lt;-</span> <span class="va">z</span><span class="op">[[</span><span class="fl">3</span><span class="op">]</span><span class="op">]</span></span>
<span><span class="va">df</span> <span class="op">&lt;-</span> <span class="fu"><a href="https://rdrr.io/r/base/data.frame.html" class="external-link">data.frame</a></span><span class="op">(</span>x <span class="op">=</span> <span class="va">x</span><span class="op">[</span><span class="fl">1</span>, <span class="op">]</span>, y <span class="op">=</span> <span class="va">y</span><span class="op">[</span><span class="fl">1</span>, <span class="op">]</span>, sw <span class="op">=</span> <span class="va">sw</span><span class="op">[</span><span class="fl">1</span>, <span class="op">]</span>, position <span class="op">=</span> <span class="fl">1</span><span class="op">:</span><span class="fu"><a href="https://rdrr.io/r/base/nrow.html" class="external-link">ncol</a></span><span class="op">(</span><span class="va">x</span><span class="op">)</span><span class="op">)</span></span>
<span><span class="fu"><a href="https://rdrr.io/r/utils/head.html" class="external-link">head</a></span><span class="op">(</span><span class="va">df</span><span class="op">)</span></span></code></pre></div>
<pre><code><span><span class="co">##    x y sw position</span></span>
<span><span class="co">## 1  1 1  0        1</span></span>
<span><span class="co">## 2  1 1  0        2</span></span>
<span><span class="co">## 3  1 1  0        3</span></span>
<span><span class="co">## 4 39 1  1        4</span></span>
<span><span class="co">## 5 48 1  1        5</span></span>
<span><span class="co">## 6 13 1  1        6</span></span></code></pre>
<div class="sourceCode" id="cb166"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/utils/head.html" class="external-link">tail</a></span><span class="op">(</span><span class="va">df</span><span class="op">)</span></span></code></pre></div>
<pre><code><span><span class="co">##     x  y sw position</span></span>
<span><span class="co">## 93 65 64  1       93</span></span>
<span><span class="co">## 94 64 64  0       94</span></span>
<span><span class="co">## 95 64 64  0       95</span></span>
<span><span class="co">## 96 64 64  0       96</span></span>
<span><span class="co">## 97 64 64  0       97</span></span>
<span><span class="co">## 98 64 64  0       98</span></span></code></pre>
<p>We can check that sample weights appear only in blocks.</p>
<div class="sourceCode" id="cb168"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="fu"><a href="https://rdrr.io/r/base/which.html" class="external-link">which</a></span><span class="op">(</span><span class="va">sw</span> <span class="op">==</span> <span class="fl">1</span><span class="op">)</span></span></code></pre></div>
<pre><code><span><span class="co">##  [1]  4  5  6 13 14 15 22 23 24 40 41 42 52 53 54 79 80 81 82 83 84 91 92 93</span></span></code></pre>
<p>Here 65 is the mask token (4^3 + 1 = size of the vocabulary + 1).</p>
<div class="sourceCode" id="cb170"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">df</span> <span class="op"><a href="../reference/pipe.html">%&gt;%</a></span> <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/filter.html" class="external-link">filter</a></span><span class="op">(</span><span class="va">sw</span> <span class="op">==</span> <span class="fl">1</span> <span class="op">&amp;</span> <span class="va">x</span> <span class="op">==</span> <span class="fl">65</span><span class="op">)</span> <span class="co"># 10% masked part</span></span></code></pre></div>
<pre><code><span><span class="co">##     x  y sw position</span></span>
<span><span class="co">## 1  65 22  1       40</span></span>
<span><span class="co">## 2  65 22  1       41</span></span>
<span><span class="co">## 3  65 22  1       42</span></span>
<span><span class="co">## 4  65 64  1       79</span></span>
<span><span class="co">## 5  65 64  1       80</span></span>
<span><span class="co">## 6  65 64  1       81</span></span>
<span><span class="co">## 7  65 64  1       82</span></span>
<span><span class="co">## 8  65 64  1       83</span></span>
<span><span class="co">## 9  65 64  1       84</span></span>
<span><span class="co">## 10 65 64  1       91</span></span>
<span><span class="co">## 11 65 64  1       92</span></span>
<span><span class="co">## 12 65 64  1       93</span></span></code></pre>
<div class="sourceCode" id="cb172"><pre class="downlit sourceCode r">
<code class="sourceCode R"><span><span class="va">df</span> <span class="op"><a href="../reference/pipe.html">%&gt;%</a></span> <span class="fu">dplyr</span><span class="fu">::</span><span class="fu"><a href="https://dplyr.tidyverse.org/reference/filter.html" class="external-link">filter</a></span><span class="op">(</span><span class="va">sw</span> <span class="op">==</span> <span class="fl">1</span> <span class="op">&amp;</span> <span class="va">x</span> <span class="op">!=</span> <span class="fl">65</span><span class="op">)</span> <span class="co"># 5% identity part and 5% random part (can randomly be the true value)</span></span></code></pre></div>
<pre><code><span><span class="co">##     x  y sw position</span></span>
<span><span class="co">## 1  39  1  1        4</span></span>
<span><span class="co">## 2  48  1  1        5</span></span>
<span><span class="co">## 3  13  1  1        6</span></span>
<span><span class="co">## 4   1  1  1       13</span></span>
<span><span class="co">## 5   1  1  1       14</span></span>
<span><span class="co">## 6   1  1  1       15</span></span>
<span><span class="co">## 7   1  1  1       22</span></span>
<span><span class="co">## 8   1  1  1       23</span></span>
<span><span class="co">## 9   2  2  1       24</span></span>
<span><span class="co">## 10 56 43  1       52</span></span>
<span><span class="co">## 11  4 43  1       53</span></span>
<span><span class="co">## 12 24 43  1       54</span></span></code></pre>
</div>
</div>
  </main><aside class="col-md-3"><nav id="toc"><h2>On this page</h2>
    </nav></aside>
</div>



    <footer><div class="pkgdown-footer-left">
  <p>Developed by Philipp Münch, René Mreches, Martin Binder, Hüseyin Anil Gündüz, Xiao-Yin To, Alice McHardy.</p>
</div>

<div class="pkgdown-footer-right">
  <p>Site built with <a href="https://pkgdown.r-lib.org/" class="external-link">pkgdown</a> 2.0.9.</p>
</div>

    </footer>
</div>

  

  

  </body>
</html>