% Generated by roxygen2: do not edit by hand

% Please edit documentation in R/evaluation.R

\name{evaluate_model}

\alias{evaluate_model}

\title{Evaluates a trained model on fasta, fastq or rds files}

\usage{

evaluate_model(

  path_input,

  model = NULL,

  batch_size = 100,

  step = 1,

  padding = FALSE,

  vocabulary = c("a", "c", "g", "t"),

  vocabulary_label = list(c("a", "c", "g", "t")),

  number_batches = 10,

  format = "fasta",

  target_middle = FALSE,

  mode = "lm",

  output_format = "target_right",

  ambiguous_nuc = "zero",

  evaluate_all_files = FALSE,

  verbose = TRUE,

  max_iter = 20000,

  target_from_csv = NULL,

  max_samples = NULL,

  proportion_per_seq = NULL,

  concat_seq = NULL,

  seed = 1234,

  auc = FALSE,

  auprc = FALSE,

  path_pred_list = NULL,

  exact_num_samples = NULL,

  activations = NULL,

  shuffle_file_order = FALSE,

  include_seq = FALSE,

  ...

)

}

\arguments{

\item{path_input}{Input directory where fasta, fastq or rds files are located.}

\item{model}{A keras model.}

\item{batch_size}{Number of samples per batch.}

\item{step}{How often to take a sample.}

\item{padding}{Whether to pad sequences too short for one sample with zeros.}

\item{vocabulary}{Vector of allowed characters. Character outside vocabulary get encoded as specified in ambiguous_nuc.}

\item{vocabulary_label}{List of labels for targets of each output layer.}

\item{number_batches}{How many batches to evaluate.}

\item{format}{File format, \code{"fasta"}, \code{"fastq"} or \code{"rds"}.}

\item{target_middle}{Whether model is language model with separate input layers.}

\item{mode}{Either \code{"lm"} for language model or \code{"label_header"}, \code{"label_csv"} or \code{"label_folder"} for label classification.}

\item{output_format}{Determines shape of output tensor for language model.

Either \code{"target_right"}, \code{"target_middle_lstm"}, \code{"target_middle_cnn"} or \code{"wavenet"}.

Assume a sequence \code{"AACCGTA"}. Output correspond as follows

\itemize{

\item \verb{"target_right": X = "AACCGT", Y = "A"}

\item \verb{"target_middle_lstm": X = (X_1 = "AAC", X_2 = "ATG"), Y = "C"} (note reversed order of X_2)

\item \verb{"target_middle_cnn": X = "AACGTA", Y = "C"}

\item \verb{"wavenet": X = "AACCGT", Y = "ACCGTA"}

}}

\item{ambiguous_nuc}{How to handle nucleotides outside vocabulary, either \code{"zero"}, \code{"discard"}, \code{"empirical"} or \code{"equal"}.

\itemize{

\item If \code{"zero"}, input gets encoded as zero vector.

\item If \code{"equal"}, input is repetition of \code{1/length(vocabulary)}.

\item If \code{"discard"}, samples containing nucleotides outside vocabulary get discarded.

\item If \code{"empirical"}, use nucleotide distribution of current file.

}}

\item{evaluate_all_files}{Boolean, if \code{TRUE} will iterate over all files in \code{path_input} once. \code{number_batches} will be overwritten.}

\item{verbose}{Boolean.}

\item{max_iter}{Stop after \code{max_iter} number of iterations failed to produce a new batch.}

\item{target_from_csv}{Path to csv file with target mapping. One column should be called "file" and other entries in row are the targets.}

\item{max_samples}{Maximum number of samples to use from one file. If not \code{NULL} and file has more than \code{max_samples} samples, will randomly choose a

subset of \code{max_samples} samples.}

\item{proportion_per_seq}{Numerical value between 0 and 1. Proportion of sequence to take samples from (use random subsequence).}

\item{concat_seq}{Character string or \code{NULL}. If not \code{NULL} all entries from file get concatenated to one sequence with \code{concat_seq} string between them.

Example: If 1.entry AACC, 2. entry TTTG and \code{concat_seq = "ZZZ"} this becomes AACCZZZTTTG.}

\item{seed}{Sets seed for \code{set.seed} function for reproducible results.}

\item{auc}{Whether to include AUC metric. If output layer activation is \code{"softmax"}, only possible for 2 targets. Computes the average if output layer has sigmoid

activation and multiple targets.}

\item{auprc}{Whether to include AUPRC metric. If output layer activation is \code{"softmax"}, only possible for 2 targets. Computes the average if output layer has sigmoid

activation and multiple targets.}

\item{path_pred_list}{Path to store list of predictions (output of output layers) and corresponding true labels as rds file.}

\item{exact_num_samples}{Exact number of samples to evaluate. If you want to evaluate a number of samples not divisible by batch_size. Useful if you want

to evaluate a data set exactly ones and know the number of samples already. Should be a vector if \code{mode = "label_folder"} (with same length as \code{vocabulary_label})

and else an integer.}

\item{activations}{List containing output formats for output layers (\verb{softmax, sigmoid} or \code{linear}). If \code{NULL}, will be estimated from model.}

\item{shuffle_file_order}{Logical, whether to go through files randomly or sequentially.}

\item{include_seq}{Whether to store input. Only applies if \code{path_pred_list} is not \code{NULL}.}

\item{...}{Further generator options. See \code{\link{get_generator}}.}

}

\value{

A list of evaluation results. Each list element corresponds to an output layer of the model.

}

\description{

Returns evaluation metric like confusion matrix, loss, AUC, AUPRC, MAE, MSE (depending on output layer).

}

\examples{

\dontshow{if (reticulate::py_module_available("tensorflow")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf}

# create dummy data

path_input <- tempfile()

dir.create(path_input)

create_dummy_data(file_path = path_input,

                  num_files = 3,

                  seq_length = 11, 

                  num_seq = 5,

                  vocabulary = c("a", "c", "g", "t"))

# create model

model <- create_model_lstm_cnn(layer_lstm = 8, layer_dense = 4, maxlen = 10, verbose = FALSE)

# evaluate

evaluate_model(path_input = path_input,

  model = model,

  step = 11,

  vocabulary = c("a", "c", "g", "t"),

  vocabulary_label = list(c("a", "c", "g", "t")),

  mode = "lm",

  output_format = "target_right",

  evaluate_all_files = TRUE,

  verbose = FALSE)

\dontshow{\}) # examplesIf}

}