diff --git a/NAMESPACE b/NAMESPACE
index 2d61f27fa5aeb1fb8aff4ca690584377bf9495a1..5c9032da3aa504c8693756eb9a242093f3e48612 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -164,6 +164,7 @@ exportClasses(performance)
exportClasses(prediction)
importFrom(MASS,mvrnorm)
importFrom(car,Anova)
+importFrom(data.table,.SD)
importFrom(data.table,data.table)
importFrom(data.table,setDT)
importFrom(data.table,setorderv)
diff --git a/NEWS.md b/NEWS.md
index a33c64539cf8ce8e6458e08effa1c0b735007950..3a51308ba40c82a1c172d540f8d0df689bb4e96e 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,10 +1,19 @@
-# v1.0.0 : End of core development, Ready for beta testing
+# CHANGES IN HTRfit VERSION 2.0.0
-* The functions, tests, and vignettes are fully functional and operational!
-* To do: 1) conducting installation tests , 2)seeking out beta testers for further evaluation.
+* Improved Documentation: Clearer and more comprehensive information on HTRfit functionalities and usage.
+* Additional Metrics: Added PR curve, precision, recall, accuracy, and R2 for enhanced model evaluation.
+* Reduced Dependencies: Streamlined installation by removing the need for plotROC.
+
+# CHANGES IN HTRfit VERSION 1.0.2
+* R files reorganization, log file enhancement and versioning
+* R files have been restructured to enhance clarity.
+* The log file location has been relocated to the R temporary directory.
+* A bug that was causing issues with writing the 'group' information in log files during model update has been fixed.
+* Fix NOTE "no visible binding for global variable" linked to ggplot2
+* Fix bug with packageVersion("HTRfit")
-# v1.0.1 : few enhancements
+# CHANGES IN HTRfit VERSION 1.0.1
* The metrics_plot function has been updated to display histograms instead of density plots.
* The identity plot's point shape has been modified, enhancing the visual clarity of comparisons between HTRfit and DESeq2 results.
@@ -13,18 +22,11 @@
* The vignette section now includes a new topic, "About mixed model evaluation".
-# v1.0.2 : R files reorganization, log file enhancement and versioning
-
-* R files have been restructured to enhance clarity.
-* The log file location has been relocated to the R temporary directory.
-* A bug that was causing issues with writing the 'group' information in log files during model update has been fixed.
-* Fix NOTE "no visible binding for global variable" linked to ggplot2
-* Fix bug with packageVersion("HTRfit")
+# CHANGES IN HTRfit VERSION 1.0.0
+* End of core development, Ready for beta testing
+* The functions, tests, and vignettes are fully functional and operational!
+* To do: 1) conducting installation tests , 2) seeking out beta testers for further evaluation.
-# v2.0.0 : Enhanced evaluation metrics
-* Improved Documentation: Clearer and more comprehensive information on HTRfit functionalities and usage.
-* Additional Metrics: Added PR curve, precision, recall, accuracy, and R2 for enhanced model evaluation.
-* Reduced Dependencies: Streamlined installation by removing the need for plotROC.
diff --git a/R/mock_rnaseq.R b/R/mock_rnaseq.R
index 7fc440aedddce49d047fdee2e3a127f914e1aaad..d604233f417621187d45a7598e394c2580b0e3d8 100644
--- a/R/mock_rnaseq.R
+++ b/R/mock_rnaseq.R
@@ -93,16 +93,23 @@ mock_rnaseq <- function(list_var, n_genes, min_replicates, max_replicates, seque
## same order as mu_ij_matx_rep
matx_dispersion <- matx_dispersion[ order(row.names(matx_dispersion)), ]
matx_dispersion_rep <- replicateMatrix(matx_dispersion, matx_bool_replication)
+
+ if (!is.null(sequencing_depth)) {
+ message("Scaling count table according to sequencing depth.")
+ mu_ij_dtf_rep <- scaleCountsTable(mu_ij_matx_rep, sequencing_depth)
+ mu_ij_matx_rep <- as.matrix(mu_ij_dtf_rep)
+ }
+
matx_countsTable <- generateCountTable(mu_ij_matx_rep, matx_dispersion_rep)
message("Counts simulation: Done")
dtf_countsTable <- matx_countsTable %>% as.data.frame()
- if (!is.null(sequencing_depth)) {
- message("Scaling count table according to sequencing depth.")
- dtf_countsTable <- scaleCountsTable(dtf_countsTable, sequencing_depth)
- }
+ #if (!is.null(sequencing_depth)) {
+ # message("Scaling count table according to sequencing depth.")
+ # dtf_countsTable <- scaleCountsTable(dtf_countsTable, sequencing_depth)
+ #}
checkFractionOfZero(dtf_countsTable)
diff --git a/R/na-na.R b/R/na-na.R
deleted file mode 100644
index 72d65fe5ac370bd3d2ef754583daf56a0f49251e..0000000000000000000000000000000000000000
--- a/R/na-na.R
+++ /dev/null
@@ -1,931 +0,0 @@
-# WARNING - Generated by {fusen} from /dev/flat_full.Rmd: do not edit by hand
-
-
-#' @name prediction-class
-#' @aliases prediction-class
-#'
-#' @title Class \code{prediction}
-#'
-#' @description
-#' Object to encapsulate numerical predictions together with the
-#' corresponding true class labels, optionally collecting predictions and
-#' labels for several cross-validation or bootstrapping runs.
-#'
-#' @section Objects from the Class:
-#' Objects can be created by using the \code{prediction} function.
-#'
-#' @note
-#' Every \code{prediction} object contains information about the 2x2
-#' contingency table consisting of tp,tn,fp, and fn, along with the
-#' marginal sums n.pos,n.neg,n.pos.pred,n.neg.pred, because these form
-#' the basis for many derived performance measures.
-#'
-#' @slot predictions A list, in which each element is a vector of predictions
-#' (the list has length > 1 for x-validation data.
-#' @slot labels Analogously, a list in which each element is a vector of true
-#' class labels.
-#' @slot cutoffs A list in which each element is a vector of all necessary
-#' cutoffs. Each cutoff vector consists of the predicted scores (duplicates
-#' removed), in descending order.
-#' @slot fp A list in which each element is a vector of the number (not the
-#' rate!) of false positives induced by the cutoffs given in the corresponding
-#' 'cutoffs' list entry.
-#' @slot tp As fp, but for true positives.
-#' @slot tn As fp, but for true negatives.
-#' @slot fn As fp, but for false negatives.
-#' @slot n.pos A list in which each element contains the number of positive
-#' samples in the given x-validation run.
-#' @slot n.neg As n.pos, but for negative samples.
-#' @slot n.pos.pred A list in which each element is a vector of the number of
-#' samples predicted as positive at the cutoffs given in the corresponding
-#' 'cutoffs' entry.
-#' @slot n.neg.pred As n.pos.pred, but for negatively predicted samples.
-#'
-#'
-#' @author
-#' Tobias Sing \email{tobias.sing@gmail.com}, Oliver Sander
-#' \email{osander@gmail.com}
-#'
-#' @seealso
-#' \code{\link{prediction}},
-#' \code{\link{performance}},
-#' \code{\link{performance-class}},
-#' \code{\link{plot.performance}}
-#'
-#' @export
-setClass("prediction",
- representation(predictions = "list",
- labels = "list",
- cutoffs = "list",
- fp = "list",
- tp = "list",
- tn = "list",
- fn = "list",
- n.pos = "list",
- n.neg = "list",
- n.pos.pred = "list",
- n.neg.pred = "list"))
-
-setMethod("show","prediction",
- function(object){
- cat("A ", class(object), " instance\n", sep = "")
- if(length(object@predictions) > 1L){
- cat(" with ", length(object@predictions)," cross ",
- "validation runs ", sep = "")
- if(length(unique(vapply(object@predictions,length,integer(1))))){
- cat("(equal lengths)", sep = "")
- } else {
- cat("(different lengths)", sep = "")
- }
- } else {
- cat(" with ", length(object@predictions[[1L]]),
- " data points", sep = "")
- }
- })
-
-#' @name performance-class
-#' @aliases performance-class
-#'
-#' @title Class \code{performance}
-#'
-#' @description
-#' Object to capture the result of a performance evaluation, optionally
-#' collecting evaluations from several cross-validation or bootstrapping runs.
-#'
-#' @section Objects from the Class:
-#' Objects can be created by using the \code{performance} function.
-#'
-#' @details
-#' A \code{performance} object can capture information from four
-#' different evaluation scenarios:
-#' \itemize{
-#' \item The behaviour of a cutoff-dependent performance measure across
-#' the range of all cutoffs (e.g. \code{performance( predObj, 'acc' )} ). Here,
-#' \code{x.values} contains the cutoffs, \code{y.values} the
-#' corresponding values of the performance measure, and
-#' \code{alpha.values} is empty.\cr
-#' \item The trade-off between two performance measures across the
-#' range of all cutoffs (e.g. \code{performance( predObj,
-#' 'tpr', 'fpr' )} ). In this case, the cutoffs are stored in
-#' \code{alpha.values}, while \code{x.values} and \code{y.values}
-#' contain the corresponding values of the two performance measures.\cr
-#' \item A performance measure that comes along with an obligatory
-#' second axis (e.g. \code{performance( predObj, 'ecost' )} ). Here, the measure values are
-#' stored in \code{y.values}, while the corresponding values of the
-#' obligatory axis are stored in \code{x.values}, and \code{alpha.values}
-#' is empty.\cr
-#' \item A performance measure whose value is just a scalar
-#' (e.g. \code{performance( predObj, 'auc' )} ). The value is then stored in
-#' \code{y.values}, while \code{x.values} and \code{alpha.values} are
-#' empty.
-#' }
-#'
-#' @slot x.name Performance measure used for the x axis.
-#' @slot y.name Performance measure used for the y axis.
-#' @slot alpha.name Name of the unit that is used to create the parametrized
-#' curve. Currently, curves can only be parametrized by cutoff, so
-#' \code{alpha.name} is either \code{none} or \code{cutoff}.
-#' @slot x.values A list in which each entry contains the x values of the curve
-#' of this particular cross-validation run. \code{x.values[[i]]},
-#' \code{y.values[[i]]}, and \code{alpha.values[[i]]} correspond to each
-#' other.
-#' @slot y.values A list in which each entry contains the y values of the curve
-#' of this particular cross-validation run.
-#' @slot alpha.values A list in which each entry contains the cutoff values of
-#' the curve of this particular cross-validation run.
-#'
-#' @references
-#' A detailed list of references can be found on the ROCR homepage at
-#' \url{http://rocr.bioinf.mpi-sb.mpg.de}.
-#'
-#' @author
-#' Tobias Sing \email{tobias.sing@gmail.com}, Oliver Sander
-#' \email{osander@gmail.com}
-#'
-#' @seealso
-#' \code{\link{prediction}}
-#' \code{\link{performance}},
-#' \code{\link{prediction-class}},
-#' \code{\link{plot.performance}}
-#'
-#' @export
-setClass("performance",
- representation(x.name = "character",
- y.name = "character",
- alpha.name = "character",
- x.values = "list",
- y.values = "list",
- alpha.values = "list" ))
-
-setMethod("show","performance",
- function(object){
- cat("A ", class(object), " instance\n", sep = "")
- if(length(object@y.values[[1L]]) > 1L){
- cat(" '", object@x.name, "' vs. '", object@y.name,
- "' (alpha: '",object@alpha.name,"')\n", sep = "")
- } else {
- cat(" '", object@y.name, "'\n", sep = "")
- }
- if(length(object@y.values) > 1L){
- cat(" for ", length(object@y.values)," cross ",
- "validation runs ", sep = "")
- } else {
- if(length(object@y.values[[1L]]) > 1L){
- cat(" with ", length(object@y.values[[1L]])," data points",
- sep = "")
- }
- }
- })
-
-
-
-#' @name prediction
-#'
-#' @title Function to create prediction objects
-#'
-#' @description
-#' Every classifier evaluation using ROCR starts with creating a
-#' \code{prediction} object. This function is used to transform the input data
-#' (which can be in vector, matrix, data frame, or list form) into a
-#' standardized format.
-#'
-#' @details
-#' \code{predictions} and \code{labels} can simply be vectors of the same
-#' length. However, in the case of cross-validation data, different
-#' cross-validation runs can be provided as the *columns* of a matrix or
-#' data frame, or as the entries of a list. In the case of a matrix or
-#' data frame, all cross-validation runs must have the same length, whereas
-#' in the case of a list, the lengths can vary across the cross-validation
-#' runs. Internally, as described in section 'Value', all of these input
-#' formats are converted to list representation.
-#'
-#' Since scoring classifiers give relative tendencies towards a negative
-#' (low scores) or positive (high scores) class, it has to be declared
-#' which class label denotes the negative, and which the positive class.
-#' Ideally, labels should be supplied as ordered factor(s), the lower
-#' level corresponding to the negative class, the upper level to the
-#' positive class. If the labels are factors (unordered), numeric,
-#' logical or characters, ordering of the labels is inferred from
-#' R's built-in \code{<} relation (e.g. 0 < 1, -1 < 1, 'a' < 'b',
-#' FALSE < TRUE). Use \code{label.ordering} to override this default
-#' ordering. Please note that the ordering can be locale-dependent
-#' e.g. for character labels '-1' and '1'.
-#'
-#' Currently, ROCR supports only binary classification (extensions toward
-#' multiclass classification are scheduled for the next release,
-#' however). If there are more than two distinct label symbols, execution
-#' stops with an error message. If all predictions use the same two
-#' symbols that are used for the labels, categorical predictions are
-#' assumed. If there are more than two predicted values, but all numeric,
-#' continuous predictions are assumed (i.e. a scoring
-#' classifier). Otherwise, if more than two symbols occur in the
-#' predictions, and not all of them are numeric, execution stops with an
-#' error message.
-#'
-#' @param predictions A vector, matrix, list, or data frame containing the
-#' predictions.
-#' @param labels A vector, matrix, list, or data frame containing the true class
-#' labels. Must have the same dimensions as \code{predictions}.
-#' @param label.ordering The default ordering (cf.details) of the classes can
-#' be changed by supplying a vector containing the negative and the positive
-#' class label.
-#'
-#' @return An S4 object of class \code{prediction}.
-#'
-#' @author
-#' Tobias Sing \email{tobias.sing@gmail.com}, Oliver Sander
-#' \email{osander@gmail.com}
-#' @export
-prediction <- function(predictions, labels, label.ordering=NULL) {
-
- ## bring 'predictions' and 'labels' into list format,
- ## each list entry representing one x-validation run
-
- ## convert predictions into canonical list format
- if (is.data.frame(predictions)) {
- names(predictions) <- c()
- predictions <- as.list(predictions)
- } else if (is.matrix(predictions)) {
- predictions <- as.list(data.frame(predictions))
- names(predictions) <- c()
- } else if (is.vector(predictions) && !is.list(predictions)) {
- predictions <- list(predictions)
- } else if (!is.list(predictions)) {
- stop("Format of predictions is invalid. It couldn't be coerced to a list.",
- call. = FALSE)
- }
-
- ## convert labels into canonical list format
- if (is.data.frame(labels)) {
- names(labels) <- c()
- labels <- as.list( labels)
- } else if (is.matrix(labels)) {
- labels <- as.list( data.frame( labels))
- names(labels) <- c()
- } else if ((is.vector(labels) ||
- is.ordered(labels) ||
- is.factor(labels)) &&
- !is.list(labels)) {
- labels <- list( labels)
- } else if (!is.list(labels)) {
- stop("Format of labels is invalid. It couldn't be coerced to a list.",
- call. = FALSE)
- }
-
-
- if(any(vapply(predictions,anyNA, logical(1)))){
- warnings("'predictions' contains NA. These missing predictions will be removed from evaluation")
- nonNA_pred <- !is.na(predictions)
- predictions <- predictions[nonNA_pred]
- labels <- labels[nonNA_pred]
- }
-
-
- ## Length consistency checks
- if (length(predictions) != length(labels))
- stop(paste("Number of cross-validation runs must be equal",
- "for predictions and labels."))
- if (! all(sapply(predictions, length) == sapply(labels, length)))
- stop(paste("Number of predictions in each run must be equal",
- "to the number of labels for each run."))
-
- ## replace infinite numbers by max values
- ## avoid prob with infinite values
- #for (i in 1:length(predictions)) {
- # epsilon <- max(predictions[[i]][is.finite(predictions[[i]] )])
- # idx_inf_values <- !is.finite( predictions[[i]] )
- # predictions[[i]][idx_inf_values] <- epsilon
- #}
-
- ## only keep prediction/label pairs that are finite numbers
- for (i in 1:length(predictions)) {
- finite.bool <- is.finite( predictions[[i]] )
- predictions[[i]] <- predictions[[i]][ finite.bool ]
- labels[[i]] <- labels[[i]][ finite.bool ]
- }
-
-
-
-
- ## abort if 'labels' format is inconsistent across
- ## different cross-validation runs
- label.format="" ## one of 'normal','factor','ordered'
- if (all(sapply( labels, is.factor)) &&
- !any(sapply(labels, is.ordered))) {
- label.format <- "factor"
- } else if (all(sapply( labels, is.ordered))) {
- label.format <- "ordered"
- } else if (all(sapply( labels, is.character)) ||
- all(sapply( labels, is.numeric)) ||
- all(sapply( labels, is.logical))) {
- label.format <- "normal"
- } else {
- stop(paste("Inconsistent label data type across different",
- "cross-validation runs."))
- }
-
- ## abort if levels are not consistent across different
- ## cross-validation runs
- if (! all(sapply(labels, levels)==levels(labels[[1]])) ) {
- stop(paste("Inconsistent factor levels across different",
- "cross-validation runs."))
- }
-
- ## convert 'labels' into ordered factors, aborting if the number
- ## of classes is not equal to 2.
- levels <- c()
- if ( label.format == "ordered" ) {
- if (!is.null(label.ordering)) {
- stop(paste("'labels' is already ordered. No additional",
- "'label.ordering' must be supplied."))
- } else {
- levels <- levels(labels[[1]])
- }
- } else {
- if ( is.null( label.ordering )) {
- if ( label.format == "factor" ) levels <- sort(levels(labels[[1]]))
- else levels <- sort( unique( unlist( labels)))
- } else {
- ## if (!setequal( levels, label.ordering)) {
- if (!setequal( unique(unlist(labels)), label.ordering )) {
- stop("Label ordering does not match class labels.")
- }
- levels <- label.ordering
- }
- for (i in 1:length(labels)) {
- if (is.factor(labels))
- labels[[i]] <- ordered(as.character(labels[[i]]),
- levels=levels)
- else labels[[i]] <- ordered( labels[[i]], levels=levels)
- }
-
- }
- #print(levels)
- #print(labels)
- if (length(levels) != 2) {
- message <- paste("Number of classes is not equal to 2.\n",
- "HTRfit currently supports only evaluation of ",
- "binary classification tasks.",sep="")
- stop(message)
- }
-
- ## determine whether predictions are continuous or categorical
- ## (in the latter case stop
- if (!is.numeric( unlist( predictions ))) {
- stop("Currently, only continuous predictions are supported by HTRfit")
- }
-
- ## compute cutoff/fp/tp data
-
- cutoffs <- list()
- fp <- list()
- tp <- list()
- fn <- list()
- tn <- list()
- n.pos <- list()
- n.neg <- list()
- n.pos.pred <- list()
- n.neg.pred <- list()
- for (i in 1:length(predictions)) {
- n.pos <- c( n.pos, sum( labels[[i]] == levels[2] ))
- n.neg <- c( n.neg, sum( labels[[i]] == levels[1] ))
- ans <- .compute.unnormalized.roc.curve( predictions[[i]], labels[[i]] )
- cutoffs <- c( cutoffs, list( ans$cutoffs ))
- fp <- c( fp, list( ans$fp ))
- tp <- c( tp, list( ans$tp ))
- fn <- c( fn, list( n.pos[[i]] - tp[[i]] ))
- tn <- c( tn, list( n.neg[[i]] - fp[[i]] ))
- n.pos.pred <- c(n.pos.pred, list(tp[[i]] + fp[[i]]) )
- n.neg.pred <- c(n.neg.pred, list(tn[[i]] + fn[[i]]) )
- }
-
-
- return( new("prediction", predictions=predictions,
- labels=labels,
- cutoffs=cutoffs,
- fp=fp,
- tp=tp,
- fn=fn,
- tn=tn,
- n.pos=n.pos,
- n.neg=n.neg,
- n.pos.pred=n.pos.pred,
- n.neg.pred=n.neg.pred))
-}
-
-## fast fp/tp computation based on cumulative summing
-.compute.unnormalized.roc.curve <- function( predictions, labels ) {
- ## determine the labels that are used for the pos. resp. neg. class :
- pos.label <- levels(labels)[2]
- neg.label <- levels(labels)[1]
-
- pred.order <- order(predictions, decreasing=TRUE)
- predictions.sorted <- predictions[pred.order]
- tp <- cumsum(labels[pred.order]==pos.label)
- fp <- cumsum(labels[pred.order]==neg.label)
-
- ## remove fp & tp for duplicated predictions
- ## as duplicated keeps the first occurrence, but we want the last, two
- ## rev are used.
- ## Highest cutoff (Infinity) corresponds to tp=0, fp=0
- dups <- rev(duplicated(rev(predictions.sorted)))
- tp <- c(0, tp[!dups])
- fp <- c(0, fp[!dups])
- cutoffs <- c(Inf, predictions.sorted[!dups])
-
- return(list( cutoffs=cutoffs, fp=fp, tp=tp ))
-}
-
-#' @name performance
-#'
-#' @title Function to create performance objects
-#'
-#' @description
-#' All kinds of predictor evaluations are performed using this function.
-#'
-#' @details
-#' Here is the list of available performance measures. Let Y and
-#' \eqn{\hat{Y}}{Yhat} be random variables representing the class and the prediction for
-#' a randomly drawn sample, respectively. We denote by
-#' \eqn{\oplus}{+} and \eqn{\ominus}{-} the positive and
-#' negative class, respectively. Further, we use the following
-#' abbreviations for empirical quantities: P (\# positive
-#' samples), N (\# negative samples), TP (\# true positives), TN (\# true
-#' negatives), FP (\# false positives), FN (\# false negatives).
-#' \describe{
-#' \item{\code{acc}:}{accuracy. \eqn{P(\hat{Y}=Y)}{P(Yhat = Y)}. Estimated
-#' as: \eqn{\frac{TP+TN}{P+N}}{(TP+TN)/(P+N)}.}
-#' \item{\code{err}:}{Error rate. \eqn{P(\hat{Y}\ne Y)}{P(Yhat !=
-#' Y)}. Estimated as: \eqn{\frac{FP+FN}{P+N}}{(FP+FN)/(P+N)}.}
-#' \item{\code{fpr}:}{False positive rate. \eqn{P(\hat{Y}=\oplus | Y =
-#' \ominus)}{P(Yhat = + | Y = -)}. Estimated as:
-#' \eqn{\frac{FP}{N}}{FP/N}.}
-#' \item{\code{fall}:}{Fallout. Same as \code{fpr}.}
-#' \item{\code{tpr}:}{True positive
-#' rate. \eqn{P(\hat{Y}=\oplus|Y=\oplus)}{P(Yhat = + | Y = +)}. Estimated
-#' as: \eqn{\frac{TP}{P}}{TP/P}.}
-#' \item{\code{rec}:}{recall. Same as \code{tpr}.}
-#' \item{\code{sens}:}{sensitivity. Same as \code{tpr}.}
-#' \item{\code{fnr}:}{False negative
-#' rate. \eqn{P(\hat{Y}=\ominus|Y=\oplus)}{P(Yhat = - | Y =
-#' +)}. Estimated as: \eqn{\frac{FN}{P}}{FN/P}.}
-#' \item{\code{miss}:}{Miss. Same as \code{fnr}.}
-#' \item{\code{tnr}:}{True negative rate. \eqn{P(\hat{Y} =
-#' \ominus|Y=\ominus)}{P(Yhat = - | Y = -)}.}
-#' \item{\code{spec}:}{specificity. Same as \code{tnr}.}
-#' \item{\code{ppv}:}{Positive predictive
-#' value. \eqn{P(Y=\oplus|\hat{Y}=\oplus)}{P(Y = + | Yhat =
-#' +)}. Estimated as: \eqn{\frac{TP}{TP+FP}}{TP/(TP+FP)}.}
-#' \item{\code{prec}:}{precision. Same as \code{ppv}.}
-#' \item{\code{npv}:}{Negative predictive
-#' value. \eqn{P(Y=\ominus|\hat{Y}=\ominus)}{P(Y = - | Yhat =
-#' -)}. Estimated as: \eqn{\frac{TN}{TN+FN}}{TN/(TN+FN)}.}
-#' \item{\code{pcfall}:}{Prediction-conditioned
-#' fallout. \eqn{P(Y=\ominus|\hat{Y}=\oplus)}{P(Y = - | Yhat =
-#' +)}. Estimated as: \eqn{\frac{FP}{TP+FP}}{FP/(TP+FP)}.}
-#' \item{\code{pcmiss}:}{Prediction-conditioned
-#' miss. \eqn{P(Y=\oplus|\hat{Y}=\ominus)}{P(Y = + | Yhat =
-#' -)}. Estimated as: \eqn{\frac{FN}{TN+FN}}{FN/(TN+FN)}.}
-#' \item{\code{rpp}:}{Rate of positive predictions. \eqn{P( \hat{Y} =
-#' \oplus)}{P(Yhat = +)}. Estimated as: (TP+FP)/(TP+FP+TN+FN).}
-#' \item{\code{rnp}:}{Rate of negative predictions. \eqn{P( \hat{Y} =
-#' \ominus)}{P(Yhat = -)}. Estimated as: (TN+FN)/(TP+FP+TN+FN).}
-#' \item{\code{phi}:}{Phi correlation coefficient. \eqn{\frac{TP \cdot
-#' TN - FP \cdot FN}{\sqrt{ (TP+FN) \cdot (TN+FP) \cdot (TP+FP)
-#' \cdot (TN+FN)}}}{(TP*TN -
-#' FP*FN)/(sqrt((TP+FN)*(TN+FP)*(TP+FP)*(TN+FN)))}. Yields a
-#' number between -1 and 1, with 1 indicating a perfect
-#' prediction, 0 indicating a random prediction. Values below 0
-#' indicate a worse than random prediction.}
-#' \item{\code{mat}:}{Matthews correlation coefficient. Same as \code{phi}.}
-#' \item{\code{mi}:}{Mutual information. \eqn{I(\hat{Y},Y) := H(Y) -
-#' H(Y|\hat{Y})}{I(Yhat, Y) := H(Y) - H(Y | Yhat)}, where H is the
-#' (conditional) entropy. Entropies are estimated naively (no bias
-#' correction).}
-#' \item{\code{chisq}:}{Chi square test statistic. \code{?chisq.test}
-#' for details. Note that R might raise a warning if the sample size
-#' is too small.}
-#' \item{\code{odds}:}{Odds ratio. \eqn{\frac{TP \cdot TN}{FN \cdot
-#' FP}}{(TP*TN)/(FN*FP)}. Note that odds ratio produces
-#' Inf or NA values for all cutoffs corresponding to FN=0 or
-#' FP=0. This can substantially decrease the plotted cutoff region.}
-#' \item{\code{lift}:}{Lift
-#' value. \eqn{\frac{P(\hat{Y}=\oplus|Y=\oplus)}{P(\hat{Y}=\oplus)}}{P(Yhat = + |
-#' Y = +)/P(Yhat = +)}.}
-#' \item{\code{f}:}{precision-recall F measure (van Rijsbergen, 1979). Weighted
-#' harmonic mean of precision (P) and recall (R). \eqn{F =
-#' \frac{1}{\alpha \frac{1}{P} + (1-\alpha)\frac{1}{R}}}{F = 1/
-#' (alpha*1/P + (1-alpha)*1/R)}. If
-#' \eqn{\alpha=\frac{1}{2}}{alpha=1/2}, the mean is balanced. A
-#' frequent equivalent formulation is
-#' \eqn{F = \frac{(\beta^2+1) \cdot P \cdot R}{R + \beta^2 \cdot
-#' P}}{F = (beta^2+1) * P * R / (R + beta^2 * P)}. In this formulation, the
-#' mean is balanced if \eqn{\beta=1}{beta=1}. Currently, ROCR only accepts
-#' the alpha version as input (e.g. \eqn{\alpha=0.5}{alpha=0.5}). If no
-#' value for alpha is given, the mean will be balanced by default.}
-#' \item{\code{rch}:}{ROC convex hull. A ROC (=\code{tpr} vs \code{fpr}) curve
-#' with concavities (which represent suboptimal choices of cutoff) removed
-#' (Fawcett 2001). Since the result is already a parametric performance
-#' curve, it cannot be used in combination with other measures.}
-#' \item{\code{auc}:}{Area under the ROC curve. This is equal to the value of the
-#' Wilcoxon-Mann-Whitney test statistic and also the probability that the
-#' classifier will score are randomly drawn positive sample higher than a
-#' randomly drawn negative sample. Since the output of
-#' \code{auc} is cutoff-independent, this
-#' measure cannot be combined with other measures into a parametric
-#' curve. The partial area under the ROC curve up to a given false
-#' positive rate can be calculated by passing the optional parameter
-#' \code{fpr.stop=0.5} (or any other value between 0 and 1) to
-#' \code{performance}.}
-#' \item{\code{aucpr}:}{Area under the precision/recall curve. Since the output
-#' of \code{aucpr} is cutoff-independent, this measure cannot be combined
-#' with other measures into a parametric curve.}
-#' \item{\code{prbe}:}{precision-recall break-even point. The cutoff(s) where
-#' precision and recall are equal. At this point, positive and negative
-#' predictions are made at the same rate as their prevalence in the
-#' data. Since the output of
-#' \code{prbe} is just a cutoff-independent scalar, this
-#' measure cannot be combined with other measures into a parametric curve.}
-#' \item{\code{cal}:}{Calibration error. The calibration error is the
-#' absolute difference between predicted confidence and actual reliability. This
-#' error is estimated at all cutoffs by sliding a window across the
-#' range of possible cutoffs. The default window size of 100 can be
-#' adjusted by passing the optional parameter \code{window.size=200}
-#' to \code{performance}. E.g., if for several
-#' positive samples the output of the classifier is around 0.75, you might
-#' expect from a well-calibrated classifier that the fraction of them
-#' which is correctly predicted as positive is also around 0.75. In a
-#' well-calibrated classifier, the probabilistic confidence estimates
-#' are realistic. Only for use with
-#' probabilistic output (i.e. scores between 0 and 1).}
-#' \item{\code{mxe}:}{Mean cross-entropy. Only for use with
-#' probabilistic output. \eqn{MXE :=-\frac{1}{P+N}( \sum_{y_i=\oplus}
-#' ln(\hat{y}_i) + \sum_{y_i=\ominus} ln(1-\hat{y}_i))}{MXE := - 1/(P+N) \sum_{y_i=+}
-#' ln(yhat_i) + \sum_{y_i=-} ln(1-yhat_i)}. Since the output of
-#' \code{mxe} is just a cutoff-independent scalar, this
-#' measure cannot be combined with other measures into a parametric curve.}
-#' \item{\code{rmse}:}{Root-mean-squared error. Only for use with
-#' numerical class labels. \eqn{RMSE:=\sqrt{\frac{1}{P+N}\sum_i (y_i
-#' - \hat{y}_i)^2}}{RMSE := sqrt(1/(P+N) \sum_i (y_i -
-#' yhat_i)^2)}. Since the output of
-#' \code{rmse} is just a cutoff-independent scalar, this
-#' measure cannot be combined with other measures into a parametric curve.}
-#' \item{\code{sar}:}{Score combinining performance measures of different
-#' characteristics, in the attempt of creating a more "robust"
-#' measure (cf. Caruana R., ROCAI2004):
-#' SAR = 1/3 * ( accuracy + Area under the ROC curve + Root
-#' mean-squared error ).}
-#' \item{\code{ecost}:}{Expected cost. For details on cost curves,
-#' cf. Drummond&Holte 2000,2004. \code{ecost} has an obligatory x
-#' axis, the so-called 'probability-cost function'; thus it cannot be
-#' combined with other measures. While using \code{ecost} one is
-#' interested in the lower envelope of a set of lines, it might be
-#' instructive to plot the whole set of lines in addition to the lower
-#' envelope. An example is given in \code{demo(ROCR)}.}
-#' \item{\code{cost}:}{Cost of a classifier when
-#' class-conditional misclassification costs are explicitly given.
-#' Accepts the optional parameters \code{cost.fp} and
-#' \code{cost.fn}, by which the costs for false positives and
-#' negatives can be adjusted, respectively. By default, both are set
-#' to 1.}
-#' }
-#'
-#' @note
-#' Here is how to call \code{performance()} to create some standard
-#' evaluation plots:
-#' \describe{
-#' \item{ROC curves:}{measure="tpr", x.measure="fpr".}
-#' \item{precision/recall graphs:}{measure="prec", x.measure="rec".}
-#' \item{sensitivity/specificity plots:}{measure="sens", x.measure="spec".}
-#' \item{Lift charts:}{measure="lift", x.measure="rpp".}
-#' }
-#'
-#' @param prediction.obj An object of class \code{prediction}.
-#' @param measure Performance measure to use for the evaluation. A complete list
-#' of the performance measures that are available for \code{measure} and
-#' \code{x.measure} is given in the 'Details' section.
-#' @param x.measure A second performance measure. If different from the default,
-#' a two-dimensional curve, with \code{x.measure} taken to be the unit in
-#' direction of the x axis, and \code{measure} to be the unit in direction of
-#' the y axis, is created. This curve is parametrized with the cutoff.
-#' @param ... Optional arguments (specific to individual performance measures).
-#'
-#' @return An S4 object of class \code{performance}.
-#'
-#' @author
-#' Tobias Sing \email{tobias.sing@gmail.com}, Oliver Sander
-#' \email{osander@gmail.com}
-#'
-#' @export
-performance <- function(prediction.obj,
- measure,
- x.measure="cutoff",
- ...) {
-
- ## define the needed environments
- envir.list <- .define.environments()
- long.unit.names <- envir.list$long.unit.names
- function.names <- envir.list$function.names
- obligatory.x.axis <- envir.list$obligatory.x.axis
- optional.arguments <- envir.list$optional.arguments
- default.values <- envir.list$default.values
-
- ## abort in case of misuse
- if (class(prediction.obj) != 'prediction' ||
- !exists(measure, where=long.unit.names, inherits=FALSE) ||
- !exists(x.measure, where=long.unit.names, inherits=FALSE)) {
- stop(paste("Wrong argument types: First argument must be of type",
- "'prediction'; second and optional third argument must",
- "be available performance measures!"))
- }
-
- ## abort, if attempt is made to use a measure that has an obligatory
- ## x.axis as the x.measure (cannot be combined)
- if (exists( x.measure, where=obligatory.x.axis, inherits=FALSE )) {
- message <- paste("The performance measure",
- x.measure,
- "can only be used as 'measure', because it has",
- "the following obligatory 'x.measure':\n",
- get( x.measure, envir=obligatory.x.axis))
- stop(message)
- }
-
- ## if measure is a performance measure with obligatory x.axis, then
- ## enforce this axis:
- if (exists( measure, where=obligatory.x.axis, inherits=FALSE )) {
- x.measure <- get( measure, envir=obligatory.x.axis )
- }
-
- if (x.measure == "cutoff" ||
- exists( measure, where=obligatory.x.axis, inherits=FALSE )) {
-
- ## fetch from '...' any optional arguments for the performance
- ## measure at hand that are given, otherwise fill up the default values
- optional.args <- list(...)
- argnames <- c()
- if ( exists( measure, where=optional.arguments, inherits=FALSE )) {
- argnames <- get( measure, envir=optional.arguments )
- default.arglist <- list()
- for (i in 1:length(argnames)) {
- default.arglist <- c(default.arglist,
- get(paste(measure,":",argnames[i],sep=""),
- envir=default.values, inherits=FALSE))
- }
- names(default.arglist) <- argnames
-
- for (i in 1:length(argnames)) {
- templist <- list(optional.args,
- default.arglist[[i]])
- names(templist) <- c('arglist', argnames[i])
-
- optional.args <- do.call('.farg', templist)
- }
- }
- optional.args <- .select.args( optional.args, argnames )
-
- ## determine function name
- function.name <- get( measure, envir=function.names )
-
- ## for each x-validation run, compute the requested performance measure
- x.values <- list()
- y.values <- list()
- for (i in 1:length( prediction.obj@predictions )) {
- argumentlist <- .sarg(optional.args,
- predictions= prediction.obj@predictions[[i]],
- labels= prediction.obj@labels[[i]],
- cutoffs= prediction.obj@cutoffs[[i]],
- fp= prediction.obj@fp[[i]],
- tp= prediction.obj@tp[[i]],
- fn= prediction.obj@fn[[i]],
- tn= prediction.obj@tn[[i]],
- n.pos= prediction.obj@n.pos[[i]],
- n.neg= prediction.obj@n.neg[[i]],
- n.pos.pred= prediction.obj@n.pos.pred[[i]],
- n.neg.pred= prediction.obj@n.neg.pred[[i]])
-
- ans <- do.call( function.name, argumentlist )
-
- if (!is.null(ans[[1]])) x.values <- c( x.values, list( ans[[1]] ))
- y.values <- c( y.values, list( ans[[2]] ))
- }
-
- if (! (length(x.values)==0 || length(x.values)==length(y.values)) ) {
- stop("Consistency error.")
- }
-
- ## create a new performance object
- return( new("performance",
- x.name = get( x.measure, envir=long.unit.names ),
- y.name = get( measure, envir=long.unit.names ),
- alpha.name = "none",
- x.values = x.values,
- y.values = y.values,
- alpha.values = list() ))
- } else {
- perf.obj.1 <- performance( prediction.obj, measure=x.measure, ... )
- perf.obj.2 <- performance( prediction.obj, measure=measure, ... )
- return( .combine.performance.objects( perf.obj.1, perf.obj.2 ) )
- }
-}
-
-#' @importFrom stats approxfun
-.combine.performance.objects <- function( p.obj.1, p.obj.2 ) {
- ## some checks for misusage (in any way, this function is
- ## only for internal use)
- if ( p.obj.1@x.name != p.obj.2@x.name ) {
- stop("Error: Objects need to have identical x axis.")
- }
- if ( p.obj.1@alpha.name != "none" || p.obj.2@alpha.name != "none") {
- stop("Error: At least one of the two objects has already been merged.")
- }
- if (length(p.obj.1@x.values) != length(p.obj.2@x.values)) {
- stop(paste("Only performance objects with identical number of",
- "cross-validation runs can be combined."))
- }
-
- x.values <- list()
- x.name <- p.obj.1@y.name
- y.values <- list()
- y.name <- p.obj.2@y.name
- alpha.values <- list()
- alpha.name <- p.obj.1@x.name
-
- for (i in 1:length( p.obj.1@x.values )) {
- x.values.1 <- p.obj.1@x.values[[i]]
- y.values.1 <- p.obj.1@y.values[[i]]
- x.values.2 <- p.obj.2@x.values[[i]]
- y.values.2 <- p.obj.2@y.values[[i]]
-
- ## cutoffs of combined object = merged cutoffs of simple objects
- cutoffs <- sort( unique( c(x.values.1, x.values.2)), decreasing=TRUE )
-
- ## calculate y.values at cutoffs using step function
- y.values.int.1 <- stats::approxfun(x.values.1, y.values.1,
- method="constant",f=1,rule=2)(cutoffs)
- y.values.int.2 <- stats::approxfun(x.values.2, y.values.2,
- method="constant",f=1,rule=2)(cutoffs)
-
- ## 'approxfun' ignores NA and NaN
- objs <- list( y.values.int.1, y.values.int.2)
- objs.x <- list( x.values.1, x.values.2 )
- na.cutoffs.1.bool <- is.na( y.values.1) & !is.nan( y.values.1 )
- nan.cutoffs.1.bool <- is.nan( y.values.1)
- na.cutoffs.2.bool <- is.na( y.values.2) & !is.nan( y.values.2 )
- nan.cutoffs.2.bool <- is.nan( y.values.2)
- bools <- list(na.cutoffs.1.bool, nan.cutoffs.1.bool,
- na.cutoffs.2.bool, nan.cutoffs.2.bool)
- values <- c(NA,NaN,NA,NaN)
-
- for (j in 1:4) {
- for (k in which(bools[[j]])) {
- interval.max <- objs.x[[ ceiling(j/2) ]][k]
- interval.min <- -Inf
- if (k < length(objs.x[[ ceiling(j/2) ]])) {
- interval.min <- objs.x[[ ceiling(j/2) ]][k+1]
- }
- objs[[ ceiling(j/2) ]][cutoffs <= interval.max &
- cutoffs > interval.min ] <- values[j]
- }
- }
-
- alpha.values <- c(alpha.values, list(cutoffs))
- x.values <- c(x.values, list(objs[[1]]))
- y.values <- c(y.values, list(objs[[2]]))
- }
-
- return( new("performance",
- x.name=x.name, y.name=y.name,
- alpha.name=alpha.name, x.values=x.values,
- y.values=y.values, alpha.values=alpha.values))
-}
-
-.define.environments <- function() {
- ## There are five environments: long.unit.names, function.names,
- ## obligatory.x.axis, optional.arguments, default.values
-
- ## Define long names corresponding to the measure abbreviations.
- long.unit.names <- new.env()
- assign("none","None", envir=long.unit.names)
- assign("cutoff", "Cutoff", envir=long.unit.names)
- assign("acc", "accuracy", envir=long.unit.names)
- assign("err", "Error Rate", envir=long.unit.names)
- assign("fpr", "False positive rate", envir=long.unit.names)
- assign("tpr", "True positive rate", envir=long.unit.names)
- assign("rec", "recall", envir=long.unit.names)
- assign("sens", "sensitivity", envir=long.unit.names)
- assign("fnr", "False negative rate", envir=long.unit.names)
- assign("tnr", "True negative rate", envir=long.unit.names)
- assign("spec", "specificity", envir=long.unit.names)
- assign("ppv", "Positive predictive value", envir=long.unit.names)
- assign("prec", "precision", envir=long.unit.names)
- assign("npv", "Negative predictive value", envir=long.unit.names)
- assign("fall", "Fallout", envir=long.unit.names)
- assign("miss", "Miss", envir=long.unit.names)
- assign("pcfall", "Prediction-conditioned fallout", envir=long.unit.names)
- assign("pcmiss", "Prediction-conditioned miss", envir=long.unit.names)
- assign("rpp", "Rate of positive predictions", envir=long.unit.names)
- assign("rnp", "Rate of negative predictions", envir=long.unit.names)
- assign("auc","Area under the ROC curve", envir=long.unit.names)
- assign("aucpr","Area under the precision/recall curve", envir=long.unit.names)
- assign("cal", "Calibration error", envir=long.unit.names)
- assign("mwp", "Median window position", envir=long.unit.names)
- assign("prbe","precision/recall break-even point", envir=long.unit.names)
- assign("rch", "ROC convex hull", envir=long.unit.names)
- assign("mxe", "Mean cross-entropy", envir=long.unit.names)
- assign("rmse","Root-mean-square error", envir=long.unit.names)
- assign("phi", "Phi correlation coefficient", envir=long.unit.names)
- assign("mat","Matthews correlation coefficient", envir=long.unit.names)
- assign("mi", "Mutual information", envir=long.unit.names)
- assign("chisq", "Chi-square test statistic", envir=long.unit.names)
- assign("odds","Odds ratio", envir=long.unit.names)
- assign("lift", "Lift value", envir=long.unit.names)
- assign("f","precision-recall F measure", envir=long.unit.names)
- assign("sar", "SAR", envir=long.unit.names)
- assign("ecost", "Expected cost", envir=long.unit.names)
- assign("cost", "Explicit cost", envir=long.unit.names)
-
- ## Define function names corresponding to the measure abbreviations.
- function.names <- new.env()
- assign("acc", ".performance.accuracy", envir=function.names)
- assign("err", ".performance.error.rate", envir=function.names)
- assign("fpr", ".performance.false.positive.rate", envir=function.names)
- assign("tpr", ".performance.true.positive.rate", envir=function.names)
- assign("rec", ".performance.true.positive.rate", envir=function.names)
- assign("sens", ".performance.true.positive.rate", envir=function.names)
- assign("fnr", ".performance.false.negative.rate", envir=function.names)
- assign("tnr", ".performance.true.negative.rate", envir=function.names)
- assign("spec", ".performance.true.negative.rate", envir=function.names)
- assign("ppv", ".performance.positive.predictive.value",
- envir=function.names)
- assign("prec", ".performance.positive.predictive.value",
- envir=function.names)
- assign("npv", ".performance.negative.predictive.value",
- envir=function.names)
- assign("fall", ".performance.false.positive.rate", envir=function.names)
- assign("miss", ".performance.false.negative.rate", envir=function.names)
- assign("pcfall", ".performance.prediction.conditioned.fallout",
- envir=function.names)
- assign("pcmiss", ".performance.prediction.conditioned.miss",
- envir=function.names)
- assign("rpp", ".performance.rate.of.positive.predictions",
- envir=function.names)
- assign("rnp", ".performance.rate.of.negative.predictions",
- envir=function.names)
- assign("auc", ".performance.auc", envir=function.names)
- assign("aucpr", ".performance.aucpr", envir=function.names)
- assign("cal", ".performance.calibration.error", envir=function.names)
- assign("prbe", ".performance.precision.recall.break.even.point",
- envir=function.names)
- assign("rch", ".performance.rocconvexhull", envir=function.names)
- assign("mxe", ".performance.mean.cross.entropy", envir=function.names)
- assign("rmse", ".performance.root.mean.squared.error",
- envir=function.names)
- assign("phi", ".performance.phi", envir=function.names)
- assign("mat", ".performance.phi", envir=function.names)
- assign("mi", ".performance.mutual.information", envir=function.names)
- assign("chisq", ".performance.chisq", envir=function.names)
- assign("odds", ".performance.odds.ratio", envir=function.names)
- assign("lift", ".performance.lift", envir=function.names)
- assign("f", ".performance.f", envir=function.names)
- assign("sar", ".performance.sar", envir=function.names)
- assign("ecost", ".performance.expected.cost", envir=function.names)
- assign("cost", ".performance.cost", envir=function.names)
-
- ## If a measure comes along with an obligatory x axis (including "none"),
- ## list it here.
- obligatory.x.axis <- new.env()
- assign("mxe", "none", envir=obligatory.x.axis)
- assign("rmse", "none", envir=obligatory.x.axis)
- assign("prbe", "none", envir=obligatory.x.axis)
- assign("auc", "none", envir=obligatory.x.axis)
- assign("aucpr", "none", envir=obligatory.x.axis)
- assign("rch","none", envir=obligatory.x.axis)
- ## ecost requires probability cost function as x axis, which is handled
- ## implicitly, not as an explicit performance measure.
- assign("ecost","none", envir=obligatory.x.axis)
-
- ## If a measure has optional arguments, list the names of the
- ## arguments here.
- optional.arguments <- new.env()
- assign("cal", "window.size", envir=optional.arguments)
- assign("f", "alpha", envir=optional.arguments)
- assign("cost", c("cost.fp", "cost.fn"), envir=optional.arguments)
- assign("auc", "fpr.stop", envir=optional.arguments)
-
- ## If a measure has additional arguments, list the default values
- ## for them here. Naming convention: e.g. "cal" has an optional
- ## argument "window.size" the key to use here is "cal:window.size"
- ## (colon as separator)
- default.values <- new.env()
- assign("cal:window.size", 100, envir=default.values)
- assign("f:alpha", 0.5, envir=default.values)
- assign("cost:cost.fp", 1, envir=default.values)
- assign("cost:cost.fn", 1, envir=default.values)
- assign("auc:fpr.stop", 1, envir=default.values)
-
- list(long.unit.names=long.unit.names, function.names=function.names,
- obligatory.x.axis=obligatory.x.axis,
- optional.arguments=optional.arguments,
- default.values=default.values)
-}
-
diff --git a/R/receiver_operating_characteristic.R b/R/receiver_operating_characteristic.R
index 27854c7b7297a7a0fdae61c850954b8b705493c1..bb3754e4844133b5a3e185ae8881084702c2e853 100644
--- a/R/receiver_operating_characteristic.R
+++ b/R/receiver_operating_characteristic.R
@@ -84,6 +84,7 @@ compute_roc_auc <- function(dt) Area_Under_Curve(x = dt$`False positive rate`,
#' @param col_truth Column name for binary ground truth values.
#' @param col_score Column name for predicted scores.
#' @return A list containing ROC curves and AUCs for each group and an aggregate ROC curve and AUC.
+#' @importFrom data.table setDT .SD
#' @export
get_roc_object <- function(evaldata_params, col_param = "description", col_truth = "isDE", col_score = "p.adj" ) {
diff --git a/R/simulation_report.R b/R/simulation_report.R
index 18baf350a21f9fcebd4e9410b9fa9b2325f49421..37b71865c6e9f3ba613d8fe929dc09481324334c 100644
--- a/R/simulation_report.R
+++ b/R/simulation_report.R
@@ -300,7 +300,7 @@ compute_metrics_summary <- function(dt) {
#' and for aggregated classification metrics.
#'
#' @examples
-#' @importFrom data.table setDT
+#' @importFrom data.table setDT .SD
#' @export
get_ml_metrics_obj <- function(evaldata_params, alpha_risk = 0.05, col_param = "description"){
diff --git a/R/waldtest.R b/R/waldtest.R
index 9728ef5a35ba68e066d0f9c14d923669bf3912d7..af7b2d086bd985b7b857fa03599013e3e4730141 100644
--- a/R/waldtest.R
+++ b/R/waldtest.R
@@ -47,7 +47,7 @@ wald_test <- function(estimation, std_error, reference_value = 0, alternative =
#'
#' @param list_tmb A list of glmmTMB objects representing the fitted models.
#' @param coeff_threshold A non-negative value which specifies a ln(fold change) threshold. The Threshold is used for the Wald test to determine whether the coefficient (β) is significant or not, depending on \code{alt_hypothesis} parameter. Default is 0, ln(FC = 1).
-#' @param alt_hypothesis Alternative hypothesis for the Wald test (default is "greaterAbs").
+#' @param alternative_hypothesis Alternative hypothesis for the Wald test (default is "greaterAbs").
#' Possible choice:
#' "greater"
#' - β > coeff_threshold,
diff --git a/README.md b/README.md
index 02306e2bd79443df44d8f2b9289607deda166052..f89e5ef4a8845f59039504777616fe1f0da204fc 100644
--- a/README.md
+++ b/README.md
@@ -101,7 +101,7 @@ In the realm of RNAseq analysis, various key experimental parameters play a cruc
<div id="bg" align="center">
- <img src="./vignettes/figs/htrfit_workflow.png" width="500" height="300">
+ <img src="./man/figures/htrfit_workflow.png" width="500" height="300">
</div>
@@ -157,7 +157,7 @@ p <- diagnostic_plot(list_tmb = l_tmb)
```
<div id="bg" align="center">
- <img src="./vignettes/figs/diagnostic_plot.png" width="600" height="360">
+ <img src="./man/figures/diagnostic_plot.png" width="600" height="360">
</div>
The diagnostic metrics show that the fit is not as good for all genes.
diff --git a/dev/flat_full.Rmd b/dev/flat_full.Rmd
index 94bb04c335dea7ce0c30fd2f1bfdfeb4ae15775c..dc94f1eb94b5bf4ce1a34e7082ea851ea10579f7 100644
--- a/dev/flat_full.Rmd
+++ b/dev/flat_full.Rmd
@@ -2,7 +2,7 @@
title: "flat_full.Rmd for working package"
output: html_document
editor_options:
- chunk_output_type: console
+chunk_output_type: console
---
<!-- Run this 'development' chunk -->
@@ -303,7 +303,7 @@ clear_memory <- function(except_obj){
```
-```{r test-utils}
+```{r tests-utils}
# Test unitaires pour la fonction join_dtf
test_that("join_dtf réalise la jointure correctement", {
# Création de données de test
@@ -2014,16 +2014,23 @@ mock_rnaseq <- function(list_var, n_genes, min_replicates, max_replicates, seque
## same order as mu_ij_matx_rep
matx_dispersion <- matx_dispersion[ order(row.names(matx_dispersion)), ]
matx_dispersion_rep <- replicateMatrix(matx_dispersion, matx_bool_replication)
+
+ if (!is.null(sequencing_depth)) {
+ message("Scaling count table according to sequencing depth.")
+ mu_ij_dtf_rep <- scaleCountsTable(mu_ij_matx_rep, sequencing_depth)
+ mu_ij_matx_rep <- as.matrix(mu_ij_dtf_rep)
+ }
+
matx_countsTable <- generateCountTable(mu_ij_matx_rep, matx_dispersion_rep)
message("Counts simulation: Done")
dtf_countsTable <- matx_countsTable %>% as.data.frame()
- if (!is.null(sequencing_depth)) {
- message("Scaling count table according to sequencing depth.")
- dtf_countsTable <- scaleCountsTable(dtf_countsTable, sequencing_depth)
- }
+ #if (!is.null(sequencing_depth)) {
+ # message("Scaling count table according to sequencing depth.")
+ # dtf_countsTable <- scaleCountsTable(dtf_countsTable, sequencing_depth)
+ #}
checkFractionOfZero(dtf_countsTable)
@@ -5477,7 +5484,7 @@ wald_test <- function(estimation, std_error, reference_value = 0, alternative =
#'
#' @param list_tmb A list of glmmTMB objects representing the fitted models.
#' @param coeff_threshold A non-negative value which specifies a ln(fold change) threshold. The Threshold is used for the Wald test to determine whether the coefficient (β) is significant or not, depending on \code{alt_hypothesis} parameter. Default is 0, ln(FC = 1).
-#' @param alt_hypothesis Alternative hypothesis for the Wald test (default is "greaterAbs").
+#' @param alternative_hypothesis Alternative hypothesis for the Wald test (default is "greaterAbs").
#' Possible choice:
#' "greater"
#' - β > coeff_threshold,
@@ -5665,6 +5672,7 @@ compute_roc_auc <- function(dt) Area_Under_Curve(x = dt$`False positive rate`,
#' @param col_truth Column name for binary ground truth values.
#' @param col_score Column name for predicted scores.
#' @return A list containing ROC curves and AUCs for each group and an aggregate ROC curve and AUC.
+#' @importFrom data.table setDT .SD
#' @export
get_roc_object <- function(evaldata_params, col_param = "description", col_truth = "isDE", col_score = "p.adj" ) {
@@ -7897,7 +7905,7 @@ compute_metrics_summary <- function(dt) {
#' and for aggregated classification metrics.
#'
#' @examples
-#' @importFrom data.table setDT
+#' @importFrom data.table setDT .SD
#' @export
get_ml_metrics_obj <- function(evaldata_params, alpha_risk = 0.05, col_param = "description"){
@@ -9273,624 +9281,6 @@ test_that("calculate_actualMixed calculates actual mixed effects as expected", {
```
-
-# High-Throughput RNA-seq model fit
-
-In the realm of RNAseq analysis, various key experimental parameters play a crucial role in influencing the statistical power to detect expression changes. Parameters such as sequencing depth, the number of replicates, and others are expected to impact this statistical power. To help with the selection of optimal values for these experimental parameters, we introduce a comprehensive statistical framework known as **HTRfit**, underpinned by computational simulation. **HTRfit** serves as a versatile tool, not only for simulation but also for conducting differential expression analysis. It facilitates this analysis by fitting Generalized Linear Models (GLMs) with multiple variables, which could encompass genotypes, environmental factors, and more. These GLMs are highly adaptable, allowing the incorporation of fixed effects, mixed effects, and interactions between variables.
-
-
-
-# Initializing variables for simulation
-
-The `init_variable()` function is used for defining the variables in your experimental design. Sizes of effects for each variable and interaction term can be defined in two different ways: 1) The user can manually sets values of all levels of a variable, in which case the effects are necessarily considered fixed in the model; 2) The effects can be randomly picked in a normal distribution with mean and standard deviation defined by the user, in which case the user can decide whether the effects are considered fixed or random in the model.
-
-
-## Manually init a single variable
-
-The `init_variable()` function allows for precise control over the variables in your experimental design.
-In this example, we manually initialize **varA** with specific size effects (mu) for three levels.
-
-
-```{r example-init_variable_man, warning=FALSE, message=FALSE}
-list_var <- init_variable( name = "varA", mu = c(0.2, 4, -3), level = 3)
-```
-
-
-## Randomly init a single variable
-
-Alternatively, you can randomly initialize **varA** by specifying a mean (mu) and standard deviation (sd).
-This introduces variability into **varA**, making it either a fixed or random effect in your design.
-
-```{r example-init_variable_rand, warning=FALSE, message=FALSE}
-list_var <- init_variable( name = "varA", mu = 10, sd = 0.2, level = 5)
-```
-
-
-## Randomly init several variables
-
-You can also initialize multiple variables, such as **varA** and **varB**, with random values.
-This flexibility allows you to create diverse experimental designs.
-
-
-```{r example-init_variable_mult, warning=FALSE, message=FALSE}
-list_var <- init_variable( name = "varA", mu = 10, sd = 0.2, level = 5) %>%
- init_variable( name = "varB", mu = -3, sd = 0.34, level = 2)
-```
-
-
-## Add interaction between variable
-
-Similarly to `init_variable()`, `add_interaction()` allows to define an interaction between variable.
-
-In this example, we initialize **varA** and **varB**, and create an interaction between **varA**, and **varB** using `add_interaction()`.
-
-
-```{r example-add_interaction, warning=FALSE, message=FALSE}
-list_var <- init_variable( name = "varA", mu = 3, sd = 0.2, level = 2) %>%
- init_variable( name = "varB", mu = 2, sd = 0.43, level = 2) %>%
- add_interaction( between_var = c("varA", "varB"), mu = 0.44, sd = 0.2)
-```
-
-
-## Complex design
-
-Interactions can involve a maximum of three variables, such as **varA**, **varB**, and **varC**.
-
-
-```{r example-add_interaction_complex, eval= FALSE, message=FALSE, warning=FALSE, include=TRUE}
-## /!\ not evaluated in Vignette
-list_var <- init_variable( name = "varA", mu = 5, sd = 0.2, level = 2) %>%
- init_variable( name = "varB", mu = 1, sd = 0.78, level = 2) %>%
- init_variable( name = "varC", mu = c(2, 3), sd = NA, level = 2) %>%
- add_interaction( between_var = c("varA", "varC"), mu = 0.44, sd = 0.2) %>%
- add_interaction( between_var = c("varA", "varB"), mu = 0.43, sd = 0.37) %>%
- add_interaction( between_var = c("varB", "varC"), mu = -0.33, sd = 0.12) %>%
- add_interaction( between_var = c("varA", "varB" ,"varC"), mu = 0.87, sd = 0.18)
-```
-
-
-## Structure of list_var object
-
-```{r example-str_obj_init, warning=FALSE, message=FALSE}
-str(list_var)
-```
-
-
-# Simulation of RNAseq data
-
-In this section, you will explore how to generate RNAseq data based on the previously defined input variables. The `mock_rnaseq()` function enables you to manage parameters in your RNAseq design, including number of genes, minimum and maximum number of replicates within your experimental setup, sequencing depth, basal expression of each gene, and dispersion of gene expression used for simulating counts.
-
-
-## Minimal example
-
-```{r example-mock_rnaseq_min, warning=FALSE, message=FALSE}
-## -- Required parameters
-N_GENES = 6000
-MIN_REPLICATES = 2
-MAX_REPLICATES = 10
-########################
-
-## -- simulate RNAseq data based on list_var, minimum input required
-## -- number of replicate randomly defined between MIN_REP and MAX_REP
-mock_data <- mock_rnaseq(list_var, N_GENES,
- min_replicates = MIN_REPLICATES,
- max_replicates = MAX_REPLICATES)
-
-## -- simulate RNAseq data based on list_var, minimum input required
-## -- Same number of replicates between conditions
-mock_data <- mock_rnaseq(list_var, N_GENES,
- min_replicates = MAX_REPLICATES,
- max_replicates = MAX_REPLICATES)
-```
-
-
-## Scaling genes counts based on sequencing depth
-
-Sequencing depth is a critical parameter affecting the statistical power of an RNAseq analysis. With the `sequencing_depth` option in the `mock_rnaseq()` function, you have the ability to control this parameter.
-
-```{r example-mock_rnaseq_seqDepth, warning=FALSE, message=FALSE}
-## -- Required parameters
-N_GENES = 6000
-MIN_REPLICATES = 2
-MAX_REPLICATES = 10
-########################
-
-SEQ_DEPTH = c(100000, 5000000, 10000000)## -- Possible number of reads/sample
-SEQ_DEPTH = 10000000 ## -- all samples have same number of reads
-mock_data <- mock_rnaseq(list_var, N_GENES,
- min_replicates = MIN_REPLICATES,
- max_replicates = MAX_REPLICATES,
- sequencing_depth = SEQ_DEPTH)
-```
-
-
-## Set dispersion of gene expression
-
-The dispersion parameter ($dispersion_i$), characterizes the relationship between the variance of the observed read counts and its mean value. In simple terms, it quantifies how much we expect observed counts to deviate from the mean value. You can specify the dispersion for individual genes using the dispersion parameter.
-
-
-```{r example-mock_rnaseq_disp, warning = FALSE, message = FALSE}
-
-## -- Required parameters
-N_GENES = 6000
-MIN_REPLICATES = 2
-MAX_REPLICATES = 4
-########################
-
-DISP = 0.1 ## -- Same dispersion for each genes
-DISP = 1000 ## -- Same dispersion for each genes
-DISP = runif(N_GENES, 0, 1000) ## -- Dispersion can vary between genes
-mock_data <- mock_rnaseq(list_var, N_GENES,
- min_replicates = MIN_REPLICATES,
- max_replicates = MAX_REPLICATES,
- dispersion = DISP )
-
-```
-
-## Set basal gene expression
-
-The basal gene expression parameter, defined using the basal_expression option, allows you to control the fundamental baseline of expression level of genes. When a single value is specified, the basal expression of all genes is the same and the effects of variables defined in list_var are applied from this basal expression level. When several values are specified, the basal expression of each gene is picked randomly among these values (with replacement). The effects of variables are then applied from the basal expression of each gene. This represents the fact that expression levels can vary among genes even when not considering the effects of the defined variables (for example, some genes are constitutively more highly expressed than others because they have stronger basal promoters).
-
-```{r example-mock_rnaseq_bexpr, warning = FALSE, message = FALSE}
-## -- Required parameters
-N_GENES = 6000
-MIN_REPLICATES = 10
-MAX_REPLICATES = 10
-########################
-
-BASAL_EXPR = -3 ## -- Value can be negative to simulate low expressed gene
-BASAL_EXPR = 2 ## -- Same basal gene expression for the N_GENES
-BASAL_EXPR = c( -3, -1, 2, 8, 9, 10 ) ## -- Basal expression can vary between genes
-mock_data <- mock_rnaseq(list_var, N_GENES,
- min_replicates = MIN_REPLICATES,
- max_replicates = MAX_REPLICATES,
- basal_expression = BASAL_EXPR)
-
-```
-
-
-
-
-## Mock RNAseq object
-
-```{r example-str_obj_mock, warning=FALSE, message=FALSE}
-str(mock_data, max.level = 1)
-```
-
-
-# Theory behind HTRfit
-
-<div id="bg" align="center">
- <img src="./figs/htrfit_workflow.png" width="500" height="300">
-</div>
-
-
-In this modeling framework, counts denoted as $K_{ij}$ for gene i and sample j are generated using a negative binomial distribution. The negative binomial distribution considers a fitted mean $\mu_{ij}$ and a gene-specific dispersion parameter $dispersion_i$.
-
-The fitted mean $\mu_{ij}$ is determined by a parameter, $q_{ij}$, which is proportionally related to the sum of all effects specified using `init_variable()` or `add_interaction()`. If basal gene expressions are provided, the $\mu_{ij}$ values are scaled accordingly using the gene-specific basal expression value ($bexpr_i$).
-
-Furthermore, the coefficients $\beta_i$ represent the natural logarithm fold changes for gene i across each column of the model matrix X. The dispersion parameter $dispersion_i$ plays a crucial role in defining the relationship between the variance of observed counts and their mean value. In simpler terms, it quantifies how far we expect observed counts to deviate from the mean value for each genes.
-
-
-# Fitting models
-
-## Prepare data for fitting
-
-The `prepareData2fit()` function serves the purpose of converting the counts matrix and sample metadata into a dataframe that is compatible with downstream **HTRfit** functions designed for model fitting. This function also includes an option to perform median ratio normalization on the data counts.
-
-
-```{r example-prepareData, warning=FALSE, message=FALSE}
-## -- simulate small example to prevent excessively lengthy vignette construction
-list_var <- init_variable( name = "varA", mu = 3, sd = 0.2, level = 2) %>%
- init_variable( name = "varB", mu = 2, sd = 0.43, level = 2) %>%
- add_interaction( between_var = c("varA", "varB"), mu = 0.44, sd = 0.2)
-N_GENES = 30
-MIN_REPLICATES = 4
-MAX_REPLICATES = 4
-BASAL_EXPR = rnorm(N_GENES , 0 , 1 )
-mock_data <- mock_rnaseq(list_var, N_GENES,
- min_replicates = MIN_REPLICATES,
- max_replicates = MAX_REPLICATES,
- basal_expression = BASAL_EXPR)
-########################
-
-
-## -- data from simulation or real data
-count_matrix <- mock_data$counts
-metaData <- mock_data$metadata
-##############################
-
-## -- convert counts matrix and samples metadatas in a data frame for fitting
-data2fit = prepareData2fit(countMatrix = count_matrix,
- metadata = metaData,
- normalization = F,
- response_name = "kij")
-
-
-## -- median ratio normalization
-data2fit = prepareData2fit(countMatrix = count_matrix,
- metadata = metaData,
- normalization = T,
- response_name = "kij")
-```
-
-
-## Fit model from your data
-
-The `fitModelParallel()` function enables independent model fitting for each gene. The number of threads used for this process can be controlled by the `n.cores` parameter.
-
-```{r example-fitModelParallel, warning=FALSE, message=FALSE}
-l_tmb <- fitModelParallel(formula = kij ~ varA,
- data = data2fit,
- group_by = "geneID",
- family = glmmTMB::nbinom2(link = "log"),
- n.cores = 1)
-```
-
-
-## Use mixed effect in your model
-
-**HTRfit** uses the **glmmTMB** functions for model fitting algorithms. This choice allows for the utilization of random effects within your formula design. For further details on how to specify your model, please refer to the [mixed model documentation](https://rdrr.io/cran/glmmTMB/man/glmmTMBControl.html).
-
-
-```{r example-fitModelParallel_mixed, warning=FALSE, message=FALSE}
-l_tmb <- fitModelParallel(formula = kij ~ varA + ( 1 | varB ),
- data = data2fit,
- group_by = "geneID",
- family = glmmTMB::nbinom2(link = "log"),
- n.cores = 1)
-```
-
-## Additional settings
-
-The function provides precise control over model settings for fitting optimization, including options for specifying the [model family](https://www.rdocumentation.org/packages/stats/versions/3.6.2/topics/family) and [model control setting](https://rdrr.io/cran/glmmTMB/man/glmmTMBControl.html). By default, a Gaussian family model is fitted, but for RNA-seq data, it is highly recommended to specify `family = glmmTMB::nbinom2(link = "log")`.
-
-
-```{r example-fitModelParallel_addSet, warning=FALSE, message=FALSE}
-l_tmb <- fitModelParallel(formula = kij ~ varA,
- data = data2fit,
- group_by = "geneID",
- n.cores = 1,
- family = glmmTMB::nbinom2(link = "log"),
- control = glmmTMB::glmmTMBControl(optCtrl=list(iter.max=1e5,
- eval.max=1e5)))
-```
-
-## Not only RNAseq data
-
-As the model family can be customized, HTRfit is not exclusively tailored for RNA-seq data.
-
-```{r example-fitModelParallel_nonRNA, warning=FALSE, message=FALSE}
-data("iris")
-l_tmb_iris <- fitModelParallel(formula = Sepal.Length ~ Sepal.Width + Petal.Length + Petal.Width ,
- data = iris,
- group_by = "Species",
- family = gaussian(),
- n.cores = 1)
-```
-
-## Extracts a tidy result table from a list tmb object
-
-The tidy_results function extracts a data frame containing estimates of ln(fold changes), standard errors, test statistics, p-values, and adjusted p-values for fixed effects. Additionally, it provides access to correlation terms and standard deviations for random effects, offering a detailed view of HTRfit modeling results.
-
-```{r example-tidyRes, warning=FALSE, message=FALSE}
-## -- get tidy results
-tidy_results(l_tmb_iris, coeff_threshold = 0.1, alternative_hypothesis = "greaterAbs")
-```
-
-
-## Update fit
-
-The `updateParallel()` function updates and re-fits a model for each gene. It offers options similar to those in `fitModelParallel()`.
-
-```{r example-update, warning=FALSE, message=FALSE}
-## -- update your fit modifying the model family
-l_tmb <- updateParallel(formula = kij ~ varA,
- list_tmb = l_tmb ,
- family = gaussian(),
- n.cores = 1)
-
-## -- update fit using additional model control settings
-l_tmb <- updateParallel(formula = kij ~ varA ,
- list_tmb = l_tmb ,
- family = gaussian(),
- n.cores = 1,
- control = glmmTMB::glmmTMBControl(optCtrl=list(iter.max=1e3,
- eval.max=1e3)))
-
-
-## -- update your model formula and your family model
-l_tmb <- updateParallel(formula = kij ~ varA + varB + varA:varB ,
- list_tmb = l_tmb ,
- family = glmmTMB::nbinom2(link = "log"),
- n.cores = 1)
-```
-
-#### Struture of list tmb object
-
-
-```{r example-str_obj_l_tmb, warning=FALSE, message=FALSE}
-str(l_tmb$gene1, max.level = 1)
-```
-
-
-## Plot fit metrics
-
-Visualizing fit metrics is essential for evaluating your models. Here, we show you how to generate various plots to assess the quality of your models. You can explore all metrics or focus on specific aspects like dispersion and log-likelihood.
-
-```{r example-plotMetrics , warning=FALSE, message=FALSE, fig.align='center', fig.height=4, fig.width=8}
-## -- plot all metrics
-diagnostic_plot(list_tmb = l_tmb)
-```
-
-
-```{r example-plotMetricsFocus, warning=FALSE, message=FALSE, fig.align='center', fig.height=3, fig.width=8}
-## -- Focus on metrics
-diagnostic_plot(list_tmb = l_tmb, focus = c("dispersion", "logLik"))
-```
-
-## Anova to select the best model
-
-
-Utilizing the `anovaParallel()` function enables you to perform model selection by assessing the significance of the fixed effects. You can also include additional parameters like type. For more details, refer to [car::Anova](https://rdrr.io/cran/car/man/Anova.html).
-
-```{r example-anova, warning=FALSE, message=FALSE}
-## -- update your fit modifying the model family
-l_anova <- anovaParallel(list_tmb = l_tmb)
-
-## -- additional settings
-l_anova <- anovaParallel(list_tmb = l_tmb, type = "III" )
-
-```
-
-
-# Simulation evaluation report
-
-In this section, we delve into the evaluation of your simulation results. The `evaluation_report()` function provide valuable insights into the performance of your simulated data and models.
-
-```{r example-evaluation_report, warning = FALSE, message = FALSE}
-## -- get simulation/fit evaluation
-resSimu <- evaluation_report(list_tmb = l_tmb,
- dds = NULL,
- mock_obj = mock_data,
- coeff_threshold = 0.4,
- alpha_risk = 0.05,
- alt_hypothesis = "greaterAbs")
-```
-
-
-## Identity plot
-
-The `evaluation_report()` function produces an identity plot, offering a visual tool to juxtapose the simulated effects (actual effects) with the model-inferred effects. This graphical representation simplifies assessing how well the model aligns with the values of the simulated effects, enabling a visual analysis of the model's goodness of fit to the simulated data. For a more quantitative evaluation of inference quality, the R-squared (R2) metric can be employed.
-
-#### Model parameters
-
-```{r example-outputIdentity_params, warning = FALSE, message = FALSE, fig.align='center', fig.height=4, fig.width=5}
-## -- Model params
-resSimu$identity$params
-```
-
-#### Dispersion parameter
-
-```{r example-outputIdentity_disp, warning = FALSE, message = FALSE, fig.align='center', fig.height=4, fig.width=5}
-## -- Dispersion params
-resSimu$identity$dispersion
-```
-
-The dispersion plot, generated by the `evaluation_report()` function, offers a visual comparison of the dispersion parameters used in the simulation \(\alpha_i\) with those estimated by the model. This graphical representation provides an intuitive way to assess the alignment between the simulated dispersion values and the model-inferred values, enabling a visual evaluation of how well the model captures the underlying data characteristics.
-
-
-## ROC curve
-
-The Receiver Operating Characteristic (ROC) curve is a valuable tool for assessing the performance of classification models, particularly in the context of identifying differentially expressed genes. It provides a graphical representation of the model's ability to distinguish between genes that are differentially expressed and those that are not, by varying the `coeff_threshold` and the `alt_hypothesis` parameters.
-
-
-```{r example-outputROC, warning = FALSE, message = FALSE, fig.align='center', fig.height=4, fig.width=7}
-## -- ROC curve
-resSimu$roc$params
-```
-
-
-## Precision-recall curve
-
-
-The precision-recall curve (PR curve) illustrates the relationship between precision and recall at various classification thresholds. It is particularly valuable in the context of imbalanced data, where one class is significantly more prevalent than the other. Unlike the ROC curve, which can be influenced by class distribution, the PR curve focuses on the model's ability to correctly identify examples of the minority class while maintaining high precision.
-
-
-```{r example-outputPR, warning = FALSE, message = FALSE, fig.align='center', fig.height=4, fig.width=7}
-## -- precision-recall curve
-resSimu$precision_recall$params
-```
-
-## AUC and classification performance metrics
-
-The area under the ROC curve (AUC) provides a single metric that summarizes the model's overall performance in distinguishing between differentially expressed and non-differentially expressed genes. A higher AUC indicates better model performance. In the case of the ROC curve, this AUC should be compared to the value of 0.5, representing a random classifier. On the other hand, for the PR curve, we compute the pr_AUC_random, serving as a baseline for comparison.
-
-
-In addition to evaluating model performance through the AUC for both ROC and PR curves, we provide access to key classification metrics, including Accuracy, Precision, Recall (or Sensitivity), and Specificity. These metrics offer a comprehensive view of the model's classification capabilities. These metrics are computed for each parameter (excluding the intercept when skip_eval_intercept = TRUE), providing detailed insights into individual parameter contributions. Furthermore, an aggregated assessment is performed, considering all parameters (except the intercept by default), offering a perspective on the model's overall classification effectiveness.
-
-
-```{r example-outputPerf, warning = FALSE, message = FALSE, fig.align='center', fig.height=4, fig.width=7}
-## -- precision-recall curve
-resSimu$performances
-```
-
-
-## Compare HTRfit with DESeq2
-
-**HTRfit** offers a wrapper for **DESeq2** outputs. This functionality allows users to seamlessly integrate the results obtained from **DESeq2** into the **HTRfit** analysis pipeline. By doing so, you can readily compare the performance of **HTRfit** with **DESeq2** on your RNAseq data. This comparative analysis aids in determining which tool performs better for your specific research goals and dataset
-
-
-```{r example-ddsComparison, warning = FALSE, message = FALSE, results='hide', fig.keep='none'}
-## -- DESeq2
-library(DESeq2)
-dds <- DESeq2::DESeqDataSetFromMatrix(
- countData = count_matrix,
- colData = metaData,
- design = ~ varA + varB + varA:varB )
-dds <- DESeq2::DESeq(dds, quiet = TRUE)
-
-
-## -- get simulation/fit evaluation
-resSimu <- evaluation_report(list_tmb = l_tmb,
- dds = dds,
- mock_obj = mock_data,
- coeff_threshold = 0.4,
- alt_hypothesis = "greaterAbs")
-```
-
-
-```{r example-outputResSimu_id, warning = FALSE, message = FALSE, fig.align='center', fig.height=4, fig.width=5}
-## -- identity plot
-###### 1) Model params
-resSimu$identity$params
-###### Dispersion params
-resSimu$identity$dispersion
-```
-
-```{r example-outputResSimu_metric, warning = FALSE, message = FALSE, fig.align='center', fig.height=4, fig.width=7}
-## -- precision-recall curve
-resSimu$precision_recall$params
-## -- ROC curve
-resSimu$roc$params
-## -- Performances metrics
-resSimu$performances
-```
-
-## Focus evaluation on a subset of genes
-
-In this section, we showcase the assessment of model performance on a subset of genes. Specifically, we focus on evaluating genes with low expression levels, identified by their basal expression ($bexpr_i$) initialized below 0 during the simulation.
-
-```{r example-subsetGenes, warning = FALSE, message = FALSE}
-## -- Focus on low expressed genes
-low_expressed_df <- mock_data$groundTruth$effects[ mock_data$groundTruth$effects$basalExpr < 0, ]
-l_genes <- unique(low_expressed_df$geneID)
-mock_lowExpressed <- subsetGenes(l_genes, mock_data)
-
-
-## -- get simulation/fit evaluation
-resSimu <- evaluation_report(list_tmb = l_tmb,
- dds = dds,
- mock_obj = mock_lowExpressed,
- coeff_threshold = 0.4,
- alt_hypothesis = "greaterAbs")
-```
-
-As we compare this evaluation to the previous one, we observe a reduction in performances for both **HTRfit** and **DESeq2** inferences.
-
-```{r example-subsetGenes_id, warning=FALSE, message=FALSE, fig.align='center', fig.height=4, fig.width=5}
-## -- identity plot
-###### 1) Model params
-resSimu$identity$params
-###### Dispersion params
-resSimu$identity$dispersion
-```
-
-```{r example-subsetGenes_metrics, warning=FALSE, message=FALSE, fig.align='center', fig.height=4, fig.width=7}
-## -- precision-recall curve
-resSimu$precision_recall$params
-## -- ROC curve
-resSimu$roc$params
-## -- Performances metrics
-resSimu$performances
-```
-
-
-## Modifying your alpha risk
-
-```{r example-alphaRisk_1, warning = FALSE, message = FALSE}
-## -- get simulation/fit evaluation
-resSimu <- evaluation_report(list_tmb = l_tmb,
- dds = dds,
- mock_obj = mock_data,
- coeff_threshold = 0.4,
- alpha_risk = 0.05, ## -- default
- alt_hypothesis = "greaterAbs")
-## -- Performances metrics
-resSimu$performances
-```
-
-
-```{r example-alphaRiskk_2, warning = FALSE, message = FALSE}
-## -- get simulation/fit evaluation
-resSimu <- evaluation_report(list_tmb = l_tmb,
- dds = dds,
- mock_obj = mock_data,
- coeff_threshold = 0.4,
- alpha_risk = 0.01,
- alt_hypothesis = "greaterAbs")
-## -- Performances metrics
-resSimu$performances
-```
-
-
-## Evaluate model inference involving mixed effects
-
-For certain experimental scenarios, such as those involving a high number of levels or longitudinal data, the utilization of mixed effects within your design formula can be beneficial. The **HTRfit** simulation framework also offers the capability to assess this type of design formula.
-
-```{r example-evalMixed, warning = FALSE, message = FALSE}
-## -- init a design with a high number of levels
-input_var_list <- init_variable( name = "varA", mu = 0.2, sd = 0.74, level = 60) %>%
- init_variable( name = "varB", mu = 0.27, sd = 0.6, level = 2) %>%
- add_interaction( between_var = c("varA", "varB"), mu = 0.44, sd = 0.89)
-## -- simulate RNAseq data
-mock_data <- mock_rnaseq(input_var_list,
- n_genes = 30, ## small number to prevent excessively lengthy vignette construction
- min_replicates = 10,
- max_replicates = 10,
- basal_expression = 5 )
-## -- prepare data & fit a model with mixed effect
-data2fit = prepareData2fit(countMatrix = mock_data$counts,
- metadata = mock_data$metadata,
- normalization = F,
- response_name = "kij")
-l_tmb <- fitModelParallel(formula = kij ~ varB + (varB | varA),
- data = data2fit,
- group_by = "geneID",
- family = glmmTMB::nbinom2(link = "log"),
- n.cores = 1)
-## -- evaluation
-resSimu <- evaluation_report(list_tmb = l_tmb,
- dds = NULL,
- mock_obj = mock_data,
- coeff_threshold = 0.27,
- alt_hypothesis = "greater")
-```
-
-```{r example-outputResSimuMixed_id, warning = FALSE, message = FALSE, fig.align='center', fig.height=4, fig.width=5}
-## -- identity plot
-###### 1) Model params
-resSimu$identity$params
-###### Dispersion params
-resSimu$identity$dispersion
-```
-
-```{r example-outputResSimuMixed_metric, warning = FALSE, message = FALSE, fig.align='center', fig.height=4, fig.width=7}
-## -- precision-recall curve
-resSimu$precision_recall$params
-## -- ROC curve
-resSimu$roc$params
-## -- Performances metrics
-resSimu$performances
-```
-
-## Structure of evaluation report object
-
-
-```{r example-str_eval_report, warning=FALSE, message=FALSE}
-str(resSimu, max.level = 1)
-```
-
-## About mixed model evaluation
-
-**HTRfit** offers a versatile simulation framework capable of fitting various types of models involving mixed effects, thanks to its implementation of **glmmTMB**. By combining the functionalities of `init_variable()` and `add_interaction()`, **HTRfit** enables the simulation of complex experimental designs. As of now, HTRfit supports the evaluation of *Type I* mixed models. In this context, *Type I* models are defined as those that follow the structure:
-- `~ varA + (1 | varB)` : where `varA` is defined as fixed effect and `varB` as random effect
-- `~ varA + (varA | varB)`: where `varA` is defined as mixed effect (fixed + random) and `varB` as random effect.
-
-Models not conforming to this specific form cannot be evaluated using **HTRfit's** current implementation. Nonetheless, you are welcome to extend its capabilities by implementing support for additional model types.
-
-
-
```{r development-inflate, eval=FALSE}
setwd("/Users/ex_dya/Documents/LBMC/HTRfit/")
#usethis::create_package(path = "/Users/ex_dya/Documents/LBMC/HTRfit/")
@@ -9900,5 +9290,5 @@ usethis::use_pipe(export = TRUE)
devtools::document()
# Keep eval=FALSE to avoid infinite loop in case you hit the knit button
# Execute in the console directly
-fusen::inflate(flat_file = "dev/flat_full.Rmd", vignette_name = "HTRfit", open_vignette = T, overwrite = T)
+fusen::inflate(flat_file = "dev/flat_full.Rmd", vignette_name = NA, open_vignette = F, overwrite = T)
```
diff --git a/inst/extdata/pub_dispersion.tsv b/inst/extdata/pub_dispersion.tsv
new file mode 100644
index 0000000000000000000000000000000000000000..b8ebda2a05648b4912962800ceb19e07cd063747
--- /dev/null
+++ b/inst/extdata/pub_dispersion.tsv
@@ -0,0 +1,6714 @@
+"x"
+37.8427993696801
+31.2856655852123
+42.032197881148
+27.4624951174308
+22.8325803444508
+35.8040686835639
+15.4971019418059
+37.0388279573191
+32.0836659984417
+19.0548313723579
+32.3771905526506
+11.698588678423
+39.1526668546084
+31.9463859555901
+29.0365888325296
+86238462.8492578
+31.4476841216861
+26.6318394150871
+1.11191327464907
+29.0382898047083
+7.76737069468528
+21.561612889668
+37.792630747452
+30.4291530712683
+37.0902904949586
+30.3168294181269
+17.5350846316716
+33.6574326835447
+11.8414557967471
+33.0246124429406
+11.1785923649251
+41.4477704361052
+30.1321428236014
+35.3147807179994
+24.4815530275555
+24.5481959830618
+25.1913001052172
+23.2795161596676
+0.624672673958161
+9.66149982980897
+24.5922920419177
+13.2779627756285
+32.9218048410707
+34.7266881120149
+19.7001737240332
+32.0274318818039
+33.9060209431704
+29.8300642988537
+34.160267051618
+17.2320192058086
+31.7994996892081
+33.6186186321725
+40.3734529053925
+22.4819079009401
+9.77926865644718
+67.7752513024965
+30.2673552481187
+3.91729231339585
+25.4926033812851
+32.1208662490517
+29.1307221332816
+42.4631834234753
+9.88364371024151
+24.3490310281853
+8.6230053224972
+28.9408531012858
+31.6118990004065
+25.5202185899204
+27.0885677474453
+26.2300052458668
+14.5279752754036
+19.0277915636652
+4.5412573050251
+26718084.3216598
+6.35064021207415
+1.78655797428262
+3.37873464068241
+5.52400312690798
+457026389.786359
+4.2461420621457
+0.536587686911298
+0.581890797297058
+0.556889385840369
+0.710868592951412
+25.9027524628601
+37.0834384162723
+21.5857867983343
+38.0287730544066
+11.027116682266
+0.59575556933132
+4.51125585413206
+31.4742937509023
+48.5463177175202
+24.3905217241699
+28.6124074915094
+3.04678184119045
+12212561.3302985
+7.60492976804269
+30.4612953461576
+39.6992744064719
+27.8219879534706
+18.7641638792196
+24.2707038203311
+32.7074893042977
+4.29127727639631
+4.58539679451823
+28.8790785754166
+25.4942048268384
+4.46197268000784
+12.4090754718394
+68901851.4569408
+3.17928294942723
+8.71517538306772
+2.92876459625869
+0.767449961054901
+12.8155386593655
+1.21321269639579
+1.20139774746413
+1.10922436350843
+0.483197352975866
+40.6208486303299
+37.4420601177297
+30.5411225093477
+37.945561346799
+35.0420619785842
+0.71755339713453
+2.33645409654591
+35.0737478677216
+4.45391451945391
+33.4036133422662
+20.0301420870944
+6955811.25894087
+17.8453097233381
+14.0341842107143
+32.2002134316303
+34.602918937072
+19.4968581178613
+33.5789688034266
+30.9732192975424
+51.0798660429665
+37.8766466731176
+52.8716902668589
+30.8595277025067
+40.3385767210174
+19.3316335171852
+33.5101874642059
+34.6123297147358
+35.9536375393239
+23.7788307764269
+28.5306834573443
+50.5671838935037
+32.3788884063895
+33.7187308390234
+27.5403758160484
+37.279181369731
+17.0155676406899
+46.4886883865975
+29.1838541273851
+31.6215061373201
+16.9443800268987
+22.4046289517716
+35.0688242344065
+37.5315296872297
+33.7322071163381
+13977350.0484038
+48759553.1679531
+32.6549179141334
+42.4369938917099
+30.1092430688624
+31.4119164715802
+2.33640928406143
+40.2174038217169
+17.2367894193533
+30.8203394518005
+37.1224457977338
+39.3526082506282
+38.0220366896912
+35.8498069707139
+24.2387695462358
+7.52269324342092
+30.1589300424729
+42.2086781438566
+37.8882427887546
+36.2410803480059
+33.8174271101283
+19.7120140714025
+43.9804684843795
+19.7459223945394
+35.1362562221077
+0.303837744876619
+16.2901444624037
+40.1665502977226
+10.1752207807246
+19.6152355997536
+34.1736894259529
+70.6228888240681
+13977350.0484038
+24.6543801811421
+48.0794703112534
+29.839725233399
+8.86423093179446
+19.5796583484825
+45.8608041574001
+29.1448502605742
+19.680015244528
+28.0858976831472
+38.1164265568799
+37.6211874874595
+27.5055164235607
+45.4751938006052
+24.6292483650246
+25.9910561727445
+16.0682009287906
+1.83260620243428
+31.6342428344695
+26.8657377992664
+25.1810643106993
+44.7354450661063
+42.6552663969192
+21.6081044382874
+35.1009350018459
+35.8398729759641
+15.590018758494
+36.7491359656839
+39.0129677415725
+0.486520366979168
+41.2378167866223
+39.110138221852
+10.6651412055934
+34.3664873474828
+40.712507832208
+25.9473181745581
+22.9230167299642
+21.6751487232266
+25.1323623862755
+33.311294249926
+40.140997060666
+30.0728287122417
+41.9374508500777
+30.6253999404081
+25.9862204428596
+23.6059023183048
+168560291.228752
+2.17978742854369
+1.07475960607323
+1.949923362552
+0.0976127432860106
+1.63510416627701
+7.04523695883286
+7.63554140931478
+19.6896985246046
+43.6211077074279
+30.4698871412975
+47.0345712928402
+38.9216070282687
+22.7347897702465
+39.8856221185045
+17.4387295071814
+25.9981214008678
+27.6589794273925
+39.4553133139172
+24.3690345998971
+0.39963622476262
+2.66482687498686
+67.8831017693594
+37.0625146394382
+38.5814571273566
+41.7452097496688
+28.1715344733255
+4.8236085249996
+13.5033219174041
+39.2445607267324
+34.7759973739069
+43.565002604313
+33.6623336403446
+28.4158163367948
+44.1087231652484
+30.1789752542639
+17.4869852657323
+44.9108563054691
+38.0948660310777
+42.8456829419573
+42.2320143912857
+152272980.653734
+19.0902777307728
+54.3977661580812
+29.8378523529574
+31.5702087392497
+21.4849675090643
+17.6527721544713
+41.2573563070808
+2.05928053953613
+30.5731871749972
+26.6343737571769
+30.2638301850396
+35.1287990666118
+23.0038129846893
+23.1963246984538
+52.4682166051907
+39.5110663647122
+31.7862972245817
+12.3625570755165
+3.13108248204649
+37.2054175324364
+26.9555255358453
+21.2600228341597
+44.1860081581894
+13977350.0484038
+22.4083052309206
+32.1666606008729
+30.7759254128717
+48.4675819649484
+13.4007481212629
+31.0378592294983
+40.0051366501018
+45.3846012260302
+63.9672790753831
+47.6950317906255
+40.4500757405353
+36.4649233458062
+27.2581885401739
+28.7982688448733
+27.3446880966518
+21.8636487934234
+13.6124727699965
+37.9327559429301
+36472052.6452294
+10.2935353328519
+31.147824432336
+36.3360308427473
+14.3709528420249
+38.3643575124379
+36.2100618994781
+33.203749379735
+38.5451054021957
+39.9234628832733
+22.3574224136143
+39.0482811796037
+35.9715959614322
+37.9048833207805
+45.9753994714814
+29.8854459873961
+3.52678683398681
+28.495811093873
+31.4260413853923
+23.7745089794647
+22.4832090961688
+45.2281584007311
+22.9005410753194
+45.6148734626702
+33.660117118869
+43.847188488625
+28.2684557241369
+38.2056782040973
+46.571024384319
+34.199891490338
+27.635176092949
+13977350.0484038
+34.1493128730679
+39.9728357011908
+7028680769.23331
+38.3527450171583
+39.9261728810051
+36.1403552554792
+49.7469627851326
+74.1293594815777
+18.4744529731146
+49.5610047291128
+10098176.6570578
+29.4645601417935
+37.3909540609696
+22.8979988974558
+28.2992220886309
+42.884411486601
+24.996314914695
+34.6006479167946
+39.3924275247334
+20.3074689351193
+7.57860615300159
+44.2476393015585
+32.5800487202509
+39.2365480039566
+13977350.0484038
+32.2720639625412
+37.670115221426
+47.6249191111727
+40.5826244256785
+21.7839462853213
+23.2048634709779
+378.399609772393
+32.6347265983154
+17.6746997153693
+37.9143804842096
+38.3383549809273
+18575516.3018149
+37.2032465951336
+30.0022191071669
+31.8110555063669
+534463385.800564
+63.2036545529137
+40.3864380276536
+29.6960616025056
+14.1490830090371
+21.5399618365961
+39.497904541368
+40.9072519994784
+3.55728395213788
+35.7078315206065
+24.9999918669942
+32.0242808696127
+27.9082483643063
+53.7099188640038
+23.0764544176107
+24.0303056592028
+40.3590325605983
+32.801787271592
+43.0325529356491
+28.9367612630674
+41.120550105237
+41.3675685489902
+30.57445263358
+34.5886694852526
+30.994256746535
+46.829223477529
+29.8492920919713
+34.0815270306616
+39.4889914411054
+42.0581246479642
+26.6642960381212
+36.7688217462518
+26.0458935480571
+32.1016458205373
+41.2810520665602
+40.7208641093327
+28.8303878198044
+19.9141826627983
+38.3152205455064
+41.8551407601772
+28.0575556899054
+39.6568453304582
+4.43590659238505
+27.9910302381521
+30.6765086398896
+28.8850598998898
+201.755861199896
+34.4418416674774
+5.33745450690643
+48.9968102212698
+33.530940517341
+25166978620.5355
+17.9208022248254
+11.1636344238096
+38.2539027594171
+13.6442239082815
+43.8237329346446
+26.2013977081874
+43.9820921382393
+13.7224795911491
+40.3562613019278
+14437994.3569558
+29.9814612956584
+38.7906809903306
+39.6706470771192
+22.2232515087272
+33.0638131548268
+3.98837166418314
+36.3594204692226
+33.6975424455924
+32.2511843561331
+22.0535329993865
+45.5361215712495
+1.3031427733143
+32.4027824861392
+29.8040266203849
+49.5167723274761
+38.7548784855763
+28.9796830686029
+46.4996003239773
+23.5680220175146
+48.0927565543626
+35.9138793675779
+102.538170854391
+62.9357499672224
+23.9975218779058
+30.7665922916528
+37.3613489747158
+21.4665298705679
+28.6357413322887
+32.0906431392384
+26.2554471432018
+37.6376572551813
+0.952632862727883
+33.7355886225953
+44.5382338292367
+16.3810671671954
+33.4881853340241
+26.4268037469854
+71562.8388643887
+1.92571678398498
+35.4513784773021
+36.5045946067448
+25.787078223848
+35.9277621937569
+31.2491662669833
+32.8787859612975
+29.1750408014049
+10.1696563393951
+23.0834734496963
+23.3934906137918
+17.1309347315065
+31.4598572529121
+11.3343472947253
+26.3666760940358
+19.2145215792144
+34.2156665054586
+27.8235886063218
+38.2525108514619
+12.2722008973138
+40.6389930994949
+33.3537605944611
+32.2786882723797
+31.0389675665066
+32.170013462773
+29.0371189597027
+46.5799403354286
+32.0559775119085
+6.41120286899541
+16.1737484382404
+55.2872303392841
+35.8423336050036
+25.773388524995
+27.5207193262414
+18.3549760849218
+33.2880009529017
+53.953262660794
+13.8634380745797
+33.4669624901926
+44.9916317762766
+34.076549643384
+43.3777602430081
+43.5524553334165
+34.2876889617985
+31.5527561452062
+3.16671081055754
+35.7505971245199
+39.5862636532053
+38.8960925195029
+7.23440344280139
+14.5909693828406
+26.7933632255268
+32.8496623537221
+23.8735832318387
+24.3369419155955
+33.0999836427488
+29.6674415351897
+23.0934397690273
+20.0518890981712
+13.3870548032607
+27.4656434398411
+20.9172687069554
+39.0735151508203
+7.50747205537908
+11.8568353464116
+32.6903436961963
+30.4708312948483
+32.0755407952782
+17.931376845585
+26.0721748963099
+34.269418288669
+25.9915576232548
+20.8417300655824
+26.1047893700728
+32.4721847286305
+1.23023289990303
+34954343.1092898
+11.8255331992533
+17.3687070375438
+3.64054812956867
+5.09582558746858
+0.592131732553557
+0.308371889598473
+31.4363737521167
+41.6697140927486
+7540023.98859911
+28.8342467286759
+43.8531646574824
+39.6556451552146
+22.7174117744028
+25.4722046657203
+48.9501153992677
+35.4911167128208
+38.4551862633777
+31.056348577301
+29.155186110007
+33.5754124081479
+30.6003920884047
+37.1057534774466
+33.3520141820276
+45.5480844561932
+25.7744760241663
+4.30290426854718
+153.452067684826
+4.97984587725805
+3.04865777509475
+20.9061841300352
+26.0645550742444
+24.4455521482513
+52.1514909777765
+32.0631410058578
+30.3171943320949
+34.2348250535937
+32.3199780444291
+43.8605052814283
+14.2938807405213
+26.8648155268047
+30.0738843462478
+35.3012158157359
+20.3538322618409
+52.3226876044059
+22.5531316055766
+19.0579253201865
+20.2788601760448
+29.184009881309
+17.7558135680354
+28.5595417051396
+34.9322354858418
+50.2423731498393
+30.2869058970026
+19.0978597789232
+2.85270081534698
+33.1591973422004
+32.4011682151106
+33.477036773018
+33.1565560094014
+37.9824142472703
+16.1210888659891
+27.5038670243843
+37.3058583251859
+9.72899824495196
+33.5431575681638
+36.1447529958572
+21.4387352165111
+10391869.3003199
+19.8697148549595
+19.1800125720225
+23.9107382335843
+12.5734612634277
+0.459741169984335
+4248129.14411132
+42.5350599788271
+0.412172630910715
+0.51006714259007
+0.337218341223237
+0.266758554657188
+0.518722746409414
+0.319377512488561
+134.321044447138
+24.0273644651717
+38.9402852525264
+47.0749274016941
+24.7592551523295
+10.0034311942021
+32.9152222798089
+32.480388981937
+39.944787964256
+21.6505707394089
+20.1661241054564
+30.0307098735173
+26.3071222960841
+17.4945268890264
+45.9854072944937
+26.2872819003806
+30.2698687375169
+31.3022333107707
+2.75815648702343
+19.6887680947043
+39.7462799511746
+23.8231049428361
+21.7668134551272
+10.7300953480039
+26.3187599910227
+38.0250511854043
+19.4310794411821
+25.8965145139133
+30.9509218961583
+55.6529637926497
+38.7097453538989
+17.408810032132
+39.2632262638321
+33.6966427656236
+25.9267393170092
+39.206614666807
+33.9416673586457
+32.9453416788771
+41.3666754165989
+44.0292489489642
+31.985625348413
+37.7770169815808
+28.5824542494326
+902936.452163437
+42.5350599788271
+4248129.14411132
+8.02872747288127
+26.6380604852984
+40.9058410454348
+32.646690100148
+11.8309750519584
+19.9551698646338
+34.4922028170171
+25.7313698696302
+1.24553301151105
+33.6454587566294
+40.4668123332628
+87150038.30677
+27.4751557570334
+40.4961032626771
+41.2118970246272
+41.3196120044776
+36.1521447881054
+54.4100902566258
+67.2038664318294
+24.7632281005611
+51.2444651401978
+24.9701885011973
+20.459068298473
+21.6460219839818
+29.1565629564898
+20.0357036711348
+37.3395318560753
+20.0931620645882
+40.8218485328161
+34.9934492799111
+23.3908808668698
+10.3033858993598
+36.2333833580151
+25.1064108009697
+35.2847319960968
+27.0946553523289
+2.61002385438888
+37.1809054778545
+33.0585817024298
+31.6106543781039
+30.3108553958268
+35.7115201521618
+9.41962807673698
+0.173855216855544
+26.2151066296022
+33.8578065827067
+11.2977696891155
+64.6304091325432
+18.4591480911134
+26.8189356018427
+43.1459492373546
+37.1129602455093
+50.9300028045617
+34.2882101720077
+13977350.0484038
+21.6699994638642
+0.385129395280831
+5.89463267273697
+17.59594877318
+55.802389168834
+30.5090329411343
+5.80094626960229
+94475300.6560519
+0.428719321281576
+12.4365012558877
+21.3119171213564
+34.1783897394686
+0.636500831307867
+30.1867316694353
+30.5372736222086
+13.9334890235914
+39.7289555120467
+25.432561922048
+26.5499332199435
+1.0647634892295
+24.5576321624497
+47.5230638422016
+27.353850175799
+18.9122466051847
+5.14248750506665
+34.8408083400282
+28.7181676974182
+30.4270917654116
+39.6957196456999
+20007997.5426627
+27.6226756251968
+26.4195845327138
+20.9349774014158
+28.9662165086294
+24.7090798231355
+9.67481384426106
+22.9764464788963
+22.3176076330414
+9.45135559928173
+38.7541332377461
+83610645.7722864
+28.6261208648674
+44.7707478657031
+40.312730071139
+31.0384339006439
+43.7694744882073
+35.2880330402121
+21.5759907159116
+37.2221956015938
+9.52725395167155
+42.7076023418234
+35.7879657680068
+3.18259617715646
+12.9503749378036
+40.8702681512395
+25.8604857928929
+28.6087318652447
+25.5354917765967
+34.0954912301677
+35.3755405316017
+40.8174149805438
+38.2820391099467
+52.2491199547271
+37.1490578005019
+40.682084331167
+13.0786691800362
+43.5667725603398
+36.0952458376245
+37.0958679294517
+49.863663130244
+41.6537306482558
+31.1028016321823
+30.1200663451782
+18.5821768873351
+31.6550404531746
+28.3215518756383
+37.0797089663872
+12.7487243353784
+39.2904425703659
+56.1397787248443
+10.2763841416488
+52.6234911923095
+32.4019843454038
+34.1613323075735
+8.29884054599425
+25.8683377891012
+18.965959194158
+19.4733647713614
+30.6655046335371
+33.2509126123251
+39.7063092235138
+26.3158260118199
+27.5282046759728
+36.0285237476643
+29.4025432439208
+51.1882538023793
+40.3303870719445
+48.4134309576173
+52.1639662078314
+33.1646516487041
+24.4732490029655
+23.8426234547847
+13.5510786295053
+38.4917636728447
+39.8781445368588
+39.4650514828378
+46.7826893363101
+20.9155619923143
+27.3199089164546
+39.9855390480549
+34.3858776655388
+45.0023198909117
+31.5866452939791
+5.58367959067439
+35.3765136533918
+59.0692427558932
+9.92622952105549
+41.1174655797719
+36.6639732218748
+35.621058220984
+36.2941256907257
+21.7533636245434
+23.5229300897085
+30.995687294675
+48.880849026323
+44.8009139467961
+41.9157581898412
+20.016097446916
+34.6283027317069
+39.8197496702098
+32.9814429734474
+2.3295108998387
+13340148.3683228
+20.7367189453407
+40.0211041786257
+45.213296023603
+33.366226965924
+25.1245081452976
+32.5606939796855
+24.8939872661515
+30.3080182505882
+31.7486281409082
+30.3974790164524
+28.5695340171888
+30.9105546225386
+49.0683769269498
+37.4255287536896
+33.1533549827849
+46.4007533913743
+33.4529911569043
+38.6829154623371
+36.9899494255057
+10.3845845739217
+32.9048465640125
+44.8141881699564
+41.7455810936188
+38.616271596487
+38.9015420120973
+31.8709822060122
+41.7133939966656
+38.8568519871273
+37.6013286284185
+47.2190936214083
+35.9285360145026
+25.1924532477359
+40.5632494507635
+30.3541387210836
+28.2347004356952
+23.3163690096073
+25.352003032244
+30.5435813852652
+17.2129141801711
+9.18665099074863
+8.96656080717138
+13.2865160151946
+34.5731136604106
+26.9817279852959
+33.370672553418
+40.490311207671
+13977350.0484038
+23.3913073275746
+218.690843960201
+35.4068114525221
+199864904125.488
+23.3853303181267
+1.54076901742729
+0.526243649948137
+33.7794809801528
+48.0865133899091
+29.0196249335234
+32.4795923774263
+29.1203083521687
+15.7329256836346
+38.7373707094705
+39.2366106003674
+7.32663606289362
+61.9676711569911
+39.0174393734961
+25.7145530690519
+35.4359836394091
+22.6795662543737
+38.709100603045
+30.6856809220663
+28.5744170108901
+36.0346747079238
+42.4438977937315
+374.672053511246
+9.28087589770267
+17.8275848980953
+31.5830286236072
+4.57404556184564
+22.9528747894806
+35.6809628852928
+25.6107828812665
+33.8409589456805
+39.5755955043815
+44.0500929507267
+34.9814145605309
+16.7643232933277
+33.8021540619844
+11.1029539951536
+22.0232961259709
+33.1962743343839
+18.7786402556273
+18.2500014344173
+35.2995317926506
+48.3635261376995
+31.6075746624161
+18.4824481788823
+55.5655501417499
+22.8436102304121
+28.6152164372141
+43.7802153599016
+22.0139122375397
+2.315258457722
+10.5950049457559
+43.0602708534466
+16.593453106305
+10.7934117061337
+31.5267392306108
+21.1882040127719
+33.0863788686146
+7.6488063623107
+47.4903779697468
+14.1523935531794
+12.1411037389566
+25.3237906444109
+28.3460641849388
+27.8053309715999
+33.4920674272036
+33.5937765977799
+7.57178043512086
+17.4444625150802
+38.3420005290127
+26.8806553376181
+42.3579390514074
+55.4682813248903
+37.3469485687773
+43.1381205814504
+45.301736367419
+37.4754818710239
+31.912169644026
+22.1233176791828
+18.5690624887339
+2.7683575649499
+31.196259290113
+18.0970440833592
+5.58049040478737
+9.77761756953175
+3.73419094186275
+3.01378686781534
+0.600957549900766
+0.884542987386167
+11962737.4160346
+30.7027025403964
+24.8561198639821
+38.3049973721033
+28.2748462284033
+31.9907310111904
+32.7758069729799
+33.0240001457823
+48.4176419389954
+38.9631077239647
+13.9875803641461
+11.0112835465054
+6.9225205887968
+31.7417437184928
+38.8492715501904
+42.6277825121995
+32.1422993748926
+2.98666932518043
+1.49656650100489
+28.8942334195929
+41.8263306966472
+24.4802840362331
+52.1707146600987
+44.1119050499222
+29.4958720153723
+23.5923549530495
+50.3938974540954
+8.03479554201123
+40.2553751731208
+30.732424641823
+23.4576009575492
+30.4208523726775
+24.4629906292072
+20.4825793285944
+21.5099545126332
+26.4359389971523
+36.5982560864501
+40.6962296390088
+25.5309453202191
+2.69215283064479
+36.711745997865
+171.32656819926
+31.3372921319549
+37.7908852338158
+46.7882420317485
+23.8113792738642
+0.915150511383077
+20.3781061203974
+42.9112420588764
+26.4477621876139
+22.4984222736393
+36.4534862562862
+31.2560904947843
+25.7315191861878
+42.6039757322924
+7.35306269484577
+30.4191071280703
+38.0905589226591
+20.7264784292378
+27.9540242918247
+8.08609350643451
+28.5234956293906
+32.6933274965536
+24.6519155137711
+32.6332295461142
+28.6047171510224
+35.1732758054588
+32.9726776718694
+27.3012010532668
+47.313171061696
+39.0667246369709
+41.8018259468489
+55.9146693785161
+37.8630112214352
+39.0460846369196
+23.5701110585377
+30.9979825620375
+21.2242967985966
+34.3211275913366
+41.3257162859912
+58.171005203521
+20.1444529821509
+25.0177038652041
+48.4901060853958
+29.9671003645091
+49.8223811115434
+42.8726008317623
+29.9297867675104
+31.040774388512
+36.6303958653821
+33.8442617530677
+24.8442536534417
+37.365557966452
+23.2962317847817
+31.8695524840541
+25.724565611764
+26.1936084641064
+26.0363994283117
+40.989298133764
+41.0570467580379
+43.0839414670601
+29.3101412536132
+0.812663651389194
+2.39811983437828
+25.7207633611787
+26.3038843834242
+37.7231227777228
+2.36008096740847
+0.654263178557623
+40.7163139472569
+30.2759793226199
+47.3574252872056
+43.2278974307855
+46.9138266107506
+25.6792546377187
+21.6104906586083
+13.4717813058937
+29.2914672279985
+41.395030036278
+26.7561690120696
+30.9286129842036
+68.7038492700447
+29.1849407686859
+17.4504538377262
+19.5172015996081
+41.5831648813663
+37.1487121595095
+49.868319206617
+26.0885083649652
+4.08197147097951
+40.3920954670136
+8.63194102249929
+43.1323059875195
+35.2367475223852
+27.338830499904
+25.4690733037199
+37.7493377297538
+52.5955630937242
+41.7332791290587
+36.3523974031456
+35.0382685625348
+31.7643447111435
+26.4194694311478
+51.5081793584606
+33.5450020618873
+44.6968049638842
+34.3756519421662
+39.820599306081
+7.87983971231427
+28.2791074860946
+36.8649193915132
+41.8439340883604
+32.2913405771875
+33.1112578878172
+35.3038288303163
+31.5039313550492
+18.1406522702716
+37.6861421067879
+30.997486706486
+40.2196044225461
+30.8550025934481
+22.0923251136763
+36.3043685343467
+39.1191825513108
+29.738050349086
+33.348428797003
+17.5740625876559
+24.8980283341801
+59.5040123200642
+55.2369412881536
+33.358550753137
+40.3894373728008
+31.5107501585158
+29.9468608984548
+22.5375264923312
+12.7487369951388
+40.363163614484
+51.3787826907992
+37.2882038881516
+33.1163501009093
+23.8818900446435
+40.0312936372859
+7.62199780134344
+35.7981584336729
+24.8623541179504
+10.7555289482279
+40.6255809503688
+28.5183032494347
+44.8497037113953
+31.4259995816256
+20.697401220502
+41.2549298640108
+38.237812057968
+19.6136481231503
+40.512613888492
+42.9588988895189
+38.8124743329864
+28.348953144745
+7.16235311358279
+10338292.623542
+19.5581202082551
+40.0869338331975
+39.3023313285953
+37.9462997389404
+8.18781165563155
+46.2839771967358
+44.2661548620445
+41.2857023623663
+8.55193917492402
+36.9889921562451
+14.6345196794168
+36.7078492619151
+1.88973458189371
+38.5532466650136
+27.9624195965126
+43.1287780169929
+34.4184083997751
+6.2768201892334
+26.3317460709224
+13.4032939122308
+47.3848048062729
+53.5398422817479
+23.0747659939574
+32.2130747590015
+47.2929889273137
+18.0530397357272
+44.9410911053628
+25.7847172951848
+4.10504032255386
+1.40855775825664
+17.5188352292733
+0.456319793779537
+26.600958814502
+53.9672770771558
+34.7401376933241
+26.4607083120559
+23.4131573166946
+67.9503097029401
+35.516176833087
+32.6797577015122
+6.14116619349116
+30.9676698111459
+8.44336867851584
+37.4106525417771
+33.0812826168546
+14.9888356473955
+37.0091996258769
+38.4854109541192
+56.9585549606439
+33.6199143846131
+31.6191923162564
+25.6340645696989
+20.6602642576714
+19.9043849600169
+33.5833244430583
+35.738178984717
+37.7512028259787
+44.2850148113991
+24.5598143482525
+42.5262804497963
+42.6738227090113
+22.706187706826
+32.2153664361468
+11.4749971635173
+52.8088214023566
+34.7740756962879
+24.3088926632222
+35.9563601977043
+29.2005606360267
+13.277372511886
+24.709606654732
+36.1987736516003
+41.1915183652569
+5070077.74780865
+37.2030570402409
+28.3216942469803
+11.7485374227126
+47.8064501619046
+27.4102187252824
+30.1487870185563
+38.5844867983858
+15.9455578596789
+16.1124679810943
+32.158073270806
+36.8689971880901
+5070890499.79616
+1.71165416992076
+10.6070273608262
+28.2966547560379
+32.6621454814671
+53.052156612581
+33.4963517073272
+45.4336679538329
+47.4346803023357
+18.536227081362
+49.0428332792934
+46.9506287686994
+27.9044220908609
+3.61128503664804
+31.4431665917377
+13.275846710613
+14.62074921179
+25.3536586147593
+27.0352481472337
+10.9826420812399
+11421671.5426095
+24.215249878052
+33.9494508743837
+9.59377023646409
+40.3790530354213
+34.9697519275203
+33.4290110800416
+30.2278791125055
+41.648485811593
+37.0315451950578
+29.8885038705643
+25.87731301355
+4.2241531383581
+39.7255366155341
+55.5855178593682
+34.7044486073295
+32.8686628972317
+33.1048271620867
+20.3497543026582
+49.3658634835171
+31.1733397441881
+21.104870710581
+43.3213894019731
+41.0803074734851
+28.8249011714028
+44.5901655054006
+32.4385293535425
+24.1030673566598
+50.750043776486
+30.7994165785394
+30.2682536973942
+28.5294512012633
+30.0411609461527
+30.0137412785519
+36.8794901187237
+27.831169869311
+24.6280900629517
+87.1143364490959
+17.5612758440968
+1.33995541166531
+3214256.15251886
+24.1326843853287
+21.5920239826963
+20.4686343067537
+41.3372416128337
+48.7859046240215
+16.4905295652523
+37.7450450210634
+20.718854928724
+49.5106352865676
+22.0898669899245
+26.825810437494
+27.8578701146666
+33.7603052417549
+21.7961592267752
+43.8849485944009
+36.5641725765473
+25.3103514103091
+41.1308175155481
+52.8776401491215
+46.2659767477703
+28.6633423557792
+27.5680981730915
+35.9028632567121
+34.797607395622
+25.2956405566149
+23.0950484563466
+28.1363966869424
+52.4080305696746
+16.0384204610253
+2.6125068810972
+22.0984637571253
+25.6018570219304
+40.4212002288063
+50.6105984941949
+36.7834052539497
+37.7266791833808
+41.5005893915123
+48.455270524693
+32.2069852394612
+42.5071294271644
+13977350.0484038
+33.4427263886744
+11.0602676549498
+14.8117791101883
+53.4999289539488
+19.406013908486
+34.7202680960753
+655.257165067846
+43.5603561887387
+26.0615622947178
+45.9996471978317
+26.8064854470419
+46.3950016485819
+21.009129618626
+18546642.9817036
+0.741023501766935
+22.6550320251851
+48.538330874126
+37.0790793467782
+28.8342490258817
+56.1213525393786
+1.26693745666903
+25.9278696375961
+45.7214915883674
+35.606260463625
+3.77383580380936
+34.936438028536
+36.6913279421014
+35.8295236233276
+33.1949117233289
+38.5318655166549
+34.7520093926896
+39.937806806649
+14.8101233168166
+29.5336767680489
+44.6174036532817
+49.7183751947543
+32.1881780045669
+21.264099074564
+34.1322693875718
+28.1076187493157
+31.0696468860409
+24.3961164525092
+42.1251124769687
+24.470479773485
+35.6639419793521
+37.6898611937738
+30.6069548188967
+42.4336904881346
+47.7991655366411
+39.5563712826842
+25.1987267287711
+35.196282830684
+54.7260012628185
+46.9369637250067
+2.35181608145567
+14.0475979354895
+16.9378712732983
+37.1355026678126
+28.3551287637472
+19.9766850873625
+19438712.8921626
+39.4636635285269
+55.0849019894504
+41.5874429170337
+34.2315560557907
+22.5828589595003
+10.1020154304832
+0.472914044009679
+32.4436705065646
+32.6342071454212
+44.1515225458533
+37.9809581075025
+45.0848045304045
+24.5946902126171
+37.4926640432173
+23.9975547004567
+40.2936692318319
+30.1036803150359
+21.5613917317815
+40.8940797615004
+6.71286466204983
+39.159567205304
+42.3183185124288
+41.2730538027902
+38.7751367287802
+2.40082436989107
+40.5289005598838
+41.9362729849756
+35.0818207205128
+19.3860882871156
+23.0253440242915
+38.8155654488427
+31.1753029546142
+19.0009777277651
+41.2408009822284
+38.502524498705
+14.12571623309
+33.0298305591342
+23.6239942971925
+4.27484509265761
+1.90613657568308
+37.8522318971877
+34.0271479450885
+26.7755128763208
+33.7216487186737
+24.5953272295772
+30.7389179751902
+31.5361700248502
+55.7809380493489
+5.44857575746843
+31.6920749188371
+35.2708031925744
+31.0694605814981
+29.2620406305866
+20.0525009909663
+22.9096193397101
+39.4634977251956
+33.91525352348
+35.8562208477179
+13977350.0484038
+26.5898684919505
+23.6353630867344
+36.4453975287536
+7.47758244027946
+24.0988741585016
+23.2043638120853
+30.0820813824135
+51.2501202634786
+16.9527157554776
+6.7613773295501
+29.5310037653016
+30.0909067510576
+30.1524651238234
+35.6227960146345
+19.0559112037321
+29.011471257996
+37.4168573570272
+36.5067961047689
+40.7009237207133
+46.827575538861
+18.3629060618411
+36.9830410462506
+51.0387472445273
+18.9468507309855
+18.9681982509812
+21.0411253808167
+49.2522422561415
+21.5819615225908
+46.7305030831504
+39.4041559730778
+29.323719078812
+34.2295197765455
+35.4209812895514
+37.8758609566719
+28.7190068831828
+44.0538815807947
+17.8330784566131
+41.5613063983162
+34.396189488868
+14.7417747177776
+38.6074462002858
+22.4453770428559
+19.256009821689
+8.32615194612231
+24.6580951906766
+5.74245956453106
+25.763782105123
+29.2987134129608
+9.64061252319533
+32.2188309632601
+27.6222319357205
+28.986617407302
+20.0175047420505
+31.8190947094941
+29.1574034009623
+33.9196165466073
+43.1862032732241
+26.6343857500071
+4.91571983065433
+14.2849191831428
+41.4469820495902
+3.82352334220456
+30.6155947754823
+50.9235468171047
+50.4606664961012
+43.7545110862706
+16.0287602530291
+35.4283491279646
+18.9821454140412
+43.4472924634393
+18.158910409155
+43.1808379837817
+16.7180010102809
+42.2519330393447
+2.50391160396197
+13966107.880201
+47.4990466427903
+10.3102633794366
+43.3511165089849
+36.0855896578467
+17.8412392453792
+42.519508102375
+1.19589412202667
+0.528667623383205
+8.45202068191407
+69697.2881362708
+10.4024023742832
+42.7216164312657
+40.2558080581198
+24.3934598175672
+55.227474300579
+34.7810144904611
+35.7732313759959
+21.3888395632818
+273781307.101014
+9.70175066438836
+50.112958148279
+16.5331359362801
+1.11648362883388
+11.7781929312037
+7.83310802801648
+34.2737064150507
+18736399.3914894
+35.1364821088235
+50.38210429501
+29.8629385439349
+24.2433705151971
+6.54952948722135
+19.3622205014083
+25.8706718369257
+34.7876021630958
+16.3184300879573
+4.63461214934258
+26.4375638293367
+34.6415358546856
+33.6487409818608
+28.5719569437438
+35.1464913100059
+43.1017472314386
+30.5240784048165
+42.2121247363189
+66.2294597821733
+9.27240598328606
+11.9900475541906
+29.1066140075004
+21.3241137578935
+31.7958698756757
+19.02482285724
+19.9482994655832
+13977350.0484038
+6.59102813926003
+36.1975934240865
+39.8667276480757
+29.0722758502468
+8.33267136959981
+22.9082398497784
+25.2463079067756
+32.2425136326999
+28.7982151370873
+22.4735225344965
+23.151485524813
+35.690590768192
+37.4106471724501
+380.650395671952
+4.72563007484376
+17.9446420074808
+6.7746079878841
+25.4993495024204
+31.8698672634806
+50.7386299000442
+13.5459397567427
+45.7634726239595
+32.9228308034558
+33.2420532817559
+9.98223466527326
+36.0969198627105
+49.0696052150222
+19.7588017549026
+31.4086668571968
+34.8423288477679
+40.9011871520547
+30.2436150598731
+35.9267660621456
+9.73883857816531
+34.1310863539421
+16.7536313123844
+43.0268866992338
+18.1845199349668
+2.3849842903977
+40.4044005422702
+47.6977885061499
+26.1199499153341
+5.13187185804465
+0.141895336558189
+0.202630744434686
+1.61852490341437
+13977350.0484038
+7.05652900070872
+0.884955794963516
+34.4795234618584
+5.89455578203909
+36.902176300233
+33.7271867469901
+36.3852086249269
+6.61820896406256
+33.7019258764587
+69.5085211525998
+4.10288368126164
+22.0593798238121
+45.7050343478256
+19.374234517427
+1.04930248670423
+42.1783855593667
+33.6505876035522
+16.0086953110835
+35.7515182929202
+47.4766934623265
+16.6302171270581
+29.6838449562594
+22.2487547618283
+26.3112821416222
+24.7728350479671
+40.3580523246915
+45.8430017925554
+32.0835286022607
+13977350.0484038
+41.7403732441306
+40.4876606557778
+48.9101721469447
+39.3212301367453
+43.2485274615442
+24.9666894957287
+25.7056140900636
+12.4658438549985
+32.9735463165826
+19.0360123618601
+28.902230097006
+34.2346314369843
+38.3649779482919
+41.9893919286064
+13.7474669510683
+31.3241984408824
+3.62178533534587
+10.7145701443839
+32.3183265421747
+46.3524596125222
+30.7613371704822
+30.608167189777
+36.9118024482249
+41.6256994726688
+22.8347698610351
+39.3387003078463
+183713887.870188
+3.59395468974628
+30.961970106798
+21.4906560968575
+36.2314488346879
+45.5293075830251
+42.0854045391662
+25.376682563022
+37.1515567038112
+21.1335742308761
+14.8472283440425
+11.1175819591839
+37.2588522106941
+38.6536845055344
+48.0632704221906
+40.2638032667754
+13.7844080119408
+26.0906991042761
+31.1690656292376
+30.548061211785
+56.4295046032992
+37.4071636466525
+33.7146470470601
+26.206325070263
+40.3758131069042
+14.6678731706398
+23.4602332570507
+0.908989800738359
+16.1083119703526
+33.4955106987223
+45.5822931506823
+37.7174196328757
+24.2028478732744
+48.1929812741194
+33.8853808954801
+50.0227305914521
+43.8221071990992
+28.0019633799927
+33.476164192526
+14.5250686701548
+64.882099017324
+27.274716731889
+45.6624454705434
+9.34393826626194
+60.6452195436971
+31.9309022189017
+33.9270560397046
+47.228549188269
+30.487770402725
+30.9790386005677
+9.82121213936299
+1.77208509852591
+0.114107653740331
+36.7477091155776
+25.107119563763
+37.6854962741069
+35.6811665016268
+38.7338493872385
+21.763217689866
+28645.7853819284
+38.6186985480385
+13977350.0484038
+44.1105739886359
+37.7550940126298
+3.35837123448861
+27.7971941388379
+37.9718830157161
+59.2706282151071
+38.3843833824605
+16.7258453567694
+40.8901742794512
+106508605.048333
+24.964417025334
+42.1979301062442
+42.2114462183524
+18.6895764379083
+63.3953004089302
+17.460961769237
+30.3650460702373
+38.8452100061462
+41.136065017472
+31.0385319677639
+23.0529310841347
+32.8413478735198
+37.4111991269951
+26.3780759578324
+28.6091612101886
+28.9680200828274
+35.1069001968076
+38.729768512186
+25.7070683872795
+41.999279130927
+35.6041781773739
+34.2119878704464
+25.2344947799199
+39.0952143216535
+19.1603761586309
+29.1906257101371
+45.3486171048477
+41.8018678105634
+25.5854104493558
+25.097214774474
+26.53549393241
+29.95453786578
+30.14159044771
+21.5340829900154
+49.2565142577834
+43.78726836216
+39.3313937761615
+17.9301005336103
+36.5730004109761
+9552346.54390344
+42.3225021037968
+31.5849748549339
+18546642.9817036
+10.9025761457708
+1.46814080533449
+2.23616593851394
+42.1775214715708
+35.7250403406964
+26.6963765026075
+24.9253130684466
+37.5719591884202
+31.2119850387192
+27.7891820080243
+55.1157382863402
+31.1942751810976
+28.2522729789951
+31.3210307046389
+22.0143010496113
+0.804341497326488
+48.2921132553386
+23.4745953215877
+35.1674221590112
+27.2990595859325
+19.6481685972764
+23.5091111421118
+21.8262857583927
+34.4349033708441
+31.609225795636
+48.182326344005
+20.8279788362623
+7.6690876311338
+13.6427432338257
+1.08069065872359
+0.397142628749932
+22.867647768558
+14.9577233855069
+27.6099256683184
+28.6685654283127
+31.8405811160777
+37.7381540222436
+31.6108085169835
+38.3677818703466
+27.5165866462214
+41.1937692063423
+20.4963567058155
+26.3879898048277
+37.8170138357705
+5.05157848383865
+29.252778743863
+70.8231020430782
+36.5918598450416
+26.6366157650049
+29.1369745402171
+43.6877229964656
+36.3454014793373
+3.93764266410359
+29.6636471525556
+22.1957368898849
+1.4992837575299
+32.0359004762364
+56.2224636465722
+44.7608504044981
+24.1684027292766
+39.078572203973
+14.7532133193606
+1.23277296777236
+5.18236633994323
+0.668048128250868
+13977350.0484038
+10.9819660789254
+0.818427489830056
+29.3238602802953
+15.7440459592368
+42.0307092496291
+5.06048800318245
+11.1047614284716
+23.7510369799806
+31.5756474597161
+31.2476871445795
+15.8909344311451
+37.3328987825809
+38.7675257978022
+38.3024826864371
+1.33622662505005
+3.85029436443114
+4.41209928068221
+18.3903202369152
+20.898900823506
+22.1067085123069
+2.0470343667629
+0.980992374730775
+17.6716749456469
+19.2026412486758
+19.6422466135704
+37.5076112886779
+28.0454037563286
+4.73102383301571
+13.4854123522917
+29.6429289393712
+45.5657396164536
+16.1424857709595
+35.2806483327343
+35.4255100292459
+41.8193028726463
+31.9206091784091
+38.7359052170489
+35.2865549211791
+26.5688197705125
+17.2008666489563
+36.7074880708269
+27.6137902663084
+25.1966653525953
+83.2493613713893
+41.4950498401082
+25.6328901453748
+31.5034257127436
+37.3102560697251
+31.3724106086076
+38.6671987418688
+4.46371761937544
+33.3532263221138
+6.80848943089073
+17.5278481725714
+28.4329180320225
+36.017881881246
+16.4879817655281
+33.2599212926299
+33.6438762284603
+37.2566773932682
+15.1393788544521
+2.34780766961699
+5.66070745257303
+3.98361467495687
+13.2012403830223
+5.71633554556479
+19.3081200651797
+0.934396116058395
+12.5014725320836
+7.62560247279933
+0.631416700401504
+27.8264972999968
+0.978181235167078
+0.843765460678624
+2.45818415299116
+2.10511722004545
+0.687136097681027
+1.45259834940614
+11.0761818540861
+40.4795583588034
+20.6120537755451
+15.7737031661196
+55.0699988278788
+24.2213736970319
+42.4975520613615
+37.9597722929145
+43.5497284556533
+13977350.0484038
+32.0190599676375
+0.239813128227607
+10.4712572295892
+3.82742710641232
+1.63554416438171
+36.6420095241488
+22.7685652708046
+16.4469101096871
+14.5556165256355
+22.9517869944255
+17.4295650927654
+24.1645910672371
+36.4189379771191
+27.9177665771025
+20.3231462984709
+24.4909613103166
+21.252071159583
+36.0478393209934
+19.9914725673746
+3.88378507244899
+33.743035750026
+27.8746740517685
+33.2700089954594
+24.4635529194637
+41.9322087104708
+18.9842917802482
+22.0767171736898
+49.5180676939444
+19.91228520083
+37.1858023365316
+38.4644084055618
+80.1380141573092
+28.9351635573603
+29.0788855187613
+28.1817604600282
+31.7079652433101
+26.0302596764175
+33.9674626830965
+35.3587110880735
+24.4573452509348
+65.5711124090819
+21.7384930937461
+39.0451269403982
+23.4766108187412
+25.2242246663757
+29.0075120572252
+30.4338246419708
+46.271305882316
+31.2418440884147
+28.6587927628908
+40.5108221042574
+17.7352718185842
+6.05246799718703
+43.1225739179118
+13368023.9070603
+1.29316248408503
+38.2733026911677
+24.859320109566
+36.7511142161856
+19.4332224912402
+26.7157812226725
+26.4802858382795
+24.3776360325335
+60183398.9974598
+24.6667260235673
+39.5583479955568
+47.1504199474761
+57.3329378573964
+37.9291770926557
+36.967610005482
+57.1527786752832
+13977350.0484038
+41.4640571030799
+9.00878612095264
+38.1327405856139
+25.0859915649785
+23.3240124913666
+27.1681126724785
+34.5238623041045
+33.8571185953761
+34.210938825284
+18.699260384631
+13.7181990218193
+41.8433233881068
+37.7024275350335
+29.8869917617051
+40.7975465200245
+12.3459251341278
+43.4123661508124
+47.8090629214379
+49.7706973972183
+18.7508126452454
+16.9046162435632
+39.0682483606866
+25.3195571689486
+24.4236610077898
+33.8046203494858
+32.3719984593135
+38.0481845236703
+13977350.0484038
+64.8122288816983
+24.9293300004875
+21.3772925121981
+38.3842586595964
+29.8115920505627
+33.1656435739962
+26.7535123674923
+33.3714628427129
+32.2444993698723
+29.9659833936945
+41.1669291297306
+51.688480133157
+19.2298828204307
+34.3030228973078
+50.7235816711727
+46.6217305951317
+35.3486100460148
+24.5800638405965
+92.0030284589873
+45.2778211184525
+19.8381627249295
+31.1237480430385
+13977350.0484038
+17.7796935657342
+29.6546066207545
+36.1024201883283
+38.5752302484406
+42.6470495469081
+18.5813879838445
+34.0202161193352
+41.1845925076962
+40.4742760073207
+21.6419215218115
+31.3700751670741
+5.25353661487157
+34.0229638704592
+39.1467886425502
+40.282891701679
+26.820074317219
+35.7203685322528
+53.230162939958
+25.0151686925473
+28.9371676385325
+25.187901690535
+35.3288135670082
+42.7250915864414
+32.1985276550758
+43.230418936278
+16.6413905304444
+23.3214093379614
+10.4795223613531
+18.1891886106198
+41.7598252680382
+24.9639203840221
+29.8632525973472
+38.3603481438052
+22.1519802386022
+32.3508754420291
+27.5917134703802
+40.1776544978968
+38.640243796056
+30.238979584406
+37.339890374928
+43.3300093877955
+25.3176768046338
+48.7009498669659
+33.3833939427183
+37.5047600506871
+21.068319122458
+24.1398664413662
+31.6171359381092
+31.9166965929879
+52.831860734273
+38.1939945073174
+34.8680920411804
+34.4086251889388
+47.1187460372186
+58.8499267616658
+16390787.943119
+65.4346181535458
+47.6528649323035
+34.6320322926278
+32.0893523468202
+39.9596370261596
+41.5141511244937
+26.2254611300455
+43.1669687439311
+28.3683444523263
+38.5227526352545
+36.1709371991534
+42.2386727515596
+31.419351390255
+46.8632149623547
+16.5869893821625
+24.5512699709122
+41.4150702425419
+34.8277558631835
+33.8539955243975
+37.0678778344454
+0.994087724085718
+42.6311854162352
+30.9294494014936
+25.1149159512945
+33.8383268143662
+32.4594045565076
+47.402519026767
+42.9854321141883
+26.162766623581
+37.9807311718598
+51.3254449369053
+9.11743897990548
+26.293289571948
+39.2361269951649
+1.11920799591286
+25.7014531242203
+31.7839298659311
+46.0643023952569
+30.3746457019941
+43.1246115831622
+3.02883806315157
+38.3820312619908
+36.8511223522369
+26.2056283385185
+11.980637244268
+27.9819822115072
+14.4902632004259
+31.992068184774
+15.6103150866852
+37.9301387845025
+56.8996778239479
+31.5355383424095
+22.9011956333176
+23.84580316492
+37.5742120872819
+39.4956048365955
+19.9290992152624
+23.9715791793464
+30.291788540678
+26.1605751248012
+30.8919461755811
+28.6389654703345
+28.9249715474629
+35.7723576566266
+23.62043515033
+37.5138268205676
+33.0367154517013
+57.723334467586
+31.7196880754462
+36.8906045548941
+27.2792326434315
+31.6817705488565
+30.9963195884848
+26.5848084122815
+44.8160121521118
+45.4074231399945
+44.6366918064769
+38.2465435789766
+20.8314562488002
+35.8040895829966
+24.8592620042849
+28.0026620924416
+14.5858147161927
+24.6927752206282
+37.5061597720247
+23.6625899522932
+30.3851197921683
+19.4639628974391
+4.50993930514178
+8.84712588393801
+32.8630364394671
+38.0131625471841
+28.2889168022305
+29.442412232283
+26.5872154354029
+22.0625892467973
+15.2646277499472
+31.0787245735751
+7.15342395081151
+42.0736176058463
+19.88743072983
+7.18840977951079
+35.8971029616346
+26.2953153942962
+49.6838063236134
+34.5836178162274
+39.2881737754568
+42.4579180493644
+36.3777592300792
+23.4756455637265
+27.6132312483532
+15.7735680945525
+14.5508658408134
+40.8955134771224
+14.0936025791738
+149.432933170327
+30.9855814774464
+30.3318414072562
+36.708049520115
+39.4518185667722
+8.44666966733904
+34.1233871675547
+27.2669796289459
+32.3685772667083
+10714.3843719295
+17.2485544669737
+54.7188126003319
+25.2821230027697
+24.0978710182093
+10.6107094792413
+40.4225012560144
+24.5355275926899
+42.746785786998
+25.0331302248381
+17.3259268971024
+39.9142116438122
+28.5664435631563
+34.1493161770599
+37.8910649525685
+32.8399452718233
+20.574431634752
+33.3817460913508
+45.1191170124524
+8.72264059311323
+42316943.7411469
+1.90224542493111
+0.72495652875797
+0.395122827114411
+2.09280851474571
+2.85045069181937
+45.4764463912588
+53.5491694035912
+58.3959348636725
+34.7028245806358
+23.5882768751327
+30.8747797445626
+47.6121781673585
+17.0729549541159
+15.9224109249557
+27.0329689538694
+21.3577634992471
+40.6528426617323
+23.4662404240996
+31.7418498910284
+46.6092913122502
+32.2595118076838
+27.422245457878
+14.3710211513536
+26.0076101090879
+41.0078572335793
+37.3775138717997
+0.96939521358221
+42.2944826920667
+46.5760008757122
+34.7396571161278
+30.1684851704701
+40.5267377223044
+0.826202248719635
+0.86098557818284
+53.7125787245415
+40.5504848805159
+35.3499358987945
+13977350.0484038
+40.8538122167532
+31.4170585504437
+59.558683064425
+45.339942258718
+15.5003081031366
+72.6388261614006
+40.7487112154336
+36.8407160920179
+2.3920617562137
+1.08947974150538
+68.679781979573
+26.7900271769211
+23.0913459735662
+45.8602045753101
+34.6395746707758
+32.1119060682374
+40.5918909688109
+27.7645667998215
+34.9529077141087
+27.8546104335585
+37.7187694555482
+35.3557310263534
+25.3014787919245
+34.0464638717767
+35.1609912439768
+31.2609784302286
+37.4846299956387
+30.7489981747416
+37.9330522840713
+120.926414132675
+43.1843776807676
+17.1947931552832
+33.8386604023232
+39.4558735416203
+34.471634770979
+4.10729517730362
+8.95226608771919
+44.7684053914792
+17.5092557542389
+40.5037508948934
+28.4344777960216
+13977350.0484038
+45.9728488832854
+21.4285424571928
+30.6589997229812
+16.1819682109682
+4.85066628381335
+41.0043424437408
+78.1748807962137
+39.5132794347749
+25.7470534477474
+33.3191099298073
+35.9660597958246
+39.1875140984365
+37.1452404847555
+26.2773640352111
+23.5651758068915
+37.0413887839019
+62.1116941732232
+36.1512181757825
+8.89253139489167
+17.5849251970308
+37.2261008039412
+49.326762984637
+34.0161132555133
+33.2373923424271
+38.0909617922522
+48.1707448275709
+43.2571656921623
+37.3051923266051
+36.3096635414558
+41.5376240557277
+25.0434745440696
+29.4772306974482
+24.3532357542651
+21.7779332905998
+31.5387942349693
+45.1863966064795
+48.5581379193497
+31.2888923631164
+14.9167809691839
+31.746803075091
+6.78294308503418
+13977350.0484038
+5.11137832946135
+50.209126004822
+38.4166011618058
+34.8123651987623
+34.5574591143725
+13.1392519719695
+8.62615419169973
+28.6307266513782
+52.6870682991026
+20.3934495010962
+38.7397266150998
+31.345688524009
+25.6370712755596
+8.6235322902259
+1.15437591133832
+34.3249356777657
+49.0255674973355
+39.3446373590797
+34.5746323138308
+33.3021167015323
+28.6580315863763
+45.9319430280427
+19.9647544248793
+23.4471447942559
+18.905853040166
+38.1458965662988
+31.4074320651537
+34.3502520133715
+27.1721192120538
+25.5911932000386
+23.1017324066061
+33.3347733099476
+24.7204106794616
+32.5556143977938
+12.9604277368786
+7.28593262015047
+30.7293430015369
+26.0940812521209
+20.2632492076447
+31.5850682727187
+63.2092665337156
+38.8426188197876
+40.7340308263785
+22.7786115777343
+43.1005875797882
+12.232992670859
+32.8372520167997
+16.1000166786886
+26.509982268354
+38.7978528025571
+18.0779541919835
+32.3583254414545
+38.1071914819377
+45.4836459772614
+19.3908004084739
+27.6114973221571
+1.33995541166531
+2.2038547861442
+5.06048800318245
+42.0307092496291
+27.0573723476253
+36.8464101192599
+21.0788370360147
+5.16877602683573
+47.7188128883244
+41.8295356142687
+19.8472430318053
+383.83902056179
+24.1168611784139
+9.5157730042344
+40.8208164338071
+113.295407746502
+43.1127261457609
+48.8739119387422
+29.5779225512841
+78775338766519.6
+31.9456239124119
+1.61645152805112
+17.2920882861226
+27.9761497577518
+30.3322602121333
+39.1357216539015
+59.7382169537697
+45.974361421466
+24.7336627242328
+32.6861798599589
+37.0023277723091
+27.8269404225703
+13.0876286839603
+18.603171882695
+37.3514685247835
+31.6729638043121
+35.9180771795425
+28.0153205470911
+35.2924406897295
+18.7909650223101
+26.9436744970425
+19.5022045636244
+38.1013182078794
+33.5633749255537
+21.6140950076124
+38.5750116899971
+28.0462413327444
+28.7150902104553
+49.3409645866843
+48.4178424263533
+32.5218968501824
+25.3307024510771
+35.3263527157161
+30.1341415651858
+37.4519685875061
+43.1945439688739
+43.4777261609817
+34.8359964961753
+60.3266989251496
+25.4630892472892
+34.7687943187447
+10.7902475640076
+25.0917592906784
+42.4599174412395
+37.7435585514427
+7.14986869581495
+48.5902207583318
+52.5381309861779
+46.0999604670807
+35.9357902886833
+26.7937058154518
+3.48125427326837
+3.30465518059296
+32.9256948237276
+11.3598779008427
+41.3581460862245
+44.239370243059
+29.8707393487648
+65.4957696335849
+37.4211041516546
+29.7263719885209
+28.6871771178419
+21.4500647939523
+23.0222167893351
+26.9119602793852
+31.828171652415
+33.8832847337395
+53.9573854422679
+29.0466874298978
+9388992.55572318
+28.6192992120968
+35.0397497115554
+20.1444974334009
+33.5096105733623
+25.738296283427
+20.4814474399968
+11.7779553475602
+28.5532580733464
+22.1921284405897
+21.3313560430045
+27.922736531971
+32.2710083431114
+26.3715783547658
+19.1309660761277
+44.4647403780402
+20.0491299886508
+24.5886821849994
+28.8009957947868
+33.104799427589
+23.0717684465089
+34.8557951466876
+44.9553987199073
+33.07939967957
+22.4961162529128
+42.5567271718533
+33.9113686132194
+9.15947972386052
+0.235844827296549
+27.7110344021597
+18.0596939744646
+35.1159006927327
+1.98051554286878
+28.2935024803485
+29.7706434790539
+23.1274461247331
+15.2068096693728
+22.7190251065208
+34.424430441073
+9.96116274955704
+31.6817485379611
+33.3837141885822
+57.5279157293253
+39.5709615128221
+41.1526362101293
+31.4789300698001
+6.87662586296662
+34.4259510206642
+4.55871807535917
+17.8960775798645
+92376658.1279866
+0.515164934517254
+3.73994952027193
+0.436756123130318
+42.0524106488084
+13977350.0484038
+10.9819660789254
+15.575806164191
+44.4934954148785
+20.6547742376289
+32.8667080728792
+36.9170227694485
+47.6203175153281
+11.2270832028108
+21.4156890906292
+19.4503159409248
+53.6898848454515
+33.4066067092394
+35.1199387923911
+1.31601233528104
+14.8852241046703
+18.2158395065047
+50.6490487954957
+30.7781635127867
+20.3653435041579
+30.9169574164426
+34.9530946426445
+355617746.924986
+29.9521938714911
+21.8401899682598
+12.033242371225
+18.3760521132106
+13.3613771051778
+43.6445949423488
+14.8249747161363
+17.3570807417047
+34.9120515425353
+11.2501907720313
+41.0021327416131
+22.8813416749063
+42.8590911297637
+14.5705724064319
+33.265753767292
+27.2661469316918
+30.6127942763735
+34.6484535261321
+26.4016293704754
+52.51576074954
+31.2993593501692
+0.856528416630756
+31.9897411623576
+24.3230944592138
+1.71367551258243
+24.8131887467355
+42.3755501075821
+11.8592986724607
+1.35778384766452
+10.50123825001
+29.2034089668999
+10.8806144701924
+1.78522342525135
+0.377831387856728
+5.03394685404332
+10.0801658257676
+1.96516467024898
+21.808149456336
+8.68401004168718
+5.27008702665155
+7.45059557341838
+31.3433006635234
+34.4259114077721
+46.5457502018531
+35.0636916724155
+32.5408876052932
+36.44590872037
+33.2689267402042
+50.5018321355441
+40.5914533558707
+25.1001985613479
+28.4744778305295
+32.7013096061917
+49.8162237278419
+18.7627879144129
+35.9229305632712
+102.936865143707
+8.71966162249695
+70.0366224072481
+19.2694991470423
+39.8778487208692
+31.3234582456841
+35.7605593598738
+41.3395541632046
+51.9877187498696
+46.5737061297376
+67.8755711051396
+6.87511446746529
+25.5914036258205
+36.5812150697448
+38.3183969802897
+39.4338540692069
+25.0144223890951
+46.977794286825
+21.3894985687689
+70.8387214327055
+33.125341863671
+34.6400167141899
+29886137.0147137
+31.6030902590684
+145.773065976345
+33.1022490553704
+29.7618261193092
+37.0757788774937
+27.8366737625603
+39.0601667042264
+35.3366896865552
+30.4992090156229
+24.545182635692
+23.8647912914213
+54.0263751654402
+35.9191994909539
+22.7801771595021
+22.8395769045274
+40.1024095472121
+43.3386337688787
+34.6091612329768
+25.1222514354226
+8.68008864649244
+54.8304206526353
+29.3473234153224
+21.5096275057683
+32.6747010470592
+41.8512227125284
+10.2425579122523
+16.4592784165767
+3.12145048374693
+10.2425579122523
+16.4592784165767
+41.7771704017063
+2.74327395559204
+28.3216180838451
+124.303328805093
+30.1236239123394
+15.7360194100583
+26.6228551606069
+37.4371230612079
+36.2451274339593
+21.5825679906399
+42.644261457442
+20.4627848381135
+32.8492642615876
+45.1845361809297
+35.7358304097034
+47.3195589330039
+12.3886299174016
+41.4134968728163
+35.6662584005344
+56.4300092136573
+25.2179412881146
+42.8995599719641
+47.77827028219
+13977350.0484038
+34.1956675039675
+13977350.0484038
+33.0071609271414
+32.7405259315763
+42.5914152800165
+24.1363847510966
+27.8299171033243
+41.3552897976989
+41.0479270752333
+25.30209362801
+13.8162015382358
+28.4039686763771
+40.8693170101408
+36.7636513734852
+27.5286023281843
+17.001045541622
+4.90482285485612
+18.8126872092325
+14.9259004833629
+33.9050100141081
+30.2579832303777
+40.1966984734978
+12.1561659899698
+6605921.483488
+29.4423124155783
+23036675.8767402
+23.1876898306629
+23.1458594374224
+32.6154171215067
+42.4449837876356
+44.2669177628436
+30.323009224203
+36.4985355848287
+46.5893679822493
+21.8915549705847
+25.273561838622
+33.8544974358356
+41.967090252636
+37.1126112200028
+36.2117411886511
+34.8005435144456
+23.730844792101
+38.9029279596226
+17.5626635790243
+27.5319058844376
+38.0819666016389
+9.61968503465974
+36.9319453831721
+27.0418175643998
+37.2307178422919
+43.241693533678
+46.9437016562251
+37.7928942203511
+24.5915272646975
+49.7419642779795
+0.804246550156864
+0.72546545283266
+27.8810556571477
+27.0563291816703
+15.0123573091233
+16.6925388855118
+28.2837158442786
+1.62235640174799
+48.1429829223298
+34.7992115463176
+34.6998155364051
+21.5297114007311
+38.0990569783976
+2.34837776776228
+0.209305889443705
+11.015080948068
+23.5088898037287
+35.9328295901126
+13.0943930689946
+51.8495765376319
+35.7364384714873
+49.727201908374
+39.6969146152705
+51.9368370773263
+26.4398373573853
+7.7452286585045
+36.1976415787063
+27.4008667834578
+25.6423702561939
+32.0252057902286
+46.1212700268543
+10.8293590613411
+21.0944285419431
+22.854972980199
+18.1719581611301
+39.7610326378966
+21.7925702181557
+3.85162241239947
+32.750510331793
+37.955208538189
+11.0426164018926
+32.1076053982293
+47.433987889504
+33.7669685396853
+29.9012918689718
+39.4815102237844
+22.3595557958008
+40.6743333248395
+36.9489901251316
+11.0394055627395
+26.0494247989487
+31.7658462150955
+17.9955868711975
+27.1269884188693
+32.9167759194383
+32.3003247688344
+34.0702717532174
+290.930628247474
+18.2695044198922
+7.99403878616016
+12.3809873428768
+43.2205424452975
+11156746.9290026
+20.2752478799436
+0.457585912445884
+67.6183458429164
+12.7413453258641
+27.6309968338939
+37.6973843462191
+85.3642441876101
+0.658405863959313
+32.5045039012237
+29.9358767518165
+31.8139268718423
+22.9298741852907
+30.9821833150411
+16.8478469429038
+41.295416995925
+29.4798897126727
+10.3425346631303
+23.783220621441
+30.7027431200073
+34.6749676076399
+26.3326745697335
+31.4730012065274
+45.5668752037461
+33.8210374380607
+40.0374209848597
+27.8329868277485
+38.5464398176933
+42.1941842349072
+43.5477652992155
+34.8826492494771
+40.5330663345997
+27.1092511349378
+46.2064384172898
+49.2026224955351
+8.00533587787517
+11.345315880487
+68901851.4569408
+0.777784991669674
+0.877240953599882
+14.0562293109382
+0.637192978775141
+0.798466836137843
+0.135236192864461
+12.8155386593655
+5.5640011667671
+5.5877660353451
+1.17080869902446
+17.07894632682
+23.5203550936522
+8.56799822182619
+8.56597155575074
+1.63510416627701
+0.420723080868737
+0.290297829013867
+24.888603506968
+26.0986649489171
+20.3839009268927
+32.9736388381014
+22.3802154506428
+41.5604716371846
+24.3250975480445
+30.717763051592
+15.0652030274121
+64.0942761956537
+35.7366616899157
+26.0782933970484
+3.54125095076615
+1.69226299071453
+15.4146104143082
+26.2827319782945
+33.946142523289
+33.7176423977316
+30.8192945301263
+30.6963624476217
+40.5047462280864
+16.5108468436072
+16.6230745306277
+16.4898426285764
+18.4134816367584
+36.4303340948615
+39.2968214202123
+44.036722455058
+25.9216109357625
+8.38393751179493
+24.3316741938317
+56.4669542726967
+78.0746597299543
+17.2607776268374
+6.75841465017868
+33.2050251323124
+1.80070194060268
+42.1149557640538
+4.06732674732188
+32.1889313212252
+37.0858289129873
+28.185111278607
+34.2329986225706
+24.5717847892755
+42.9606678022471
+49.9144397659536
+36.079203202674
+37.6303792008601
+29.8608157835177
+37.7571902914248
+45.2312796703632
+24.9784201390383
+31.3350225041684
+5.10577480139619
+40.3212138555014
+11.9853314914162
+30.5956018845786
+50.3959716838962
+33.1983176796757
+28.1939025599413
+47.7613274570594
+49.6426968313025
+5.46430192496694
+8.57562682706695
+32.1478537028016
+1.7396338705446
+7.25582051042279
+16.3915729004265
+5.66800526591347
+24.569325874915
+19.730462877699
+23.9470642325196
+25.1278129232068
+32.3085060847393
+15.3959142977049
+4867.54852063787
+38.2221394189901
+24.7672708674383
+0.691905166409812
+47.9829549495218
+36.2351085415547
+29.7489775241707
+9.31093092943653
+8.87652405627169
+11.803043157423
+31.3510130842819
+35.1720217007215
+38.5375425139911
+24.4823577255883
+50.4445132390407
+30.4956246871565
+4.27000396057347
+3.9446953531675
+38.1999973707447
+29.5681100514342
+32.9118575508297
+20.9856882909361
+25.6894869534645
+29.1427945827766
+41.9922351427673
+24.6252914774747
+22.2104622283372
+27.3471521777703
+33.9590110402006
+46.4504237049725
+40.8794178346312
+35.747398539839
+22.2591892353761
+23.5176162038673
+5.60211637931424
+0.920437901626324
+1.87356974344242
+20.7032319912592
+13.9713643736656
+36.697688190366
+47.6838862299777
+41.8556509002552
+37.679502416912
+13977350.0484038
+33.005875962399
+27.0588212387956
+28.9935054005514
+31.4456064379897
+12.3163809656126
+45.9278531115478
+23.9834662939805
+12.8743629908345
+47.2620513744394
+32.3685780609256
+15.5582371861187
+47.6543031160682
+26.0448072238759
+35.5317082534009
+26.7341600801201
+24.6845684396785
+33.1753470917313
+34.2771617989275
+24.4181821489208
+25.8258062455368
+35.4816337315239
+25.5092876449994
+11.9220899117834
+37.7486621982142
+36.4263433900488
+38.4266213374141
+11.5273529039692
+12.8939095206711
+27.5994235519876
+8.28002720741761
+29.2233460987756
+27.838406026313
+7.07940642614986
+31.3126385843685
+24.660666751377
+15.8858735238271
+21.7058133430804
+18.1693800691177
+13.6639748026127
+52.5220901451113
+31.3493580611406
+18.1690295910919
+41.354620335345
+43.700729787567
+42.6898107090135
+35.9964707734104
+20.3285767987533
+31.4212076204326
+34.2570675696806
+43.0065102311702
+25.371825204717
+15.997475611216
+25.4414688848667
+22.766467107682
+9.48544821420969
+29.8502947929018
+38.5771223568699
+37.0994663952198
+16.303810751289
+6.52472423168818
+24.7582386808091
+27.9122305274662
+1.87707865380906
+26.4559632152115
+38.9983564301249
+18.1120424030774
+9.623161659609
+19.1710646139158
+4.76370678594645
+6.89032621024524
+1.85654082194386
+8.28076877701046
+0.953641454449983
+0.609950645929941
+852449354535.966
+0.809980060443781
+0.264835499721821
+3.40068525728326
+43.0831175830121
+26.9784039148317
+29.4192050949078
+25.308303728185
+49.3428390312144
+33.468311285956
+24.6101933750589
+13.5663653160373
+37.3165484176794
+34.384874766101
+33.2695002934011
+27.9219190012668
+6.32989711587534
+9.96702696359105
+24.9960013910658
+26.723553290326
+15.3851937541217
+0.125833870198229
+47.2532422555423
+23.1273073243981
+2.26113821065918
+4.77566710600096
+37.8936830243005
+2.46676584773718
+19.7674723477458
+3.49427350979727
+24.0347398974783
+20.1693719269536
+26.6094634894971
+14.3497705343921
+7.73161233032457
+34.3390900946131
+4.30236231437763
+8.11112316321517
+22.543309118123
+17.2500609545442
+512746151.25768
+49.2784918312657
+35.6935483638658
+33.8745003667273
+35.9354933615314
+32.0341337554174
+27.5828479667784
+15.8451446154293
+33.5237023573615
+30.9822773380288
+6.43636449591398
+0.502341998886978
+0.339468830572101
+13.4158129399884
+6.7671088744393
+32.4094955462953
+33.6758909382193
+37.5637569180768
+35.4903599774927
+28.7991582458153
+36.3953762249218
+2.91687587856334
+34.7292033495956
+1.8192945744633
+29.5316285295418
+43.946430485093
+44.8583076600475
+37.9445845732464
+35.302982477964
+2.11775053872048
+25.4055512658334
+32.5018067608492
+30.0882639599008
+13977350.0484038
+42654250.0953321
+37.4892125072376
+48.726895593921
+34.4573985087662
+2.25499515418524
+37.5650724466844
+4.81609970804315
+4.74729104952633
+44.1084724261679
+32.149467729292
+36.1296619236515
+12728134.2867151
+27.9090472841318
+37.3707196215858
+36.0441353825766
+33.2715093917389
+2.62885124217521
+9.00237879135732
+36.5087125034738
+44.7290759300182
+31.0126519123257
+3.92155978652743
+29.3019457782721
+5.44972757751103
+46.4686056231477
+31.4438319037107
+99.4136774727912
+39.1886007978987
+18.4276068579756
+26.5564162322355
+19.6294243980554
+3.39742421784017
+30.3601727952734
+46.1990043856094
+24.4214911821953
+34.7065733126239
+56.2999358443788
+21.0460336376005
+35.2004317119735
+19.418258131477
+25.0551137879957
+48.585340173806
+30.6004685391225
+29.0741372867411
+45.0396139904988
+13.537667177648
+40.729686345806
+26.8221602833047
+15.733727591321
+26.321746270029
+35.246169068975
+19.7264315743616
+57.1425909990853
+29.4586636820684
+34.8112127930069
+18.9782598911938
+45.046657820685
+39.7738701369682
+42.2603833424234
+143773720.346788
+11.9544951478178
+21.3856787759326
+36.0154439755549
+39.97204910095
+48.3422331135383
+23.9658728105628
+36.415712702039
+38.3563733252669
+31.2586718318742
+25064190.5242835
+39.6347646023672
+36.1693393335095
+23.301559422252
+36.9284608102834
+32.1019022770678
+31.4492976425165
+41.3936236278178
+27.0768905438233
+37.5213659581085
+48.5030239440638
+16.7991992742367
+31.9304434354339
+34.8098861176968
+36.616897805123
+29.7245971219699
+19.1156643949482
+22.8955973404003
+27.4514783480677
+9.41217030983124
+45.9845976120974
+22.867135673552
+16.144987681448
+38.3620039090516
+44.4420510973338
+27.9313690765888
+27.2183313254193
+28.2863344619739
+5652311.13534306
+7.01039857395475
+31.4047341765768
+40.2425958756735
+27.0446772683167
+8.90423190230294
+23.3981946775916
+32.5709295441675
+38.685119191063
+10.2392835417168
+23.9912856865612
+27.9566365311251
+35.8583278227655
+30.6135824722932
+21.7074348785331
+44.3506031743398
+36.1295295737163
+37.7050188207289
+37.1320941837512
+14.102710848997
+13.6707360706487
+22.4667288514779
+34.6413070972346
+17.7364532376352
+11.5766533600499
+30.3165824615899
+16651037.0734857
+25.8835820903982
+33.0236055708566
+43.022308710909
+26.6447889805308
+33.7073285124759
+104442769477.485
+7.21794106977097
+9.29826974888613
+58.84286385968
+24.4309508312803
+51.7102034285681
+17.8155035330883
+36.0878274830259
+4.55805172085081
+19.4273588579216
+22.3454834704807
+42.8943014377371
+43.666158379884
+29.7182615717274
+32.8537086749902
+19.7872980137785
+39.3152903127428
+30.0971335993419
+28.1856658845114
+100.53584720258
+20.6275980752194
+32.9079886533409
+11.4474350577222
+18.3875512359694
+36.4367276538444
+40.4531906431614
+37.1773017456471
+24.4993627339721
+4538452.21615208
+44.4531426807743
+39.223939081971
+36.7640124704143
+50.016359125358
+29.9047089479872
+15.1591651839049
+23.6215479230109
+46.4018666821242
+50.8381983202908
+39.297899526591
+24.2979732716836
+23.5342136596645
+87.4361598729417
+26.7534818568672
+33.0195407696497
+20.2397158080483
+30.0005187539118
+27.8076385503764
+22.1494823488327
+22.9762279882842
+44.1556734691057
+35.6224080272315
+40.6025795366563
+51.9075208969586
+16.0989990140863
+6.42889090360025
+24.8631793120403
+13977350.0484038
+28.4492707695178
+35.8533263463923
+11.1648055019365
+40.4543402589717
+34.3938615296851
+5.24253411839834
+12.504953277666
+44.8691109410284
+23.8620304871712
+32.3084466044922
+45.4117464129923
+38.4208873020866
+34.9165370600516
+21.4861944920132
+44.3962371042042
+22.5569145938929
+15.0583177123996
+5.62913650371326
+2.6555589175359
+14.0871949676726
+5.81155738147113
+3.03729483929329
+1.15006382305493
+1.85654082194386
+8.28076877701046
+2.48920691010775
+0.609950645929941
+1089988.19279076
+0.809980060443781
+9.72939479590344
+3.40068525728326
+34.9347319083032
+24.403512056925
+50.5841660794832
+23.2092703818092
+1.59090875618416
+45.0901475763779
+30.8550556297457
+71.5668891723527
+29.2544251604661
+35.7511048582599
+23.093226254728
+36.152508989437
+80.463283778772
+21.5787085692134
+38.8833064546279
+30.0306144172323
+22.95610673048
+34.3075724495582
+37.9988944604212
+24.0358845980551
+34.2933103463338
+8200.54883100381
+25.2806627426033
+37.3013776074189
+14.2180092842715
+31.4851614649169
+39.7196889834103
+0.383725428973623
+1.19986292090531
+110957.111753701
+0.705925401031634
+16.9736787251673
+35.8307685538159
+39.1510423482883
+28.5882013321308
+15.148794522868
+22.9367569517397
+5.33361224229708
+5.30796684352081
+17.3502688066919
+23.6810218433878
+30.7646387668755
+36.725645575965
+26.8431661887748
+26.213818686576
+34.3111787179304
+36.4820346745167
+26.6406645627757
+7.85386246995926
+36.5877710476113
+25.5506945932063
+38.2606302331875
+28.3898598762626
+25.9606111614668
+29.5755399485669
+58.4034535393446
+27.3358952872382
+54.0834413252109
+34.2650265165467
+17.6265485684527
+24.6289162088743
+26.4139348692294
+35.7313859733015
+31.0029925210763
+85.5386982942743
+34.6942525012958
+33.9945571513146
+37.9534174402609
+30.5751294376643
+37.5291973439705
+38.6527899803092
+44.3652304940273
+42.2123634944513
+33.3022702499004
+24.504972603679
+28.3214025082234
+48.9896802176216
+26.2079001881463
+47.8531031971011
+12.6460961819778
+46790204.7012859
+35.6914628214584
+28.2049893451435
+28.832139906742
+22.7771783327717
+23.7903155938849
+37.2887190340597
+21.6086525905451
+31.1571092718814
+31.1743060105204
+28.4596145599863
+35.5564863004687
+21.7385377142809
+29.5170902845881
+21.1650942604519
+30.5431041502766
+12.2590226209702
+36.7305117372214
+18.0257184315477
+26.4843352357765
+20.5480961271098
+36.3412108469891
+61.6326092529624
+33.6814479716593
+37.2738784884163
+44.7024413172193
+43.2740414163674
+35.5580870043303
+28.512986543162
+24.3755288263481
+34.8476363475629
+29.7739752886595
+54.94277656474
+28.6244616037236
+30.9412088830388
+38.674281174544
+23.6343934440482
+18.4628394322955
+7.65638338273643
+37.6328700408937
+21.9000107081013
+46.4205661558808
+29.7519509119782
+34.874111453939
+30.69175358398
+39.0191752351768
+35.6992019975138
+27.9993314478777
+34.1324389121427
+32.8055213274598
+18.7317911728739
+26.0996494233194
+30.6889588978278
+26.8819519368363
+42.6631462784511
+35.8397775835992
+28.6801325775481
+4.55513012143272
+38.8434512427922
+33.6918549412473
+35.9465005754222
+33.7405737338375
+43.0628701337223
+26.317437115969
+3.8204210190833
+16.6357447900123
+36.7809575618183
+41.1189221489311
+61.4532946379722
+42.3533111249246
+5.57458278654788
+37.7959938998427
+39.1281636508604
+22.1848352232838
+0.74979535187292
+8.42462190504044
+129709490446.953
+31.9129286547688
+48.8153806483089
+40.5738275953405
+25.5853896132672
+14.0218592216529
+3.02434928263837
+0.122002825764583
+1.99050021148375
+0.345424410858514
+30.0586747245242
+0.365919804606805
+40.9796823935792
+27.7662999782475
+39.5394164014351
+32.1702249058772
+41.5780180152054
+24.4928097457135
+51.6700233773547
+36.7764993476825
+31.2349475164338
+21.1023950690075
+27.872979543481
+31.9540504881319
+29.6692883151888
+30.6819801425088
+35.9686968244961
+52.3416357699713
+42.8817508383207
+46.6135124865851
+8.721130423315
+57.5629648637833
+22.7372510616675
+39.4711320461141
+22.2070086792131
+29.8133462237963
+55.4132123354858
+16.9562232574162
+76.1004319200109
+31.5577863529802
+25.2327101294169
+31.1383177550117
+38.6528959517708
+47.2682335793386
+27.1124832784455
+11.9577411377716
+36.9074936125901
+35.4696852310222
+45.1960245819563
+30.0027611469473
+21.867356602212
+20.5405890409518
+4.71631490458933
+35.2333898075473
+28.5501764397734
+52.2500906564379
+8.46945561527714
+27.2610378216656
+28.0133093184993
+28.0928625138968
+34.1740968964281
+33.8431941446277
+60.6563903250598
+34.18017676849
+29.6547998036384
+20.8226297707482
+39.41530498586
+36.1745489923785
+27.4008797500177
+3.09776326210105
+28.5423432783743
+33.2401194979336
+33.638673605145
+34.9081839122902
+31.2638201620396
+33.6185461539638
+36.3700585996319
+24.9050926223345
+41.6008442036061
+48.0833845930445
+25.11254878207
+41.4916306773703
+25.8564270883323
+55.0900756840631
+30.6850087592578
+50.334901058418
+22.416057061932
+33.2063671390399
+43.5797308793025
+30.7670849880166
+69.0695957442609
+51.8581042928574
+37.6982983784956
+24.2380305649458
+19.1220873540827
+42.0067749783754
+13.6336029528397
+24.8883578384606
+36.8298813616337
+44.1403588505849
+9.17576663280493
+0.598568271781823
+22.0292906679247
+30.5219966672674
+26.9712644468675
+21.3906352893102
+19.6319402049475
+15.5060544526294
+43.6896447965698
+20.6594948364596
+23.5375687822234
+33.8045203239478
+38.7853905241193
+26.3428005555369
+33833045.8491425
+24.0748998665159
+38.616210570954
+1.24999309248004
+32.5938822779928
+33.5714853125821
+29.6704243040262
+23528187.5814939
+35.9408689110691
+26.8230924765811
+28.7778206137352
+32.8968158910688
+24.1621081421086
+18573897.2849132
+43.4776535297768
+12.8659843059807
+16.9032988898219
+33.7482912895611
+38.8869968519872
+33.4788333579059
+25.0340934630735
+23.3991702064521
+37.0258338276186
+10.5046477247983
+33.4431035703608
+31.0672527338925
+22.5209624115078
+36.387205142609
+44.4621966148847
+34.0956812324028
+32.6446637252871
+11.6246631650587
+42.1590584287627
+28.9002611399085
+41.5781732224721
+37.8693933768018
+45.0085338977867
+29.0079783308361
+45.4250025208665
+25.5540558432703
+8.17838538637053
+25.6821541626856
+23.4447614909686
+33.4487133968413
+23.0486003432911
+31.0726761696849
+52.7333934973886
+31.8479287913476
+33.6656134591451
+25.1339271069383
+35.9460133670539
+34.1763933042942
+18.2285123010502
+32.8545890942072
+31.651273042047
+13977350.0484038
+31.6614939706906
+13.4906031520306
+27.8204958870243
+22.9154548732348
+35.5027848414072
+31.8649662479023
+38.4330683746969
+40.7500240464195
+29.3466232450349
+26.0879157891014
+1.54565150428621
+18.5421442509271
+30.3201775851159
+17.3004371485733
+19.0228867112188
+35.8782281426181
+17.6487580474833
+18.8436335689606
+27.208921760356
+24.996952697344
+42.2424293837486
+26.1607531532549
+24.9126103748673
+55.1392470556206
+31.7986675345359
+0.44355657478609
+13.7670202800146
+29.7704051233935
+11.1196228589902
+33.3450171252499
+36.9935018361125
+43.20239824372
+32.1484394444379
+0.241916801995084
+32.8187280085908
+26.7314264450802
+54.845875943209
+46.2656106798285
+40.015911832836
+5.38472000070858
+39.3289024765932
+37.7426486144387
+37.3147081332774
+49.1998470135401
+8.54602289897121
+19.7796899063563
+34.7195359431172
+30.7874078789197
+38.570049457912
+39.368295447516
+30.8800564576476
+46.1442516779132
+21.7315888063171
+49.475638640791
+29.0910498589487
+22.8128296117478
+22.0932079240408
+0.940244031377013
+36.9209017170066
+35.46684293804
+37.5318041057681
+27.1564995947756
+38.2879381660442
+16.9987371593571
+38.8676159965691
+34.1568173641338
+56.2247402449941
+35.5436319228333
+25.4328309975078
+39.9561536869598
+23.7647197787522
+34.7641387466851
+21.4109056496994
+35.0088778159254
+32.1165260532987
+5.61217265552069
+3.45485882594059
+36.9754022534252
+0.525574009166839
+0.656565137499207
+0.387646175677794
+41.289895808111
+43.0618296875281
+38.6109796347208
+33.2905822931668
+12.875187665314
+30.409311014427
+38.5089928737771
+54.9248321423508
+25.0749831228432
+37.4993500670676
+35.2637027537727
+74.2423109887316
+27.3660538833418
+21.4223389435937
+7.4000056113993
+29.7702903376152
+23.7251264086936
+39.101245361798
+22.9739571290987
+35.0809758806479
+41.2360621761285
+40.1406561215145
+23.3616114683605
+25.9212428159604
+18.0022675665999
+32.3169781681914
+35.1804659748926
+30.1319851760049
+25.0868688474064
+29.3845088546346
+30.0390293522103
+13.5083673687055
+15.691859247829
+24.0035967571639
+27.8500532138903
+0.129586563294739
+46.2548354392148
+25.4003484194835
+28.1512960649319
+33.9202278534901
+21.1448731374849
+11.0299604309151
+43.1779413957579
+40.7594616166647
+52.563798162609
+24.4073506098383
+37.7309090453271
+20.3234854757053
+26.5584645871531
+58.6520792867763
+24.4018685942693
+29.3604981860246
+38.8401608972672
+41.9066706484317
+28.8237386001604
+20.3768202602088
+30.4305277458735
+42.4562924848277
+26.5712432102936
+28.6398279365086
+44.236499467947
+40.2426368090055
+38.205570857593
+32.4321701082506
+32.1485871626813
+33.7919478197338
+30.4416637367383
+24.5022119191714
+35.9917554470797
+29.6638014473806
+37.9718119479521
+37.2559036774197
+24.9054317919015
+15.5667909672831
+50.6015703191226
+5.80867149045721
+13977350.0484038
+35.726656083853
+17.9145881113312
+29.3757215572423
+34.702038362338
+41.277677537522
+22.4818849841785
+38.5924251034469
+21.1641312932427
+36.6612190465419
+21.9326535378747
+29.9453306824483
+47.0500271716294
+26.4314293091882
+37.3026714709829
+26.0138103604453
+24.1895596472756
+29.6823568117724
+33.7414798459775
+43.718809467635
+19.6092981090979
+28.0375944992277
+30.2586939896192
+4.01806551739352
+38.1356083516015
+40.9289435767587
+27.0989329091842
+40.1354077795329
+3.17191141650271
+28.3226765175688
+19.2916922136909
+4.03467022142895
+3.20213813249474
+29.0766835923757
+17.6845855530041
+30.9389785795742
+31.5781805437657
+29.513257010381
+25.1230710532466
+2927410.54238139
+25.9262985471601
+22.426618058587
+19.7728323454043
+35.3259686026583
+23.6747972601786
+20.7537029039816
+33.6723888675645
+61.5468924729357
+29.806543191521
+25.6799541809528
+139.366182441648
+35.1380410299931
+46.1823610135141
+39.1051081422576
+13977350.0484038
+1.90092670206067
+20.4608907218011
+19.1829237258091
+30.0786802631232
+34.3959827199981
+9.05774524416531
+15.2123058577905
+48.2078177442353
+36.9271836460186
+37.9207213579204
+16030394.5947326
+29.3330047839311
+23.9684483745996
+40.3465906616863
+33.5704249589577
+29.8850187974314
+30.9585617988714
+2.76010334361047
+34.0941750870124
+35.6419439430772
+33.0365242136008
+27.4551183602607
+1001.59145402466
+40.712932654808
+43.9852295570211
+52.5267480449302
+36.2668547460443
+0.45335822662913
+36.1399055696468
+11.6155350764313
+24.710670053739
+42.3467847935165
+13.6830361524248
+17.3598783431382
+32.385232868794
+17.1327727177709
+8.35569658220547
+8.04788586987622
+22.1110999026775
+33.2306730725808
+10.0880593691537
+17.529995357809
+10.2920644472294
+16.4330913381881
+6.85549199969592
+0.477630854314789
+3.43759560861726
+1.45259834940614
+13.6711382054615
+0.790237922679681
+1.45259834940614
+29.165802326826
+36.7252680368762
+42.2544410250314
+13.8283867327522
+37.8397170636655
+31.7487472443611
+52.4983402805199
+50.0818703201497
+29.5791189593679
+11.3770976043095
+27.0893901093723
+1.33188435313927
+1.92610413166707
+33.4361646281085
+17.7173610586123
+31.7381172202191
+31.9770360556454
+43.8737057263419
+33.5888007494765
+31.0579699326455
+29.1090849807949
+33.5741698139357
+46.5368580757375
+30.6828394184039
+13.7564615676057
+47.6650830485228
+38.1526042366503
+38.1335355305253
+41.617721536121
+58.3623188464912
+15.2982316400164
+19.4349025853739
+27.9860149456906
+26.566544573066
+24.5516417471343
+7.6457474756922
+22.7017156109223
+5.03288615126301
+35.7953130215635
+36.8674809431756
+15.1514585233171
+2.16612021258683
+21.4776595235342
+41.3426225719895
+36.4169314157977
+17.064395644819
+31.9674766578324
+27.3742616679325
+36.4981864351504
+34.7396483099923
+34.5629629650918
+15.836752360531
+30.6781814610878
+23.8279247671227
+23.0334534623987
+17.5617241956547
+41.4575073923732
+43.5078309190083
+39.285886834532
+32.0391471201945
+35.4476651218765
+46.76059184796
+37.3608976787313
+34.8665073450696
+38.9757113253374
+12.8867740192467
+50.2973553484895
+48.7123711364092
+10.4363584924736
+23.7083513842874
+10.4118603238735
+41.3626936925124
+27.8631998080232
+49.2376700641527
+42.3858205041322
+49.7529491911077
+4.7452617979234
+27.9897828610226
+47.3168398677013
+26.1845900782982
+11.9694218532842
+6.53090258861864
+33.8456356577765
+45.29716698046
+31.3199309136108
+30.1770635414707
+18.9293042448408
+37.5380383303925
+36.3238861971552
+35.4993387658613
+31.2911180554418
+25.2341273915534
+25.8940682135084
+29.9413099019964
+45.9204143649265
+24.7358192070533
+41.4031685480926
+40.8479684785782
+26.2760311903129
+44.8743875748211
+37.8608521284334
+28.347212784465
+18.6450031592226
+13.162353015793
+17.0229065254753
+42.3794739393292
+25.1480845551901
+30.3730200551311
+62.7527014929492
+24.1308096410414
+33.8818588333117
+42.4915684577676
+18.3493312476925
+58.325927531553
+33.9437699044681
+31.0306513419631
+32.471547100286
+28.147736083959
+21.7361770877171
+33.7970147594084
+29.814396010407
+39.6793642818069
+46.5070747165707
+39.5514751678353
+18.5114272548538
+0.511325269169588
+73.8553889482748
+47.2788951604524
+43.063744562703
+38.90721107516
+23.3092584838428
+36.3862106592017
+49.2124874228233
+14.656935186239
+37.6521739985487
+31.6079828882323
+17.2374549669218
+29.1128591104795
+21.4463851568572
+24.4089625225875
+22.1077167908015
+27.5486627018947
+29.5966657144747
+36.0013068246709
+24.7043548328518
+34.992628093494
+37.6882006254031
+15.943857135255
+29.7386014078562
+80038984.4098503
+48.5627195921172
+44.2848135185412
+16.3727881017532
+29.2524610203203
+40.6094149696097
+12.845013458273
+25.7698217978081
+38.8071532939805
+40.4428294666848
+3.00514898660135
+1.02411295931548
+1.71284991584431
+1.54531856036047
+2.40804337063142
+5.87841613887784
+1.99375941705468
+47.2122987082365
+1.02411295931548
+59.451789196934
+47.2122987082365
+18546642.9817036
+0.759298286854351
+1.02411295931548
+54.7118390925087
+2.09207929575943
+47.2122987082365
+1.02411295931548
+59.451789196934
+47.2122987082365
+59.451789196934
+9.00533811997021
+0.692257891431008
+33.3949251073181
+0.109566167355218
+38.5327916974647
+56.3017537263438
+25.6735092993129
+47.0709183000709
+39.4975713439279
+26.155997790693
+34.4954992541306
+18.1209651536442
+44.8998565916253
+39.6282388629568
+32.8327831073054
+21.9733154012458
+21.0001586670194
+19.2151754002577
+21.4153746683238
+36.6679985925572
+35.5228802574781
+17.6721469835946
+29.7456124024864
+35.7663546792605
+0.841055072458091
+36.1137869298842
+43.7958803503564
+30.6500705566384
+31.8183412706192
+27.7958559699267
+32.970706537178
+39.9005487842582
+37.0307114838475
+57.5744811901255
+45.74611787193
+32.7635715486737
+30.9368365442778
+40.2529077562141
+23.2547969801355
+37.3752133375384
+16.9909509394143
+34.631095186175
+25.7937128733509
+34.8832350185687
+26.2419773398345
+19.2684118806
+31.467838654184
+27.3293719323313
+53.9113182562985
+37.3655998446547
+22.3854289909184
+20.8358870643067
+16.0658778969871
+17.0712343262433
+50.9090715184291
+39.9209618697947
+29.4011683849186
+18.8229254313811
+11.9794472768688
+30.3763407140806
+29.6749649500278
+28.0997186570494
+43.0157699033347
+88421875.7609773
+22.5447418022815
+43.0528583191784
+33.7486762561202
+28.1370848628023
+48.6103759041165
+0.383725428973623
+1.23213194438608
+31.1928189576877
+54.8750848251775
+25.4631699783553
+35.7625880348504
+11.8294217143295
+32.530515184897
+10.4327237400837
+5.50777247155383
+13.4439977123275
+36.2198364953945
+35.9860985729221
+17.6542731945284
+29.402893633532
+33.2359992656051
+67.9719432781308
+42.4881226627896
+54.6097313890379
+25.2952017848353
+32.5122445441717
+32.5540994045924
+35.220922502085
+40.0948204870021
+30.8426274488183
+32.4977907659659
+45.604565489206
+35.6916737788919
+30.7564851057099
+12.8814282767321
+33.6206612783183
+1.09502878341178
+28.3652560491422
+18.867722974515
+28.5834120324024
+26.6735141927419
+16.4993564544469
+16.9104636547648
+22.4389641134496
+31.0371496566894
+14.5068562822907
+24.738272613636
+67.0078728263865
+30.6931704180311
+11.6279211417303
+44.0288186455014
+45.4090933987964
+31.9587297022409
+31.7470065937923
+23.9095794473511
+10.7112303327107
+36.5909279662329
+23.838970055569
+34.0105902727472
+31.9255851251278
+35.0380688718405
+14.2471809516591
+0.458354399333767
+15.2909337618152
+29.4276604411513
+39.3537839094204
+18.9580686872738
+66.7448911559223
+29.6747902968801
+39.6411498074975
+13977350.0484038
+51.2442231042306
+37.7046893649483
+22.3134819357983
+26.0155070777367
+43.6546633297423
+62.4280580775739
+38.3543098160543
+53.1860178944904
+68.8558639540071
+48.2720594531025
+7726316.17157914
+14.5813436630444
+23.3313084948913
+13977350.0484038
+24.6858624039979
+38.0474328170975
+37.8751733898735
+11.0309158236107
+25.9302763108093
+29.7535068992773
+38.6304605076375
+10.8402199710361
+0.813971424858357
+1.82913182769708
+1.38242717427042
+16.5259817388752
+34.4026444178318
+35.8838820566916
+31.8442628117777
+44.3103867437682
+10.6154685817222
+23.4851187507738
+18.8733602797338
+46.738143096294
+7.76077903955759
+43.4101473375102
+20.9355129100431
+26.9952800235505
+27.837086563109
+20.4078117688625
+36.5205502316229
+20.0355268013058
+17.9381404672094
+20.9032232561263
+17.9261106273195
+47.4129915670186
+27.289028021586
+30.3423600997595
+12.4986740731017
+33.2956389561597
+38.9408239756993
+17.3285453875839
+39.364863822022
+19.8554215123488
+16.225954793536
+5.77979804877956
+14.690697392518
+31.8095612803193
+24.6462617882113
+31.7416548951997
+8.25712345399962
+56.1771295162628
+28.8577763750814
+19.7041045054799
+6.99275951289857
+29.3890987053293
+13977350.0484038
+41.3912865244432
+12.5079183937719
+41.366872216846
+48.3444175110319
+35.2820246841718
+28.538827916443
+44.6843330980642
+27.5030950829254
+22.707314115997
+30.927720354547
+39.4166253967792
+46.3368843093063
+39.1125257363789
+21.3501907065345
+10.899838081188
+30.0098948607943
+51.7785717370768
+62.9033211570821
+2837896278.43157
+28.8859728581117
+259.205823748273
+65774959.3736969
+50.2348506874416
+36.0022327324117
+45.0192782248462
+39.6301329133256
+40.8543884470702
+38.0351067980726
+28.8528681158382
+21.6518681274614
+35.6760268333582
+26.4747046168149
+22.5521326984306
+37.3486871568361
+20.2852915131677
+34.8092400482416
+25.3556053724261
+31.7610928697795
+14.9361481562822
+34.1198545020045
+44.5605325615309
+29.6295789436949
+44.0287944452804
+22.9406575934949
+41.5451164509089
+34.5958309186917
+32.51860226523
+25.7948752403505
+29.5781800546433
+25.9898195605398
+29.7651685682743
+31.1856904024394
+39.8639433808595
+24.88806359395
+24.8036153978201
+13977350.0484038
+9.37234908309569
+24.5767037779215
+62.7847683613967
+39.3254630152694
+32.2604613552203
+19.83035838997
+43.9768258350553
+14061654.1599225
+47.2888004216229
+23.4875838087923
+34.6412912308199
+37.7325284285814
+25.3444306523845
+11.7056269658569
+15.8609451347086
+2.4302934297616
+44.6629923777073
+17.1168367972885
+38.0626533487846
+14.7965153273215
+5.88267035951691
+35.6729903759457
+34.2723972139753
+36.9283794587777
+34.8991919654069
+20.1132853597604
+36.9586221209528
+12.8581823120837
+10.9282507134973
+31.8166072905244
+24.4759317401436
+38.2866535991523
+13.9236677118666
+36.8954131251131
+37.1509674003639
+24.3181778439935
+39.6180450223295
+29.6313669225709
+29.4851997140318
+35.192086030272
+30.0311507089389
+52.01724282813
+14.8850367075262
+20.1954991563583
+6.69952270696684
+29.8389846619887
+32.5168824139449
+36.2804621738019
+24.3031767117313
+19.4922948679045
+0.529600677421187
+5.44495699658736
+16.5871741491724
+37.2903388831061
+50.2809663400872
+24.0310293012029
+29.3417984375784
+37.08982362939
+35.9960180599131
+28.0080043156181
+30.2849166176236
+39.2520807143745
+33.6974836366665
+12.3011057013159
+4.43893430123861
+32.1453490773468
+9.54414341023424
+1.10684919525672
+0.306079764676306
+9.96698876617963
+13977350.0484038
+4.73258502747918
+9.94707131285581
+3.95689517136643
+5.50546095906938
+9.94707131285581
+10.4024023742832
+43.4140875754943
+55.6041021656354
+39.5928862964634
+37.7233816449153
+32.5502158489547
+40.6297581941672
+36.6986537451374
+36.8812448589372
+43.5901982838052
+24.9624328792155
+1.19097966042543
+16.572578298018
+26.9162661139219
+42.1591019811324
+3.33818648464598
+33.3111311812083
+33.6780542666589
+30.0945587585335
+33.0489781095417
+44.9822736083521
+32.4189190377984
+27.7636714246626
+33.859106300156
+19.1941788271789
+37.1065162954039
+34.3073315148777
+38.6180471899804
+49.4167177179638
+41.891824780981
+44.4522942786533
+10.9532302406639
+32.9246501108975
+25.4373909520031
+14.0195294942017
+3.30165740638342
+35.7308206582313
+39.0082747939634
+15.2376232706618
+28.1533589522188
+36.0570833142336
+38.6379711311634
+35.2188612580716
+29.4338466294954
+0.13462553396059
+0.383725428973623
+40.7936228330017
+23.3063111130659
+25.9837176171165
+2.81535340214758
+32119648.4687603
+49.3569160858516
+1095523564.73605
+25.5658447225818
+33.1185237909722
+30.126742911615
+33.3948215961814
+34.8389948388195
+21.0946304932099
+29.2514639850162
+27.8539615719806
+8.88929479341417
+33.4513990602745
+40.8330101237607
+21.9429659468876
+32.7683790003058
+39.5651964999063
+4.61294781481709
+19.6711710700899
+13.5965596525663
+27.3468819681798
+10.3015047231749
+44.8244087671301
+30.2723213741453
+19.8833717189345
+15.6110889646646
+26.807769912321
+24.7086150627646
+26.2559230576506
+33.625652419863
+10.9872956571895
+32.3562795422105
+51.5188538376272
+18.0000531902548
+38.6457063476855
+28.4426929903226
+59.2587781736644
+31.8890202428906
+38.2400889980267
+31.6150705738455
+33.2115825997864
+35.2749799381425
+29.5924317678874
+2.98558095839672
+1.19322036450538
+24.634222975664
+39.5755740742081
+34.4062813057515
+33.3447509019535
+3.26482222284687
+25.0666097012966
+25.8958512683509
+46.9649750996562
+14.3712945553706
+92.8219480572526
+44.8820333546023
+16.8952807099244
+40.8743363192188
+20.1053217208171
+18.393269744003
+35.7113281947273
+9.12588187112507
+12.2862634047521
+8.98862473778308
+38.0184004931391
+7.91711080830294
+10.0334247628605
+39.7980173805335
+39.3594721417873
+37.7490564031412
+35.0484301597615
+35.4885328156058
+96.172797609449
+14.1881056062285
+34.5131155146517
+32.0901536601045
+31.2964882294108
+33.0582500998794
+27.7659412055788
+35.1729817521195
+42.392554923702
+14.5661888193449
+31.4857072886968
+17.2311976000803
+23.4389922859163
+24.3940689042147
+32.501751786044
+18.9774828169353
+31.142617920724
+39.876226704436
+30.1110829666868
+38.0337155577065
+34.1983310543461
+23.6502428153123
+38.4047503455365
+26.8818027919874
+46.6011556567004
+31.4363737521167
+15.9973878547123
+13977350.0484038
+0.420723080868737
+21.4382787499604
+14.7133088583612
+48.9833267076975
+13.8321694897141
+27.4382668568541
+39.769654011996
+14.0539124847035
+25.7987317734887
+35.6425507473734
+31.6873162161766
+33.9585853607446
+4.98417733816304
+45.9084313561205
+36.1154723768756
+13977350.0484038
+48263748.8957624
+15.3317948489078
+33.0901397002386
+27.7187528485987
+3.74765842469633
+86.2363857703924
+46.1402944516386
+37.7322013207516
+67.7444616457448
+23.3324683795495
+29.3971499190656
+36.092922388039
+17.0123914422995
+44.0640857281676
+25.6835072923696
+20.1668289143763
+34.6929704125148
+37.2694125025509
+18048284.8013571
+23.0287892560699
+1.36112089031274
+18.3523356832382
+33.2626972416405
+21.749483889377
+36.2538894586606
+27.1143656577517
+32.8277284314719
+36.9571604985923
+10.6020756993836
+25.171784979051
+28.308225578712
+33.1660356864944
+31.5353826132577
+14.3015300862941
+1.82307103110487
+11.6758836924267
+15.3851323474253
+32.8472211270661
+20.0653811104109
+40.1893471480611
+0.790958298854322
+1.08069065872359
+45.7153717017839
+28.6747060096022
+14.1035862153092
+32.4269386873595
+25.3891016311732
+15.262213476399
+8.87687736843828
+25.7446913788546
+25.9999303863002
+21.9749898603948
+35.3004266651001
+41.8873226392802
+27.6977469579984
+40.1945608596041
+109.088516207961
+36.4544351523933
+36.9031455500742
+10.82679049784
+15.5148487373655
+40.0722035271351
+18.3165991768023
+49.5176977243502
+32.1979851361163
+29.9089136166428
+29.5793485716164
+26.2811243892428
+16.6610494763423
+24.3985848223457
+17.5878963491245
+41.7508424154068
+41.6986201472721
+13.6972577437807
+14056542.6533999
+45.2509571100866
+33.2952131339719
+16.1892637914181
+37.9949628045214
+30.0884071219045
+23.7220983543066
+28.9515939129563
+28.6572731760655
+37.8072986574424
+28.8509713141458
+29.0956882755412
+29.6884905994098
+15.3849605073431
+50.5002399651364
+35.6940143171367
+30.9384251986543
+33.910165793981
+27.5636992956555
+43.4215699550103
+13.4705018738848
+25.989548354785
+35.3607992635759
+27.6184111342253
+16.2762790814512
+5.01943902993404
+34.2935021051654
+7.75500195340761
+36.1135107185299
+29.2509017005721
+26.7884464845244
+20.2295392885416
+42.2004730135643
+34.2566723372243
+18.6567353460199
+40.518205604832
+39.3972891655422
+30.1277382683275
+29.0080149707202
+32.1786041481573
+10.5989548053685
+42.6373298821231
+39.9681015163369
+9.84674328095803
+38.6398030531668
+42.6063373410213
+33.6055314024821
+72.3832416624042
+41.7336429122028
+29.444359693052
+24.8856856759997
+50.4017518074281
+82.8927486813359
+31.7621109881019
+38.8489442472601
+8.08564977020309
+37.2348249171856
+25.7447110025031
+21.2997524197691
+25.9725803456153
+16.426789254331
+33.2586818078599
+28.1786812792793
+33.8209701712649
+23.1645225726384
+303.049544503442
+43.3800216111066
+38.4033560590703
+24.6104099684397
+31.624521127369
+33.9374789363346
+30.4293502932099
+47.4032210197959
+58.6089178518916
+14.9586272297139
+26.4096928609122
+39.666565102051
+1.73596706629096
+40.809015580164
+30.0261435122183
+35.5663611272526
+35.6678084616834
+40.751896614316
+49.5869334446677
+33.2143111445896
+1.36778458671768
+46.6998907269993
+27.3114712702763
+25.2865821070854
+36.0463984585971
+40.9629548465205
+38.4358622999902
+36.1797011448651
+36.8817358518479
+32.9340052553516
+34.2514172997588
+20.5728070932139
+24.2867441819747
+34.0883735873178
+14.7686013832293
+33.6238215181336
+15.4417172948152
+9.92778096871947
+21.3862278983355
+16.0666594188815
+2.19368280827428
+35.7750015475985
+47.9442377980571
+36.3642145108211
+16.1670804997349
+30.9268210991669
+29.0930890397466
+9.39787198861618
+12.0603347517275
+23.3581378708581
+31.8595606747619
+46.5754586647756
+32.1273020408239
+48.0293300275147
+25.023141802455
+37.3455406804807
+25.4373628171179
+43.9943092473673
+39.4490570431121
+30.3737445401353
+29.3471817962517
+21.4317166186066
+43.3350523665373
+42.2690119588096
+63.0914783221061
+30.5868667966429
+28.7244061642163
+16.4210310923514
+33.173053108916
+30.5810949146549
+27.0659860423377
+36.8855598056681
+34.1401772457208
+42.6960549938338
+31.7249717828601
+5.67727292080443
+20.6992060939493
+53.215386555044
+57.5228280554584
+28.7892984601347
+23.7408745043356
+42.1298253368303
+35.2163733789351
+44.6187279627215
+25.9796601548142
+30.5484602621591
+41.0689513419417
+29.1978249590735
+17.4520877029339
+36.2238810183006
+36.0106456867899
+54.2061773626609
+33.7231137731149
+25.6749559184246
+32.1320680809629
+46.3015683784878
+10.3348909087495
+81.2678432406593
+37.683904173068
+49.5824087814669
+37242701.0046648
+39.2931237983963
+35.7219665803705
+24.0457468566445
+33.9400187893278
+33.0545953850552
+62.9884805379852
+33.9909131144785
+32.9864746200592
+21.9017590125759
+15.7607141981801
+42.4362226701732
+46.826814329534
+12.2846859012982
+42.3432325763782
+28.8380652093929
+2.68051395039606
+14.4920525027964
+37.2749147059792
+46.9149414645619
+12.2687993530248
+30.4734217867899
+35.0879717232569
+30.3559130719233
+32.2068898028628
+25.6725042311609
+62.7769417048622
+29.9854565292902
+12.5746622619217
+41.2320011098253
+24.2647329497656
+45.927599272008
+31.3216514334173
+28.204987694028
+23.6785616262475
+26.8343561973186
+36.4549490535093
+31.1780823271314
+37.3327562124132
+18.719766791379
+29.2770744070072
+6.47813854892405
+32.745141145454
+31.0119204539338
+43.8198998251529
+34.3763476725833
+5.36318714631069
+20.9921026007633
+18.6706018443156
+36.199579842962
+27.1478823479648
+31.8476971471815
+31.8405025037715
+17.3072793737646
+12.6888007256633
+16.2732924029737
+28.4571004815834
+25.9993895405824
+15.7281364012695
+43.5209259261065
+28.603084098523
+31.6137425834132
+36.9931289310869
+30.7735139639244
+34.5019530201537
+16.6487419189013
+28.1746563786102
+33.6800588564062
+14.9466662941247
+16.7808562087401
+4.33514709351285
+25.2262594932451
+29.014019431901
+33.5000154762296
+38.4702216634443
+26.8505564818935
+41.7492742620753
+46.6967606198392
+23.5765460054117
+6.67312296464655
+31.4997710809271
+30.6920040677714
+35.7883265580235
+2.15314454656326
+10.2464213715088
+13977350.0484038
+35.8637821017554
+47.4153654257425
+37.696917993202
+35.5931416466205
+11.6343953761453
+38.3997825954662
+44.328838349166
+49.7587497571937
+27.3548139644805
+77.2721547837098
+6.00515769251615
+23.2573996200025
+29.4463663664461
+8.64509007736179
+29.3647250197592
+25.6352394407551
+38.1040386265982
+1.57282189593704
+7.5000508377507
+1.90649115174886
+4.94592194709969
+1.50050174826265
+0.33578274444701
+33.1055371094089
+25.3396569639201
+31.8916912834412
+31.9485192605044
+24.5283677284966
+39.0250107792152
+20.2469597191722
+20.8076148662871
+23.2782556250707
+29.1213370411914
+25.7994667560662
+44.2280255529052
+32.227068587326
+28.6616510684596
+39.4572113245522
+34.6112475563304
+15.1114866286368
+8.6368117267197
+15.3719975735809
+41.0486670883178
+31.9294804939604
+31.3271639317971
+26.5445417085818
+15.6004713470743
+59.2269452410631
+19.4193060591509
+36.8788199452948
+41.7213382766497
+37483019.6377987
+39.3518765131612
+36.0816788063388
+43.4564541724341
+46.684376916799
+1.08516544085766
+6.65408678086186
+40.2079575412273
+27.4907195968078
+27.8086514553242
+19.8023723950145
+14.1263989484612
+28.4606874532878
+39.0943450307898
+28.8389416847119
+6.65893567978582
+2.28703078159797
+35.9566335987926
+28.959195763906
+62.789578480917
+40.6004272131859
+31.1387598656041
+35.0509804112574
+28.4673467149618
+44.7073209766234
+38.8507418902988
+35.9301864166999
+31.9544742031004
+0.926413591258883
+9.19654685000719
+36.9589530794359
+46.9274317147032
+7.30383927663766
+25.4407169943222
+28.2602226990245
+26.7628533137403
+40.3213049087107
+37.3126310995381
+42.7682038903999
+34.1858658276769
+43.56566487483
+47.397424213494
+12830459.7173501
+2.79208275505412
+32.2734790595955
+31.9514556956345
+31.5690565837722
+32.5436616201812
+12.7431935531197
+29.4320485741876
+33.2872596550898
+30.5144790383578
+45.5088149382716
+22.9815796099247
+35.5844839652218
+18.8381484968467
+36.3775307081611
+43.0810319319706
+25.311413982231
+34.1508428667737
+34.4795567702849
+37.524577096799
+20.9096225532361
+42.7572226466475
+21.2592159570835
+35.5753171876458
+11.5107938047218
+29.03399334872
+15.764604387829
+29.2060020160819
+31.4734493156368
+43.5272828214643
+45.0464087161133
+29.197968008167
+10864495.4666306
+13977350.0484038
+28.2586440932207
+32.6864659165647
+54.0070556759158
+41.2747706214253
+22.8233435208195
+37.2901692498888
+88.2719399849318
+29.5014878909296
+86708928.9021844
+26.7108247651938
+25.730233233073
+27.922335064203
+27.9218020461618
+26.6419192954522
+53.7719838092185
+44.0605468262766
+52.7481429209776
+17.0534294616657
+24.5486578906439
+43.4608888023014
+10.0918952885953
+41.9883677766318
+34.9207174176165
+40.5137788530363
+34.9850722836867
+21.8435695181715
+38.1145470079375
+37.9022703400474
+28.6923418818745
+32.896327519371
+33.4802004261642
+40.5582201568987
+32.0201594316324
+42.1219040870262
+12.6170014956213
+32.2745476439908
+45.3247913120664
+25.7270716954706
+32.179683860433
+44.1478193314584
+19.8524867744129
+29.6125715180622
+37.4518418177527
+20.3686908682218
+35.8155523465257
+33.0000016148061
+38.5977915712078
+33.994081997638
+28.5480176055392
+20.1269818478051
+13977350.0484038
+35.5297041471319
+7340.15683964806
+100.272640334672
+22.004701340793
+22.4846193557941
+25.0751065668305
+1.29200490884964
+25.0973652511914
+48.0562096419139
+19.5801883446373
+37.7101915095331
+37.8532499299659
+20.988164867089
+51.0504525772545
+34.9203665644575
+23.8020402307289
+23.824255510188
+39.231800998052
+43.7969619050658
+27.2678431351111
+28.892957464015
+24.6265143344788
+30.2556083663084
+22.2910909813762
+33.0552109324791
+51.085614388267
+34.1065179048703
+14.3858460066628
+26.0631018263913
+31.6948433211225
+29.1664960232469
+23.7769108669018
+40.9246650828645
+33.5293078918986
+29.3433565650766
+58.2159767632631
+29.227720484844
+23.0640845576241
+41.0808943395671
+34.090694176252
+42.5797457713762
+44.2675229131497
+27.9271617180627
+42.5870702941033
+25.2936485365987
+31.6981204316386
+47.4302976304271
+33.6829209753143
+33.2620051361909
+45.7061111347244
+33.9381577934741
+18.5810010384807
+23.8740697821465
+17.2143058262112
+28.1504963698318
+29.8642581842716
+41.3611632104984
+20.4301254021114
+35.149255148805
+23.2213979708167
+18.5677557730123
+31.1714946355561
+4.95253228668827
+30.4800525375467
+36.7445166395861
+37.2150598523514
+36.9814797600468
+27.4537648481074
+48.5116667244451
+37.421144911506
+31.5861806446995
+32.024819583937
+7.45173656728783
+35.4602901053597
+44.9798132115994
+35.1032856406347
+41.4491299735083
+32.9451187685279
+47.3585928612907
+17.5438250102995
+16.9178957693034
+34.8294807900943
+33.8431646254822
+36.5868323835929
+11.9482359197467
+27.7687698256752
+36.5199857471255
+34.7769508500776
+34.5027041963426
+32.7790786022242
+37.743935223749
+49.4021386415785
+1.36859433777071
+36.0597072450073
+25.7504223225185
+38.1810236701755
+37.831181537821
+30.6884867657452
+29.6744548025864
+18.7958592459552
+29.854405262605
+23.0167516734473
+36.6980637161065
+34.0699025962033
+35.8126086766098
+47.4356050329254
+31.0042196202943
+27.8575679653269
+22.4006101512056
+35.0173978277475
+41.7354628505566
+56.6873119628197
+41.9412286968403
+44.1657516352348
+34.2911508170756
+36.2798864195224
+56.3939776150129
+115.91251911103
+43.4974516411407
+21.6833878462757
+40.6571266883792
+34.0886309596211
+39.7677845578583
+6.58590290616879
+29.5307536794009
+25.1668753581615
+18.6824096332293
+6.79146146291725
+35.0804036388514
+34.2713606414752
+16.28281237678
+26.401002871313
+16.530103335045
+14.9546804482448
+24.9501940422455
+28.5645097396033
+22.1526574037367
+16.6223460326935
+33.7747572061901
+35.8408097066604
+24.1417848443503
+23.4192670960185
+36.5189427253349
+1.04207592622475
+7.97679483941489
+9.65426230352734
+34.6389121412788
+37.3901997952551
+34.0615881817529
+15.6508208100533
+32.3506224714721
+39.083283757733
+24.3123854330969
+31.8324017871462
+33.0746164214577
+34.7598996961786
+28.1604599122424
+35.796714745061
+38.9772290208026
+23.391285319131
+28.4174666259203
+57.858228118982
+62.7558861096459
+926239778.562071
+27.1741919565029
+29.4006187858472
+37.7616576044623
+36.3699056848601
+14.0296105202725
+25.2106187565497
+24.6314225925546
+41.3872566238397
+23.9102927332768
+39.2159262268178
+42.0088036062556
+38.9921266380431
+30.2573940791447
+23.1321463197272
+1.20493551593686
+1.51295259186131
+20.2119927403806
+24.052966132721
+38.7959178538923
+24.7565524010677
+17.8646093750419
+27.7513428835127
+21.2612795316943
+35.4543420099944
+27.4107683712145
+30.9354717366274
+36.5491658757006
+19.9338050739788
+1.98013419466167
+6.60367734978909
+27.2508048503281
+0.224619453992646
+33.4749975618983
+0.982667462186012
+11.4877470412642
+0.451932730959467
+13977350.0484038
+0.420723080868737
+22.3507559066567
+2.80123414733681
+24.2133665123195
+12.1658454555376
+576263202.9537
+33.5371551406693
+15.0202501940901
+26.2456443904647
+23.0795121904705
+33.5166809988265
+19.3570157998631
+64.0195858914668
+40.5385539273732
+24.415447260235
+32.0324345194811
+10.9438420165274
+38.9413194499419
+45.6973073860831
+51.3573507629927
+31.3815866952027
+31.087420324255
+14.8357231056379
+35.3811183889258
+36.4638649129154
+36.8560426254394
+22.1988155370329
+12.8210793922258
+38.113951439351
+40.2542299244052
+33.7658357237586
+32.0189144208838
+27.0783558108834
+25.4768863968619
+33.1788074278894
+22.7067494457582
+27.6238622304609
+26.0264182839885
+14.1380316043589
+37.6917062554553
+15.8771574592563
+40.5361109987776
+35.0641537098871
+33.4995721296901
+31.8460599953461
+10.5774044039653
+14.3918529905268
+35.3729384602488
+15.970328425516
+31.9442845681941
+43.7812500836841
+38.0244561618714
+27.8284708404924
+36.7760777254267
+39.3477807181393
+39.0304791483558
+43.8117564408778
+41.5842304387908
+13.0201134222344
+36.5502167540738
+17.2022652466439
+31.0195373756488
+32.7259995432786
+38.6301603454024
+12.1422848248274
+20.4021069816149
+15.9294532492086
+17.0146884032145
+4.03061755955935
+29.415314298117
+50.4451616779553
+36.9927399494038
+34.6691955657998
+54.3678456339834
+20.2698823067175
+2.91861998908434
+24.8622644854597
+2.54139970934847
+21.864639148398
+6623952.83445384
+35.8478132365437
+6.85549199969592
+0.763908349185302
+50.0619046203271
+25.4769233653912
+28.0287868258658
+33.9297572725704
+36.7437022808779
+24.2914808840552
+26.289569544095
+40.8710677180471
+28.6046714580993
+41.8681256000139
+39.3226444725995
+27.2878018133409
+30.430157530743
+6.63026430465366
+0.908735752909271
+25.1934950003071
+17.7001084080952
+18.7855650663856
+21.1617114529072
+36.1421201213694
+36.4796700019641
+4.98875714680303
+46.2021487795787
+41.4453617853728
+46.3097907027973
+28.1682346148739
+91.0137473641365
+30.9106824737581
+42.6448399301417
+32.8806273318372
+31.7645296234328
+45.286613683859
+33.4880235117972
+30.1518792572358
+22.1679007303366
+28.9088000217569
+23.4546053805118
+21.4634356059371
+45.2231931653783
+29.5595096917371
+19.4285362160089
+31.3706017059948
+28.2293311905946
+41.4227178854777
+34.936588976069
+32.5067025496945
+37.5857399545564
+36.0893990873515
+44.5461822471565
+42.3471193426381
+18.5742667844412
+26.1378765934042
+37.6163493214784
+20908681.0102167
+40.4161959961656
+40.7513328785523
+24.3193879576893
+33.0820330049274
+10.2598044707616
+39.4767551439231
+38.4936052322783
+45.2046054762952
+34.6946087563183
+34.3471397542748
+30.3209392986562
+47.0277368363015
+42.39851278476
+36.4737072106451
+47.9643430200449
+40.0170988989088
+30.3737812894074
+42.3721678056733
+27.2114484917143
+39.3817746846944
+24.5123613270143
+22.7633518638937
+35.1669157365397
+18.1061194291251
+24.8316570118396
+39.3235141443929
+38.3226238474642
+35.6238650746155
+28.8210498484599
+23.317331475889
+33.4729912602786
+30.8179970891687
+33.9771992123437
+13977350.0484038
+28.3111566013979
+17.3641777077474
+8.6979482265938
+37.3652217718071
+30.74120476868
+38.0435136397926
+28.0406836845952
+23.3277272327126
+48.2690525479752
+31.7648250549487
+22.8995750317211
+75.5512935245717
+14.1855311048376
+30.7962993045752
+41.5944023692066
+27.8425853478054
+45.3548799949754
+646.920273704167
+38.2965953428343
+21.815831352117
+29.3227366420636
+18.5562445863763
+28.5191068728122
+35.0964589025
+1.47424997309179
+0.438601803442839
+8.11814298523033
+20.9141400869598
+16.1498194143429
+24.2150247319169
+16.7117156582896
+30.328168702947
+24.1910931638411
+28.8890771549188
+14.7144775846542
+36.6074849572892
+19.4092735489259
+28.9080877080471
+27.7875780482114
+18.6736251627784
+22.1441533474256
+16.2281317952072
+40.8855838976911
+54.2243070429419
+24.5341436014248
+27.8946264930403
+22.8485856991501
+48.6326573287224
+23.4493568084606
+43.419277588865
+17.7001404825573
+38.4800944590826
+43.6366292075894
+1.58363707366378
+10.6370416143017
+31.4297400222906
+37.2985988060288
+22.7038250130334
+35.013696318282
+24.4228739692669
+28.6008979069196
+46.0073129052734
+42.3742194409422
+31.8538725283038
+30.6472064028641
+39.5494528173431
+20.102822152769
+28.5595803767075
+33.68472422613
+27.5017979912897
+30.0163270304916
+43.1128056705852
+6961003.37827996
+38.2432185093398
+4.97017857199807
+19.2359836003409
+43.3939989132751
+24.07502202711
+23.271628310384
+5.14853905756257
+3.58535156600441
+23.4036028680036
+8.89861721717545
+27.6767439526001
+1.92799905339914
+0.599624514940552
+3.39804513781593
+90.4023704220546
+27.6181079313155
+1.25279479220705
+0.373714060719851
+3.77809301455099
+0.432910222302043
+33.2873927671937
+26.7732451297114
+22.7201995651403
+37.1898231299043
+29.2417675043945
+34.5735835606066
+39.0339665939189
+33.7659019259088
+26.1117856155347
+3.50405088420971
+1.5962169506134
+2.79955878814202
+21.8644886399069
+20908571.3821656
+15.9858709522968
+41.9286104073888
+43.283037427901
+29.6711601995029
+41.1650033405257
+35.1726128369263
+28.9748629209706
+29.0388322365057
+25.5307151711759
+28.6292876051402
+46.2029156383219
+31.52087253549
+37.0898809868955
+1.2783397416144
+24.2744129708803
+28.8554188899716
+24.0505463281475
+18.242034424438
+4.42150818046276
+40.8190735883924
+23.5344977954061
+32.0703158966606
+7.65031781121754
+44.6438257744181
+51.9875876369312
+7.77702335170923
+34.2686482164792
+18.3747857672486
+20.66832393067
+19.511866067104
+43.045821611598
+10.9474718923931
+40.8114987175443
+38.014904959763
+30.6790703706538
+31.8881477647453
+31.4977015923556
+46.1397810163426
+35.4054032630978
+27.8891708215208
+22.8306338731264
+175.801382187653
+40.4395522591724
+28.1393601528214
+6.18202168969741
+17.8349456005274
+10.4177384720951
+22.0862814684146
+56.6333018599047
+32.1940549899885
+28.3880637810846
+17.946286665711
+48.5241319361293
+32.6638979809607
+41.941539137827
+31.0894471826773
+35.3027324577599
+31.7871798643027
+37.4636176384041
+13977350.0484038
+47.6799777581046
+34.7155542777447
+39.7518842261833
+5.34578061980031
+17546875.8187895
+13863839.0228732
+36.7417700244308
+15.8101325785815
+10.731227590222
+23.2224271935664
+38.8573915589576
+31.5331323131999
+21.8329483249101
+40.9807609581475
+26.3480651917681
+32.1976791297458
+10.9951251760613
+26.3643560369741
+23.4308289387187
+29.4202623837525
+23.8791228788152
+28.1122669210832
+10.7058795708762
+38.0423300255999
+24.9599638710793
+30.4288579793575
+38.7418236900368
+27.4783598944998
+23.7546421787607
+52.7684035361227
+33.9863754426253
+38.7958120101163
+44.236637759698
+6.63026350747073
+46.7516388548805
+20.0201028911306
+34.2914305995422
+33.0704609073708
+27.9343859644338
+50.3292935566003
+41.9374612925669
+20.4380599908798
+44.8253705659434
+34.8585466764705
+16.8189943375
+64.7343901269352
+30.5208477352892
+35.7379840149831
+24.1106029662646
+26.8031191000448
+36.9730578773842
+39.4865431907991
+27.4840498614294
+24.6688091213475
+23.4424386250456
+59.5433995890627
+40.2638082454703
+14.4967220489669
+31.0181576383926
+43.6008192365034
+32.5537432061599
+47.1938154465259
+47.844753212916
+38.1813251246458
+33.9182400756837
+43.8621810780604
+27.6513759688512
+32.3593691958892
+24.9378368389127
+21.9065226303801
+26.6455282772353
+26.2460880374402
+36.99925671845
+34.785009263491
+34.2619519958999
+21.0359397084978
+32.1324253973426
+94615590.7164947
+0.720213001230412
+50.3053496774116
+23.6992670071222
+48.1890402501272
+35.7153150771293
+53.0005943257387
+17.9492717506624
+50.4901323237774
+34.5998374499202
+30.0826161569223
+25.4202119983068
+28.7492898291723
+36.3442330078522
+29.5292287286739
+27.3961368946789
+50.1427459065268
+19.2521518733508
+31.004935280391
+25.3939238873365
+23.3932235171773
+8.04834934873451
+13977350.0484038
+13977350.0484038
+25.7750256384046
+31.781029412828
+37.7290075865128
+37.5094888190165
+35.1881215928904
+46.5378913146994
+50.4251174362123
+1.57159061862723
+49606715.9226324
+40.8678730838554
+37.0519038355418
+16.6148566972054
+45.1195216964856
+36.3051088939117
+28.592000130667
+17.8803522439709
+18.1121669812433
+27.343721936736
+15.3824212535977
+18.0662542572464
+29.0038914631079
+3.01145809491305
+45.3574183395931
+39.3505728275179
+8.65613664332881
+12.0666811397287
+28.7955112734047
+31.4639263980793
+23.214977754213
+21.3482819397581
+37.7008843231126
+12.3834698561113
+3.34405115359555
+0.519780290971291
+8.91596755644697
+40.9079724474751
+9.23933717507765
+16.7914016713641
+48.6647458020714
+34.9934684309461
+35.5710629134555
+87.5315846940941
+4.76089091970972
+49.9254562700112
+39.1743041548751
+20.1299494099174
+37.9264387546781
+29.4345087612851
+31.6663631334131
+32.1733931012544
+46.2087766262244
+33.5421732317311
+63.2754348687071
+38.2843328609199
+34.7161321807644
+42.5667636742578
+10.6806707634362
+26.0995555660539
+23.4165030067614
+34.3824146717523
+17.9235288698499
+36.9852904743407
+44.8377757495584
+31.4594608405067
+40.4824337465477
+24.0509172534302
+51.38584746065
+11029039.584798
+29.5476657926558
+25.9290802305564
+24.7610471642983
+43.5400538213138
+32.1695443619599
+13977350.0484038
+39.3384762883315
+40.0665266271343
+37.7684610785527
+22.4748405867818
+33.3241631630118
+49.8291683534406
+6.96668701394689
+47.6132223916447
+34.6171335723686
+33.4474868816703
+70.6772630479127
+28.8414957668918
+38.2574885747212
+38.3867421582094
+29.3121111621344
+26.0350446443477
+0.641027773279853
+31.7461398697594
+28.0937460656421
+44.146035392986
+26.9054661488417
+27.3676231007525
+36.3599607143846
+2.16623291878558
+35.6433520550493
+11.6158721958198
+196.706488488684
+26.5759181153674
+41.2871838578409
+45.6382106953019
+32.0322004933961
+30.4772911640363
+36.0202827838398
+18.7062977346528
+40.9451066769013
+36.2124733535486
+2.14839793590611
+35.1454381797488
+36.6690224957431
+41.1676817233653
+34.9951008946983
+30.1329576975727
+34.5186176868265
+31.790054899312
+33.5310259567253
+0.230682066453569
+17.634184559346
+20.2072163574777
+42.9860762923541
+28.7442554140013
+26.5295148229347
+26.9779905122516
+14.1584954200912
+23.6886569698671
+44.5818141358027
+9.93331645237466
+16.8437432150787
+25.4976030645675
+28.6477516299656
+31.8428281179343
+23.3204977386105
+10979037.3105326
+32.4148509811882
+28.8095147902496
+49.6840918749075
+42.2731690641418
+40.8550065434845
+28.568048793816
+24.8711014183907
+43.1533614825984
+24852233.3872532
+28.3907201759214
+19.7614945015429
+42.4889771578327
+18.3290211783841
+32.595427223468
+41.6442900139403
+41.6602321880765
+45.557700911005
+23.5764254329542
+4.80356041764629
+35.645354746045
+41.4050861901971
+40.9636091328541
+8.17790216546685
+99.8582217860346
+66368880.3137019
+18.9419691352355
+28.4549733516972
+50.1649410162261
+27.5413344680448
+7.28117894348174
+36.2887957574261
+54.7368257029306
+11.4950717111279
+42.4319033524067
+43.7964448007202
+35.0243738686217
+57.7610362826402
+36.4174859617843
+42.1714030535971
+19.7816120954027
+34.8840433463611
+1.53273330661443
+29.3952752675613
+33.9847333937035
+46.9231707128587
+24.9401152131652
+42.6885135739105
+49.9239998263142
+13977350.0484038
+32.2795258079783
+44.1357268140107
+9.62953097299516
+409.011896836639
+36.3829068666951
+42.5957931386094
+54.487760342205
+4.05196607766844
+0.225129854082691
+36.781264569682
+45.4764148788769
+3356622522.55061
+37.5965345119379
+24.4549903492956
+12.0878425873838
+25.7791542879922
+38.5733542218995
+13.4088069783215
+37.2987471651612
+45.2781251872479
+39.4151273890826
+36.4402318642929
+27.8724899967408
+46.5081359135628
+16.7024643259285
+32.0940217070764
+33.8885036675864
+45.6339928164944
+37.3179954276464
+36.6186391486541
+21.8047462036861
+12.9753379750695
+0.923445477695757
+33.1161829825136
+35.1331232341784
+44.2115094876053
+35.388914404576
+30.0820145987799
+38.3904846652041
+14.8306172729216
+34.0491074637193
+32.4603520661509
+4.53451859900154
+22.314457957076
+27.7628065088212
+20.4749629953709
+0.662055340802244
+18.4260942492341
+28.1513698865658
+4.71030256576021
+7.44384563842085
+22.4138654784509
+28.6430343913758
+48.6833809584011
+19.9602940245223
+6.94102385042886
+1.5228186455947
+14.491778080978
+8.14108312923291
+2.2982287055937
+14.6074886607564
+3.45519204324254
+0.767922995426798
+2.12946264604828
+9.94707131285581
+10.4024023742832
+41.9076288875276
+26.7056988251151
+24.3444099057814
+33.5564542980192
+29.4738965512071
+40.9084415707867
+37.254066790509
+31.37726921731
+31.4639123304396
+52.5064288592256
+33.5656188620129
+32.3420339630847
+17.8200091778976
+15.2996816154736
+35.6925424012768
+38.3912364190481
+21.5324623381424
+37.2806826308597
+43.8291605691156
+26.7985209218357
+1.48257229658125
+47.6673517075039
+36.7865530376807
+28.1244593424018
+18.4573035215853
+22.7740564149997
+23.9323495010103
+40.1886305985108
+38.0891317072153
+37.0658230098884
+38.4991001836215
+36.316583506296
+11711462.131437
+34.6610376321034
+34.3735214041034
+24.3427522263964
+28.8272297395414
+44.3038831807068
+6251917690563.78
+24.3795980290854
+32.0041889543731
+30.4590058269368
+44.2169012850302
+12.5074115948072
+19.5066120433835
+32.2453526640287
+43.2306731367693
+37.5223572725536
+33.4749001190099
+32.9381560316588
+35.0117606473822
+34.5633712776492
+58.2860244232489
+37.0709220703068
+25.9525999538736
+22.3289210367343
+50.5286754692538
+36.6154762438231
+35.8893891416291
+27.7915548297465
+12.3354948558481
+18.9605438501163
+26.2037880831687
+5851412.39140027
+40.8887651026613
+30.4066842176316
+30.2476728934183
+52.6114672640846
+30.9869986164827
+39.3341283776267
+29.3677797545875
+34.2089752526683
+36.0489641113408
+33.2515495199823
+1.29032021386657
+34.5098057811042
+35.3936223036581
+38.1882565910211
+49.1872960998442
+47.9177139751389
+64.5793597458067
+35.5220933702293
+36.1036388335062
+33.5759022128953
+39.3229205168644
+36.1310443658245
+39.8053413400007
+35.7638282897531
+29.9322088389147
+39.9325397237535
+45.5748862355622
+44.3473867108346
+40.2748832293088
+44.8164563309911
+46.8921377456135
+32.3017383099547
+38.812022163564
+6620628.54096118
+16.622505615106
+42.2600759182169
+48.1521211785885
+38.9623261711735
+36.3433554465308
+47.1406336294551
+1.33250275017664
+32.2891968870192
+34.1118304011489
+35.1126585288855
+32.317292975928
+20.1195477312348
+27.0092042821796
+27.6915738661577
+33.0949214219743
+19.2289655776855
+49.5190085371707
+24.2757654083812
+2.72966203205154
+36.546260626375
+25.829457451958
+47.6716739472665
+38.8907152046973
+15.4960124756754
+23.3380055746639
+25.9681074396123
+0.864758624761094
+43.6385057299888
+21.8505084699333
+27.7701427868739
+34.1219201245282
+48.7250632018545
+35.433582227074
+27.9130747743473
+26.0101022261342
+13.7849328914604
+44.70956687321
+37.0642216782934
+27.9862559891879
+40.0339817091432
+13977350.0484038
+32.2306901338843
+7.76895679619592
+37.6593727936144
+33.1341042689794
+23.2736311759283
+45.020519006844
+25.4977446301837
+3.93121268337524
+49.2760215274189
+25.1365648583212
+28.6178575220726
+26.2436329871337
+21.783688959894
+13.6128356071164
+13.9168343742084
+26.7480895197312
+15.176127108794
+17.5187896893898
+17557999.8456998
+8.0281731026293
+35.5823712242967
+17.5142477265312
+27.5496703709902
+36.0231566197328
+25.0377158083675
+37.0221760642021
+39.2374428161872
+22.7406258401619
+47.6495517851156
+28.3007373916268
+26.5756596669712
+18.1266002227421
+26.5720938151246
+33.8194896140887
+25.5455425427772
+52.846801850335
+23.9930561886823
+7.23128165830833
+33.8620156520515
+48.2553547524862
+37.0736106412726
+36.173836592547
+28.6981027335435
+30.5006311615756
+50.7200846280409
+25.621749013002
+33.9186170974431
+20.5659973863638
+10.0290353483974
+44.6885785661148
+46.5251058026869
+39.6105206661894
+19.0398763280657
+18.6210575420237
+29.1438307158458
+41.2785347574052
+33.4855266540753
+8.57707327096338
+25.6063748042803
+21.638238565385
+33.4094077446564
+28.9709773865443
+23.3704306905155
+31.4183814876062
+20.4329310710196
+36.8708674181053
+31.2504477308114
+18.8245051365826
+12.5420540886471
+26.7895845800867
+42.4271833003392
+29.3438027572875
+35.5229696088008
+35.4684228642275
+23.938442547344
+25.8205749583446
+40.6320759898991
+14.233107102591
+35.9224848670381
+33.616622337266
+37.4067845963457
+41.5234399348938
+40.4311420763669
+32.3675680046822
+24.4324339368618
+19.797605621928
+32.6881542893517
+26.3584396673102
+23.7831625773258
+37.9466078245368
+23.7305302372652
+23.3634546880184
+28.0996081892615
+35.0316842793788
+29.463324322833
+40.3602453823618
+23.6533035538471
+40.0468067477237
+41.7368480431801
+24.3640430132526
+45.2969043148026
+39.4940151641692
+23.1819161434377
+35.6411265837976
+39.9808909059354
+22.6519434750025
+35.4910963684441
+38.8830778227719
+23.9595724286965
+15.8320736107142
+27.0044464755834
+34.2251512435331
+46.5464389067917
+34.1499779810251
+27.1672666333971
+38.7904885077987
+18.1733341867935
+8.72444016289162
+26.7442222214645
+41.2062347257427
+29.851733728389
+4.09694293798991
+41.3794063602514
+37.810308771284
+23.2972814416132
+32.8057210898312
+17.7489848714991
+26.3004503593866
+35.5287693589231
+18546642.9817036
+0.116689157811337
+36.6060005523983
+48.1840064614227
+33.8251663099339
+12802198.5579437
+24.7582398096826
+22.1665183551676
+56.245156109465
+29.5033598776371
+38.7256595531803
+17.6866549176143
+29.2486033633741
+37.1759159959501
+36.8077165185489
+29.8945813592804
+29.0016651158792
+66.0666601707392
+28.626455775775
+2.46461930518608
+4.03340344303888
+25.9012627258872
+29.4144497679891
+0.816106459543285
+1.28727000696954
+3.45519204324254
+2.12946264604828
+15.575806164191
+13977350.0484038
+10.9819660789254
+15.3832332889304
+36.908659806868
+13.3014543450243
+28.8054723691431
+35.5820787511207
+29.4892967965488
+18.7366265315046
+4.23095372475381
+36.5461204390755
+91.8466882783262
+36.3216566462849
+40.5230014693434
+14.866945773376
+18103981.1918851
+37.8371629343035
+23.1480871935661
+7.9581554095954
+28.5087754611757
+51.979123264394
+34.3984078729228
+25.3329603470731
+34.3258624713803
+57.1518076486115
+31.3365314879359
+37.2939065158685
+37.6083073664129
+42.260966433802
+29.7273479899413
+16.6720176749964
+22.5744059139134
+35.5728819858538
+35.6364948943941
+14.3921023462561
+24.5539521222329
+34.2465247155976
+43.6842034387914
+32.708010579166
+45.4668937810174
+48.2860912208819
+34.6383691255887
+23.0335981990911
+62.4528516081463
+8.38575000120012
+34.2808710836571
+47.3886566706874
+33.6869639102788
+35.8148506604478
+16.6832253412346
+28.4430994171647
+19.5508467243164
+34.6238378387166
+54.2798308475743
+32.9006287171527
+38.8050733696037
+17.1654940650987
+26.9752867745807
+23.4268336474036
+11.5836347117569
+34.1495589527954
+42.5102338017301
+26.9697084168469
+46.3404010230758
+8.9912240409956
+41.9840378646563
+39.8686153874667
+19.0887609109638
+36.4774317483157
+12320560.2517341
+29.124904679471
+68.2018994064665
+27.6450551078873
+26.1224101421628
+29.0060168914152
+51.1921969048322
+67.1705248854237
+35.9468637109047
+41.5516013966921
+36.1458386868696
+2.93191134045604
+23.9928133313198
+27.2250052979721
+0.315815664212775
+43.6835586399131
+31.4579920197339
+35.2455470259667
+28.4911136297892
+43.5395954084198
+42.8615783998988
+28.2062963029849
+23.0647632278069
+33.4484537760435
+0.823156377835705
+50.291498749248
+40.5169002675345
+34.7431586589808
+26.9466714642242
+15.4265750987732
+42.6332669758591
+36.7016625725177
+23.49014918503
+42.4372330296947
+32.2670050350322
+10.5456978379563
+20.7296722031044
+51.7602663419917
+50.80068461974
+41.0035841501562
+37.570640968856
+33.7678852616622
+31.5441830389906
+28.2385501146988
+32.8585779888351
+2.22083537035111
+44.9497185274281
+39.7205813966118
+39.53725434886
+23.9344069603005
+27.1884550340384
+40.6958608537376
+36.3456839186948
+32.6187541849626
+36.1813720535803
+34.3452258412304
+30.7238241000072
+16.9544432842326
+51.7108432760167
+36.1479567472607
+48.1920019041162
+23.8140725371052
+28.5884978003868
+17.2919860593651
+35.6353131625628
+37.4188424249264
+42.7552599131468
+17.1479874209072
+30.7966251848126
+30.6256078404971
+22.2948949920994
+46.2732499247292
+38.3329793929881
+27.3375699173509
+25.9610612527182
+4.51125585413206
+0.493080270438857
+32.2024958957812
+28.7588809547962
+28.1760170868437
+31.910515121459
+18.0176980168779
+11.3982787494776
+27.5454269896919
+38.3036163532455
+11.573232109311
+42.698549208263
+25743607.1707683
+36.4746839452661
+30.0743967437084
+28.0083858043855
+14.8764288866505
+13.4974553306539
+17.3364564806221
+13.0755008374797
+23.7516326461706
+29.6750973242849
+32.3131922383326
+22.3915335843711
+1.76260972572515
+0.795483854023975
+16.2151742546337
+94615590.7164947
+0.952983267547465
+4.10832956664437
+41.9017058095861
+13977350.0484038
+24.3896739375475
+13977350.0484038
+25.3182315398841
+30.9051129725501
+35.1163564256488
+53.332756470087
+24.871238372248
+39.0537278211782
+29.9450276604715
+43.657076688755
+35.3185352634128
+0.428985556166751
+33499555.6509129
+28377964.1192869
+45.5291200580358
+36.9196666784061
+40.2377474059723
+36.0885228972739
+17.2061387436347
+19.5478128143902
+30.1927262650063
+6.88427413026192
+22.5579267910941
+17.1111308551667
+49.0856664444993
+36.3134419049975
+20.1329907782521
+26.0219999696411
+21.1709996269255
+22.465904417591
+34.5526023439062
+36.8723991431319
+26.3664111919829
+32.3558460996707
+34.4682065489642
+38.2020167693969
+13.4424812828707
+13.1792783400502
+24.5039625933776
+4.47823222837778
+20.4632674824453
+1.7661905542463
+21.3276074821004
+38.6638726679685
+15.3636789014107
+13.9872816697543
+138.172038922239
+4.84810335803077
+10.9819660789254
+0.69740528293011
+6.95799894806845
+10319568.1886302
+13977350.0484038
+22.5525118938596
+1.99235066747005
+47.3335590780915
+23.2583325839921
+21.8896302908319
+17.5868557211962
+43.1948267436262
+6.82912380741596
+26.0448266644018
+13977350.0484038
+31.9873890425852
+12.7465669319934
+17.8826161954662
+47.0957468873644
+50.3410931348339
+102.423389946734
+4.60221491851754
+66368880.3137019
+552619554.403846
+14.090563755069
+14.403121412507
+22.5370889630594
+17.5969933083643
+35.6462921254033
+7.18545679354382
+2.79171174018668
+2.69331825200681
+2.79043419505352
+2.27533076591777
diff --git a/vignettes/figs/diagnostic_plot.png b/man/figures/diagnostic_plot.png
similarity index 100%
rename from vignettes/figs/diagnostic_plot.png
rename to man/figures/diagnostic_plot.png
diff --git a/vignettes/figs/htrfit_workflow.png b/man/figures/htrfit_workflow.png
similarity index 100%
rename from vignettes/figs/htrfit_workflow.png
rename to man/figures/htrfit_workflow.png
diff --git a/man/performance-class.Rd b/man/performance-class.Rd
index 9e8e4d53da0679f529019eefb02826695c77afe2..3d3bc11f542186cdbd9bd6ffef695ab637853778 100644
--- a/man/performance-class.Rd
+++ b/man/performance-class.Rd
@@ -1,41 +1,14 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/fake-section-title.R, R/na-na.R
+% Please edit documentation in R/fake-section-title.R
\docType{class}
\name{performance-class}
\alias{performance-class}
\title{Class \code{performance}}
\description{
-Object to capture the result of a performance evaluation, optionally
-collecting evaluations from several cross-validation or bootstrapping runs.
-
Object to capture the result of a performance evaluation, optionally
collecting evaluations from several cross-validation or bootstrapping runs.
}
\details{
-A \code{performance} object can capture information from four
-different evaluation scenarios:
-\itemize{
-\item The behaviour of a cutoff-dependent performance measure across
-the range of all cutoffs (e.g. \code{performance( predObj, 'acc' )} ). Here,
-\code{x.values} contains the cutoffs, \code{y.values} the
-corresponding values of the performance measure, and
-\code{alpha.values} is empty.\cr
-\item The trade-off between two performance measures across the
-range of all cutoffs (e.g. \code{performance( predObj,
- 'tpr', 'fpr' )} ). In this case, the cutoffs are stored in
-\code{alpha.values}, while \code{x.values} and \code{y.values}
-contain the corresponding values of the two performance measures.\cr
-\item A performance measure that comes along with an obligatory
-second axis (e.g. \code{performance( predObj, 'ecost' )} ). Here, the measure values are
-stored in \code{y.values}, while the corresponding values of the
-obligatory axis are stored in \code{x.values}, and \code{alpha.values}
-is empty.\cr
-\item A performance measure whose value is just a scalar
-(e.g. \code{performance( predObj, 'auc' )} ). The value is then stored in
-\code{y.values}, while \code{x.values} and \code{alpha.values} are
-empty.
-}
-
A \code{performance} object can capture information from four
different evaluation scenarios:
\itemize{
@@ -79,50 +52,20 @@ other.}
\item{\code{y.values}}{A list in which each entry contains the y values of the curve
of this particular cross-validation run.}
-\item{\code{alpha.values}}{A list in which each entry contains the cutoff values of
-the curve of this particular cross-validation run.}
-
-\item{\code{x.name}}{Performance measure used for the x axis.}
-
-\item{\code{y.name}}{Performance measure used for the y axis.}
-
-\item{\code{alpha.name}}{Name of the unit that is used to create the parametrized
-curve. Currently, curves can only be parametrized by cutoff, so
-\code{alpha.name} is either \code{none} or \code{cutoff}.}
-
-\item{\code{x.values}}{A list in which each entry contains the x values of the curve
-of this particular cross-validation run. \code{x.values[[i]]},
-\code{y.values[[i]]}, and \code{alpha.values[[i]]} correspond to each
-other.}
-
-\item{\code{y.values}}{A list in which each entry contains the y values of the curve
-of this particular cross-validation run.}
-
\item{\code{alpha.values}}{A list in which each entry contains the cutoff values of
the curve of this particular cross-validation run.}
}}
\section{Objects from the Class}{
-Objects can be created by using the \code{performance} function.
-
-
Objects can be created by using the \code{performance} function.
}
\references{
-A detailed list of references can be found on the ROCR homepage at
-\url{http://rocr.bioinf.mpi-sb.mpg.de}.
-
A detailed list of references can be found on the ROCR homepage at
\url{http://rocr.bioinf.mpi-sb.mpg.de}.
}
\seealso{
-\code{\link{prediction}}
-\code{\link{performance}},
-\code{\link{prediction-class}},
-\code{\link{plot.performance}}
-
\code{\link{prediction}}
\code{\link{performance}},
\code{\link{prediction-class}},
diff --git a/man/performance.Rd b/man/performance.Rd
index aebc6c484d4db008f3185db80769c84430f9631f..886aee2b7bccdf04404eb1f60278cd14d4fabf34 100644
--- a/man/performance.Rd
+++ b/man/performance.Rd
@@ -1,11 +1,9 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/fake-section-title.R, R/na-na.R
+% Please edit documentation in R/fake-section-title.R
\name{performance}
\alias{performance}
\title{Function to create performance objects}
\usage{
-performance(prediction.obj, measure, x.measure = "cutoff", ...)
-
performance(prediction.obj, measure, x.measure = "cutoff", ...)
}
\arguments{
@@ -23,13 +21,9 @@ the y axis, is created. This curve is parametrized with the cutoff.}
\item{...}{Optional arguments (specific to individual performance measures).}
}
\value{
-An S4 object of class \code{performance}.
-
An S4 object of class \code{performance}.
}
\description{
-All kinds of predictor evaluations are performed using this function.
-
All kinds of predictor evaluations are performed using this function.
}
\details{
@@ -178,163 +172,8 @@ Accepts the optional parameters \code{cost.fp} and
negatives can be adjusted, respectively. By default, both are set
to 1.}
}
-
-Here is the list of available performance measures. Let Y and
-\eqn{\hat{Y}}{Yhat} be random variables representing the class and the prediction for
-a randomly drawn sample, respectively. We denote by
-\eqn{\oplus}{+} and \eqn{\ominus}{-} the positive and
-negative class, respectively. Further, we use the following
-abbreviations for empirical quantities: P (\# positive
-samples), N (\# negative samples), TP (\# true positives), TN (\# true
-negatives), FP (\# false positives), FN (\# false negatives).
-\describe{
-\item{\code{acc}:}{accuracy. \eqn{P(\hat{Y}=Y)}{P(Yhat = Y)}. Estimated
-as: \eqn{\frac{TP+TN}{P+N}}{(TP+TN)/(P+N)}.}
-\item{\code{err}:}{Error rate. \eqn{P(\hat{Y}\ne Y)}{P(Yhat !=
- Y)}. Estimated as: \eqn{\frac{FP+FN}{P+N}}{(FP+FN)/(P+N)}.}
-\item{\code{fpr}:}{False positive rate. \eqn{P(\hat{Y}=\oplus | Y =
- \ominus)}{P(Yhat = + | Y = -)}. Estimated as:
-\eqn{\frac{FP}{N}}{FP/N}.}
-\item{\code{fall}:}{Fallout. Same as \code{fpr}.}
-\item{\code{tpr}:}{True positive
-rate. \eqn{P(\hat{Y}=\oplus|Y=\oplus)}{P(Yhat = + | Y = +)}. Estimated
-as: \eqn{\frac{TP}{P}}{TP/P}.}
-\item{\code{rec}:}{recall. Same as \code{tpr}.}
-\item{\code{sens}:}{sensitivity. Same as \code{tpr}.}
-\item{\code{fnr}:}{False negative
-rate. \eqn{P(\hat{Y}=\ominus|Y=\oplus)}{P(Yhat = - | Y =
- +)}. Estimated as: \eqn{\frac{FN}{P}}{FN/P}.}
-\item{\code{miss}:}{Miss. Same as \code{fnr}.}
-\item{\code{tnr}:}{True negative rate. \eqn{P(\hat{Y} =
- \ominus|Y=\ominus)}{P(Yhat = - | Y = -)}.}
-\item{\code{spec}:}{specificity. Same as \code{tnr}.}
-\item{\code{ppv}:}{Positive predictive
-value. \eqn{P(Y=\oplus|\hat{Y}=\oplus)}{P(Y = + | Yhat =
- +)}. Estimated as: \eqn{\frac{TP}{TP+FP}}{TP/(TP+FP)}.}
-\item{\code{prec}:}{precision. Same as \code{ppv}.}
-\item{\code{npv}:}{Negative predictive
-value. \eqn{P(Y=\ominus|\hat{Y}=\ominus)}{P(Y = - | Yhat =
- -)}. Estimated as: \eqn{\frac{TN}{TN+FN}}{TN/(TN+FN)}.}
-\item{\code{pcfall}:}{Prediction-conditioned
-fallout. \eqn{P(Y=\ominus|\hat{Y}=\oplus)}{P(Y = - | Yhat =
- +)}. Estimated as: \eqn{\frac{FP}{TP+FP}}{FP/(TP+FP)}.}
-\item{\code{pcmiss}:}{Prediction-conditioned
-miss. \eqn{P(Y=\oplus|\hat{Y}=\ominus)}{P(Y = + | Yhat =
- -)}. Estimated as: \eqn{\frac{FN}{TN+FN}}{FN/(TN+FN)}.}
-\item{\code{rpp}:}{Rate of positive predictions. \eqn{P( \hat{Y} =
- \oplus)}{P(Yhat = +)}. Estimated as: (TP+FP)/(TP+FP+TN+FN).}
-\item{\code{rnp}:}{Rate of negative predictions. \eqn{P( \hat{Y} =
- \ominus)}{P(Yhat = -)}. Estimated as: (TN+FN)/(TP+FP+TN+FN).}
-\item{\code{phi}:}{Phi correlation coefficient. \eqn{\frac{TP \cdot
- TN - FP \cdot FN}{\sqrt{ (TP+FN) \cdot (TN+FP) \cdot (TP+FP)
- \cdot (TN+FN)}}}{(TP*TN -
- FP*FN)/(sqrt((TP+FN)*(TN+FP)*(TP+FP)*(TN+FN)))}. Yields a
-number between -1 and 1, with 1 indicating a perfect
-prediction, 0 indicating a random prediction. Values below 0
-indicate a worse than random prediction.}
-\item{\code{mat}:}{Matthews correlation coefficient. Same as \code{phi}.}
-\item{\code{mi}:}{Mutual information. \eqn{I(\hat{Y},Y) := H(Y) -
- H(Y|\hat{Y})}{I(Yhat, Y) := H(Y) - H(Y | Yhat)}, where H is the
-(conditional) entropy. Entropies are estimated naively (no bias
-correction).}
-\item{\code{chisq}:}{Chi square test statistic. \code{?chisq.test}
-for details. Note that R might raise a warning if the sample size
-is too small.}
-\item{\code{odds}:}{Odds ratio. \eqn{\frac{TP \cdot TN}{FN \cdot
- FP}}{(TP*TN)/(FN*FP)}. Note that odds ratio produces
-Inf or NA values for all cutoffs corresponding to FN=0 or
-FP=0. This can substantially decrease the plotted cutoff region.}
-\item{\code{lift}:}{Lift
-value. \eqn{\frac{P(\hat{Y}=\oplus|Y=\oplus)}{P(\hat{Y}=\oplus)}}{P(Yhat = + |
- Y = +)/P(Yhat = +)}.}
-\item{\code{f}:}{precision-recall F measure (van Rijsbergen, 1979). Weighted
-harmonic mean of precision (P) and recall (R). \eqn{F =
- \frac{1}{\alpha \frac{1}{P} + (1-\alpha)\frac{1}{R}}}{F = 1/
- (alpha*1/P + (1-alpha)*1/R)}. If
-\eqn{\alpha=\frac{1}{2}}{alpha=1/2}, the mean is balanced. A
-frequent equivalent formulation is
-\eqn{F = \frac{(\beta^2+1) \cdot P \cdot R}{R + \beta^2 \cdot
- P}}{F = (beta^2+1) * P * R / (R + beta^2 * P)}. In this formulation, the
-mean is balanced if \eqn{\beta=1}{beta=1}. Currently, ROCR only accepts
-the alpha version as input (e.g. \eqn{\alpha=0.5}{alpha=0.5}). If no
-value for alpha is given, the mean will be balanced by default.}
-\item{\code{rch}:}{ROC convex hull. A ROC (=\code{tpr} vs \code{fpr}) curve
-with concavities (which represent suboptimal choices of cutoff) removed
-(Fawcett 2001). Since the result is already a parametric performance
-curve, it cannot be used in combination with other measures.}
-\item{\code{auc}:}{Area under the ROC curve. This is equal to the value of the
-Wilcoxon-Mann-Whitney test statistic and also the probability that the
-classifier will score are randomly drawn positive sample higher than a
-randomly drawn negative sample. Since the output of
-\code{auc} is cutoff-independent, this
-measure cannot be combined with other measures into a parametric
-curve. The partial area under the ROC curve up to a given false
-positive rate can be calculated by passing the optional parameter
-\code{fpr.stop=0.5} (or any other value between 0 and 1) to
-\code{performance}.}
-\item{\code{aucpr}:}{Area under the precision/recall curve. Since the output
-of \code{aucpr} is cutoff-independent, this measure cannot be combined
-with other measures into a parametric curve.}
-\item{\code{prbe}:}{precision-recall break-even point. The cutoff(s) where
-precision and recall are equal. At this point, positive and negative
-predictions are made at the same rate as their prevalence in the
-data. Since the output of
-\code{prbe} is just a cutoff-independent scalar, this
-measure cannot be combined with other measures into a parametric curve.}
-\item{\code{cal}:}{Calibration error. The calibration error is the
-absolute difference between predicted confidence and actual reliability. This
-error is estimated at all cutoffs by sliding a window across the
-range of possible cutoffs. The default window size of 100 can be
-adjusted by passing the optional parameter \code{window.size=200}
-to \code{performance}. E.g., if for several
-positive samples the output of the classifier is around 0.75, you might
-expect from a well-calibrated classifier that the fraction of them
-which is correctly predicted as positive is also around 0.75. In a
-well-calibrated classifier, the probabilistic confidence estimates
-are realistic. Only for use with
-probabilistic output (i.e. scores between 0 and 1).}
-\item{\code{mxe}:}{Mean cross-entropy. Only for use with
-probabilistic output. \eqn{MXE :=-\frac{1}{P+N}( \sum_{y_i=\oplus}
- ln(\hat{y}_i) + \sum_{y_i=\ominus} ln(1-\hat{y}_i))}{MXE := - 1/(P+N) \sum_{y_i=+}
- ln(yhat_i) + \sum_{y_i=-} ln(1-yhat_i)}. Since the output of
-\code{mxe} is just a cutoff-independent scalar, this
-measure cannot be combined with other measures into a parametric curve.}
-\item{\code{rmse}:}{Root-mean-squared error. Only for use with
-numerical class labels. \eqn{RMSE:=\sqrt{\frac{1}{P+N}\sum_i (y_i
- - \hat{y}_i)^2}}{RMSE := sqrt(1/(P+N) \sum_i (y_i -
- yhat_i)^2)}. Since the output of
-\code{rmse} is just a cutoff-independent scalar, this
-measure cannot be combined with other measures into a parametric curve.}
-\item{\code{sar}:}{Score combinining performance measures of different
-characteristics, in the attempt of creating a more "robust"
-measure (cf. Caruana R., ROCAI2004):
-SAR = 1/3 * ( accuracy + Area under the ROC curve + Root
-mean-squared error ).}
-\item{\code{ecost}:}{Expected cost. For details on cost curves,
-cf. Drummond&Holte 2000,2004. \code{ecost} has an obligatory x
-axis, the so-called 'probability-cost function'; thus it cannot be
-combined with other measures. While using \code{ecost} one is
-interested in the lower envelope of a set of lines, it might be
-instructive to plot the whole set of lines in addition to the lower
-envelope. An example is given in \code{demo(ROCR)}.}
-\item{\code{cost}:}{Cost of a classifier when
-class-conditional misclassification costs are explicitly given.
-Accepts the optional parameters \code{cost.fp} and
-\code{cost.fn}, by which the costs for false positives and
-negatives can be adjusted, respectively. By default, both are set
-to 1.}
-}
}
\note{
-Here is how to call \code{performance()} to create some standard
-evaluation plots:
-\describe{
-\item{ROC curves:}{measure="tpr", x.measure="fpr".}
-\item{precision/recall graphs:}{measure="prec", x.measure="rec".}
-\item{sensitivity/specificity plots:}{measure="sens", x.measure="spec".}
-\item{Lift charts:}{measure="lift", x.measure="rpp".}
-}
-
Here is how to call \code{performance()} to create some standard
evaluation plots:
\describe{
diff --git a/man/prediction-class.Rd b/man/prediction-class.Rd
index 87a05dabbb5e3ec149315fdcda6b6f2d22bb43fb..472b9af76bed7e16d3fa0ae6e855ed89da7c76e5 100644
--- a/man/prediction-class.Rd
+++ b/man/prediction-class.Rd
@@ -1,14 +1,10 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/fake-section-title.R, R/na-na.R
+% Please edit documentation in R/fake-section-title.R
\docType{class}
\name{prediction-class}
\alias{prediction-class}
\title{Class \code{prediction}}
\description{
-Object to encapsulate numerical predictions together with the
-corresponding true class labels, optionally collecting predictions and
-labels for several cross-validation or bootstrapping runs.
-
Object to encapsulate numerical predictions together with the
corresponding true class labels, optionally collecting predictions and
labels for several cross-validation or bootstrapping runs.
@@ -45,46 +41,10 @@ samples in the given x-validation run.}
samples predicted as positive at the cutoffs given in the corresponding
'cutoffs' entry.}
-\item{\code{n.neg.pred}}{As n.pos.pred, but for negatively predicted samples.}
-
-\item{\code{predictions}}{A list, in which each element is a vector of predictions
-(the list has length > 1 for x-validation data.}
-
-\item{\code{labels}}{Analogously, a list in which each element is a vector of true
-class labels.}
-
-\item{\code{cutoffs}}{A list in which each element is a vector of all necessary
-cutoffs. Each cutoff vector consists of the predicted scores (duplicates
-removed), in descending order.}
-
-\item{\code{fp}}{A list in which each element is a vector of the number (not the
-rate!) of false positives induced by the cutoffs given in the corresponding
-'cutoffs' list entry.}
-
-\item{\code{tp}}{As fp, but for true positives.}
-
-\item{\code{tn}}{As fp, but for true negatives.}
-
-\item{\code{fn}}{As fp, but for false negatives.}
-
-\item{\code{n.pos}}{A list in which each element contains the number of positive
-samples in the given x-validation run.}
-
-\item{\code{n.neg}}{As n.pos, but for negative samples.}
-
-\item{\code{n.pos.pred}}{A list in which each element is a vector of the number of
-samples predicted as positive at the cutoffs given in the corresponding
-'cutoffs' entry.}
-
\item{\code{n.neg.pred}}{As n.pos.pred, but for negatively predicted samples.}
}}
\note{
-Every \code{prediction} object contains information about the 2x2
-contingency table consisting of tp,tn,fp, and fn, along with the
-marginal sums n.pos,n.neg,n.pos.pred,n.neg.pred, because these form
-the basis for many derived performance measures.
-
Every \code{prediction} object contains information about the 2x2
contingency table consisting of tp,tn,fp, and fn, along with the
marginal sums n.pos,n.neg,n.pos.pred,n.neg.pred, because these form
@@ -92,18 +52,10 @@ the basis for many derived performance measures.
}
\section{Objects from the Class}{
-Objects can be created by using the \code{prediction} function.
-
-
Objects can be created by using the \code{prediction} function.
}
\seealso{
-\code{\link{prediction}},
-\code{\link{performance}},
-\code{\link{performance-class}},
-\code{\link{plot.performance}}
-
\code{\link{prediction}},
\code{\link{performance}},
\code{\link{performance-class}},
diff --git a/man/prediction.Rd b/man/prediction.Rd
index 9d9848629d46cbad846609a5b79319dd363eae78..1d20b7865e3971ae541a0d0062d7d3fdd3186dcb 100644
--- a/man/prediction.Rd
+++ b/man/prediction.Rd
@@ -1,11 +1,9 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/fake-section-title.R, R/na-na.R
+% Please edit documentation in R/fake-section-title.R
\name{prediction}
\alias{prediction}
\title{Function to create prediction objects}
\usage{
-prediction(predictions, labels, label.ordering = NULL)
-
prediction(predictions, labels, label.ordering = NULL)
}
\arguments{
@@ -20,16 +18,9 @@ be changed by supplying a vector containing the negative and the positive
class label.}
}
\value{
-An S4 object of class \code{prediction}.
-
An S4 object of class \code{prediction}.
}
\description{
-Every classifier evaluation using ROCR starts with creating a
-\code{prediction} object. This function is used to transform the input data
-(which can be in vector, matrix, data frame, or list form) into a
-standardized format.
-
Every classifier evaluation using ROCR starts with creating a
\code{prediction} object. This function is used to transform the input data
(which can be in vector, matrix, data frame, or list form) into a
@@ -57,38 +48,6 @@ FALSE < TRUE). Use \code{label.ordering} to override this default
ordering. Please note that the ordering can be locale-dependent
e.g. for character labels '-1' and '1'.
-Currently, ROCR supports only binary classification (extensions toward
-multiclass classification are scheduled for the next release,
-however). If there are more than two distinct label symbols, execution
-stops with an error message. If all predictions use the same two
-symbols that are used for the labels, categorical predictions are
-assumed. If there are more than two predicted values, but all numeric,
-continuous predictions are assumed (i.e. a scoring
-classifier). Otherwise, if more than two symbols occur in the
-predictions, and not all of them are numeric, execution stops with an
-error message.
-
-\code{predictions} and \code{labels} can simply be vectors of the same
-length. However, in the case of cross-validation data, different
-cross-validation runs can be provided as the \emph{columns} of a matrix or
-data frame, or as the entries of a list. In the case of a matrix or
-data frame, all cross-validation runs must have the same length, whereas
-in the case of a list, the lengths can vary across the cross-validation
-runs. Internally, as described in section 'Value', all of these input
-formats are converted to list representation.
-
-Since scoring classifiers give relative tendencies towards a negative
-(low scores) or positive (high scores) class, it has to be declared
-which class label denotes the negative, and which the positive class.
-Ideally, labels should be supplied as ordered factor(s), the lower
-level corresponding to the negative class, the upper level to the
-positive class. If the labels are factors (unordered), numeric,
-logical or characters, ordering of the labels is inferred from
-R's built-in \code{<} relation (e.g. 0 < 1, -1 < 1, 'a' < 'b',
-FALSE < TRUE). Use \code{label.ordering} to override this default
-ordering. Please note that the ordering can be locale-dependent
-e.g. for character labels '-1' and '1'.
-
Currently, ROCR supports only binary classification (extensions toward
multiclass classification are scheduled for the next release,
however). If there are more than two distinct label symbols, execution
diff --git a/man/tidy_results.Rd b/man/tidy_results.Rd
index 03479bb5ba409d3eaf954dc1eb5c2e6572c64f93..9e0cda692dfc52857488024a8bcb3fe0a1f69bae 100644
--- a/man/tidy_results.Rd
+++ b/man/tidy_results.Rd
@@ -16,10 +16,7 @@ tidy_results(
\item{coeff_threshold}{A non-negative value which specifies a ln(fold change) threshold. The Threshold is used for the Wald test to determine whether the coefficient (β) is significant or not, depending on \code{alt_hypothesis} parameter. Default is 0, ln(FC = 1).}
-\item{correction_method}{a character string indicating the correction method to apply to p-values. Possible values are:
-"holm", "hochberg", "hommel", #' "bonferroni", "BH", "BY", "fdr", and "none".}
-
-\item{alt_hypothesis}{Alternative hypothesis for the Wald test (default is "greaterAbs").
+\item{alternative_hypothesis}{Alternative hypothesis for the Wald test (default is "greaterAbs").
Possible choice:
"greater"
\itemize{
@@ -30,6 +27,9 @@ or two-tailed alternative:
"greaterAbs"
\item |β| > coeff_threshold
}}
+
+\item{correction_method}{a character string indicating the correction method to apply to p-values. Possible values are:
+"holm", "hochberg", "hommel", #' "bonferroni", "BH", "BY", "fdr", and "none".}
}
\value{
A tidy data frame containing the results of statistical tests for the estimated coefficients.
diff --git a/pkgdown/_pkgdown.yml b/pkgdown/_pkgdown.yml
index a6e61fa36069b8b4e5c7322996e7afdb67e8e656..364b42953243ccec0a60b43224c5ee690e10c54f 100644
--- a/pkgdown/_pkgdown.yml
+++ b/pkgdown/_pkgdown.yml
@@ -3,22 +3,18 @@ repo:
home: https://gitbio.ens-lyon.fr/LBMC/yvertlab/vortex/plasticity_mutation/HTRfit
navbar:
structure:
- left: [home, intro, reference, articles, tutorials, news]
- right: [gitlab]
+ left: [home, intro, reference, articles, news]
+ right: [search , gitlab]
components:
gitlab:
icon: fab fa-gitlab fa-lg
- href: https://gitbio.ens-lyon.fr/LBMC/yvertlab/vortex/plasticity_mutation/HTRfit
- news:
- releases:
- - text: "HTRfit 2.0.0"
- href: https://gitbio.ens-lyon.fr/LBMC/yvertlab/vortex/plasticity_mutation/HTRfit/-/releases/v1.0.2
- - text: "HTRfit 1.0.2"
- href: https://gitbio.ens-lyon.fr/LBMC/yvertlab/vortex/plasticity_mutation/HTRfit/-/releases/v1.0.2
- - text: "HTRfit 1.0.1"
- href: https://gitbio.ens-lyon.fr/LBMC/yvertlab/vortex/plasticity_mutation/HTRfit/-/releases/v1.0.1
- - text: "HTRfit 1.0.0"
- href: https://gitbio.ens-lyon.fr/LBMC/yvertlab/vortex/plasticity_mutation/HTRfit/-/releases/v1.0.0
-
-
-
\ No newline at end of file
+ href: https://gitbio.ens-lyon.fr/LBMC/yvertlab/vortex/plasticity_mutation/HTRfits
+template:
+ bootstrap: 5
+ bslib:
+ primary: "#0054AD"
+ border-radius: 0.5rem
+ btn-border-radius: 0.25rem
+ includes:
+ in_header:
+ <script defer data-domain="pkgdown.r-lib.org,all.tidyverse.org" src="https://plausible.io/js/plausible.js"></script>
diff --git a/vignettes/htrfit.Rmd b/vignettes/htrfit.Rmd
deleted file mode 100644
index b376ffac4126e99d1b8535da066abb1797b03e60..0000000000000000000000000000000000000000
--- a/vignettes/htrfit.Rmd
+++ /dev/null
@@ -1,603 +0,0 @@
----
-title: "HTRfit"
-output: rmarkdown::html_vignette
-vignette: >
- %\VignetteIndexEntry{htrfit}
- %\VignetteEngine{knitr::rmarkdown}
- %\VignetteEncoding{UTF-8}
----
-
-```{r, include = FALSE}
-knitr::opts_chunk$set(
- collapse = TRUE,
- comment = "#>"
-)
-```
-
-```{r setup}
-library(HTRfit)
-```
-
-<!-- WARNING - This vignette is generated by {fusen} from /dev/flat_full.Rmd: do not edit by hand -->
-
-<!-- Run this 'development' chunk -->
-<!-- Store every call to library() that you need to explore your functions -->
-
-
-<!--
- You need to run the 'description' chunk in the '0-dev_history.Rmd' file before continuing your code there.
-
-If it is the first time you use {fusen}, after 'description', you can directly run the last chunk of the present file with inflate() inside.
--->
-
-
-# High-Throughput RNA-seq model fit
-
-In the realm of RNAseq analysis, various key experimental parameters play a crucial role in influencing the statistical power to detect expression changes. Parameters such as sequencing depth, the number of replicates, and others are expected to impact this statistical power. To help with the selection of optimal values for these experimental parameters, we introduce a comprehensive statistical framework known as **HTRfit**, underpinned by computational simulation. **HTRfit** serves as a versatile tool, not only for simulation but also for conducting differential expression analysis. It facilitates this analysis by fitting Generalized Linear Models (GLMs) with multiple variables, which could encompass genotypes, environmental factors, and more. These GLMs are highly adaptable, allowing the incorporation of fixed effects, mixed effects, and interactions between variables.
-
-
-
-
-# Initializing variables for simulation
-
-The `init_variable()` function is used for defining the variables in your experimental design. Sizes of effects for each variable and interaction term can be defined in two different ways: 1) The user can manually sets values of all levels of a variable, in which case the effects are necessarily considered fixed in the model; 2) The effects can be randomly picked in a normal distribution with mean and standard deviation defined by the user, in which case the user can decide whether the effects are considered fixed or random in the model.
-
-
-
-## Manually init a single variable
-
-The `init_variable()` function allows for precise control over the variables in your experimental design.
-In this example, we manually initialize **varA** with specific size effects (mu) for three levels.
-
-
-
-```{r example-init_variable_man, warning = FALSE, message = FALSE}
-list_var <- init_variable( name = "varA", mu = c(0.2, 4, -3), level = 3)
-```
-
-## Randomly init a single variable
-
-Alternatively, you can randomly initialize **varA** by specifying a mean (mu) and standard deviation (sd).
-This introduces variability into **varA**, making it either a fixed or random effect in your design.
-
-
-```{r example-init_variable_rand, warning = FALSE, message = FALSE}
-list_var <- init_variable( name = "varA", mu = 10, sd = 0.2, level = 5)
-```
-
-## Randomly init several variables
-
-You can also initialize multiple variables, such as **varA** and **varB**, with random values.
-This flexibility allows you to create diverse experimental designs.
-
-
-
-```{r example-init_variable_mult, warning = FALSE, message = FALSE}
-list_var <- init_variable( name = "varA", mu = 10, sd = 0.2, level = 5) %>%
- init_variable( name = "varB", mu = -3, sd = 0.34, level = 2)
-```
-
-## Add interaction between variable
-
-Similarly to `init_variable()`, `add_interaction()` allows to define an interaction between variable.
-
-In this example, we initialize **varA** and **varB**, and create an interaction between **varA**, and **varB** using `add_interaction()`.
-
-
-
-```{r example-add_interaction, warning = FALSE, message = FALSE}
-list_var <- init_variable( name = "varA", mu = 3, sd = 0.2, level = 2) %>%
- init_variable( name = "varB", mu = 2, sd = 0.43, level = 2) %>%
- add_interaction( between_var = c("varA", "varB"), mu = 0.44, sd = 0.2)
-```
-
-## Complex design
-
-Interactions can involve a maximum of three variables, such as **varA**, **varB**, and **varC**.
-
-
-
-```{r example-add_interaction_complex, eval = FALSE, message = FALSE, warning = FALSE, include = TRUE}
-## /!\ not evaluated in Vignette
-list_var <- init_variable( name = "varA", mu = 5, sd = 0.2, level = 2) %>%
- init_variable( name = "varB", mu = 1, sd = 0.78, level = 2) %>%
- init_variable( name = "varC", mu = c(2, 3), sd = NA, level = 2) %>%
- add_interaction( between_var = c("varA", "varC"), mu = 0.44, sd = 0.2) %>%
- add_interaction( between_var = c("varA", "varB"), mu = 0.43, sd = 0.37) %>%
- add_interaction( between_var = c("varB", "varC"), mu = -0.33, sd = 0.12) %>%
- add_interaction( between_var = c("varA", "varB" ,"varC"), mu = 0.87, sd = 0.18)
-```
-
-## Structure of list_var object
-
-```{r example-str_obj_init, warning = FALSE, message = FALSE}
-str(list_var)
-```
-
-# Simulation of RNAseq data
-
-In this section, you will explore how to generate RNAseq data based on the previously defined input variables. The `mock_rnaseq()` function enables you to manage parameters in your RNAseq design, including number of genes, minimum and maximum number of replicates within your experimental setup, sequencing depth, basal expression of each gene, and dispersion of gene expression used for simulating counts.
-
-
-
-## Minimal example
-
-```{r example-mock_rnaseq_min, warning = FALSE, message = FALSE}
-## -- Required parameters
-N_GENES = 6000
-MIN_REPLICATES = 2
-MAX_REPLICATES = 10
-########################
-
-## -- simulate RNAseq data based on list_var, minimum input required
-## -- number of replicate randomly defined between MIN_REP and MAX_REP
-mock_data <- mock_rnaseq(list_var, N_GENES,
- min_replicates = MIN_REPLICATES,
- max_replicates = MAX_REPLICATES)
-
-## -- simulate RNAseq data based on list_var, minimum input required
-## -- Same number of replicates between conditions
-mock_data <- mock_rnaseq(list_var, N_GENES,
- min_replicates = MAX_REPLICATES,
- max_replicates = MAX_REPLICATES)
-```
-
-## Scaling genes counts based on sequencing depth
-
-Sequencing depth is a critical parameter affecting the statistical power of an RNAseq analysis. With the `sequencing_depth` option in the `mock_rnaseq()` function, you have the ability to control this parameter.
-
-
-```{r example-mock_rnaseq_seqDepth, warning = FALSE, message = FALSE}
-## -- Required parameters
-N_GENES = 6000
-MIN_REPLICATES = 2
-MAX_REPLICATES = 10
-########################
-
-SEQ_DEPTH = c(100000, 5000000, 10000000)## -- Possible number of reads/sample
-SEQ_DEPTH = 10000000 ## -- all samples have same number of reads
-mock_data <- mock_rnaseq(list_var, N_GENES,
- min_replicates = MIN_REPLICATES,
- max_replicates = MAX_REPLICATES,
- sequencing_depth = SEQ_DEPTH)
-```
-
-## Set dispersion of gene expression
-
-The dispersion parameter ($dispersion_i$), characterizes the relationship between the variance of the observed read counts and its mean value. In simple terms, it quantifies how much we expect observed counts to deviate from the mean value. You can specify the dispersion for individual genes using the dispersion parameter.
-
-
-
-```{r example-mock_rnaseq_disp, warning = FALSE, message = FALSE}
-
-## -- Required parameters
-N_GENES = 6000
-MIN_REPLICATES = 2
-MAX_REPLICATES = 4
-########################
-
-DISP = 0.1 ## -- Same dispersion for each genes
-DISP = 1000 ## -- Same dispersion for each genes
-DISP = runif(N_GENES, 0, 1000) ## -- Dispersion can vary between genes
-mock_data <- mock_rnaseq(list_var, N_GENES,
- min_replicates = MIN_REPLICATES,
- max_replicates = MAX_REPLICATES,
- dispersion = DISP )
-
-```
-
-## Set basal gene expression
-
-The basal gene expression parameter, defined using the basal_expression option, allows you to control the fundamental baseline of expression level of genes. When a single value is specified, the basal expression of all genes is the same and the effects of variables defined in list_var are applied from this basal expression level. When several values are specified, the basal expression of each gene is picked randomly among these values (with replacement). The effects of variables are then applied from the basal expression of each gene. This represents the fact that expression levels can vary among genes even when not considering the effects of the defined variables (for example, some genes are constitutively more highly expressed than others because they have stronger basal promoters).
-
-
-```{r example-mock_rnaseq_bexpr, warning = FALSE, message = FALSE}
-## -- Required parameters
-N_GENES = 6000
-MIN_REPLICATES = 10
-MAX_REPLICATES = 10
-########################
-
-BASAL_EXPR = -3 ## -- Value can be negative to simulate low expressed gene
-BASAL_EXPR = 2 ## -- Same basal gene expression for the N_GENES
-BASAL_EXPR = c( -3, -1, 2, 8, 9, 10 ) ## -- Basal expression can vary between genes
-mock_data <- mock_rnaseq(list_var, N_GENES,
- min_replicates = MIN_REPLICATES,
- max_replicates = MAX_REPLICATES,
- basal_expression = BASAL_EXPR)
-
-```
-
-## Mock RNAseq object
-
-```{r example-str_obj_mock, warning = FALSE, message = FALSE}
-str(mock_data, max.level = 1)
-```
-
-# Theory behind HTRfit
-
-<div id="bg" align="center">
- <img src="./figs/htrfit_workflow.png" width="500" height="300">
-</div>
-
-
-In this modeling framework, counts denoted as $K_{ij}$ for gene i and sample j are generated using a negative binomial distribution. The negative binomial distribution considers a fitted mean $\mu_{ij}$ and a gene-specific dispersion parameter $dispersion_i$.
-
-The fitted mean $\mu_{ij}$ is determined by a parameter, $q_{ij}$, which is proportionally related to the sum of all effects specified using `init_variable()` or `add_interaction()`. If basal gene expressions are provided, the $\mu_{ij}$ values are scaled accordingly using the gene-specific basal expression value ($bexpr_i$).
-
-Furthermore, the coefficients $\beta_i$ represent the natural logarithm fold changes for gene i across each column of the model matrix X. The dispersion parameter $dispersion_i$ plays a crucial role in defining the relationship between the variance of observed counts and their mean value. In simpler terms, it quantifies how far we expect observed counts to deviate from the mean value for each genes.
-
-
-
-# Fitting models
-
-## Prepare data for fitting
-
-The `prepareData2fit()` function serves the purpose of converting the counts matrix and sample metadata into a dataframe that is compatible with downstream **HTRfit** functions designed for model fitting. This function also includes an option to perform median ratio normalization on the data counts.
-
-
-
-```{r example-prepareData, warning = FALSE, message = FALSE}
-## -- simulate small example to prevent excessively lengthy vignette construction
-list_var <- init_variable( name = "varA", mu = 3, sd = 0.2, level = 2) %>%
- init_variable( name = "varB", mu = 2, sd = 0.43, level = 2) %>%
- add_interaction( between_var = c("varA", "varB"), mu = 0.44, sd = 0.2)
-N_GENES = 30
-MIN_REPLICATES = 4
-MAX_REPLICATES = 4
-BASAL_EXPR = rnorm(N_GENES , 0 , 1 )
-mock_data <- mock_rnaseq(list_var, N_GENES,
- min_replicates = MIN_REPLICATES,
- max_replicates = MAX_REPLICATES,
- basal_expression = BASAL_EXPR)
-########################
-
-
-## -- data from simulation or real data
-count_matrix <- mock_data$counts
-metaData <- mock_data$metadata
-##############################
-
-## -- convert counts matrix and samples metadatas in a data frame for fitting
-data2fit = prepareData2fit(countMatrix = count_matrix,
- metadata = metaData,
- normalization = F,
- response_name = "kij")
-
-
-## -- median ratio normalization
-data2fit = prepareData2fit(countMatrix = count_matrix,
- metadata = metaData,
- normalization = T,
- response_name = "kij")
-```
-
-## Fit model from your data
-
-The `fitModelParallel()` function enables independent model fitting for each gene. The number of threads used for this process can be controlled by the `n.cores` parameter.
-
-
-```{r example-fitModelParallel, warning = FALSE, message = FALSE}
-l_tmb <- fitModelParallel(formula = kij ~ varA,
- data = data2fit,
- group_by = "geneID",
- family = glmmTMB::nbinom2(link = "log"),
- n.cores = 1)
-```
-
-## Use mixed effect in your model
-
-**HTRfit** uses the **glmmTMB** functions for model fitting algorithms. This choice allows for the utilization of random effects within your formula design. For further details on how to specify your model, please refer to the [mixed model documentation](https://rdrr.io/cran/glmmTMB/man/glmmTMBControl.html).
-
-
-
-```{r example-fitModelParallel_mixed, warning = FALSE, message = FALSE}
-l_tmb <- fitModelParallel(formula = kij ~ varA + ( 1 | varB ),
- data = data2fit,
- group_by = "geneID",
- family = glmmTMB::nbinom2(link = "log"),
- n.cores = 1)
-```
-
-## Additional settings
-
-The function provides precise control over model settings for fitting optimization, including options for specifying the [model family](https://www.rdocumentation.org/packages/stats/versions/3.6.2/topics/family) and [model control setting](https://rdrr.io/cran/glmmTMB/man/glmmTMBControl.html). By default, a Gaussian family model is fitted, but for RNA-seq data, it is highly recommended to specify `family = glmmTMB::nbinom2(link = "log")`.
-
-
-
-```{r example-fitModelParallel_addSet, warning = FALSE, message = FALSE}
-l_tmb <- fitModelParallel(formula = kij ~ varA,
- data = data2fit,
- group_by = "geneID",
- n.cores = 1,
- family = glmmTMB::nbinom2(link = "log"),
- control = glmmTMB::glmmTMBControl(optCtrl=list(iter.max=1e5,
- eval.max=1e5)))
-```
-
-## Not only RNAseq data
-
-As the model family can be customized, HTRfit is not exclusively tailored for RNA-seq data.
-
-
-```{r example-fitModelParallel_nonRNA, warning = FALSE, message = FALSE}
-data("iris")
-l_tmb_iris <- fitModelParallel(formula = Sepal.Length ~ Sepal.Width + Petal.Length + Petal.Width ,
- data = iris,
- group_by = "Species",
- family = gaussian(),
- n.cores = 1)
-```
-
-## Extracts a tidy result table from a list tmb object
-
-The tidy_results function extracts a data frame containing estimates of ln(fold changes), standard errors, test statistics, p-values, and adjusted p-values for fixed effects. Additionally, it provides access to correlation terms and standard deviations for random effects, offering a detailed view of HTRfit modeling results.
-
-
-```{r example-tidyRes, warning = FALSE, message = FALSE}
-## -- get tidy results
-tidy_results(l_tmb_iris, coeff_threshold = 0.1, alternative_hypothesis = "greaterAbs")
-```
-
-## Update fit
-
-The `updateParallel()` function updates and re-fits a model for each gene. It offers options similar to those in `fitModelParallel()`.
-
-
-```{r example-update, warning = FALSE, message = FALSE}
-## -- update your fit modifying the model family
-l_tmb <- updateParallel(formula = kij ~ varA,
- list_tmb = l_tmb ,
- family = gaussian(),
- n.cores = 1)
-
-## -- update fit using additional model control settings
-l_tmb <- updateParallel(formula = kij ~ varA ,
- list_tmb = l_tmb ,
- family = gaussian(),
- n.cores = 1,
- control = glmmTMB::glmmTMBControl(optCtrl=list(iter.max=1e3,
- eval.max=1e3)))
-
-
-## -- update your model formula and your family model
-l_tmb <- updateParallel(formula = kij ~ varA + varB + varA:varB ,
- list_tmb = l_tmb ,
- family = glmmTMB::nbinom2(link = "log"),
- n.cores = 1)
-```
-
-#### Struture of list tmb object
-
-```{r example-str_obj_l_tmb, warning = FALSE, message = FALSE}
-str(l_tmb$gene1, max.level = 1)
-```
-
-## Plot fit metrics
-
-Visualizing fit metrics is essential for evaluating your models. Here, we show you how to generate various plots to assess the quality of your models. You can explore all metrics or focus on specific aspects like dispersion and log-likelihood.
-
-
-```{r example-plotMetrics, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 8}
-## -- plot all metrics
-diagnostic_plot(list_tmb = l_tmb)
-```
-
-```{r example-plotMetricsFocus, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 3, fig.width = 8}
-## -- Focus on metrics
-diagnostic_plot(list_tmb = l_tmb, focus = c("dispersion", "logLik"))
-```
-
-## Anova to select the best model
-
-Utilizing the `anovaParallel()` function enables you to perform model selection by assessing the significance of the fixed effects. You can also include additional parameters like type. For more details, refer to [car::Anova](https://rdrr.io/cran/car/man/Anova.html).
-
-
-```{r example-anova, warning = FALSE, message = FALSE}
-## -- update your fit modifying the model family
-l_anova <- anovaParallel(list_tmb = l_tmb)
-
-## -- additional settings
-l_anova <- anovaParallel(list_tmb = l_tmb, type = "III" )
-
-```
-
-# Simulation evaluation report
-
-In this section, we delve into the evaluation of your simulation results. The `evaluation_report()` function provide valuable insights into the performance of your simulated data and models.
-
-
-```{r example-evaluation_report, warning = FALSE, message = FALSE}
-## -- get simulation/fit evaluation
-resSimu <- evaluation_report(list_tmb = l_tmb,
- dds = NULL,
- mock_obj = mock_data,
- coeff_threshold = 0.4,
- alpha_risk = 0.05,
- alt_hypothesis = "greaterAbs")
-```
-
-## Identity plot
-
-The `evaluation_report()` function produces an identity plot, offering a visual tool to juxtapose the simulated effects (actual effects) with the model-inferred effects. This graphical representation simplifies assessing how well the model aligns with the values of the simulated effects, enabling a visual analysis of the model's goodness of fit to the simulated data. For a more quantitative evaluation of inference quality, the R-squared (R2) metric can be employed.
-
-
-#### Model parameters
-
-```{r example-outputIdentity_params, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 5}
-## -- Model params
-resSimu$identity$params
-```
-
-#### Dispersion parameter
-
-```{r example-outputIdentity_disp, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 5}
-## -- Dispersion params
-resSimu$identity$dispersion
-```
-
-The dispersion plot, generated by the `evaluation_report()` function, offers a visual comparison of the dispersion parameters used in the simulation \(\alpha_i\) with those estimated by the model. This graphical representation provides an intuitive way to assess the alignment between the simulated dispersion values and the model-inferred values, enabling a visual evaluation of how well the model captures the underlying data characteristics.
-
-
-
-## ROC curve
-
-The Receiver Operating Characteristic (ROC) curve is a valuable tool for assessing the performance of classification models, particularly in the context of identifying differentially expressed genes. It provides a graphical representation of the model's ability to distinguish between genes that are differentially expressed and those that are not, by varying the `coeff_threshold` and the `alt_hypothesis` parameters.
-
-
-
-```{r example-outputROC, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 7}
-## -- ROC curve
-resSimu$roc$params
-```
-
-## Precision-recall curve
-
-The precision-recall curve (PR curve) illustrates the relationship between precision and recall at various classification thresholds. It is particularly valuable in the context of imbalanced data, where one class is significantly more prevalent than the other. Unlike the ROC curve, which can be influenced by class distribution, the PR curve focuses on the model's ability to correctly identify examples of the minority class while maintaining high precision.
-
-
-
-```{r example-outputPR, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 7}
-## -- precision-recall curve
-resSimu$precision_recall$params
-```
-
-## AUC and classification performance metrics
-
-The area under the ROC curve (AUC) provides a single metric that summarizes the model's overall performance in distinguishing between differentially expressed and non-differentially expressed genes. A higher AUC indicates better model performance. In the case of the ROC curve, this AUC should be compared to the value of 0.5, representing a random classifier. On the other hand, for the PR curve, we compute the pr_AUC_random, serving as a baseline for comparison.
-
-
-In addition to evaluating model performance through the AUC for both ROC and PR curves, we provide access to key classification metrics, including Accuracy, Precision, Recall (or Sensitivity), and Specificity. These metrics offer a comprehensive view of the model's classification capabilities. These metrics are computed for each parameter (excluding the intercept when skip_eval_intercept = TRUE), providing detailed insights into individual parameter contributions. Furthermore, an aggregated assessment is performed, considering all parameters (except the intercept by default), offering a perspective on the model's overall classification effectiveness.
-
-
-
-```{r example-outputPerf, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 7}
-## -- precision-recall curve
-resSimu$performances
-```
-
-## Compare HTRfit with DESeq2
-
-**HTRfit** offers a wrapper for **DESeq2** outputs. This functionality allows users to seamlessly integrate the results obtained from **DESeq2** into the **HTRfit** analysis pipeline. By doing so, you can readily compare the performance of **HTRfit** with **DESeq2** on your RNAseq data. This comparative analysis aids in determining which tool performs better for your specific research goals and dataset
-
-
-
-```{r example-ddsComparison, warning = FALSE, message = FALSE, results = 'hide', fig.keep = 'none'}
-## -- DESeq2
-library(DESeq2)
-dds <- DESeq2::DESeqDataSetFromMatrix(
- countData = count_matrix,
- colData = metaData,
- design = ~ varA + varB + varA:varB )
-dds <- DESeq2::DESeq(dds, quiet = TRUE)
-
-
-## -- get simulation/fit evaluation
-resSimu <- evaluation_report(list_tmb = l_tmb,
- dds = dds,
- mock_obj = mock_data,
- coeff_threshold = 0.4,
- alt_hypothesis = "greaterAbs")
-```
-
-```{r example-outputResSimu_id, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 5}
-## -- identity plot
-###### 1) Model params
-resSimu$identity$params
-###### Dispersion params
-resSimu$identity$dispersion
-```
-
-```{r example-outputResSimu_metric, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 7}
-## -- precision-recall curve
-resSimu$precision_recall$params
-## -- ROC curve
-resSimu$roc$params
-## -- Performances metrics
-resSimu$performances
-```
-
-## Focus evaluation on a subset of genes
-
-In this section, we showcase the assessment of model performance on a subset of genes. Specifically, we focus on evaluating genes with low expression levels, identified by their basal expression ($bexpr_i$) initialized below 0 during the simulation.
-
-
-```{r example-subsetGenes, warning = FALSE, message = FALSE}
-## -- Focus on low expressed genes
-low_expressed_df <- mock_data$groundTruth$effects[ mock_data$groundTruth$effects$basalExpr < 0, ]
-l_genes <- unique(low_expressed_df$geneID)
-mock_lowExpressed <- subsetGenes(l_genes, mock_data)
-
-
-## -- get simulation/fit evaluation
-resSimu <- evaluation_report(list_tmb = l_tmb,
- dds = dds,
- mock_obj = mock_lowExpressed,
- coeff_threshold = 0.4,
- alt_hypothesis = "greaterAbs")
-```
-
-As we compare this evaluation to the previous one, we observe a reduction in performances for both **HTRfit** and **DESeq2** inferences.
-
-
-```{r example-subsetGenes_id, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 5}
-## -- identity plot
-###### 1) Model params
-resSimu$identity$params
-###### Dispersion params
-resSimu$identity$dispersion
-```
-
-```{r example-subsetGenes_metrics, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 7}
-## -- precision-recall curve
-resSimu$precision_recall$params
-## -- ROC curve
-resSimu$roc$params
-## -- Performances metrics
-resSimu$performances
-```
-
-## Modifying your alpha risk
-
-```{r example-alphaRisk_1, warning = FALSE, message = FALSE}
-## -- get simulation/fit evaluation
-resSimu <- evaluation_report(list_tmb = l_tmb,
- dds = dds,
- mock_obj = mock_data,
- coeff_threshold = 0.4,
- alpha_risk = 0.05, ## -- default
- alt_hypothesis = "greaterAbs")
-## -- Performances metrics
-resSimu$performances
-```
-
-```{r example-alphaRiskk_2, warning = FALSE, message = FALSE}
-## -- get simulation/fit evaluation
-resSimu <- evaluation_report(list_tmb = l_tmb,
- dds = dds,
- mock_obj = mock_data,
- coeff_threshold = 0.4,
- alpha_risk = 0.01,
- alt_hypothesis = "greaterAbs")
-## -- Performances metrics
-resSimu$performances
-```
-
-
-## Structure of evaluation report object
-
-```{r example-str_eval_report, warning = FALSE, message = FALSE}
-str(resSimu, max.level = 1)
-```
-
-## About mixed model evaluation
-
-**HTRfit** offers a versatile simulation framework capable of fitting various types of models involving mixed effects, thanks to its implementation of **glmmTMB**. By combining the functionalities of `init_variable()` and `add_interaction()`, **HTRfit** enables the simulation of complex experimental designs. As of now, HTRfit supports the evaluation of *Type I* mixed models. In this context, *Type I* models are defined as those that follow the structure:
-- `~ varA + (1 | varB)` : where `varA` is defined as fixed effect and `varB` as random effect
-- `~ varA + (varA | varB)`: where `varA` is defined as mixed effect (fixed + random) and `varB` as random effect.
-
-Models not conforming to this specific form cannot be evaluated using **HTRfit's** current implementation. Nonetheless, you are welcome to extend its capabilities by implementing support for additional model types.
-
-
-
-
diff --git a/vignettes/htrfit_vs_deseq2.Rmd b/vignettes/htrfit_vs_deseq2.Rmd
new file mode 100644
index 0000000000000000000000000000000000000000..c028226350b52c55d52c1d92836bc0e12979bfae
--- /dev/null
+++ b/vignettes/htrfit_vs_deseq2.Rmd
@@ -0,0 +1,120 @@
+---
+title: "HTRfit vs DESeq2"
+output: rmarkdown::html_vignette
+vignette: >
+ %\VignetteIndexEntry{HTRfit vs DESeq2}
+ %\VignetteEngine{knitr::rmarkdown}
+ %\VignetteEncoding{UTF-8}
+---
+
+
+```{r, include = FALSE}
+knitr::opts_chunk$set(
+ collapse = TRUE,
+ comment = "#>"
+)
+```
+
+
+```{r setup, warning = FALSE, message = FALSE, results='hide'}
+devtools::load_all()
+library(HTRfit)
+library(DESeq2)
+```
+
+
+**HTRfit** offers a wrapper for **DESeq2** outputs. This functionality allows users to seamlessly integrate the results obtained from **DESeq2** into the **HTRfit** evaluation pipeline. By doing so, you can readily compare the performance of **HTRfit** with **DESeq2** on your RNAseq data. This comparative analysis aids in determining which tool performs better for your specific research goals and dataset
+
+
+## Simulation
+
+
+```{r design_init, warning = FALSE, message = FALSE}
+## -- init a design
+input_var_list <- init_variable( name = "genotype", mu = 0, sd = 0.29, level = 2) %>%
+ init_variable( name = "environment", mu = 0.27, sd = 0.6, level = 4) %>%
+ add_interaction( between_var = c("genotype", "environment"), mu = 0.44, sd = 0.89)
+
+
+N_GENES <- 100
+MIN_REPLICATES <- 4
+MAX_REPLICATES <- 4
+SEQ_DEPTH <- 5e6
+
+## -- simulate RNAseq data
+mock_data <- mock_rnaseq(input_var_list,
+ n_genes = N_GENES,
+ min_replicates = MIN_REPLICATES,
+ max_replicates = MAX_REPLICATES,
+ basal_expression = 2,
+ sequencing_depth = SEQ_DEPTH)
+```
+
+
+
+## Fit models
+
+```{r data2fit, warning = FALSE, message = FALSE, results = 'hide'}
+## -- data from simulation or real data
+count_matrix <- mock_data$counts
+metaData <- mock_data$metadata
+```
+
+
+#### HTRfit
+```{r prepareData_and_fit, warning = FALSE, message = FALSE}
+## -- convert counts matrix and samples metadatas in a data frame for fitting
+data2fit = prepareData2fit(countMatrix = count_matrix,
+ metadata = metaData,
+ normalization = T,
+ response_name = "kij")
+
+l_tmb <- fitModelParallel(formula = kij ~ genotype + environment + genotype:environment,
+ data = data2fit,
+ group_by = "geneID",
+ family = glmmTMB::nbinom2(link = "log"),
+ n.cores = 1)
+```
+
+
+#### DESeq2
+
+```{r fit_dds, warning = FALSE, message = FALSE, results = 'hide'}
+## -- DESeq2
+dds <- DESeq2::DESeqDataSetFromMatrix(
+ countData = count_matrix,
+ colData = metaData,
+ design = ~ genotype + environment + genotype:environment )
+dds <- DESeq2::DESeq(dds, quiet = TRUE)
+```
+
+
+## Evaluation
+
+```{r example-ddsComparison, warning = FALSE, message = FALSE}
+## -- get simulation/fit evaluation
+resSimu <- evaluation_report(list_tmb = l_tmb,
+ dds = dds,
+ mock_obj = mock_data,
+ coeff_threshold = 0.4,
+ alt_hypothesis = "greaterAbs")
+```
+
+
+
+```{r example-outputResSimu_id, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 5}
+## -- identity plot
+###### 1) Model params
+resSimu$identity$params
+###### Dispersion params
+resSimu$identity$dispersion
+```
+
+```{r example-outputResSimu_metric, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 7}
+## -- precision-recall curve
+resSimu$precision_recall$params
+## -- ROC curve
+resSimu$roc$params
+## -- Performances metrics
+resSimu$performances
+```
diff --git a/vignettes/list_var_object.Rmd b/vignettes/list_var_object.Rmd
new file mode 100644
index 0000000000000000000000000000000000000000..a5ddc145aa7f24d52f73b3c34a447ba8f85a34e3
--- /dev/null
+++ b/vignettes/list_var_object.Rmd
@@ -0,0 +1,116 @@
+---
+title: "List var object"
+output: rmarkdown::html_vignette
+vignette: >
+ %\VignetteIndexEntry{List var object}
+ %\VignetteEngine{knitr::rmarkdown}
+ %\VignetteEncoding{UTF-8}
+---
+
+```{r, include = FALSE}
+knitr::opts_chunk$set(
+ collapse = TRUE,
+ comment = "#>"
+)
+```
+
+
+```{r setup}
+library(HTRfit)
+```
+
+The `init_variable()` function is used for defining the variables in your experimental design. Sizes of effects for each variable and interaction term can be defined in two different ways: 1) The user can manually sets values of all levels of a variable, in which case the effects are necessarily considered fixed in the model; 2) The effects can be randomly picked in a normal distribution with mean and standard deviation defined by the user, in which case the user can decide whether the effects are considered fixed or random in the model.
+
+It is crucial to highlight that the choice of these parameters plays a critical role in the simulation process. For optimal results, we recommend basing these decisions on real data, as outlined in the [Tutorial](articles/tutorial.html)
+
+## Manually init a single variable
+
+The `init_variable()` function allows for precise control over the variables in your experimental design.
+In this example, we manually initialize **varA** with specific size effects (mu) for three levels.
+
+```{r example-init_variable_man, warning = FALSE, message = FALSE}
+list_var <- init_variable( name = "varA", mu = c(0.2, 4, -3), level = 3)
+```
+
+## Randomly init a single variable
+
+Alternatively, you can randomly initialize **varA** by specifying a mean (mu) and standard deviation (sd).
+This introduces variability into **varA**, making it either a fixed or random effect in your design.
+
+
+```{r example-init_variable_rand, warning = FALSE, message = FALSE}
+list_var <- init_variable( name = "varA", mu = 10, sd = 0.2, level = 5)
+```
+
+## Randomly init several variables
+
+You can also initialize multiple variables, such as **varA** and **varB**, with random values.
+This flexibility allows you to create diverse experimental designs.
+
+
+```{r example-init_variable_mult, warning = FALSE, message = FALSE}
+list_var <- init_variable( name = "varA", mu = 10, sd = 0.2, level = 5) %>%
+ init_variable( name = "varB", mu = -3, sd = 0.34, level = 2)
+```
+
+## Add interaction between variable
+
+Similarly to `init_variable()`, `add_interaction()` allows to define an interaction between variable.
+
+In this example, we initialize **varA** and **varB**, and create an interaction between **varA**, and **varB** using `add_interaction()`.
+
+
+
+```{r example-add_interaction, warning = FALSE, message = FALSE}
+list_var <- init_variable( name = "varA", mu = 3, sd = 0.2, level = 2) %>%
+ init_variable( name = "varB", mu = 2, sd = 0.43, level = 2) %>%
+ add_interaction( between_var = c("varA", "varB"), mu = 0.44, sd = 0.2)
+```
+
+## Complex design
+
+Interactions can involve a maximum of three variables, such as **varA**, **varB**, and **varC**.
+
+```{r example-add_interaction_complex, eval = FALSE, message = FALSE, warning = FALSE, include = TRUE}
+list_var <- init_variable( name = "varA", mu = 5, sd = 0.2, level = 2) %>%
+ init_variable( name = "varB", mu = 1, sd = 0.78, level = 2) %>%
+ init_variable( name = "varC", mu = c(2, 3), sd = NA, level = 2) %>%
+ add_interaction( between_var = c("varA", "varC"), mu = 0.44, sd = 0.2) %>%
+ add_interaction( between_var = c("varA", "varB"), mu = 0.43, sd = 0.37) %>%
+ add_interaction( between_var = c("varB", "varC"), mu = -0.33, sd = 0.12) %>%
+ add_interaction( between_var = c("varA", "varB" ,"varC"), mu = 0.87, sd = 0.18)
+```
+
+
+## Control correlation between effects
+
+
+HTRfit offers a convenient way to control the correlation between variables during simulation. Take, for example, the following scenario :
+
+```{r example-set_corr, warning = FALSE, message = FALSE}
+list_var <- init_variable( name = "varA", mu = 3, sd = 0.2, level = 2) %>%
+ init_variable( name = "varB", mu = c(1,3), level = 2) %>%
+ add_interaction( between_var = c("varA", "varB"), mu = 0.44, sd = 0.2) %>%
+ set_correlation(between_var = c("varA", "varA:varB"), corr = -0.99)
+```
+
+This code demonstrates how variables, initialized with random values using mu and sd, can be further refined by setting the correlation between these variables. If the `set_correlation()` function is not used, the default correlation value is 0.
+
+Observe the 0.99 correlation in the `mock_obj`:
+
+```{r simu_corr_var}
+## -- simulate RNAseq data
+mock_data <- mock_rnaseq(list_var,
+ n_genes = 100,
+ min_replicates = 4,
+ max_replicates = 4)
+plot(mock_data$groundTruth$effects$varA, mock_data$groundTruth$effects$`varA:varB`, xlab = "varA", ylab= "varA:varB")
+```
+
+For further details : [`mock_rnaseq` object](articles/mock_rnaseq_object.html)
+
+## Structure of list_var object
+
+```{r example-str_obj_init, warning = FALSE, message = FALSE}
+str(list_var)
+```
diff --git a/vignettes/mock_rnaseq_object.Rmd b/vignettes/mock_rnaseq_object.Rmd
new file mode 100644
index 0000000000000000000000000000000000000000..20c7383105245a7debc96dd08a400aa534ee9e4a
--- /dev/null
+++ b/vignettes/mock_rnaseq_object.Rmd
@@ -0,0 +1,130 @@
+---
+title: "Mock rnaseq object"
+output: rmarkdown::html_vignette
+vignette: >
+ %\VignetteIndexEntry{Mock rnaseq object}
+ %\VignetteEngine{knitr::rmarkdown}
+ %\VignetteEncoding{UTF-8}
+---
+
+```{r, include = FALSE}
+knitr::opts_chunk$set(
+ collapse = TRUE,
+ comment = "#>"
+)
+```
+
+
+
+```{r setup}
+library(HTRfit)
+```
+
+
+
+
+In this section, you will explore how to generate RNAseq data based on the `list_var` object. The `mock_rnaseq()` function enables you to manage parameters in your RNAseq design, including number of genes, minimum and maximum number of replicates within your experimental setup, sequencing depth, basal expression of each gene, and dispersion of gene expression used for simulating counts.
+
+## List var object
+
+For further details : [`list var` object](articles/list_var_object.html)
+
+```{r example-add_interaction, warning = FALSE, message = FALSE}
+list_var <- init_variable( name = "varA", mu = 3, sd = 0.2, level = 2) %>%
+ init_variable( name = "varB", mu = 2, sd = 0.43, level = 2) %>%
+ add_interaction( between_var = c("varA", "varB"), mu = 0.44, sd = 0.2)
+```
+
+## Minimal example
+
+```{r example-mock_rnaseq_min, warning = FALSE, message = FALSE}
+## -- Required parameters
+N_GENES = 6000
+MIN_REPLICATES = 2
+MAX_REPLICATES = 10
+########################
+
+## -- simulate RNAseq data based on list_var, minimum input required
+## -- number of replicate randomly defined between MIN_REP and MAX_REP
+mock_data <- mock_rnaseq(list_var, N_GENES,
+ min_replicates = MIN_REPLICATES,
+ max_replicates = MAX_REPLICATES)
+
+## -- simulate RNAseq data based on list_var, minimum input required
+## -- Same number of replicates between conditions
+mock_data <- mock_rnaseq(list_var, N_GENES,
+ min_replicates = MAX_REPLICATES,
+ max_replicates = MAX_REPLICATES)
+```
+
+## Scaling genes counts based on sequencing depth
+
+Sequencing depth is a critical parameter affecting the statistical power of an RNAseq analysis. With the `sequencing_depth` option in the `mock_rnaseq()` function, you have the ability to control this parameter.
+
+
+```{r example-mock_rnaseq_seqDepth, warning = FALSE, message = FALSE}
+## -- Required parameters
+N_GENES = 6000
+MIN_REPLICATES = 2
+MAX_REPLICATES = 10
+########################
+
+SEQ_DEPTH = c(100000, 5000000, 10000000)## -- Possible number of reads/sample
+SEQ_DEPTH = 10000000 ## -- all samples have same number of reads
+mock_data <- mock_rnaseq(list_var, N_GENES,
+ min_replicates = MIN_REPLICATES,
+ max_replicates = MAX_REPLICATES,
+ sequencing_depth = SEQ_DEPTH)
+```
+
+## Set dispersion of gene expression
+
+The dispersion parameter ($dispersion_i$), characterizes the relationship between the variance of the observed read counts and its mean value. In simple terms, it quantifies how much we expect observed counts to deviate from the mean value. You can specify the dispersion for individual genes using the dispersion parameter.
+
+
+
+```{r example-mock_rnaseq_disp, warning = FALSE, message = FALSE}
+
+## -- Required parameters
+N_GENES = 6000
+MIN_REPLICATES = 2
+MAX_REPLICATES = 4
+########################
+
+DISP = 0.1 ## -- Same dispersion for each genes
+DISP = 1000 ## -- Same dispersion for each genes
+DISP = runif(N_GENES, 0, 1000) ## -- Dispersion can vary between genes
+mock_data <- mock_rnaseq(list_var, N_GENES,
+ min_replicates = MIN_REPLICATES,
+ max_replicates = MAX_REPLICATES,
+ dispersion = DISP )
+
+```
+
+## Set basal gene expression
+
+The basal gene expression parameter, defined using the basal_expression option, allows you to control the fundamental baseline of expression level of genes. When a single value is specified, the basal expression of all genes is the same and the effects of variables defined in list_var are applied from this basal expression level. When several values are specified, the basal expression of each gene is picked randomly among these values (with replacement). The effects of variables are then applied from the basal expression of each gene. This represents the fact that expression levels can vary among genes even when not considering the effects of the defined variables (for example, some genes are constitutively more highly expressed than others because they have stronger basal promoters).
+
+
+```{r example-mock_rnaseq_bexpr, warning = FALSE, message = FALSE}
+## -- Required parameters
+N_GENES = 6000
+MIN_REPLICATES = 10
+MAX_REPLICATES = 10
+########################
+
+BASAL_EXPR = -3 ## -- Value can be negative to simulate low expressed gene
+BASAL_EXPR = 2 ## -- Same basal gene expression for the N_GENES
+BASAL_EXPR = c( -3, -1, 2, 8, 9, 10 ) ## -- Basal expression can vary between genes
+mock_data <- mock_rnaseq(list_var, N_GENES,
+ min_replicates = MIN_REPLICATES,
+ max_replicates = MAX_REPLICATES,
+ basal_expression = BASAL_EXPR)
+
+```
+
+## Mock RNAseq object
+
+```{r example-str_obj_mock, warning = FALSE, message = FALSE}
+str(mock_data, max.level = 1)
+```
diff --git a/vignettes/rnaseq_analysis.Rmd b/vignettes/rnaseq_analysis.Rmd
new file mode 100644
index 0000000000000000000000000000000000000000..1eb84623acb72f49b43b43f022c162202d0406a7
--- /dev/null
+++ b/vignettes/rnaseq_analysis.Rmd
@@ -0,0 +1,193 @@
+---
+title: "RNAseq analysis"
+output: rmarkdown::html_vignette
+vignette: >
+ %\VignetteIndexEntry{RNAseq analysis}
+ %\VignetteEngine{knitr::rmarkdown}
+ %\VignetteEncoding{UTF-8}
+---
+
+```{r, include = FALSE}
+knitr::opts_chunk$set(
+ collapse = TRUE,
+ comment = "#>"
+)
+```
+
+```{r setup}
+library(HTRfit)
+```
+
+
+
+In RNAseq, we employ Generalized Linear Models (GLM) to unravel how genes respond to various experimental conditions. These models assist in deciphering the specific impacts of experimental variables on gene expression.HTRfit can be utilized to analyze such RNAseq data, providing a robust framework for exploring and interpreting the intricate relationships between genes and experimental conditions.
+
+## Input data
+
+HTRfit analysis necessitates a count matrix and sample metadata, in the form of dataframes.
+Notice that gene_id have to be specified as rownames of `count_matrix`.
+
+```{r create_data , include = FALSE}
+## -- hided in vignette
+## -- simulate small example to prevent excessively lengthy vignette construction
+list_var <- init_variable( name = "genotype", mu = 3, sd = 0.2, level = 2) %>%
+ init_variable( name = "environment", mu = 2, sd = 0.43, level = 2) %>%
+ add_interaction( between_var = c("genotype", "environment"), mu = 0.44, sd = 0.2)
+N_GENES = 30
+MIN_REPLICATES = 4
+MAX_REPLICATES = 4
+BASAL_EXPR = 3
+mock_data <- mock_rnaseq(list_var, N_GENES,
+ min_replicates = MIN_REPLICATES,
+ max_replicates = MAX_REPLICATES,
+ basal_expression = BASAL_EXPR)
+########################
+
+## -- data from simulation or real data
+count_matrix <- mock_data$counts
+metaData <- mock_data$metadata
+##############################
+```
+
+
+```{r display_input }
+## -- gene count matrix
+head(count_matrix)
+## -- samples metadata
+head(metaData)
+```
+
+
+## Prepare data for fitting
+
+The `prepareData2fit()` function serves the purpose of converting the counts matrix and sample metadata into a dataframe that is compatible with downstream **HTRfit** functions designed for model fitting. This function also includes an option to perform median ratio normalization on the data counts.
+
+
+```{r example-prepareData, warning = FALSE, message = FALSE}
+## -- convert counts matrix and samples metadatas in a data frame for fitting
+data2fit = prepareData2fit(countMatrix = count_matrix,
+ metadata = metaData,
+ normalization = F,
+ response_name = "kij")
+
+
+## -- median ratio normalization
+data2fit = prepareData2fit(countMatrix = count_matrix,
+ metadata = metaData,
+ normalization = T,
+ response_name = "kij")
+```
+
+## Fit model from your data
+
+The `fitModelParallel()` function enables independent model fitting for each gene. The number of threads used for this process can be controlled by the `n.cores` parameter.
+
+
+```{r example-fitModelParallel, warning = FALSE, message = FALSE}
+l_tmb <- fitModelParallel(formula = kij ~ genotype + environment + genotype:environment,
+ data = data2fit,
+ group_by = "geneID",
+ family = glmmTMB::nbinom2(link = "log"),
+ n.cores = 1)
+```
+
+## Use mixed effect in your model
+
+**HTRfit** uses the **glmmTMB** functions for model fitting algorithms. This choice allows for the utilization of random effects within your formula design. For further details on how to specify your model, please refer to the [mixed model documentation](https://rdrr.io/cran/glmmTMB/man/glmmTMBControl.html).
+
+```{r example-fitModelParallel_mixed, warning = FALSE, message = FALSE}
+l_tmb <- fitModelParallel(formula = kij ~ genotype + ( 1 | environment ),
+ data = data2fit,
+ group_by = "geneID",
+ family = glmmTMB::nbinom2(link = "log"),
+ n.cores = 1)
+```
+
+## Additional settings
+
+The function provides precise control over model settings for fitting optimization, including options for specifying the [model family](https://www.rdocumentation.org/packages/stats/versions/3.6.2/topics/family) and [model control setting](https://rdrr.io/cran/glmmTMB/man/glmmTMBControl.html). By default, a Gaussian family model is fitted, but for RNA-seq data, it is highly recommended to specify `family = glmmTMB::nbinom2(link = "log")`.
+
+
+
+```{r example-fitModelParallel_addSet, warning = FALSE, message = FALSE}
+l_tmb <- fitModelParallel(formula = kij ~ genotype + environment + genotype:environment,
+ data = data2fit,
+ group_by = "geneID",
+ n.cores = 1,
+ family = glmmTMB::nbinom2(link = "log"),
+ control = glmmTMB::glmmTMBControl(optCtrl=list(iter.max=1e5,
+ eval.max=1e5)))
+```
+
+## Extracts a tidy result table from a list tmb object
+
+The tidy_results function extracts a data frame containing estimates of ln(fold changes), standard errors, test statistics, p-values, and adjusted p-values for fixed effects. Additionally, it provides access to correlation terms and standard deviations for random effects, offering a detailed view of HTRfit modeling results.
+
+
+```{r example-tidyRes, warning = FALSE, message = FALSE}
+## -- get tidy results
+tidy_results(l_tmb, coeff_threshold = 0.1, alternative_hypothesis = "greaterAbs")
+```
+
+## Update fit
+
+The `updateParallel()` function updates and re-fits a model for each gene. It offers options similar to those in `fitModelParallel()`.
+
+
+```{r example-update, warning = FALSE, message = FALSE}
+## -- update your fit modifying the model family
+l_tmb <- updateParallel(formula = kij ~ genotype + environment + genotype:environment,
+ list_tmb = l_tmb ,
+ family = gaussian(),
+ n.cores = 1)
+
+## -- update fit using additional model control settings
+l_tmb <- updateParallel(formula = kij ~ genotype + environment + genotype:environment ,
+ list_tmb = l_tmb ,
+ family = gaussian(),
+ n.cores = 1,
+ control = glmmTMB::glmmTMBControl(optCtrl=list(iter.max=1e3,
+ eval.max=1e3)))
+
+
+## -- update your model formula and your family model
+l_tmb <- updateParallel(formula = kij ~ genotype + environment ,
+ list_tmb = l_tmb ,
+ family = glmmTMB::nbinom2(link = "log"),
+ n.cores = 1)
+```
+
+#### Struture of list tmb object
+
+```{r example-str_obj_l_tmb, warning = FALSE, message = FALSE}
+str(l_tmb$gene1, max.level = 1)
+```
+
+## Plot fit metrics
+
+Visualizing fit metrics is essential for evaluating your models. Here, we show you how to generate various plots to assess the quality of your models. You can explore all metrics or focus on specific aspects like dispersion and log-likelihood.
+
+
+```{r example-plotMetrics, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 4, fig.width = 8}
+## -- plot all metrics
+diagnostic_plot(list_tmb = l_tmb)
+```
+
+```{r example-plotMetricsFocus, warning = FALSE, message = FALSE, fig.align = 'center', fig.height = 3, fig.width = 8}
+## -- Focus on metrics
+diagnostic_plot(list_tmb = l_tmb, focus = c("dispersion", "logLik"))
+```
+
+## Anova to select the best model
+
+Utilizing the `anovaParallel()` function enables you to perform model selection by assessing the significance of the fixed effects. You can also include additional parameters like type. For more details, refer to [car::Anova](https://rdrr.io/cran/car/man/Anova.html).
+
+
+```{r example-anova, warning = FALSE, message = FALSE}
+## -- update your fit modifying the model family
+l_anova <- anovaParallel(list_tmb = l_tmb)
+
+## -- additional settings
+l_anova <- anovaParallel(list_tmb = l_tmb, type = "III" )
+
+```
diff --git a/vignettes/rnaseq_notOnly.Rmd b/vignettes/rnaseq_notOnly.Rmd
new file mode 100644
index 0000000000000000000000000000000000000000..a898e40806943dfb806e713d9afe1c83c0babbc8
--- /dev/null
+++ b/vignettes/rnaseq_notOnly.Rmd
@@ -0,0 +1,49 @@
+---
+title: "Not only RNAseq"
+output: rmarkdown::html_vignette
+vignette: >
+ %\VignetteIndexEntry{Not only RNAseq}
+ %\VignetteEngine{knitr::rmarkdown}
+ %\VignetteEncoding{UTF-8}
+---
+
+```{r, include = FALSE}
+knitr::opts_chunk$set(
+ collapse = TRUE,
+ comment = "#>"
+)
+```
+
+
+```{r setup}
+library(HTRfit)
+```
+
+In the realm of RNAseq analysis, it's widely acknowledged that counts follow a Negative Binomial distribution. Hence, we recommend employing `family = glmmTMB::nbinom2(link = "log")` in the parameters of fitModelParallel() to align with this distribution.
+In RNAseq it is well known that counts have a Negative binomial distribution that's why we recommand using `family = glmmTMB::nbinom2(link = "log")` in `fitModelParallel()` parameters.
+
+```{r distrib_rnaseq, warning = FALSE, message = FALSE, fig.align='center', fig.height=4, fig.width=4 }
+pub_data_fn <- system.file("extdata", "rna_pub_data.tsv", package = "HTRfit")
+pub_data <- read.table(file = pub_data_fn, header = TRUE)
+plot(density(unlist(log10(pub_data[,-1]))), main = "Density of log RNAseq counts ")
+```
+
+While HTRfit is optimized for RNA-seq data, it's worth noting that the model family can be customized to accommodate other data types. In a different context, the model family can be adapted according to the nature of your response data.
+To illustrate, consider an attempt to model Iris sepal length based on sepal width using the iris dataframe. Evaluating the distribution shape of the variable to be modeled (Sepal.Width), we observe a normal distribution.
+
+
+```{r distrib_iris, warning = FALSE, message = FALSE, fig.align='center', fig.height=4, fig.width=5}
+data("iris")
+plot(density(unlist(iris$Sepal.Width)))
+```
+
+Given this observation, we opt to fit a model for each species using a Gaussian family model. Details about model family [here](https://www.rdocumentation.org/packages/stats/versions/3.6.2/topics/family).
+
+```{r example-fitModelParallel_nonRNA, warning = FALSE, message = FALSE}
+l_tmb_iris <- fitModelParallel(formula = Sepal.Width ~ Sepal.Length ,
+ data = iris,
+ group_by = "Species",
+ family = gaussian(),
+ n.cores = 1)
+l_tmb_iris
+```
diff --git a/vignettes/test_vignette.Rmd b/vignettes/test_vignette.Rmd
deleted file mode 100644
index c0d22405ae8b3de2cf36c53c81b4da8ef170be1e..0000000000000000000000000000000000000000
--- a/vignettes/test_vignette.Rmd
+++ /dev/null
@@ -1,25 +0,0 @@
----
-title: "Test vignette"
-output: rmarkdown::html_vignette
-vignette: >
- %\VignetteIndexEntry{Test vignette}
- %\VignetteEngine{knitr::rmarkdown}
- %\VignetteEncoding{UTF-8}
----
-
-```{r, include = FALSE}
-knitr::opts_chunk$set(
- collapse = TRUE,
- comment = "#>"
-)
-```
-
-```{r setup}
-library(HTRfit)
-print("oui")
-```
-
-
-[test_link](htrfit.html)
-
-[test_link2](HTRfit_simulation_pipeline.html)
\ No newline at end of file
diff --git a/vignettes/theoryBehindHtrfit.Rmd b/vignettes/theoryBehindHtrfit.Rmd
new file mode 100644
index 0000000000000000000000000000000000000000..cc635ecb82f28966d6c9377e674ebdd69651a164
--- /dev/null
+++ b/vignettes/theoryBehindHtrfit.Rmd
@@ -0,0 +1,33 @@
+---
+title: "Theory behind HTRfit"
+output: rmarkdown::html_vignette
+vignette: >
+ %\VignetteIndexEntry{Theory behind HTRfit}
+ %\VignetteEngine{knitr::rmarkdown}
+ %\VignetteEncoding{UTF-8}
+---
+
+```{r, include = FALSE}
+knitr::opts_chunk$set(
+ collapse = TRUE,
+ comment = "#>"
+)
+```
+
+# HTRfit simulation workflow
+
+In the realm of RNAseq analysis, various key experimental parameters play a crucial role in influencing the statistical power to detect expression changes. Parameters such as sequencing depth, the number of replicates, and others are expected to impact statistical power.
+To navigate the selection of optimal values for these experimental parameters, we introduce a comprehensive statistical framework known as HTRfit, underpinned by computational simulation. Moreover, HTRfit offers seamless compatibility with DESeq2 outputs, facilitating a comprehensive evaluation of RNAseq analysis.
+
+
+# Theory behind HTRfit
+
+<div id="bg" align="center">
+ <img src="man/figures/htrfit_workflow.png" width="500" height="300">
+</div>
+
+
+In this modeling framework, counts denoted as $K_{ij}$ for gene i and sample j are generated using a negative binomial distribution. The negative binomial distribution considers a fitted mean $\mu_{ij}$ and a gene-specific dispersion parameter $dispersion_i$.
+The fitted mean $\mu_{ij}$ is determined by a parameter, $q_{ij}$, which is proportionally related to the sum of all effects specified using `init_variable()` or `add_interaction()`. If basal gene expressions are provided, the $\mu_{ij}$ values are scaled accordingly using the gene-specific basal expression value ($bexpr_i$).
+Furthermore, the coefficients $\beta_i$ represent the natural logarithm fold changes for gene i across each column of the model matrix X. The dispersion parameter $dispersion_i$ plays a crucial role in defining the relationship between the variance of observed counts and their mean value. In simpler terms, it quantifies how far we expect observed counts to deviate from the mean value for each genes.
+
diff --git a/vignettes/HTRfit_simulation_pipeline.Rmd b/vignettes/tutorial.Rmd
similarity index 93%
rename from vignettes/HTRfit_simulation_pipeline.Rmd
rename to vignettes/tutorial.Rmd
index 3796e01f11d5e855e1e4f47dfc711b84db35d2ff..593b25bda082ebeacb694eb877a2f662d313dac7 100644
--- a/vignettes/HTRfit_simulation_pipeline.Rmd
+++ b/vignettes/tutorial.Rmd
@@ -1,8 +1,8 @@
---
-title: "HTRfit simulation pipeline"
+title: "Tutorial"
output: rmarkdown::html_vignette
vignette: >
- %\VignetteIndexEntry{HTRfit simulation pipeline}
+ %\VignetteIndexEntry{Tutorial}
%\VignetteEngine{knitr::rmarkdown}
%\VignetteEncoding{UTF-8}
---
@@ -16,6 +16,7 @@ knitr::opts_chunk$set(
```{r setup}
library(HTRfit)
+library(reshape2)
```
@@ -31,19 +32,21 @@ Counts data from the [SRP217588](https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?
```{r loading_pub_data, warning = FALSE, message = FALSE}
-pub_data <- read.table(file = "inst/extdata/rna_pub_data.tsv", h = T)
+pub_data_fn <- system.file("extdata", "rna_pub_data.tsv", package = "HTRfit")
+pub_data <- read.table(file = pub_data_fn, header = TRUE)
rownames(pub_data) <- pub_data$gene_id
pub_data$gene_id <- NULL
-pub_metadata <- read.table(file = "inst/extdata/rna_pub_metadata.tsv", h = T)
+pub_metadata_fn <- system.file("extdata", "rna_pub_metadata.tsv", package = "HTRfit")
+pub_metadata <- read.table(file = pub_metadata_fn, header = TRUE)
```
### Fit a model on a public dataset
To estimate plausible parameter distributions, we will fit a generalized linear model (GLM) to the publicly available data. This GLM will allow us to estimate plausible coefficients betaG, betaE, and betaGxE for 6000 genes. The `tidy_pub_fit` object contains the estimate obtain by HTRfit for each coefficient and each genes.
-```{r fit_pub_data, warning = FALSE, message = FALSE}
+```{ r fit_pub_data, eval=FALSE, message=FALSE, warning=FALSE }
-## -- prepare data & fit a model
+## -- prepare data & fit a model
data2fit = prepareData2fit(countMatrix = pub_data,
metadata = pub_metadata,
normalization = F,
@@ -55,6 +58,7 @@ l_tmb <- fitModelParallel(formula = kij ~ genotype + environment + genotype:envi
n.cores = 1)
## -- extract results
tidy_pub_fit <- tidy_results(l_tmb)
+
```
### Estimating parameter distributions
@@ -63,9 +67,19 @@ tidy_pub_fit <- tidy_results(l_tmb)
Now let's examine the distributions of the estimated parameters. Means, standard deviation and correlation between these parameters are inferred using the following lines :
-```{r fit_pub_data, warning = FALSE, message = FALSE}
+```{r load_fit_pub_data, include = FALSE}
+## -- hided in vignette
+## -- allows to not evaluate previous chunk (a bit long)
+tidy_pub_fn <- system.file("extdata", "tidy_pub_results.tsv", package = "HTRfit")
+tidy_pub_fit <- read.table(file = tidy_pub_fn, header = TRUE)
+dispersion_pub_fn <- system.file("extdata", "pub_dispersion.tsv", package = "HTRfit")
+dispersion_pub <- read.table(file = dispersion_pub_fn, header = TRUE)$x
+```
+
+
+```{r beta_pub_data, warning = FALSE, message = FALSE}
# -- Obtain the parameters
-all_beta_pub <- dcast(tidy_pub_fit, formula = ID ~ term, value.var = "estimate")
+all_beta_pub <- reshape2::dcast(tidy_pub_fit, formula = ID ~ term, value.var = "estimate")
columns_env <- c("environmentenv2","environmentenv3", "environmentenv4" )
beta_env <- rowMeans(all_beta_pub[, columns_env])
columns_interaction <- c("genotype2:environmentenv2","genotype2:environmentenv3", "genotype2:environmentenv4" )
@@ -87,7 +101,7 @@ cor(grouped_beta_pub)
We also save the dispersion parameter obtain from the public dataset.
-```{r pub_dispersion, warning = FALSE, message = FALSE }
+```{r pub_dispersion, warning = FALSE, message = FALSE, eval = FALSE}
dispersion_pub <- glance_tmb(l_tmb)$dispersion
```
@@ -160,6 +174,7 @@ resSimu <- evaluation_report(list_tmb = l_tmb,
alt_hypothesis = "greaterAbs")
```
+
#### Identity plot
The `evaluation_report()` function produces an identity plot, offering a visual tool to juxtapose the simulated effects (actual effects) with the model-inferred effects. This graphical representation simplifies assessing how well the model aligns with the values of the simulated effects, enabling a visual analysis of the model's goodness of fit to the simulated data. For a more quantitative evaluation of inference quality, the R-squared (R2) metric can be employed.
@@ -265,7 +280,7 @@ l_tmb <- fitModelParallel(formula = kij ~ genotype + environment + genotype:envi
data = data2fit,
group_by = "geneID",
family = glmmTMB::nbinom2(link = "log"),
- n.cores = 6,
+ n.cores = 1,
control = glmmTMB::glmmTMBControl(optCtrl=list(iter.max=1e5, eval.max=1e5)))
```
@@ -314,7 +329,7 @@ l_tmb <- fitModelParallel(formula = kij ~environment + ( environment | genotype
data = data2fit,
group_by = "geneID",
family = glmmTMB::nbinom2(link = "log"),
- n.cores = 6,
+ n.cores = 1,
control = glmmTMB::glmmTMBControl(optCtrl=list(iter.max=1e5, eval.max=1e5)))
```