[R] Getting error message, "LOOCV is not compatible with `resamples()` since only one resampling estimate is available. "

javed khan j@vedbtk111 @end|ng |rom gm@||@com
Tue Mar 3 12:29:40 CET 2020


Hi, I am using different validation methods for random search and grid
search. The validation methods are 10 fold CV, bootstrap and LOOCV but for
LOOCV, I get the error message when I draw boxplots for all the results.

Error is , LOOCV is not compatible with `resamples()` since only one
resampling estimate is available.

The code is below.

d=readARFF("china.arff")
index <- createDataPartition(d$Effort, p = .70,list = FALSE)
tr <- d[index, ]
ts <- d[-index, ]
index_2 <- createFolds(tr$Effort, returnTrain = TRUE, list = TRUE)




ct_rand <- trainControl(method = "repeatedcv", number=10, repeats=10,index
= index_2, search="random")
ct_grid <- trainControl(method = "repeatedcv", number=10, repeats=10,index
= index_2, search="grid")


ct_boot1 <- trainControl(method = "boot", number=100,  index = index_2,
search="random")
ct_boot2 <- trainControl(method = "boot", number=100,  index = index_2,
search="grid")

ct_locv <- trainControl(method = "LOOCV",  search="random")
ct_locv2 <- trainControl(method = "LOOCV",   search="grid")

set.seed(30218)
ran_CV <- train(Effort ~ ., data = tr,
                    method = "pls",
                    tuneLength = 15,
                    metric = "MAE",
                    preProc = c("center", "scale", "zv"),
                    trControl = ct_rand)
getTrainPerf(ran_CV)
rn <- predict(ran_CV, newdata = ts)

## ## ## ## ##grid search CV

set.seed(30218)
grid_CV <- train(Effort ~ ., data = tr,
                     method = "pls",
                     metric = "MAE",
                     preProc = c("center", "scale", "zv"),
                     trControl = ct_grid)

getTrainPerf(grid_CV)

set.seed(30218)
ran_boot <- train(Effort ~ ., data = tr,
                method = "pls",
                tuneLength = 15,
                metric = "MAE",
                preProc = c("center", "scale", "zv"),
                trControl = ct_boot1)
getTrainPerf(ran_boot)
rn <- predict(ran_search, newdata = ts)
##MAE(rn, ts$Effort)


## ## ## ## ##grid search boot

set.seed(30218)
grid_boot <- train(Effort ~ ., data = tr,
                 method = "pls",
                 metric = "MAE",
                 preProc = c("center", "scale", "zv"),
                 trControl = ct_boot2)

getTrainPerf(grid_boot)


set.seed(30218)
ran_locv <- train(Effort ~ ., data = tr,
                method = "pls",
                tuneLength = 15,
                metric = "MAE",
                preProc = c("center", "scale", "zv"),
                trControl = ct_locv)
getTrainPerf(ran_locv)
rn <- predict(ran_search, newdata = ts)
##MAE(rn, ts$Effort)


## ## ## ## ##grid search CV

set.seed(30218)
grid_locv <- train(Effort ~ ., data = tr,
                 method = "pls",
                 metric = "MAE",
                 preProc = c("center", "scale", "zv"),
                 trControl = ct_locv2)

getTrainPerf(grid_locv)


rValues <- resamples(list(Random_Search_CV=ran_CV, Grid_Search_CV=grid_CV,
Random_Search_Boot=ran_boot, Grid_Search_Boot=grid_boot ,
                          Random_Search_LOOCV=ran_locv,
Grid_Search_LOOCV=grid_locv))

bwplot(rValues,metric="MAE", scales=list(cex=1), col="Green")

	[[alternative HTML version deleted]]



More information about the R-help mailing list