Package {VeraCrop}

Extract influential points with safe FarmID mapping

Description

Internal function that extracts Cook's distance, leverage, and DFFITS with proper mapping to FarmID using row names rather than positional indices.

Usage

.extract_influential_points(model, data, na_action)

Arguments

Value

A list with influence statistics and mapped IDs

Add Missing Indicators

Description

Creates binary indicator variables for missing values

Usage

add_missing_indicators(
  data,
  vars = NULL,
  suffix = "_missing",
  as_numeric = FALSE,
  verbose = TRUE
)

Arguments

Value

Data frame with missing indicators added

Examples

df <- data.frame(Yield = rnorm(20, 5000, 500),
                  Nitrogen = c(rnorm(17, 120, 20), NA, NA, NA))
result <- add_missing_indicators(df, verbose = FALSE)
names(result)

Adjust Intercept After Back-Transformation

Description

When predictors are standardized, the intercept must be adjusted to make predictions in the original scale.

Usage

adjust_intercept(
  intercept_std,
  back_transformed_coefs,
  scaling_params,
  response_mean = NULL,
  response_sd = NULL
)

Arguments

Details

For X standardized only: intercept_original = intercept_std - sum(beta_original * mean_X)

For both X and Y standardized: intercept_original = mean_Y + intercept_std * sd_Y - sum(beta_original * mean_X)

Value

Adjusted intercept for the original scale model

Examples

df <- data.frame(Yield = rnorm(30, 5000, 500), N_rate = rnorm(30, 100, 20))
scaled <- scale_variables(df, exclude_vars = "Yield", verbose = FALSE)
model <- lm(Yield ~ N_rate_scaled, data = scaled$data)
back_coefs <- back_transform_coefficients(coef(model), scaled$scaling_params)
adjust_intercept(coef(model)["(Intercept)"], back_coefs[-1], scaled$scaling_params)

Back-transform Coefficients to Original Scale

Description

Converts coefficients from standardized predictors back to the original scale of the predictors.

Usage

back_transform_coefficients(
  coefs,
  scaling_params,
  response_sd = NULL,
  keep_names = TRUE
)

Arguments

Details

For standardized predictors only (X scaled, Y not scaled): beta_original = beta_standardized / sd(X)

For both X and Y scaled: beta_original = beta_standardized * (sd_Y / sd_X)

Value

Named vector of back-transformed coefficients

Examples

df <- data.frame(Yield = rnorm(30, 5000, 500), N_rate = rnorm(30, 100, 20))
scaled <- scale_variables(df, exclude_vars = "Yield", verbose = FALSE)
model <- lm(Yield ~ N_rate_scaled, data = scaled$data)
back_transform_coefficients(coef(model), scaled$scaling_params)

Back-transform Data

Description

Transforms scaled data back to the original scale using stored parameters.

Usage

back_transform_data(data, scaling_params, remove_scaled = FALSE)

Arguments

Value

Data frame with variables back-transformed to original scale

Examples

df <- data.frame(Yield = rnorm(30, 5000, 500), N_rate = rnorm(30, 100, 20))
scaled <- scale_variables(df, exclude_vars = "Yield", verbose = FALSE)
restored <- back_transform_data(scaled$data, scaled$scaling_params)
head(restored$N_rate)

Bootstrap Confidence Intervals for the CPA Yield Gap Decomposition

Description

Uses case resampling (nonparametric bootstrap) to estimate percentile confidence intervals for the overall yield gap and for each significant variable's contribution to it (Gap_Component and Share_Percent in the CPA table). The set of significant variables is held fixed across all resamples – by default it is determined once, from the original (non-resampled) data – so that the reported intervals describe the uncertainty of the CPA decomposition itself, not of variable selection. Optionally, variable selection is also re-run independently on each resample purely as a diagnostic, to report how often each fixed variable would be re-selected on its own (selection_stability).

Usage

bootstrap_cpa(
  data,
  response = "Yield",
  sig_vars = NULL,
  selection_method = c("ftest", "aic"),
  n_boot = 1000,
  conf_level = 0.95,
  top_percentile = NULL,
  min_top_samples = NULL,
  check_selection_stability = TRUE,
  seed = NULL,
  verbose = TRUE
)

Arguments

Value

A list with components:

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
result <- yield_gap_analysis(prep$data, response = "Yield", verbose = FALSE)
sig_vars <- result$cpa_table$Variable[result$cpa_table$Variable != "Intercept"]
boot <- bootstrap_cpa(prep$data, response = "Yield", sig_vars = sig_vars,
                       n_boot = 200, verbose = FALSE)
boot$summary

Calculate Min-Max Effects of Variables on Yield

Description

Estimates the effect of each variable on yield by comparing predictions when the variable is set to its minimum vs. maximum.

Usage

calculate_minmax_effects(
  model,
  data,
  vars,
  use_quantiles = TRUE,
  quantile_range = c(0.05, 0.95)
)

Arguments

Value

A data frame with columns: Variable, Effect, EffectAbs, EffectPct.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
sel <- select_vars_ftest(prep$data, response = "Yield", verbose = FALSE)
mm <- calculate_minmax_effects(sel$final_model, prep$data, sel$significant_vars)
mm

Calculate Optimal Values for Selected Variables

Description

Determines the optimal value for each selected predictor variable.

Usage

calculate_opt_values(
  data,
  vars,
  coefs,
  data_top,
  response_var = "Yield",
  top_percentile = 0.99,
  min_top_samples = 20
)

Arguments

Value

A list with components: var_table, used_percentile, n_top_samples, n_total.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
sel <- select_vars_ftest(prep$data, response = "Yield", verbose = FALSE)
sig_vars <- sel$significant_vars
coefs_vars <- coef(sel$final_model)[sig_vars]
yield_thr <- quantile(prep$data$Yield, 0.90, na.rm = TRUE)
data_top <- prep$data[prep$data$Yield >= yield_thr, ]
opt <- calculate_opt_values(prep$data, sig_vars, coefs_vars, data_top,
                             response_var = "Yield", top_percentile = 0.90)
opt$var_table

Calculate Relative Importance of Variables

Description

Computes the relative importance of predictors using LMG or standardized beta.

Usage

calculate_relative_importance(
  data,
  vars,
  response = "Yield",
  method = c("lmg", "std_beta"),
  verbose = TRUE
)

Arguments

Value

A list with components: method and results.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
sel <- select_vars_ftest(prep$data, response = "Yield", verbose = FALSE)
ri <- calculate_relative_importance(prep$data, sel$significant_vars,
                                     response = "Yield", verbose = FALSE)
ri$results

Check Linear Regression Assumptions

Description

Automatically checks the five main assumptions of linear regression: 1. Linearity (using Ramsey RESET test or plot) 2. Independence of residuals (Durbin-Watson with p-value) 3. Homoscedasticity (Breusch-Pagan) 4. Normality of residuals (Shapiro-Wilk with skewness/kurtosis) 5. Multicollinearity (VIF/GVIF with proper scaling)

Usage

check_assumptions(
  model,
  data = NULL,
  plot = TRUE,
  verbose = TRUE,
  max_warnings = 2,
  seed = 42,
  alpha = 0.05,
  vif_threshold = 10,
  vif_warning_threshold = 5
)

Arguments

Value

A list with all test results, summary status, data readiness flag, and (if plot=TRUE) a recorded diagnostic plot object.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
model <- lm(Yield ~ ., data = prep$data)
result <- check_assumptions(model, data = prep$data, plot = FALSE, verbose = FALSE)
result$data_ready

Quick Check for Data Readiness

Description

Quick Check for Data Readiness

Usage

check_data_ready(data, response_var = NULL, verbose = TRUE, details = FALSE)

Arguments

Value

If details=TRUE, returns a list with ready, issues, and warnings. Otherwise returns a logical.

Examples

df <- data.frame(Yield = rnorm(20, 5000, 500), Nitrogen = rnorm(20, 120, 20))
check_data_ready(df, response_var = "Yield", verbose = FALSE)

Detect variable types

Description

Detect variable types

Usage

detect_variable_types(
  data,
  response_var = "Yield",
  max_categorical_levels = 10,
  treat_numeric_as_ordinal = FALSE,
  auto_detect_ordinal_from_name = TRUE,
  auto_detect_ordinal_mode = "strict",
  ordinal_level_orders = list(),
  reference_method = "mode",
  verbose = TRUE
)

Arguments

Value

List with variable type information

Examples

df <- data.frame(
  Yield = rnorm(30, 5000, 500),
  Nitrogen = rnorm(30, 120, 20),
  Region = sample(c("North", "South", "East"), 30, replace = TRUE)
)
types <- detect_variable_types(df, response_var = "Yield", verbose = FALSE)
types$Region$type

Encode variables for statistical models

Description

Encode variables for statistical models

Usage

encode_variables(
  data,
  var_types,
  response_var = "Yield",
  max_dummy_levels = 10,
  min_freq_for_encoding = 0.05,
  min_count_for_encoding = 5,
  grouping_rule = "or",
  verbose = TRUE
)

Arguments

Value

A list with:

data

Encoded data frame

encoding_info

Metadata describing each encoding

dropped_vars

Character vector of variables dropped during encoding

Examples

df <- data.frame(
  Yield = rnorm(30, 5000, 500),
  Region = sample(c("North", "South", "East"), 30, replace = TRUE)
)
types <- detect_variable_types(df, response_var = "Yield", verbose = FALSE)
result <- encode_variables(df, types, response_var = "Yield", verbose = FALSE)
names(result$data)

Apply All Filtering Steps

Description

Convenience function that applies all filtering steps in the recommended order: 1. Remove non-predictors (IDs) 2. Remove constant variables 3. Remove near-zero variance variables 4. Remove highly correlated variables

Usage

filter_variables(
  data,
  response_var = "Yield",
  exclude_vars = NULL,
  exclude_patterns = NULL,
  freq_cut = 19,
  unique_cut = 10,
  corr_threshold = 0.9,
  keep_method = "higher_correlation",
  cor_method = "pearson",
  cor_use = "pairwise.complete.obs",
  verbose = TRUE
)

Arguments

Details

Returns a list with filtered data and detailed removal records.

Value

A list with:

Examples

data(wheat1)
result <- filter_variables(wheat1, response_var = "Yield", verbose = FALSE)
result$n_initial
result$n_final

Group levels of high-cardinality categorical variables

Description

Group levels of high-cardinality categorical variables

Usage

group_high_cardinality(
  x,
  min_freq = 0.05,
  min_count = 5,
  other_label = "..OTHER..",
  rule = "or"
)

Arguments

Value

Factor with grouped levels

Examples

x <- c(rep("A", 20), rep("B", 15), rep("C", 2), rep("D", 1))
grouped <- group_high_cardinality(x, min_freq = 0.05, min_count = 5)
table(grouped)

Handle Missing Values

Description

Imputes missing values using specified methods. Different methods can be used for different data types.

Usage

handle_missing_values(
  data,
  response_var = "Yield",
  numeric_method = "median",
  categorical_method = "mode",
  logical_method = "mode",
  missing_threshold = 0.2,
  ties_method = "first",
  add_indicators = FALSE,
  indicator_suffix = "_missing",
  verbose = TRUE
)

Arguments

Details

Percent of imputed cells is calculated relative to the original variables before indicator creation (when add_indicators = TRUE).

Note: integer64 (bit64 package) values: if bit64 is installed, values are preserved as integer64. Otherwise, they are converted to double (may lose precision for values > 2^53).

Mode appropriateness can be adjusted via options("veracrop.mode_threshold"). Default is 1.0. Increase to reduce warnings, decrease to be more strict.

Value

A list with:

Examples

df <- data.frame(
  Yield = rnorm(30, 5000, 500),
  Nitrogen = c(rnorm(27, 120, 20), NA, NA, NA)
)
result <- handle_missing_values(df, response_var = "Yield", verbose = FALSE)
sum(is.na(result$data))

Quick Missing Value Summary

Description

Provides a quick summary of missing values in a data frame

Usage

missing_summary(data, verbose = TRUE)

Arguments

Value

A list with missing value statistics (invisibly if verbose=TRUE)

Examples

df <- data.frame(Yield = rnorm(20, 5000, 500),
                  Nitrogen = c(rnorm(17, 120, 20), NA, NA, NA))
summary_result <- missing_summary(df, verbose = FALSE)
summary_result$total_missing

Quick Model Summary with Assumption Checks

Description

Combines model summary and assumption checks in one function.

Usage

model_diagnostics(
  model,
  data = NULL,
  plot = TRUE,
  verbose = TRUE,
  max_warnings = 2,
  ...
)

Arguments

Value

A list with model summary and assumption checks.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
model <- lm(Yield ~ ., data = prep$data)
result <- model_diagnostics(model, data = prep$data, plot = FALSE, verbose = FALSE)
result$summary$r.squared

Pareto Analysis of Yield Gap Contributions

Description

Identifies the most important variables contributing to the yield gap.

Usage

pareto_analysis(yg_results, top_n = NULL, verbose = TRUE)

Arguments

Value

A list with components: pareto_table, top_n, top_variables.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
results <- yield_gap_analysis(prep$data, response = "Yield", verbose = FALSE)
cpa_input <- results$cpa_table[, c("Variable", "Beta", "Mean", "Opt")]
yg <- yield_gap_components(cpa_input)
pareto <- pareto_analysis(yg, verbose = FALSE)
pareto$pareto_table

Generate All Yield Gap Plots

Description

Convenience function that calls all five plotting functions and optionally combines them into a single figure using patchwork.

Usage

plot_all_graphs(
  results,
  top_percentile = 0.99,
  save_plots = FALSE,
  output_dir = tempdir(),
  verbose = TRUE
)

Arguments

Value

Invisibly returns a named list of all five ggplot2 objects.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
results <- yield_gap_analysis(prep$data, response = "Yield", verbose = FALSE)
plot_all_graphs(results, top_percentile = 0.95,
                 save_plots = TRUE, output_dir = tempdir(), verbose = FALSE)

Plot Variable Contributions to Yield Gap

Description

Horizontal bar chart showing the percentage contribution of each selected variable to the total yield gap.

Usage

plot_contribution_shares(
  results,
  top_percentile = 0.99,
  save_plot = FALSE,
  filename = "contribution_shares.png"
)

Arguments

Value

A ggplot2 object, or NULL if no data available.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
results <- yield_gap_analysis(prep$data, response = "Yield", verbose = FALSE)
plot_contribution_shares(results)

Plot Min-Max Effects of Variables on Yield

Description

Diverging horizontal bar chart showing how much yield changes when each variable moves from its minimum to maximum value.

Usage

plot_minmax_effects(
  results,
  save_plot = FALSE,
  filename = "minmax_effects.png"
)

Arguments

Value

A ggplot2 object, or NULL if no data available.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
results <- yield_gap_analysis(prep$data, response = "Yield", verbose = FALSE)
plot_minmax_effects(results)

Plot Observed vs. Predicted Yield

Description

Scatter plot comparing observed and model-predicted yields, including a 1:1 line, regression line, and R-squared/RMSE annotations.

Usage

plot_observed_vs_predicted(
  results,
  save_plot = FALSE,
  filename = "observed_vs_predicted.png"
)

Arguments

Value

A ggplot2 object, or NULL if no data available.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
results <- yield_gap_analysis(prep$data, response = "Yield", verbose = FALSE)
plot_observed_vs_predicted(results)

Plot Relative Importance of Variables

Description

Horizontal bar chart of relative importance percentages, using either the LMG or standardized beta method.

Usage

plot_relative_importance(
  results,
  save_plot = FALSE,
  filename = "relative_importance.png"
)

Arguments

Value

A ggplot2 object, or NULL if no data available.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
results <- yield_gap_analysis(prep$data, response = "Yield", verbose = FALSE)
plot_relative_importance(results)

Plot Distribution of Yield Gap Across Fields

Description

Histogram with density curve showing the distribution of per-field yield gaps, with mean and median reference lines.

Usage

plot_yield_gap_distribution(
  results,
  save_plot = FALSE,
  filename = "yield_gap_distribution.png"
)

Arguments

Value

A ggplot2 object, or NULL if no data available.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
results <- yield_gap_analysis(prep$data, response = "Yield", verbose = FALSE)
plot_yield_gap_distribution(results)

Main Preprocessing Pipeline for Yield Gap Analysis

Description

Runs the full preprocessing pipeline end to end: removes non-predictor (ID-like) columns, imputes missing values, detects variable types, encodes variables for linear regression, removes constant variables introduced by encoding, optionally scales continuous variables, and optionally removes near-zero-variance and highly correlated variables.

Usage

prep_yield_gap(data, response_var = "Yield", ...)

Arguments

Details

This function is a thin orchestrator: all actual logic lives in the dedicated modules (remove_non_predictors, handle_missing_values, detect_variable_types, encode_variables, remove_constant_vars, scale_variables, remove_near_zero_variance, remove_highly_correlated).

Value

A list with:

data

Final preprocessed data frame.

var_types

Variable type metadata from detect_variable_types().

encoding_info

Encoding metadata from encode_variables().

scaling_params

Scaling parameters from scale_variables(), or NULL if scale_vars = FALSE.

removed_vars

Named list of character vectors: variables removed at each filtering stage (non_predictors, constant, nzv, correlated).

data_size

One of "small", "medium", "large", based on input row count.

response_var

The response variable name used throughout.

preprocessing_params

The fully-resolved settings list actually used.

Examples

df <- data.frame(
  Yield = rnorm(30, 5000, 500),
  Nitrogen = rnorm(30, 120, 20),
  Region = sample(c("North", "South", "East"), 30, replace = TRUE)
)
prep <- prep_yield_gap(df, response_var = "Yield", verbose = FALSE)
dim(prep$data)

# Using the bundled real dataset:
data(wheat1)
prep2 <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
dim(prep2$data)

Create a Preprocessing Report

Description

Create a Preprocessing Report

Usage

preprocessing_report(
  preprocessed_result,
  validation_result = NULL,
  output_file = NULL,
  verbose = TRUE
)

Arguments

Value

Character string with report (invisibly if verbose=TRUE)

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
report <- preprocessing_report(prep, verbose = FALSE)
cat(report[1:5], sep = "\n")

Remove Constant Variables

Description

Removes variables that have only one unique non-NA value (including all-NA columns). Works for ALL data types (numeric, factor, character).

Usage

remove_constant_vars(data, response_var = "Yield", verbose = TRUE)

Arguments

Value

A list with:

Examples

df <- data.frame(Yield = rnorm(20, 5000, 500), AlwaysFive = 5)
result <- remove_constant_vars(df, response_var = "Yield", verbose = FALSE)
names(result$data)

Remove Highly Correlated Variables

Description

Identifies and removes highly correlated numeric variables using an iterative greedy algorithm similar to caret::findCorrelation().

Usage

remove_highly_correlated(
  data,
  response_var = "Yield",
  threshold = 0.9,
  keep_method = "higher_correlation",
  cor_method = "pearson",
  cor_use = "pairwise.complete.obs",
  verbose = TRUE
)

Arguments

Details

The algorithm: 1. Finds the pair with the highest absolute correlation above threshold 2. Removes one variable from the pair (based on keep_method) 3. Recalculates correlations and repeats until no pair exceeds threshold

Value

A list with:

Examples

set.seed(1)
x <- rnorm(30, 100, 10)
df <- data.frame(Yield = rnorm(30, 5000, 500),
                  Nitrogen = x,
                  Nitrogen_copy = x + rnorm(30, 0, 0.01))
result <- remove_highly_correlated(df, response_var = "Yield", verbose = FALSE)
names(result$data)

Remove Near Zero Variance Variables

Description

Identifies and removes variables with near zero variance using the same logic as caret::nearZeroVar: - freqRatio: ratio of most common to second most common value - percentUnique: percentage of unique values

Usage

remove_near_zero_variance(
  data,
  response_var = "Yield",
  freq_cut = 19,
  unique_cut = 10,
  verbose = TRUE
)

Arguments

Details

A variable is considered near-zero variance if: freqRatio > freq_cut AND percentUnique < unique_cut

Value

A list with:

Examples

df <- data.frame(Yield = rnorm(30, 5000, 500),
                  AlmostConstant = c(rep(1, 28), 2, 3))
result <- remove_near_zero_variance(df, response_var = "Yield", verbose = FALSE)
names(result$data)

Remove Non-Predictor Variables

Description

Removes variables that should not be used as predictors, such as ID columns. Date and location variables are NOT removed by default.

Usage

remove_non_predictors(
  data,
  exclude_vars = NULL,
  exclude_patterns = NULL,
  verbose = TRUE
)

Arguments

Value

A list with:

Examples

df <- data.frame(Field_ID = 1:10, Yield = rnorm(10, 5000, 500),
                  Nitrogen = rnorm(10, 120, 20))
result <- remove_non_predictors(df, verbose = FALSE)
names(result$data)

Save Field-Level Yield Gap Analysis to Excel

Description

Creates a detailed field-level Excel report showing per-field predictions, yield gaps, factor contributions, limiting factors, and rankings.

Usage

save_field_level_excel(
  results,
  df_final,
  output_path = file.path(tempdir(), "field_level_analysis.xlsx"),
  verbose = TRUE
)

Arguments

Details

Requires the writexl package. Please install it with: install.packages("writexl")

Value

Invisibly returns the output path.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
results <- yield_gap_analysis(prep$data, response = "Yield", verbose = FALSE)
save_field_level_excel(results, prep$data,
                        output_path = file.path(tempdir(), "field_analysis.xlsx"))

Save Yield Gap Analysis Results to Excel

Description

Exports all analysis results (CPA table, predictions, variable statistics, relative importance, Pareto analysis, cross-validation, and model summary) to a multi-sheet Excel file.

Usage

save_results_excel(
  results,
  output_path = file.path(tempdir(), "yield_gap_analysis_full.xlsx"),
  verbose = TRUE
)

Arguments

Details

Requires the writexl package. Please install it with: install.packages("writexl")

Value

Invisibly returns the output path.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
results <- yield_gap_analysis(prep$data, response = "Yield", verbose = FALSE)
save_results_excel(results,
                    output_path = file.path(tempdir(), "yield_gap_results.xlsx"))

Scale New Data Using Existing Parameters

Description

Applies the same scaling parameters to new data (e.g., test set).

Usage

scale_new_data(data, scaling_params, keep_original = FALSE, verbose = TRUE)

Arguments

Value

Data frame with scaled variables

Examples

train <- data.frame(Yield = rnorm(30, 5000, 500), N_rate = rnorm(30, 100, 20))
scaled <- scale_variables(train, exclude_vars = "Yield", verbose = FALSE)
new_obs <- data.frame(N_rate = c(90, 110, 130))
scale_new_data(new_obs, scaled$scaling_params, verbose = FALSE)

Scale Continuous Variables

Description

Standardizes (or robustly scales) numeric variables in a data frame.

Usage

scale_variables(
  data,
  vars = NULL,
  exclude_vars = NULL,
  method = "standard",
  keep_original = FALSE,
  verbose = TRUE
)

Arguments

Value

A list with:

Examples

df <- data.frame(Yield = rnorm(30, 5000, 500), N_rate = rnorm(30, 100, 20))
result <- scale_variables(df, exclude_vars = "Yield", method = "standard", verbose = FALSE)
result$scaling_params

Variable Selection via AIC-Based Stepwise Regression

Description

Selects variables by minimizing AIC (Akaike Information Criterion) using step, as an alternative to F-test-based selection (see select_vars_ftest).

Usage

select_vars_aic(data, response = "Yield", direction = "both", verbose = TRUE)

Arguments

Details

AIC-based and F-test-based selection answer different questions and will often disagree on how many variables to keep: AIC-based stepwise selection optimizes overall model fit (penalized by model complexity) and has no fixed significance threshold, so it will often retain MORE variables than a strict F-test-based approach with typical F-to-Enter/F-to-Remove thresholds (e.g. around 4 and 3.9, as used by default in software such as SigmaPlot) - variables that improve AIC slightly may not individually clear a fixed F/p-value bar. Neither method is universally "correct"; they encode different criteria for what counts as a useful predictor.

Value

A list with:

Examples

df <- data.frame(Yield = rnorm(40, 5000, 500),
                  Nitrogen = rnorm(40, 120, 20),
                  Phosphorus = rnorm(40, 40, 10))
sel <- select_vars_aic(df, response = "Yield", verbose = FALSE)
sel$significant_vars

Forward Stepwise Variable Selection Using F-test

Description

Selects significant predictor variables using a forward stepwise algorithm based on partial F-tests. Supports both P-value threshold and F-to-Enter/F-to-Remove criteria (like SigmaPlot).

Usage

select_vars_ftest(
  data,
  response = "Yield",
  p_threshold = 0.05,
  use_bonferroni = TRUE,
  max_vars = NULL,
  f_to_enter = NULL,
  f_to_remove = NULL,
  verbose = TRUE
)

Arguments

Value

A list with components:

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
sel <- select_vars_ftest(prep$data, response = "Yield", verbose = FALSE)
sel$significant_vars

Sugarcane Yield Trial Data

Description

Field-level sugarcane yield data from 371 fields, recording yield and a broad range of crop-cycle, fertilization, pest/disease, and remote-sensing/soil variables. No missing values. A larger, cleaner third example dataset (compared to wheat2), useful for illustrating the package on a different crop and a dataset with several genuinely ordinal and perfectly collinear (aliased) variables - see Examples.

Usage

sugarcane1

Format

A data frame with 371 rows and 30 variables:

field_id

Character. Field identifier (non-predictor; removed automatically by remove_non_predictors).

cultivar

Character. Sugarcane cultivar name (nominal).

crop_type

Character. Crop cycle stage: "PC" (plant cane) followed by "R1" through "R9" (successive ratoon/regrowth cycles). This is a genuinely ORDINAL variable stored as text (PC < R1 < R2 < ... < R9) - see Examples for how to encode its order explicitly via ordinal_level_orders.

texture_class

Character. Soil texture class (nominal, e.g. "Loam", "Silty clay").

drainage_issue

Character. "Yes"/"No": whether the field has a drainage issue.

smut_incidence

Character. "Yes"/"No": presence of sugarcane smut disease.

other_weeds_presence

Character. "Yes"/"No": presence of weeds other than the main tracked species.

harvestable_area

Numeric. Harvestable area (hectares).

harvest_days_from_September23_2023

Integer. Harvest date, as days after September 23, 2023.

harvest_duration

Numeric. Duration of the harvest operation (days).

crop_cycle_days

Integer. Total length of the crop cycle (days).

first_topdressing_days_from_February20_2023

Integer. First nitrogen topdressing date, as days after February 20, 2023.

second_topdressing_days_from_February20_2023

Integer. Second nitrogen topdressing date, as days after February 20, 2023.

third_topdressing_days_from_February20_2023

Integer. Third nitrogen topdressing date, as days after February 20, 2023.

starter_urea_kg_ha

Numeric. Starter urea fertilizer rate (kg/ha).

first_topdressing_urea_kg_ha

Numeric. First topdressing urea rate (kg/ha).

second_topdressing_urea_kg_ha

Numeric. Second topdressing urea rate (kg/ha).

third_topdressing_urea_kg_ha

Numeric. Third topdressing urea rate (kg/ha).

total_urea_kg_ha

Numeric. Total urea applied (kg/ha). NOTE: this is the exact sum of starter_urea_kg_ha, first_topdressing_urea_kg_ha, second_topdressing_urea_kg_ha, and third_topdressing_urea_kg_ha - i.e. perfectly collinear (aliased) with them when all four are included in the same model. Both select_vars_aic and select_vars_ftest handle this gracefully (see Examples), but including this column alongside all four of its components in a manually-fit lm() will produce an NA coefficient for whichever is entered last.

leaf_N_at_mid_stem_elongation

Numeric. Leaf nitrogen concentration at mid stem elongation (percent).

leaf_N_at_grand_growth

Numeric. Leaf nitrogen concentration at grand growth stage (percent).

leaf_N_before_irrigation_cutoff

Numeric. Leaf nitrogen concentration before irrigation cutoff (percent).

irrigation_cutoff_days_from_September23_2023

Integer. Irrigation cutoff date, as days after September 23, 2023.

sesamia_bored_internodes_percent

Numeric. Percentage of internodes bored by Sesamia stem borer.

cogongrass_infested_m2

Integer. Area infested by cogongrass (square meters).

irrigation_interval

Numeric. Average interval between irrigation events (days).

max_LAI

Numeric. Maximum Leaf Area Index observed during the season.

MCARI

Numeric. Modified Chlorophyll Absorption in Reflectance Index (a remote-sensing vegetation index).

soil_EC

Numeric. Soil electrical conductivity.

Yield

Numeric. Cane yield. Response variable.

Source

Field trial data used during package development and testing.

Examples

data(sugarcane1)

# crop_type is genuinely ordinal (PC < R1 < ... < R9); without an
# explicit order it would conservatively be treated as nominal.

prep <- prep_yield_gap(
  sugarcane1,
  response_var = "Yield",
  ordinal_level_orders = list(
    crop_type = c("PC", paste0("R", 1:9))
  ),
  verbose = FALSE
)
dim(prep$data)

# total_urea_kg_ha is a perfect linear combination of the four
# individual urea application columns; both selection methods drop it
# automatically (with a warning) rather than erroring.
result <- yield_gap_analysis(prep$data, response = "Yield",
                              selection_method = "aic", verbose = FALSE)
result$yield_gap

Get Scaling Summary

Description

Provides a human-readable summary of scaling parameters.

Usage

summary_scaling(scaling_params, verbose = TRUE)

Arguments

Value

A data frame with scaling parameters (invisibly if verbose=TRUE)

Examples

df <- data.frame(Yield = rnorm(30, 5000, 500), N_rate = rnorm(30, 100, 20))
scaled <- scale_variables(df, exclude_vars = "Yield", verbose = FALSE)
summary_scaling(scaled$scaling_params, verbose = FALSE)

Trim leading/trailing whitespace from character columns

Description

Real-world CSV/Excel exports very commonly introduce accidental leading or trailing whitespace on text values (from copy-paste, cell formatting, trailing tabs, etc.). Left untrimmed, this silently splits what should be a single category into two distinct levels - e.g. "Sirvan" (22 rows) and "Sirvan " (4 rows) being treated as two different cultivars - with no error or warning anywhere downstream, since both are perfectly valid (if different) character strings from R's point of view. This function is called automatically and by default at the start of prep_yield_gap.

Usage

trim_whitespace(data, verbose = TRUE)

Arguments

Value

The data frame with whitespace trimmed from all character columns. Factor and non-character columns are left unchanged.

Examples

df <- data.frame(Cultivar = c("Sirvan", "Sirvan ", " Sirvan"),
                  Yield = c(5000, 4900, 5100),
                  stringsAsFactors = FALSE)
trimmed <- trim_whitespace(df, verbose = FALSE)
length(unique(trimmed$Cultivar))

k-Fold Cross-Validation for Yield Gap Model

Description

Evaluates model predictive performance using k-fold cross-validation. A NEW model is refit on each training fold (using sig_vars and response); the model argument is used only to validate that sig_vars/response are consistent with an already- fitted whole-data model, not refit directly, since cross-validation by definition requires refitting on each fold's training subset.

Usage

validate_model_with_cv(
  model,
  data,
  sig_vars,
  response = "Yield",
  k_folds = 5,
  seed = NULL
)

Arguments

Value

A list with components: by_fold and summary.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
sel <- select_vars_ftest(prep$data, response = "Yield", verbose = FALSE)
cv <- validate_model_with_cv(sel$final_model, prep$data, sel$significant_vars,
                              response = "Yield", k_folds = 5, seed = 42)
cv$summary

Validate Preprocessing Results

Description

Performs comprehensive validation of preprocessing output.

Usage

validate_preprocessing(preprocessed_result, verbose = TRUE)

Arguments

Value

List with validation results, always including: valid, issues, warnings, n_errors, n_warnings, summary, response_var, has_encoding, has_scaling.

Examples

df <- data.frame(Yield = rnorm(30, 5000, 500), Nitrogen = rnorm(30, 120, 20))
prep <- prep_yield_gap(df, response_var = "Yield", verbose = FALSE)
val <- validate_preprocessing(prep, verbose = FALSE)
val$valid

Wheat Yield Trial Data (Clean)

Description

Field-level wheat yield data from 200 fields, recording yield and agronomic management/environmental variables. Used throughout the package documentation and vignette as the primary worked example: a relatively large, clean dataset (no missing values) suitable for demonstrating the full prep_yield_gap / yield_gap_analysis workflow end to end.

Usage

wheat1

Format

A data frame with 200 rows and 13 variables:

Yield

Numeric. Grain yield (kg/ha). Response variable.

Irrigation

Numeric. Irrigation water applied.

Nitrogen

Numeric. Nitrogen fertilizer applied (kg/ha).

Phosphorus

Numeric. Phosphorus fertilizer applied (kg/ha).

Potassium

Numeric. Potassium fertilizer applied (kg/ha).

Soil_pH

Numeric. Soil pH.

Soil_Organic_Matter

Numeric. Soil organic matter (percent).

Plant_density

Numeric. Plant density (plants per unit area).

Weed_Infestation

Numeric. Weed infestation score/count.

Drought

Integer (0/1). Whether the field experienced drought stress.

Pest

Integer (0/1). Whether the field experienced pest pressure.

Cultivar

Character. Wheat cultivar name (nominal, several levels).

Sowing_Date

Numeric. Sowing date, encoded as day offset.

Source

Field trial data used during package development and testing.

Examples

data(wheat1)
str(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
dim(prep$data)

Wheat Yield Trial Data (Small, High-Dimensional, Real-World Messy)

Description

Field-level wheat yield data from 40 fields with many recorded management, socioeconomic, and pest/disease variables (29 columns). Deliberately kept in its original, unedited form (including a non-predictor ID column and a stray leading/trailing whitespace inconsistency in Cultivar, where "Sirvan" appears both as "Sirvan" and "Sirvan ") to serve as a realistic example of common real-world data-quality issues and of a small, high predictor-to-sample-ratio dataset - exactly the profile that exercises trim_whitespace, near-zero-variance filtering, and the auto-tuning logic in prep_yield_gap.

Usage

wheat2

Format

A data frame with 40 rows and 29 variables:

Field_id

Integer. Field identifier (non-predictor; removed automatically by remove_non_predictors).

Farmers_age

Integer. Age of the farmer (years).

Litracy_Level

Character. Farmer's education level (a true ordinal variable stored as text - see Examples for how to encode its order explicitly via ordinal_level_orders).

Experience_Years

Integer. Farmer's years of experience.

Cultivated_Area

Numeric. Cultivated area (hectares).

Rotation

Character. Crop rotation sequence preceding wheat.

Crop_Residue_Mgmt

Character. How previous crop residue was managed (e.g. "Burned", "Incorporated", "Retained").

Pre_Plow_Irrigation

Character. "Yes"/"No": whether a pre-plowing irrigation was applied.

Disk_Passes

Integer. Number of disk harrow passes.

Land_Leveled

Character. "Yes"/"No": whether the land was laser-leveled.

Seeder_Type

Character. Type of seeder used (e.g. "Seed Drill", "Centrifugal").

Planting_Day_From_Sep23

Integer. Planting date, as days after September 23.

Seed_Rate

Integer. Seeding rate.

Cultivar

Character. Wheat cultivar name. Contains a real leading/trailing whitespace inconsistency ("Sirvan" vs. "Sirvan ") used to illustrate trim_whitespace.

TSP_Rate

Integer. Triple superphosphate fertilizer rate.

Good_Drainage

Character. "Yes"/"No": whether the field has good drainage.

Irrigation_Count

Integer. Number of irrigation events.

Soil_texture

Character. Soil texture class (e.g. "Clay", "Clay-sand").

Weed_species

Character. Multi-value field: weed species present, concatenated with "-" separators - treated as a high-cardinality nominal variable; see group_high_cardinality.

Spraying_DAP

Integer. Herbicide spraying date, in days after planting.

Herbicide_Type

Character. Multi-value field: herbicide active ingredient(s) applied, concatenated with "_" separators.

Urea_as_adjuvant

Character. "Yes"/"No": whether urea was used as a spray adjuvant.

Nitrogen_topdressing_Number

Integer. Number of nitrogen topdressing applications.

Topdressing_N

Integer. Total topdressing nitrogen applied.

Puccinia_disease

Character. "Yes"/"No": presence of Puccinia (rust) disease.

Days_To_Harvest

Integer. Days from planting to harvest.

Selection_Pressure

Character. "Yes"/"No": indicator of herbicide selection pressure.

Herbicide_Resistance_Suspected

Character. "Yes"/"No": whether herbicide resistance was suspected in the field.

Yield

Integer. Grain yield (kg/ha). Response variable.

Source

Field trial data used during package development and testing.

Examples

data(wheat2)

# Demonstrates automatic whitespace trimming: "Sirvan " and "Sirvan"
# are correctly merged into a single category.
length(unique(wheat2$Cultivar))
trimmed <- trim_whitespace(wheat2, verbose = FALSE)
length(unique(trimmed$Cultivar))


# Full pipeline, including an explicit ordinal level order for
# Litracy_Level since it cannot be safely inferred automatically.
prep <- prep_yield_gap(
  wheat2,
  response_var = "Yield",
  ordinal_level_orders = list(
    Litracy_Level = c("Elementary School", "Middle School", "Diploma",
                       "Bachelor's Degree", "Master's Degree")
  ),
  verbose = FALSE
)
dim(prep$data)

Complete Yield Gap Analysis Using CPA Method

Description

Runs the full Comparative Performance Analysis (CPA) pipeline: variable selection, optimal value calculation, min-max effects, yield gap decomposition, cross-validation, relative importance, and Pareto analysis.

Usage

yield_gap_analysis(
  data,
  response = "Yield",
  p_threshold = NULL,
  top_percentile = NULL,
  max_vars = NULL,
  calculate_ri = TRUE,
  ri_method = c("lmg", "std_beta"),
  selection_method = c("ftest", "aic"),
  aic_direction = c("both", "forward", "backward"),
  perform_cv = TRUE,
  cv_seed = 42,
  use_bonferroni = FALSE,
  verbose = TRUE
)

Arguments

Value

A list with the following components:

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
results <- yield_gap_analysis(prep$data, response = "Yield", verbose = FALSE)
results$yield_gap
results$cpa_table

Calculate Yield Gap Components from CPA Table

Description

Computes contributions of each variable to the yield gap.

Usage

yield_gap_components(cpa_table)

Arguments

Value

A list with components: cpa_table, yield_mean, yield_opt, yield_gap, gap_positive, gap_negative.

Examples

data(wheat1)
prep <- prep_yield_gap(wheat1, response_var = "Yield", verbose = FALSE)
results <- yield_gap_analysis(prep$data, response = "Yield", verbose = FALSE)
cpa_input <- results$cpa_table[, c("Variable", "Beta", "Mean", "Opt")]
yg <- yield_gap_components(cpa_input)
yg$yield_gap