# Manuel Dahnert # User study: Looking beyond the horizon: Evaluation of four compact visualization techniques for time series in a spatial context # Evaluation of confidence # Null hypothesis testing # Non-parametric tests (Kruskal-Wallis, Friedman) for interval/ratio errors # Chi-Square test for task 9 (Dichotomy data) # Kruskal-Wallis not applicable, since study design is repeated measure! # Load common functions and constants source("eval-functions.R") # Load necessary librarys loadLibrary("RColorBrewer") loadLibrary("data.table") # Load raw study data and perform general filtering data.csv <- read.csv("rawdata_study.csv", header = TRUE, sep =";") data <- filterData(data.csv, max.skips = 10, min.time = 30) # Exclude extremely long completion times data <- data[data$duration_ms < 1800000, ] # Perform spefic filtering for timings (exclude trainings, demography and post tasks) data <- data[data$questionId == "seq", ] data <- data[, c(1, 13, 16, 18)] # Exclude unnecessary columns data$givenAnswer <- as.numeric(factor(data$givenAnswer)) data <- data.table(data) data2 <- data.table() for(index in tasks.index) { task.name <- paste("task", index, sep = "") task.dt <- data[task.1 == task.name, ] task.techs <- task.dt[, .("tech" = length(technique)), by = c("participant")] if (index == 3) { task.techs <- task.techs[tech == 1, ] } else { task.techs <- task.techs[tech == 4, ] } task.data <- task.dt[task.techs, participant : technique, on = "participant"] data2 <- rbind(data2, task.data) } data <- data2 # # Create trimmed data table # data2 <- data.table() # for(index in tasks.index) { # data.task <- data.table() # task.name <- paste("task", index, sep = "") # # for (tech in techniques.long) { # tech.data <- data[task.1 == task.name & technique == tech] # tech.data <- trimDataTable(tech.data, "givenAnswer", 0.1) # data.task <- rbind(data.task, tech.data) # } # # task.dt <- data.task[task.1 == task.name, ] # task.techs <- task.dt[, .("tech" = length(technique)), by = c("participant", "task.1")] # # # if (index == 3) { # # task.techs <- task.techs[tech == 1, ] # # } else { # # task.techs <- task.techs[tech == 4, ] # # } # # # # task.data <- task.dt[task.techs, participant : error, on = "participant"] # # data2 <- rbind(data2, task.data) # data2 <- rbind(data2, data.task) # } # data <- data2 # Define output directory export.root <- "export/difficulty/" dir.create(export.root, showWarnings = FALSE) # Create directory # ------------------------------------------------------------------------------ # NULL HYPOTHESIS TESTING for Task 1,2,4-8,10 # ------------------------------------------------------------------------------ #NOT APPLICABLE, study design is repeated measure #Perform Kruskal-Wallis test and save verbose results # output.file <- paste(export.root, "kruskal-verbose.txt", sep = "") # sink(output.file) # for (index in c(1,2,4,5,6,7,8,9,10)) { # task.name <- paste("task", index, sep = "") # task.data <- data[data$task.1 == task.name, ] # # cat(task.name, "\n") # # if (length(unique(task.data$technique)) <= 1) next # # kruskal.test <- kruskal.test(givenAnswer ~ technique, data = task.data) # kurskal.post <- pairwise.wilcox.test(task.data$givenAnswer, task.data$technique, # p.adj ="bonferroni", exact = FALSE) # print(kruskal.test) # print(kurskal.post) # cat("\n\n") # } # sink() # Perform Friedman test and save verbose results output.file <- paste(export.root, "friedman-verbose.txt", sep = "") sink(output.file) for (index in c(1,2,4,5,6,7,8,9,10)) { task.name <- paste("task", index, sep = "") task.data <- data[data$task.1 == task.name, ] task.data$technique <- factor(task.data$technique) # Combine data to a Group~Value table friedman.data <- cbind(task.data[task.data$technique==techniques.long[1],]$givenAnswer, task.data[task.data$technique==techniques.long[2],]$givenAnswer, task.data[task.data$technique==techniques.long[3],]$givenAnswer, task.data[task.data$technique==techniques.long[4],]$givenAnswer) colnames(friedman.data) <- c("hg", "chg", "bhg", "bp") # Perform actual tests friedman.test <- friedman.test(friedman.data) friedman.post <- pairwise.wilcox.test(task.data$givenAnswer, task.data$technique, p.adj="bonferroni", exact=FALSE, paired = TRUE) cat(task.name,"(", nrow(task.data)/4, ")", "\n") print(friedman.test) print(friedman.post) cat("\n\n") } sink() # NOT APPLICABLE, study design is repeated measure # Perform Kruskal test and save compact results # output.file <- paste(export.root, "kruskal-compact.txt", sep = "") # sink(output.file) # cat("Kruskal p-value bp-bhg\tbp-chg\tbp-hg\tbhg-chg\tbhg-hg\tchg-hg\n") # for (index in c(1,2,4,5,6,7,8,10)) { # task.name <- paste("task", index, sep = "") # task.data <- data[data$task.1 == task.name, ] # # if (length(unique(task.data$technique)) <= 1) next # # kruskal.test <- kruskal.test(error ~ technique, data = task.data) # kruskal.post <- pairwise.wilcox.test(task.data$error, task.data$technique, # p.adj ="bonferroni", exact = FALSE) # # # Get p-values # kruskal.matrix <- as.matrix(kruskal.post$p.value) # kruskal.pvalue <- as.vector(kruskal.matrix)[-c(4,7,8)] # # # Finally print everything # cat(sprintf("%02d",index), "->", sprintf("%.4f",kruskal.test$p.value), ":\t") # cat(paste0(sprintf("%.4f", kruskal.pvalue), sep = "\t"), "\n", sep = "") # } # sink() # Perform Friedman test and save verbose results # output.file <- paste(export.root, "friedman-verbose.txt", sep = "") # sink(output.file) # for (index in c(1,2,4,5,6,7,8,9,10)) { # task.name <- paste("task", index, sep = "") # task.data <- data[data$task.1 == task.name, ] # # # Combine data to a Group~Value table # friedman.data <- cbind(task.data[task.data$technique==techniques.long[1],]$error, # task.data[task.data$technique==techniques.long[2],]$error, # task.data[task.data$technique==techniques.long[3],]$error, # task.data[task.data$technique==techniques.long[4],]$error) # colnames(friedman.data) <- c("hg", "chg", "bhg", "bp") # # # Perform actual tests # friedman.test <- friedman.test(friedman.data) # friedman.post <- pairwise.wilcox.test(task.data$error, task.data$technique, # p.adj="bonferroni", exact=FALSE, # paired = TRUE) # cat(task.name,"(", nrow(task.data)/4, ")", "\n") # print(friedman.test) # print(friedman.post) # cat("\n\n") # } #sink() # Perform Friedman test and save compact results # output.file <- paste(export.root, "friedman-compact.txt", sep = "") # sink(output.file) # cat("Friedman p-value bp-bhg\tbp-chg\tbp-hg\tbhg-chg\tbhg-hg\tchg-hg\n") # for (index in c(1,2,4,5,6,7,8,9,10)) { # task.name <- paste("task", index, sep = "") # task.data <- data[data$task.1 == task.name, ] # # if (length(unique(task.data$technique)) <= 1) next # # # Combine data to a Group~Value table # friedman.data <- cbind(task.data[task.data$technique==techniques.long[1],]$error, # task.data[task.data$technique==techniques.long[2],]$error, # task.data[task.data$technique==techniques.long[3],]$error, # task.data[task.data$technique==techniques.long[4],]$error) # colnames(friedman.data) <- c("hg", "chg", "bhg", "bp") # # # Perform actual tests # friedman.test <- friedman.test(friedman.data) # friedman.post <- pairwise.wilcox.test(task.data$error, task.data$technique, # p.adj ="bonferroni", exact = FALSE, # paired = TRUE) # # # Get p-values # friedman.matrix <- as.matrix(friedman.post$p.value) # friedman.pvalue <- as.vector(friedman.matrix)[-c(4,7,8)] # # # Finally print everything # cat(sprintf("%02d",index), "->", sprintf("%.4f",friedman.test$p.value), ":\t") # cat(paste0(sprintf("%.4f", friedman.pvalue), sep = "\t"), "\n", sep = "") # } # sink() # Special casse task 2/3 for CHG # Check if slope increase / decrease has effect on error # output.file <- paste(export.root, "t2_3-wilcoxon.txt", sep = "") # task.increase <- data[data$task.1 == "task2" & data$technique == "colhorgraphs", ] # task.decrease <- data[data$task.1 == "task3" & data$technique == "colhorgraphs", ] # # # Wilcoxon test # sink(output.file) # cat("Task 2/3 - CHG") # wilcox.test(task.increase$givenAnswer, task.decrease$givenAnswer, paired = T) # sink() # # # Mann-Whitney's U test # output.file <- paste(export.root, "t2_3-mann-whitney.txt", sep = "") # sink(output.file) # cat("Task 2/3 - CHG") # wilcox.test(task.increase$givenAnswer, task.decrease$givenAnswer) # sink() # Special casse task 2/3 for CHG # Check if slope increase / decrease has effect on error output.file <- paste(export.root, "t2_3-wilcoxon.txt", sep = "") task.dt <- data[(task.1 == "task2" | task.1 == "task3") & technique == "colhorgraphs", ] task.techs <- task.dt[, .("tech" = length(task.1)), by = c("participant")] task.techs <- task.techs[tech == 2, ] task.data <- task.dt[task.techs, participant : technique, on = "participant"] # Wilcoxon test sink(output.file) cat("Task 2/3 - CHG") wilcox.test(task.data[task.1 == "task2"]$givenAnswer, task.data[task.1 == "task3"]$givenAnswer, paired = T, exact = F, p.adj = "bonferroni") sink()