--- title: 'Discrepancy review: A feasibility study of a novel peer review intervention to reduce undisclosed discrepancies between registrations and publications' author: | | Short title: Discrepancy review: A feasibility study | | TARG Meta-Research Group & Collaborators | (Contributor roles are detailed at the end of this document) | | Address correspondence to robert.thibault@stanford.edu date: "`r Sys.Date()`" output: pdf_document: header-includes: \usepackage[labelformat=empty]{caption} --- ```{r setup, include = FALSE} knitr::opts_chunk$set(include = FALSE) # this option stops the code chunks from being output from the knit ``` ```{r packages} library(tidyverse) # for cleaner code library(knitr) # for kable function library(kableExtra) # for kable table styling library(irr) # for inter-rater agreement calculations library(english) # to convert numbers to word equivalents via the function: words() library(statpsych) # for the precision analysis ``` ```{r originalDiscrepancyReview} # load the data from the original discrepancy review filepath <- paste0(getwd(), "/DiscrepancyReview_DataForTable1_220707.csv" ) d <- read.csv(filepath, header = TRUE) # make table with summary data from original discrepancy reviews (the 18 items) domainNames <- c("Hypotheses (primary)", "Hypotheses (secondary)", "Independent variables", "Covariates / moderators", "Outcomes (primary)", "Outcomes (seconday)", "Sample size", "Sample size justification", "Participant eligibility", "Data exclusion", "Missing data", "Randomization", "Blinding", "Preprocessing", "Analysis (primary)", "Statistical assumptions", "Inferential criteria", "Analysis (secondary)") # We also need a vector with the column names written out like this so we can index the dataframe properly items18 <- c("hypo1", "hypo2", "iv", "covar", "out1", "out2", "sampleSize", "sampleJustification", "excludeParticipant", "excludeData", "missingData", "random", "blinding", "preprocess", "analysis1", "assumptions", "inference", "analysis2") # make Table 1 column headers for summary data of original discrepancy reviews colsMain <- c("0", "1", "2", "0", "1", "2", "NA", "no", "yes", "disclosed", "NA", "none", "negligible", "minor", "major") # This function takes one item (e.g. 'hypo1') and codes how it was reported and if it was discrepant across all the studies from a single coder. In other words, it converts the raw data into the summary data to input into Table 1. itemsFormat <- function(item, d){ itemList <- c(sum(d[[grep(paste0(item, ".preregCode"), colnames(d))]]=="na") + sum(d[[grep(paste0(item, ".preregCode"), colnames(d))]]==0), sum(d[[grep(paste0(item, ".preregCode"), colnames(d))]]==1, na.rm=T), sum(d[[grep(paste0(item, ".preregCode"), colnames(d))]]==2, na.rm=T), sum(d[[grep(paste0(item, ".manuCode"), colnames(d))]]=="na") + sum(d[[grep(paste0(item, ".manuCode"), colnames(d))]]==0), sum(d[[grep(paste0(item, ".manuCode"), colnames(d))]]==1, na.rm=T), sum(d[[grep(paste0(item, ".manuCode"), colnames(d))]]==2, na.rm=T), # discrepancies sum(d[[grep(paste0(item, ".contentDisc"), colnames(d))]]=="na"), sum(d[[grep(paste0(item, ".contentDisc"), colnames(d))]]=="n"), sum(d[[grep(paste0(item, ".contentDisc"), colnames(d))]]=="y"), # discrepancy disclosed (denominator is the previous number) sum(d[[grep(paste0(item, ".discDisclosed"), colnames(d))]]=="y", na.rm=T), # sum issues of discrepancies or transparency sum(d[[grep(paste0(item, ".discIssue"), colnames(d))]]=="na"), sum(d[[grep(paste0(item, ".discIssue"), colnames(d))]]=="none", na.rm=T), sum(d[[grep(paste0(item, ".discIssue"), colnames(d))]]=="0", na.rm=T), sum(d[[grep(paste0(item, ".discIssue"), colnames(d))]]=="1", na.rm=T), sum(d[[grep(paste0(item, ".discIssue"), colnames(d))]]=="2", na.rm=T) ) return(itemList) } # Run the function above to create the summary data for coder 1 (RT) mainTable1 <- data.frame(matrix(ncol = length(colsMain), nrow = 0)) for (i in items18){ mainTable1 <- rbind(mainTable1, itemsFormat(i, filter(d, coder == "RT"))) } colnames(mainTable1) <- colsMain rownames(mainTable1) <- domainNames # same as above for data from coder 2 (not RT) mainTable2 <- data.frame(matrix(ncol = length(colsMain), nrow = 0)) for (i in items18){ mainTable2 <- rbind(mainTable2, itemsFormat(i, filter(d, coder != "RT"))) } colnames(mainTable2) <- colsMain rownames(mainTable2) <- domainNames ``` ```{r originalDiscrepancyReview_errorCheck} # this code checks if the sums for each section of Table 1 & 2 add up to the number of manuscripts that were coded. If the formatting was a bit off, it will not have been read (e.g., "None" instead of "none"). check1 <- data.frame(matrix(ncol = 4, nrow = 0)) for (i in 1:nrow(mainTable1)){ check1[i,1] <- sum(mainTable1[i,1:3])==sum(d$coder=="RT") check1[i,2] <- sum(mainTable1[i,4:6])==sum(d$coder=="RT") check1[i,3] <- sum(mainTable1[i,7:9])==sum(d$coder=="RT") check1[i,4] <- sum(mainTable1[i,11:15])==sum(d$coder=="RT") } if (F %in% check1){ stop() } check2 <- data.frame(matrix(ncol = 4, nrow = 0)) for (i in 1:nrow(mainTable2)){ check2[i,1] <- sum(mainTable2[i,1:3])==sum(d$coder!="RT") check2[i,2] <- sum(mainTable2[i,4:6])==sum(d$coder!="RT") check2[i,3] <- sum(mainTable2[i,7:9])==sum(d$coder!="RT") check2[i,4] <- sum(mainTable2[i,11:15])==sum(d$coder!="RT") } if (F %in% check2){ stop() } ``` ```{r originalDiscrepancyReview_IRR} # make dataframe with just coder 2 responses d2 <- d %>% filter(coder != "RT") # make a dataframe with just coder 1 responses d1 <- d %>% filter(coder == "RT") #make dataframe with only coder 1 response for manuscripts that overlapped with coder 2. d1double <- d1 %>% subset(idRand %in% d2$idRand) # This line stops the code if we aren't comparing the same manuscripts IDs from coder 1 and coder 2 if (F %in% (d1double$idRand == d2$idRand)){ stop("IRR comparison are not ligned up properly") } # Make data frame of TRUE and FALSES for if the cells match (i.e., the coding matches between the two coders) IRR <- as.data.frame(d1double == d2) colnames(IRR) <- colnames(d) # Initialize a table to hold the summary data about IRR IRRtable <- data.frame(matrix(ncol = 5, nrow = 0)) # this function summarizes the IRR data for one item at a time IRRtest <- function(item, d){ IRR <- c(sum((d[[grep(paste0(item, ".preregCode"), colnames(d))]])==T), sum((d[[grep(paste0(item, ".manuCode"), colnames(d))]])==T), sum((d[[grep(paste0(item, ".contentDisc"), colnames(d))]])==T), sum((d[[grep(paste0(item, ".discDisclosed"), colnames(d))]])==T), sum((d[[grep(paste0(item, ".discIssue"), colnames(d))]])==T) ) return(IRR) } # run function above on all 18 items for (i in items18){ IRRtable <- rbind(IRRtable, IRRtest(i, IRR)) } # provide IRR summary table with column names colnames(IRRtable) <- c("Registration", "Manuscript", "Discrepancy", "Disclosed", "Issues") # provide summary IRR table with row names rownames(IRRtable) <- domainNames ``` ```{r, manuscriptCharacteristics} # import all other data (everything other than the 18 items from the original discrepancy review). # I already use the variable 'd' for the previous dataframe, so I will use 'c' for this dataframe as in manuscript 'C'haracteristics. filepath <- paste0(getwd(), "/DiscrepancyReview_DataForTable2_220707.csv") c <- read.csv(filepath, header = TRUE) # calculate # of each type of registration (to be used in Table 2) nCT <- sum(c$regType == "ct") nOSF <- sum(c$regType == "osf") nPROSPERO <- sum(c$regType == "prospero") # filter the dataframe into 3 separate dataframes for clinical trial registrations, osf registrations, and prospero registration (to be used in Table 2) c_ct <- c %>% filter(regType == "ct") c_osf <- c %>% filter(regType == "osf") c_prospero <- c %>% filter(regType == "prospero") # make a vector with the times taken to perform discrepancy review for each type of registration and each discrepancy review process timeOriginal_ct <- rbind(c_ct$timeOriginal1, c_ct$timeOriginal2) %>% as.numeric(na.rm=T) timeOriginal_osf <- rbind(c_osf$timeOriginal1, c_osf$timeOriginal2) %>% as.numeric(na.rm=T) timeOriginal_prospero <- rbind(c_prospero$timeOriginal1, c_prospero$timeOriginal2) %>% as.numeric(na.rm=T) timeUpdated_ct <- rbind(c_ct$timeUpdated1, c_ct$timeUpdated2) %>% as.numeric(na.rm=T) timeUpdated_osf <- rbind(c_osf$timeUpdated1, c_osf$timeUpdated2) %>% as.numeric(na.rm=T) timeUpdated_prospero <- rbind(c_prospero$timeUpdated1, c_prospero$timeUpdated2) %>% as.numeric(na.rm=T) # make new columns to input to IRR function for when manuscripts were secondary publications related to a registration ("exp" for "exploratory" manuscripts) c <- c %>% mutate(submissionExp1 = grepl("reports a non-registered secondary or exploratory analysis", c$importanceSubmission1), submissionExp2 = grepl("reports a non-registered secondary or exploratory analysis", c$importanceSubmission2), finalExp1 = grepl("dif study", c$importanceFinal1), finalExp2 = grepl("dif study", c$importanceFinal2) ) # calculate kappa for identifying studies as exploratory ksubmissionExp <- kappa2(cbind(c$submissionExp1, c$submissionExp2))$value %>% round(2) irrSubmissionExp <- sum(c$submissionExp1 == c$submissionExp2) # repeat steps above for accepted manuscripts. # first create new dataframe for only the accepted manuscripts cAccept <- c %>% filter(editorDecisionFinal == "accept") cAccept <- cAccept %>% mutate(finalExp1 = grepl("dif study", cAccept$importanceFinal1), finalExp2 = grepl("dif study", cAccept$importanceFinal2) ) kFinalExp <- kappa2(cbind(cAccept$finalExp1, cAccept$finalExp2))$value %>% round(2) irrFinalExp <- sum(cAccept$finalExp1 == cAccept$finalExp2) #make new columns for IRR on importance ratings by replacing the categorical ratings with 0, 1, 2. c$importanceSubmission1 <- ifelse(c$importanceSubmission1 == "Not important" | c$importanceSubmission1 == "The manuscript and registration reflect the same study and there are no discrepancies, or all discrepancies are disclosed.", 0, ifelse(c$importanceSubmission1 == "Somewhat important", 1, ifelse(c$importanceSubmission1 == "Quite important", 2, NA) ) ) c$importanceSubmission2 <- ifelse(c$importanceSubmission2 == "Not important" | c$importanceSubmission2 == "The manuscript and registration reflect the same study and there are no discrepancies, or all discrepancies are disclosed.", 0, ifelse(c$importanceSubmission2 == "Somewhat important", 1, ifelse(c$importanceSubmission2 == "Quite important", 2, NA) ) ) cAccept$importanceFinal1 <- ifelse(cAccept$importanceFinal1 == "Not important", 0, ifelse(cAccept$importanceFinal1 == "Somewhat important", 1, ifelse(cAccept$importanceFinal1 == "Quite important", 2, NA) ) ) cAccept$importanceFinal2 <- ifelse(cAccept$importanceFinal2 == "Not important", 0, ifelse(cAccept$importanceFinal2 == "Somewhat important", 1, ifelse(cAccept$importanceFinal2 == "Quite important", 2, NA) ) ) # this IRR is only for manuscripts where both coders coded that the registration and manuscript were the "same study". irrImportanceSubmission <- abs(c$importanceSubmission1 - c$importanceSubmission2) %>% na.omit() importanceSubmission <- data.frame(c$importanceSubmission1, c$importanceSubmission2) %>% na.omit() kimportanceSubmission <- kappa2(importanceSubmission, weight = c("equal"), sort.levels=T)$value %>% round(2) irrImportanceFinal <- abs(cAccept$importanceFinal1 - cAccept$importanceFinal2) %>% na.omit() importanceFinal <- data.frame(cAccept$importanceFinal1, cAccept$importanceFinal2) %>% na.omit() kimportanceFinal <- kappa2(importanceFinal, weight = c("equal"), sort.levels=T)$value %>% round(2) # IRR for whether registrations were permanent kPermSubmission <- kappa2(data.frame(c$permIssue1, c$permIssue2) %>% na.omit())$value %>% round(2) kPermFinal <- kappa2(data.frame(c$permIssueFinal1, c$permIssueFinal2) %>% na.omit())$value %>% round(2) ``` ```{r manuscriptCharacteristicsTable} # input the data for each column of Table 2. See rownames() line for what each row represents. t2_ct <- c(nrow(c_ct), sum(c_ct$code1format == "original"), sum(c_ct$code2format == "original"), paste0(median(timeOriginal_ct, na.rm=T), " (", range(timeOriginal_ct, na.rm=T)[1], ", ", range(timeOriginal_ct, na.rm=T)[2], ") ", "mins"), sum(c_ct$code1format == "updated"), sum(c_ct$code2format == "updated"), paste0(median(timeUpdated_ct, na.rm=T), " (", range(timeUpdated_ct, na.rm=T)[1], ", ", range(timeUpdated_ct, na.rm=T)[2], ") ", "mins"), sum(c_ct$nonPermanent == "1"), paste0(sum(c_ct$badgeCriteria == "1"), "†"), paste0(sum(c_ct$exploratory == "2"), "/", sum(c_ct$exploratory == "1")), "", paste0(sum(c_ct$importanceSubmissionResolved == "Quite important"), "/", sum(grepl("important", c_ct$importanceSubmissionResolved))), paste0(sum(c_ct$importanceSubmissionResolved == "Somewhat important"), "/", sum(grepl("important", c_ct$importanceSubmissionResolved))), paste0(sum(c_ct$importanceSubmissionResolved == "Not important"), "/", sum(grepl("important", c_ct$importanceSubmissionResolved))), "", sum(c_ct$editorDecisionFinal == "accept"), sum(c_ct$nonPermanentFinal == "1", na.rm=T), paste0(sum(c_ct$importanceFinalResolved == "dif study (disclosed)", na.rm=T), "/", sum(grepl("dif study", c_ct$importanceFinalResolved))), "", paste0(sum(c_ct$importanceFinalResolved == "Quite important", na.rm=T), "/", sum(grepl("important", c_ct$importanceFinalResolved))), paste0(sum(c_ct$importanceFinalResolved == "Somewhat important", na.rm=T), "/", sum(grepl("important", c_ct$importanceFinalResolved))), paste0(sum(c_ct$importanceFinalResolved == "Not important", na.rm=T), "/", sum(grepl("important", c_ct$importanceFinalResolved))), paste0(sum(c$primOutDiscResolved==1, na.rm=T), "/", sum(!is.na(c$primOutDiscResolved))), paste0(sum(c$secondOutDiscResolved==1, na.rm=T), "/", sum(!is.na(c$secondOutDiscResolved))) ) t2_osf <- c(nrow(c_osf), sum(c_osf$code1format == "original"), sum(c_osf$code2format == "original"), paste0(median(timeOriginal_osf, na.rm=T), " (", range(timeOriginal_osf, na.rm=T)[1], ", ", range(timeOriginal_osf, na.rm=T)[2], ") ", "mins"), sum(c_osf$code1format == "updated"), sum(c_osf$code2format == "updated"), paste0(median(timeUpdated_osf, na.rm=T), " (", range(timeUpdated_osf, na.rm=T)[1], ", ", range(timeUpdated_osf, na.rm=T)[2], ") ", "mins"), paste0(sum(c_osf$nonPermanent == "1"), "‡"), sum(c_osf$badgeCriteria == "1"), paste0(sum(c_osf$exploratory == "2"), "/", sum(c_osf$exploratory == "1")), "", paste0(sum(c_osf$importanceSubmissionResolved == "Quite important"), "/", sum(grepl("important", c_osf$importanceSubmissionResolved))), paste0(sum(c_osf$importanceSubmissionResolved == "Somewhat important"), "/", sum(grepl("important", c_osf$importanceSubmissionResolved))), paste0(sum(c_osf$importanceSubmissionResolved == "Not important"), "/", sum(grepl("important", c_osf$importanceSubmissionResolved))), "", paste0(sum(c_osf$editorDecisionFinal == "accept"), "§"), paste0(sum(c_osf$nonPermanentFinal == "1", na.rm=T), "‡"), paste0(sum(c_osf$importanceFinalResolved == "dif study (disclosed)", na.rm=T), "/", sum(grepl("dif study", c_osf$importanceFinalResolved))), "", paste0(sum(c_osf$importanceFinalResolved == "Quite important", na.rm=T), "/", sum(grepl("important", c_osf$importanceFinalResolved))), paste0(sum(c_osf$importanceFinalResolved == "Somewhat important", na.rm=T), "/", sum(grepl("important", c_osf$importanceFinalResolved))), paste0(sum(c_osf$importanceFinalResolved == "Not important", na.rm=T), "/", sum(grepl("important", c_osf$importanceFinalResolved))), "not assessed", "not assessed" ) t2_prospero <- c(nrow(c_prospero), sum(c_prospero$code1format == "original"), sum(c_prospero$code2format == "original"), paste0(median(timeOriginal_prospero, na.rm=T)), sum(c_prospero$code1format == "updated"), sum(c_prospero$code2format == "updated"), paste0(median(timeUpdated_prospero, na.rm=T), " (", range(timeUpdated_prospero, na.rm=T)[1], ", ", range(timeUpdated_prospero, na.rm=T)[2], ") ", "mins"), sum(c_prospero$nonPermanent == "1"), sum(c_prospero$badgeCriteria == "1"), paste0(sum(c_prospero$exploratory == "2"), "/", sum(c_prospero$exploratory == "1")), "", paste0(sum(c_prospero$importanceSubmissionResolved == "Quite important"), "/", sum(grepl("important", c_prospero$importanceSubmissionResolved))), paste0(sum(c_prospero$importanceSubmissionResolved == "Somewhat important"), "/", sum(grepl("important", c_prospero$importanceSubmissionResolved))), paste0(sum(c_prospero$importanceSubmissionResolved == "Not important"), "/", sum(grepl("important", c_prospero$importanceSubmissionResolved))), "", sum(c_prospero$editorDecisionFinal == "accept"), "NA", "NA", "", "NA", "NA", "NA", "NA", "NA" ) # create table 2 with variable 't2'. The dots add spaces when knitting the table to pdf. t2 <- data.frame(t2_ct, t2_osf, t2_prospero) colnames(t2) <- c("Clinical trial", "OSF", "PROSPERO") rownames(t2) <- c("Total reviewed", "    Original discrepancy review performed (submitted to editor)", "    Original discrepancy review performed (second reviewer)", "        Time for original discrepancy review, median and range", "    Updated discrepancy review performed (submitted to editor)", "    Updated discrepancy review performed (second reviewer)", "        Time for updated discrepancy review, median and range ", "    Non-permanent registrations", "    Submitted manuscript meets criteria for preregistered badge", "    Manuscripts correctly labeled as a secondary publication", "    Importance of addressing discrepancies (submitted manuscripts)*", "        Quite important", "        Somewhat important", "        Not important or no discrepancies", "", "Accepted for publication", "    Non-permanent registrations ", "    Manuscripts correctly labeled as a secondary publication ", "    Importance of addressing discrepancies (accepted manuscripts)*", "        Quite important ", "        Somewhat important ", "        Not important or no discrepancies ", "    One or more undisclosed primary outcome discrepancy", "    One or more undisclosed secondary outcome discrepancy" ) ``` ```{r, commentsAddressed} # This chunk evaluates how well comments were addressed. full = 0 for (i in grep("com01", colnames(cAccept)):grep("com10", colnames(cAccept))){ full <- full + sum(grepl("full", cAccept[,i])) } partial = 0 for (i in grep("com01", colnames(cAccept)):grep("com10", colnames(cAccept))){ partial <- partial + sum(grepl("partial", cAccept[,i])) } none = 0 for (i in grep("com01", colnames(cAccept)):grep("com10", colnames(cAccept))){ none <- none + sum(grepl("none", cAccept[,i])) } nonIssue = 0 for (i in grep("com01", colnames(cAccept)):grep("com10", colnames(cAccept))){ nonIssue <- nonIssue + sum(grepl("nonIssue", cAccept[,i])) } # sanity check to ensure the sums were properly calculated. if ((full + partial + none + nonIssue) != sum(cAccept$numberComments)){ stop("issue in # of comments addressed") } ``` ```{r powerCalc} # power and precision calculations for Box (potential trial of discrepancy review) #### documentation available at : https://csass.ucsc.edu/powerprecision.pdf a = .05 # alpha value p1= .1 # proportion with at least one primary discrepancy in the experimental group p2= .33 # proportion with at least one primary discrepancy in the control group w = .20 # precision # n per group for inputs above for POWER analysis nPower <- power.prop.test(n = NULL, p1, p2, sig.level = .05, power = .80, alternative = "two.sided") nPower <- nPower$n %>% round(0) # n per group for inputs above for PRECISION analysis nPrecision <- size.ci.prop2(a, p1, p2, w) ``` ```{r contibutorshipTable} contribItem <- c("Conceptualization", "Data curation", "Formal Analysis", "Funding acquisition", "Investigation", "Methodology", "Project administration", "Resources", "Software", "Supervision", "Validation", "Visualization", "Writing – original draft", "Writing – review & editing" ) contribAuthor <- c("Robert T. Thibault(1,2,3), Jacqueline Thompson(2,3), Katie Drax(2,3), Marcus R. Munafò(2,3)", "Robert T. Thibault", "Robert T. Thibault", "Robert T. Thibault", "Robert T. Thibault, Tom E. Hardwicke(4), Charlotte R. Pennington(5), Robbie W. A. Clark(2,3), Aoife O'Mahony(6), Gustav Nilsonne(7,8,9)", "Robert T. Thibault, Marcus R. Munafò", "Robert T. Thibault", "not applicable", "Robert T. Thibault", "Robert T. Thibault, Marcus R. Munafò", "Robert T. Thibault", "Robert T. Thibault", "Robert T. Thibault", "Robert T. Thibault, Tom E. Hardwicke, Charlotte R. Pennignton, Robbie W. A. Clark, Gustav Nilsonne, Marcus R. Munafò" ) contrib <- cbind(contribItem, contribAuthor) aff1 <- "Meta-Research Innovation Center at Stanford (METRICS), Stanford University" aff2 <- "School of Psychological Science, University of Bristol" aff3 <- "MRC Integrative Epidemiology Unit at the University of Bristol" aff4 <- "Department of Psychology, University of Amsterdam" aff5 <- "School of Psychology, Aston University" aff6 <- "School of Psychology, Cardiff University" aff7 <- "Department of Clinical Neuroscience, Karolinska Institutet" aff8 <- "Department of Psychology, Stockholm University" aff9 <- "QUEST Center, Berlin Institute of Health at Charité – Universitätsmedizin Berlin" ``` # Written text To make the manuscript more easily reproducible, we have written four of the more data heavy paragraphs using RMarkdown. # Section 3.3 (first paragraph) Discrepancy reviewers reported preferring the updated process but noted that the less structured methodology required greater focus. The updated discrepancy review process took less time than the original process---clinical trial registrations median and range: `r median(timeUpdated_ct, na.rm=T) %>% round(0)` (`r range(timeUpdated_ct, na.rm=T) %>% round(0)`) minutes (*n* reviews = `r sum(c$regType == "ct" & c$code1format == "updated", c$regType == "ct" & c$code2format == "updated")`) versus `r median(timeOriginal_ct, na.rm=T) %>% round(0)` (`r range(timeOriginal_ct, na.rm=T) %>% round(0)`) minutes (*n* = `r sum(c$regType == "ct" & c$code1format == "original", c$regType == "ct" & c$code2format == "original")`), OSF registrations: `r median(timeUpdated_osf, na.rm=T) %>% round(0)` (`r range(timeUpdated_osf, na.rm=T) %>% round(0)`) minutes (*n* = `r sum(c$regType == "osf" & c$code1format == "updated", c$regType == "osf" & c$code2format == "updated")`) versus `r median(timeOriginal_osf, na.rm=T) %>% round(0)` (`r range(timeOriginal_osf, na.rm=T) %>% round(0)`) minutes (*n* = `r sum(c$regType == "osf" & c$code1format == "original", c$regType == "osf" & c$code2format == "original")`), PROSPERO registrations: `r median(timeUpdated_prospero, na.rm=T) %>% round(0)` (`r range(timeUpdated_prospero, na.rm=T) %>% round(0)`) minutes (*n* = `r sum(c$regType == "prospero" & c$code1format == "updated", c$regType == "prospero" & c$code2format == "updated")`) versus `r median(timeOriginal_prospero, na.rm=T) %>% round(0)` minutes (*n* = `r sum(c$regType == "prospero" & c$code1format == "original", c$regType == "prospero" & c$code2format == "original")`) (see Table 2). We feel updated discrepancy review is more feasible for journals to implement as a standard practice. For two of five manuscripts that underwent updated discrepancy review, and for one of 16 manuscripts that underwent original discrepancy review, the editors invited the discrepancy reviewer for a second round of review to ensure their comments were addressed, as is common for traditional peer review. The times reported listed in Table 2 are only for the first round of discrepancy review. # Section 3.6.1 (first part of second paragraph) The discrepancy reports contained between `r range(c$numberComments)[1]`-`r range(c$numberComments)[2]` comments (mean = `r mean(c$numberComments) %>% round(1)` SD = `r sd(c$numberComments) %>% round(1)`, median = `r median(c$numberComments)`, IQR = `r quantile(c$numberComments, 0.25)`-`r quantile(c$numberComments, 0.75)`). Of the `r sum(cAccept$numberComments)` comments in the `r sum(c$editorDecisionFinal == "accept")` manuscripts that were published, `r full` were fully addressed, `r partial` were partially addressed, `r none` were not addressed, and `r nonIssue` presented issues that the discrepancy reviewer later noticed did not need to be addressed. We categorized these `r sum(c$editorDecisionFinal == "accept")` manuscripts into four bins: `r sum(c$commentsCategory == "serious", na.rm=TRUE)` addressed all or nearly all comments, `r sum(c$commentsCategory == "ignored", na.rm=TRUE)` largely ignored discrepancy review and addressed very few comments (accounting for `r none - 4` of the `r none` comments that were not addressed), `r sum(c$commentsCategory == "mixed", na.rm=TRUE)` addressed some comments, and `r sum(c$commentsCategory == "secondary", na.rm=TRUE)` were secondary publications associated with a clinical trial registration but shied away from explicitly stating that aspects of their study were not registered. # Section 3.6.2 (second paragraph) Inter-rater agreement varied greatly across these potential trial outcomes. Coders had perfect agreement (Cohen's $\kappa$ = 1.00) on whether there were issues in the registrations for submitted manuscripts (where `r sum(c$nonPermanent == 1, na.rm=T)`/`r nrow(c)` had issues) and had one disagreement (Cohen's $\kappa$ = `r kPermFinal`) for published manuscripts (where `r sum(c$nonPermanentFinal == 1, na.rm=T)`/`r nrow(cAccept)` had issues). Coders agreed when identifying whether manuscripts were a secondary publication associated with a clinical trial registration for `r irrSubmissionExp`/`r nrow(c)` submitted manuscripts (Cohen's $\kappa$ = `r ksubmissionExp`) and `r irrFinalExp`/`r nrow(cAccept)` published manuscripts (Cohen's $\kappa$ = `r kFinalExp`). Of the six published manuscripts linked with clinical trial registrations that were published, only two of them were the main report of the clinical trial. Coders agreed that neither had a primary outcome discrepancy and that both had at least one secondary outcome discrepancy. Coders' subjective assessments of whether discrepancies were 'quite important', 'somewhat important', or 'not important' to address matched in `r sum(irrImportanceSubmission == 0)` cases, were off by one category in `r sum(irrImportanceSubmission == 1)` cases, and off by two categories in `r sum(irrImportanceSubmission == 2)` case for submitted manuscripts (Cohen's $\kappa$ = `r kimportanceSubmission`). For published manuscripts, they matched in `r sum(irrImportanceFinal == 0)` cases, were off by one category in `r sum(irrImportanceFinal == 1)` case, and off by two categories in `r sum(irrImportanceFinal == 2)` cases (Cohen's $\kappa$ = `r kimportanceFinal`). Inter-rater agreement was low for these subjective assessments, but differences in coding were easily resolved through discussion because one coder would often share information that the other coder had missed. In general, the coder who stated discrepancies were more important to address shifted the other coder’s rating. More precise instructions for these subjective assessments may improve inter-rater agreement to the point where it could be reliably used as an outcome measure in a trial. # Box 2 (last paragraph) Both power and precision analyses require assumptions about the proportion of manuscripts with discrepancies in both the control and experimental group. For the control group, we assume a base-rate of `r paste0(p2*100, "%")` of manuscripts with at least one primary outcome discrepancy, which is the point estimate from a relevant meta-analysis (TARG Meta-Research Group & Collaborators 2021). For the experimental group, we take a best guess that about `r paste0(p1*100, "%")` of publications will have at least one primary outcome discrepancy. To detect this difference between groups, a Fisher’s exact test would require `r nPower` manuscripts per group ($\alpha$ = .05, power = .80). Alternatively, to estimate the difference between the two groups within a range of `r paste0(w*100, "%")` (e.g., to show the experimental group has between 10-30% fewer manuscripts with primary outcome discrepancies), would require `r nPrecision` manuscripts per group (95% confidence interval). ```{r, outputTable1, include = TRUE, echo = FALSE, results = "asis"} knitr::kable(mainTable1, caption = "Table 1. Reporting and discrepancies across the 18 dimensions included in the original discrepancy review process. Reporting was coded as 0 (not reported), 1 (reported, but unclear), or 2 (reported clearly). This table presents data from the first coder for the 10 manuscripts that underwent the original discrepancy review process and were not a secondary publication associated with a clinical trial registration.", booktabs = T, linesep = "", align = "c") %>% kable_styling(latex_options = "striped") %>% kable_styling(font_size = 8) %>% add_footnote(c("Supplementary Material C operationalizes each dimension. Supplementary Table B2 contains the same data from a second coder on the subset of manuscripts where the second coder also performed the original discrepancy review process. We did not attempt to resolve differences between coders. Inter-rater agreement is available in Supplementary Table B3. Issues were considered NA if the dimension was irrelevant to the particular study (e.g., if neither the registration nor manuscript had secondary hypotheses). Issues were coded as present for some dimensions due to a lack of transparency rather than a discrepancy." ), notation = "none", threeparttable = T) %>% add_header_above(c("Dimension" = 1, "Registration" = 3, "Manuscript" = 3, "Discrepancy" = 4, "Issues" = 5) ) ``` ```{r, outputTable2, include = TRUE, echo = FALSE, results = "asis"} knitr::kable(t2, caption = "Table 2. Characteristics and outcomes of the manuscripts reviewed", booktabs = T, linesep = "", align = "c") %>% kable_styling(latex_options = "striped") %>% kable_styling(font_size = 8) %>% row_spec(c(1,16), bold=T) %>% # this makes the 1st and 16th lines boded (total reviewed and total accepted), for easier visualization add_footnote(c("We only assessed the importance of addressing discrepancies for manuscripts that were not secondary publications associated with a clinical trial registration.", "All the manuscripts assocated with clinical trial registrations that we reviewed were submitted to Nicotine and Tobacco Research. Although this journal does not offer preregistered badges, we nonetheless checked whether manuscripts met the badge criteria.", paste0("In addition to these numbers, ", words(sum(c$nonPermanent == 0.5)), " submitted manuscripts and ", words(sum(c$nonPermanentFinal == 0.5, na.rm=T)), " accepted manuscript had permanent registrations on the OSF REGISTRIES webpage, but only included a link to a non-permanent version on the OSF HOME website."), "One registration was very difficult to map onto its associated manuscript (e.g., the registration contained over 1000 words in the hypotheses section) and the coders judged their confidence in their ratings to be hardly more confident than chance and no more confidence than chance. Thus, we do not provide an assessment of the importance of addressing discrepancies for this study." ), notation = "symbol", threeparttable = T ) ``` ```{r, outputSupplementaryTable2, include = TRUE, echo = FALSE, results = "asis"} knitr::kable(mainTable2, caption = "Supplementary Table B2. Reporting and discrepancies across the 18 dimensions included in the original discrepancy review process (second coder, n = 6)", booktabs = T, linesep = "", align = "c") %>% kable_styling(latex_options = "striped") %>% kable_styling(font_size = 8) %>% add_footnote(c("Issues were considered NA if the dimension was irrelevant to the particular study (e.g., if neither the registration nor manuscript had secondary hypotheses). Issues were coded as present for some dimensions due to a lack of transparency rather than a discrepancy."), notation = "symbol", threeparttable = T) %>% add_header_above(c("Dimension" = 1, "Registration" = 3, "Manuscript" = 3, "Discrepancy" = 4, "Issues*" = 5) ) ``` ```{r, outputSupplementaryTable3, include = TRUE, echo = FALSE, results = "asis"} knitr::kable(IRRtable, caption = "Supplementary Table B3. Inter-rater agreement across the 18 domains included in the original discrepancy review process (n = 6)", booktabs = T, linesep = "", align = "c") %>% kable_styling(latex_options = "striped") ``` ```{r outputContributorshipTable, include = TRUE, echo = FALSE, results = "asis"} knitr::kable(contrib, caption = "Contributions according to Contributor Roles Taxonomy (CRediT) (casrai.org/credit/)", booktabs = T, linesep = "", col.names = c("","")) %>% kable_styling(latex_options = "striped") %>% kable_styling(full_width = TRUE) %>% add_footnote(c(aff1, aff2, aff3, aff4, aff5, aff6, aff7, aff8, aff9), notation = "number") %>% column_spec(2, width = "11cm") %>% kable_styling(latex_options = "hold_position") ```