#>
#> ************
#> Welcome to scienceverse For support and examples visit:
#> http://scienceverse.github.io/
#> - Get and set global package options with: scienceverse_options()
#> ************#>
#> Attaching package: 'dplyr'#> The following objects are masked from 'package:stats':
#>
#> filter, lag#> The following objects are masked from 'package:base':
#>
#> intersect, setdiff, setequal, unionThe demo below extends the example from:
Lakens, D., & DeBruine, L. M. (2020, January 27). Improving Transparency, Falsifiability, and Rigour by Making Hypothesis Tests Machine Readable. https://doi.org/10.31234/osf.io/5xcda
Set up the study with a name and any additional info you want to add.
study <- study(name = "Kinship and Prosocial Behaviour",
abstract = "A reanalysis of data from DeBruine (2002) Facial Resemblance Enhances Trust, PRSLB.")Add each author in the following format. Use the function
credit_roles() to see a list of the roles and their
descriptions. Use credit_roles("name") or
credit_roles("abbr") to just see their names or
abbreviations. You can also add any further info, like an orcid or email
address.
study <- add_author(study,
orcid = "0000-0002-7523-5539",
surname = "DeBruine",
given = "Lisa M.",
roles = c("con", "dat", "sof", "dra", "edi"),
email = "lisa.debruine@glasgow.ac.uk") %>%
add_author(orcid = "0000-0002-0247-239X",
surname = "Lakens",
given = "Daniël",
roles = c("con", "ana", "dra", "edi"))Now add a hypothesis with a verbal description. You can add more than one hypothesis, but this demo study only has one.
study <- add_hypothesis(study, id = "self_pref",
description = "Cues of kinship will increase prosocial behaviour. Cues of kinship will be manipulated by morphed facial self-resemblance. Prosocial behaviour will be measured by responses in the trust game. The prediction is that the number of trusting AND/OR reciprocating moves will be greater to self morphs than to other morphs.")Add all relevant analyses for testing this hypothesis in the order
that they should be run. Do any data prep first. Each analysis needs an
id for reference in the criteria later. The
code can be R code (wrap it in {} if you need more than one
line) or the file path for a .R file. You can also add other information
about your analysis, such as the software it’s running on.
study <- add_analysis(study, id = "trust",
code = t.test(kin$trust_self,
kin$trust_other,
paired = TRUE,
conf.level = 0.975),
software = R.version.string) %>%
add_analysis(id = "recip",
code = t.test(kin$recip_self,
kin$recip_other,
paired = TRUE,
conf.level = 0.975),
software = R.version.string)You can also test auxillary assumptions and set up more complicated decision rules for evaluation. Here, we’ll test whether the resulting data are normally distributed and use an alternative statistical test if they are not. Therefore, we need to add analyses to do the normality tests and analyses to do the non-parametric versions.
# test normality of the 4 dvs
study <- add_analysis(study, id = "norm",
code = {
ts <- shapiro.test(kin$trust_self)$p.value
to <- shapiro.test(kin$trust_other)$p.value
rs <- shapiro.test(kin$recip_self)$p.value
ro <- shapiro.test(kin$recip_other)$p.value
},
return = c("ts", "to", "rs", "ro"),
software = R.version.string)
study <- add_analysis(study,
id = "trust_wilcox",
code = wilcox.test(kin$trust_self,
kin$trust_other,
paired = TRUE,
conf.level = 0.975,
conf.int = TRUE),
software = R.version.string) %>%
add_analysis(id = "recip_wilcox",
code = wilcox.test(kin$recip_self,
kin$recip_other,
paired = TRUE,
conf.level = 0.975,
conf.int = TRUE),
software = R.version.string)N.B. It is up to you how much you want to break your analysis into separate analyses in scienceverse. At one extreme, you could include all of your analyses in a single .R file that returns a list of all relevant values, instead of separating them into different analyses. However, smaller logical groupings of analyses make it easier for readers to find and understand the code relevant to a result.
Next, add the criteria you will need to check to corroborate or
falsify each hypothesis. Each criterion needs a unique id
so you can reference it next in the evaluation. Then, optionally specify
the hypothesis_id and analysis_id where you’ll
look for the result. If you omit these it will default to the last one
added. For example, the analysis trust returns a list from
the function t.test, which includes a value for
conf.int. To see if the first number in this vector is
larger than 0, set result to “conf.int[1]”,
operator to “>” , and comparator to \(0\). The options for operator
are “>”, “<”, “=”, and “!=”.
study <- study %>%
add_criterion(id = "t_lo",
result = "conf.int[1]",
operator = ">",
comparator = 0,
hypothesis_id = "self_pref",
analysis_id = "trust") %>%
add_criterion("t_hi", "conf.int[2]", ">", 0.2, "trust") %>%
add_criterion("r_lo", "conf.int[1]", ">", 0.0, "recip") %>%
add_criterion("r_hi", "conf.int[2]", ">", 0.2, "recip") %>%
add_criterion("ts_norm", "ts", ">", 0.05, "norm") %>%
add_criterion("to_norm", "to", ">", 0.05, "norm") %>%
add_criterion("rs_norm", "rs", ">", 0.05, "norm") %>%
add_criterion("ro_norm", "ro", ">", 0.05, "norm")%>%
add_criterion("t_lo_w", "conf.int[1]", ">", 0.0, "trust_wilcox") %>%
add_criterion("t_hi_w", "conf.int[2]", ">", 0.2, "trust_wilcox") %>%
add_criterion("r_lo_w", "conf.int[1]", ">", 0.0, "recip_wilcox") %>%
add_criterion("r_hi_w", "conf.int[2]", ">", 0.2, "recip_wilcox")Add evaluation criteria for corroboration and falsification. The
logical evaluation should reference the criterion
ids above and can consist of those ids and the symbols
(, ), &, and |.
You can optionally include a verbal description of the
criterion to clarify the logical evaluation.
study <- add_eval(study, "corroboration",
"(ts_norm & to_norm & t_lo & t_hi) |
(!(ts_norm & to_norm) & t_lo_w & t_hi_w) |
(rs_norm & ro_norm & r_lo & r_hi) |
(!(rs_norm & ro_norm) & r_lo_w & r_hi_w)",
description = "The hypothesis is corroborated if the 97.5% CI lower bound is greater than 0 and the 97.5% CI upper bound is greater than 0.2 (the SESOI) for either the trust or reciprocation moves. Use t.test if the data are normally distributed, or Wilcox test otherwise.")
study <- add_eval(study, "falsification",
"( (ts_norm & to_norm & !t_hi) |
(!(ts_norm & to_norm) & !t_hi_w) ) &
( (rs_norm & ro_norm & !r_hi) |
(!(rs_norm & ro_norm) & !r_hi_w) )",
description = "The hypothesis is falsified if the 97.5% CI upper bound is smaller than 0.2 (the SESOI) for both trust and reciprocation. Use t.test if the data are normally distributed, or Wilcox test otherwise.")At this point, you can save your study to a machine-readable JSON-formatted file.
study_save(study, "ext_prereg.json", format = "json")#> Saving to /Users/debruine/rproj/scienceverse/scienceverse/vignettes/ext_prereg.jsonYou can also export a human-readable pre-registration using the built-in “prereg” template.
study_save(study, "ext_prereg.html", format = "prereg")#> Saving to /Users/debruine/rproj/scienceverse/scienceverse/vignettes/ext_prereg.htmlIf your data have a factorial design, you can add simulated data by
describing the design parameters as for the
faux::sim_design function.
study <- add_sim_data(study, data_id = "kin",
within = list(game = c("trust", "recip"),
person = c("self", "other")),
n = 24,
mu = list(trust_self = 1.2,
trust_other = 0.8,
recip_self = 1.8,
recip_other = 1.8),
sd = 0.8, r = 0.5)#> Warning in faux::codebook(data = data, name = name, vardesc = vardesc,
#> schemaVersion = schemaVersion, : Couldn't set levels for id#> id set to dataType string#> Warning in faux::codebook(data = data, name = name, vardesc = vardesc,
#> schemaVersion = schemaVersion, : Couldn't set levels for trust_self#> trust_self set to dataType float#> Warning in faux::codebook(data = data, name = name, vardesc = vardesc,
#> schemaVersion = schemaVersion, : Couldn't set levels for trust_other#> trust_other set to dataType float#> Warning in faux::codebook(data = data, name = name, vardesc = vardesc,
#> schemaVersion = schemaVersion, : Couldn't set levels for recip_self#> recip_self set to dataType float#> Warning in faux::codebook(data = data, name = name, vardesc = vardesc,
#> schemaVersion = schemaVersion, : Couldn't set levels for recip_other#> recip_other set to dataType floatThe study_power function runs the specified number of
replications: simulating your data, running the analyses, and evaluating
the criteria to produce a summary of the proportion of simulation where
each hypothesis was corroborated, falsified, or inconclusive.
study <- study_power(study, rep = 100)#> Simulating Datasets...#> Running Analyses...#> Evaluating Hypotheses...#> Hypothesis self_pref
#> corroboration: 53.0%
#> falsification: 0.0%
#> inconclusive: 47.0%You can access the values for each criterion for each simulation with
the function get_power (set values = TRUE).
This returns a list of hypotheses and their associated criteria that you
can then plot or further assess.
power <- get_power(study, values = TRUE)
# as_tibble(power$self_pref$criteria) %>%
# pivot_longer(t_lo:r_hi,
# names_to = "criteria",
# values_to = "value") %>%
# ggplot(aes(value, color = criteria)) +
# geom_density()The id you give the data can be used in the analysis
code to refer to this dataset. The data argument can be a
data frame or a file path to a data file or a PsychDS-formatted
codebook. You can optionally add column parameters such as descriptions
with the argument vardesc and other dataset parameters as
additional arguments.
kin_data <- data.frame(
trust_self = c(1,2,2,1,1,1,1,1,2,0,2,0,1,2,2,3,2,2,1,1,2,0,0,1),
trust_other = c(1,2,2,0,1,0,0,0,1,0,1,0,1,1,1,0,1,2,2,0,0,0,2,1),
recip_self = c(0,1,3,2,1,1,1,3,3,2,3,1,1,2,3,3,3,1,1,1,3,0,3,1),
recip_other = c(1,1,2,2,3,2,1,3,3,1,3,0,1,3,3,3,3,0,3,0,1,0,3,2)
)
desc <- list(
description = list(
trust_self = "Number of trusting moves towards self-morphs",
trust_other = "Number of trusting moves towards self-morphs",
recip_self = "Number of reciprocating moves towards other-morphs",
recip_other = "Number of reciprocating moves towards other-morphs"
),
dataType = rep("int", 4) # all variables are integer types
)
study <- add_data(study, id = "kin",
data = kin_data,
vardesc = desc,
url = "https://osf.io/ewfhs/")When you run study_analyse, the data are loaded as their
id names and the analyses are run in order. For example,
the data from the section above will be available to your analyses as a
dataframe called kin. The analysis is run in a separate
environment, so you won’t see any new objects in the global environment.
Therefore, changes you make to, e.g., a dataframe called
kin in your global environment won’t affect the analysis.
The evaluation of each criterion and hypothesis is printed as a
message.
study <- study_analyse(study)#> Hypothesis self_pref: Cues of kinship will increase prosocial behaviour. Cues of kinship will be manipulated by morphed facial self-resemblance. Prosocial behaviour will be measured by responses in the trust game. The prediction is that the number of trusting AND/OR reciprocating moves will be greater to self morphs than to other morphs.
#>
#> Criterion t_lo:
#> * conf.int[1] > 0 is TRUE
#> * conf.int[1] = 0.021
#>
#> Criterion t_hi:
#> * conf.int[2] > 0.2 is TRUE
#> * conf.int[2] = 0.979
#>
#> Criterion r_lo:
#> * conf.int[1] > 0 is FALSE
#> * conf.int[1] = -0.509
#>
#> Criterion r_hi:
#> * conf.int[2] > 0.2 is TRUE
#> * conf.int[2] = 0.426
#>
#> Criterion ts_norm:
#> * ts > 0.05 is FALSE
#> * ts = 0.004
#>
#> Criterion to_norm:
#> * to > 0.05 is FALSE
#> * to = 0.000
#>
#> Criterion rs_norm:
#> * rs > 0.05 is FALSE
#> * rs = 0.000
#>
#> Criterion ro_norm:
#> * ro > 0.05 is FALSE
#> * ro = 0.001
#>
#> Criterion t_lo_w:
#> * conf.int[1] > 0 is FALSE
#> * conf.int[1] = -0.000
#>
#> Criterion t_hi_w:
#> * conf.int[2] > 0.2 is TRUE
#> * conf.int[2] = 1.500
#>
#> Criterion r_lo_w:
#> * conf.int[1] > 0 is FALSE
#> * conf.int[1] = -1.500
#>
#> Criterion r_hi_w:
#> * conf.int[2] > 0.2 is TRUE
#> * conf.int[2] = 1.000
#>
#> Conclusion: inconclusive
#> * Corroborate ((ts_norm & to_norm & t_lo & t_hi) |
#> (!(ts_norm & to_norm) & t_lo_w & t_hi_w) |
#> (rs_norm & ro_norm & r_lo & r_hi) |
#> (!(rs_norm & ro_norm) & r_lo_w & r_hi_w)): FALSE
#> * Falsify (( (ts_norm & to_norm & !t_hi) |
#> (!(ts_norm & to_norm) & !t_hi_w) ) &
#> ( (rs_norm & ro_norm & !r_hi) |
#> (!(rs_norm & ro_norm) & !r_hi_w) )): FALSEAt this point, you can save the post-registration version of your study to a JSON-formatted file and also export a human-readable post-registration report.
study_save(study, "ext_postreg.json", "json")#> Saving to /Users/debruine/rproj/scienceverse/scienceverse/vignettes/ext_postreg.json
study_save(study, "ext_postreg.html", "postreg")#> Saving to /Users/debruine/rproj/scienceverse/scienceverse/vignettes/ext_postreg.htmlYou can reload and edit your study by loading the .json file with the study() function.
study2 <- study("ext_postreg.json")
study2 # displays a summary#> Kinship and Prosocial Behaviour
#> -------------------------------
#>
#> * Hypotheses: self_pref
#> * Data: kin
#> * Analyses: trust, recip, norm, trust_wilcox, recip_wilcox
#>
#> Hypothesis self_pref: Cues of kinship will increase prosocial behaviour. Cues of kinship will be manipulated by morphed facial self-resemblance. Prosocial behaviour will be measured by responses in the trust game. The prediction is that the number of trusting AND/OR reciprocating moves will be greater to self morphs than to other morphs.
#>
#> Criterion t_lo:
#> * conf.int[1] > 0 is TRUE
#> * conf.int[1] = 0.021
#>
#> Criterion t_hi:
#> * conf.int[2] > 0.2 is TRUE
#> * conf.int[2] = 0.979
#>
#> Criterion r_lo:
#> * conf.int[1] > 0 is FALSE
#> * conf.int[1] = -0.509
#>
#> Criterion r_hi:
#> * conf.int[2] > 0.2 is TRUE
#> * conf.int[2] = 0.426
#>
#> Criterion ts_norm:
#> * ts > 0.05 is FALSE
#> * ts = 0.004
#>
#> Criterion to_norm:
#> * to > 0.05 is FALSE
#> * to = 0.000
#>
#> Criterion rs_norm:
#> * rs > 0.05 is FALSE
#> * rs = 0.000
#>
#> Criterion ro_norm:
#> * ro > 0.05 is FALSE
#> * ro = 0.001
#>
#> Criterion t_lo_w:
#> * conf.int[1] > 0 is FALSE
#> * conf.int[1] = 0.000
#>
#> Criterion t_hi_w:
#> * conf.int[2] > 0.2 is TRUE
#> * conf.int[2] = 1.500
#>
#> Criterion r_lo_w:
#> * conf.int[1] > 0 is FALSE
#> * conf.int[1] = -1.500
#>
#> Criterion r_hi_w:
#> * conf.int[2] > 0.2 is TRUE
#> * conf.int[2] = 1.000
#>
#> Conclusion: inconclusive
#> * Corroborate ((ts_norm & to_norm & t_lo & t_hi) |
#> (!(ts_norm & to_norm) & t_lo_w & t_hi_w) |
#> (rs_norm & ro_norm & r_lo & r_hi) |
#> (!(rs_norm & ro_norm) & r_lo_w & r_hi_w)): FALSE
#> * Falsify (( (ts_norm & to_norm & !t_hi) |
#> (!(ts_norm & to_norm) & !t_hi_w) ) &
#> ( (rs_norm & ro_norm & !r_hi) |
#> (!(rs_norm & ro_norm) & !r_hi_w) )): FALSEIf you want to explore how the results change if you change criteria
or analyses, you can use the functions update_analysis or
update_criteria.
For example, we can change the analyses to use 95% CIs instead of 97.5% CIs. Make sure to re-analyse after changing data, analyses or criteria (this doesn’t happen automatically because some analyses can take a very long time).
study2 <- study2 %>%
update_analysis(id = "trust",
code = t.test(kin$trust_self,
kin$trust_other,
paired = TRUE,
conf.level = 0.95)) %>%
update_analysis(id = "recip",
code = t.test(kin$recip_self,
kin$recip_other,
paired = TRUE,
conf.level = 0.95)) %>%
update_analysis(id = "trust_wilcox",
code = wilcox.test(kin$trust_self,
kin$trust_other,
paired = TRUE,
conf.level = 0.95,
conf.int = TRUE)) %>%
update_analysis(id = "recip_wilcox",
code = wilcox.test(kin$recip_self,
kin$recip_other,
paired = TRUE,
conf.level = 0.95,
conf.int = TRUE)) %>%
study_analyse()#> Hypothesis self_pref: Cues of kinship will increase prosocial behaviour. Cues of kinship will be manipulated by morphed facial self-resemblance. Prosocial behaviour will be measured by responses in the trust game. The prediction is that the number of trusting AND/OR reciprocating moves will be greater to self morphs than to other morphs.
#>
#> Criterion t_lo:
#> * conf.int[1] > 0 is TRUE
#> * conf.int[1] = 0.087
#>
#> Criterion t_hi:
#> * conf.int[2] > 0.2 is TRUE
#> * conf.int[2] = 0.913
#>
#> Criterion r_lo:
#> * conf.int[1] > 0 is FALSE
#> * conf.int[1] = -0.445
#>
#> Criterion r_hi:
#> * conf.int[2] > 0.2 is TRUE
#> * conf.int[2] = 0.361
#>
#> Criterion ts_norm:
#> * ts > 0.05 is FALSE
#> * ts = 0.004
#>
#> Criterion to_norm:
#> * to > 0.05 is FALSE
#> * to = 0.000
#>
#> Criterion rs_norm:
#> * rs > 0.05 is FALSE
#> * rs = 0.000
#>
#> Criterion ro_norm:
#> * ro > 0.05 is FALSE
#> * ro = 0.001
#>
#> Criterion t_lo_w:
#> * conf.int[1] > 0 is TRUE
#> * conf.int[1] = 0.000
#>
#> Criterion t_hi_w:
#> * conf.int[2] > 0.2 is TRUE
#> * conf.int[2] = 1.500
#>
#> Criterion r_lo_w:
#> * conf.int[1] > 0 is FALSE
#> * conf.int[1] = -1.000
#>
#> Criterion r_hi_w:
#> * conf.int[2] > 0.2 is TRUE
#> * conf.int[2] = 1.000
#>
#> Conclusion: corroborate
#> * Corroborate ((ts_norm & to_norm & t_lo & t_hi) |
#> (!(ts_norm & to_norm) & t_lo_w & t_hi_w) |
#> (rs_norm & ro_norm & r_lo & r_hi) |
#> (!(rs_norm & ro_norm) & r_lo_w & r_hi_w)): TRUE
#> * Falsify (( (ts_norm & to_norm & !t_hi) |
#> (!(ts_norm & to_norm) & !t_hi_w) ) &
#> ( (rs_norm & ro_norm & !r_hi) |
#> (!(rs_norm & ro_norm) & !r_hi_w) )): FALSEYou can get all the results with the get_result()
function. If you don’t specify the result name or the analysis ID, it
defaults to all of the results of the first analysis. It returns a list
that you can use, but will display an RMarkdown-formatted list if you
print it (and set the chunk options to results='asis').
get_result(study, analysis_id = "trust_wilcox")You can also get a specific result from a specific analysis. Set
return to “value” to get the result as a number and “char”
if you want to keep trailing zeros (this returns the number as a
character string).
get_result(study = study,
result = "p.value",
analysis_id = "trust",
digits = 3,
return = "char")#> [1] "0.020"You can display the value as a link if you set return to
“html”. You can use the shorthand function get_html if you
only have one study object loaded. The digits default to the global
option, so you can set that as shown.
#> [1] "<a href='#analysis_1' title='Analysis 1 Result p.value'>0.020</a>"You’ll probably want to use get_html() inline most of
the time like below. The links go to a section at the end of this
document that is created with make_script().
For trusting judgements, people trusted self-resembling faces more than other-resembling faces (t = 2.50, df = 23.0, p = 0.020, 95% CI = [0.02, 0.98]).
For reciprocation judgments judgements, people did not reciprocate to self-resembling faces more than other-resembling faces (t = -0.21, df = 23.0, p = 0.833, 95% CI = [-0.51, 0.43]).
Add the script inline at the end with the function
make_script(). Set header to
FALSE to omit the YAML header. Set header_lvl
to change the default starting header level of 2. This function will
save the data and codebooks in a folder called “data”. Set the argument
data_path to NULL to include the actual data in the text of
the script. For large datasets, you’ll want to leave it as the default
“data” folder (or set a custom folder name).
make_script(study,
data_path = NULL,
header_lvl = 3,
header = FALSE) %>%
cat(){r} kin <- read.csv(text='"trust_self","trust_other","recip_self","recip_other" 1,1,0,1 2,2,1,1 2,2,3,2 1,0,2,2 1,1,1,3 1,0,1,2 1,0,1,1 1,0,3,3 2,1,3,3 0,0,2,1 2,1,3,3 0,0,1,0 1,1,1,1 2,1,2,3 2,1,3,3 3,0,3,3 2,1,3,3 2,2,1,0 1,2,1,3 1,0,1,0 2,0,3,1 0,0,0,0 0,2,3,3 1,1,1,2')
{r} t.test(kin$trust_self, kin$trust_other, paired = TRUE, conf.level = 0.975)
2.50454132981017
23
0.0197947772076059
0.02129457076301060.978705429236990.5
0
0.199637352376173
two.sided
Paired t-test
kin$trust_self and kin$trust_other
{r} t.test(kin$recip_self, kin$recip_other, paired = TRUE, conf.level = 0.975)
-0.213835558603476
23
0.832559599965427
-0.5089016844526920.425568351119359-0.0416666666666667
0
0.194853778944834
two.sided
Paired t-test
kin$recip_self and kin$recip_other
{r} { ts <- shapiro.test(kin$trust_self)$p.value to <- shapiro.test(kin$trust_other)$p.value rs <- shapiro.test(kin$recip_self)$p.value ro <- shapiro.test(kin$recip_other)$p.value } # return values list( `ts` = ts, `to` = to, `rs` = rs, `ro` = ro )
{r} wilcox.test(kin$trust_self, kin$trust_other, paired = TRUE, conf.level = 0.975, conf.int = TRUE)
86.5
0.0258957996037173
0
two.sided
Wilcoxon signed rank test with continuity correction
kin$trust_self and kin$trust_other
-2.46667120907145e-051.50006421487460.999956902706118
{r} wilcox.test(kin$recip_self, kin$recip_other, paired = TRUE, conf.level = 0.975, conf.int = TRUE)
36
0.836637402400589
0
two.sided
Wilcoxon signed rank test with continuity correction
kin$recip_self and kin$recip_other
-1.499971156046711.00004707946561-3.77600367583061e-05