├── docs ├── 2018 │ └── 07 │ │ └── 17 │ │ └── ears-first-post │ │ └── index.html ├── .gitkeep ├── images │ ├── logo.png │ ├── earslogo.png │ └── hugo-logo.png ├── fonts │ ├── lato-v11-latin-regular.woff │ ├── lato-v11-latin-regular.woff2 │ ├── merriweather-v13-latin-regular.woff │ └── merriweather-v13-latin-regular.woff2 ├── post │ ├── 2015-07-23-r-rmarkdown_files │ │ └── figure-html │ │ │ └── pie-1.png │ ├── index.xml │ └── index.html ├── tags │ ├── index.xml │ └── index.html ├── categories │ ├── index.xml │ └── index.html ├── js │ └── math-code.js ├── css │ ├── fonts.css │ └── main.css ├── sitemap.xml ├── about │ └── index.html ├── index.html ├── index.xml └── members │ └── index.html ├── figure1.png ├── reproducibility-study-outline.pdf ├── README.md ├── .gitignore ├── reproducibility-study-outline.Rmd └── MPI-Reproducibility.bib /docs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /figure1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rianaminocher/ears/HEAD/figure1.png -------------------------------------------------------------------------------- /docs/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rianaminocher/ears/HEAD/docs/images/logo.png -------------------------------------------------------------------------------- /docs/images/earslogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rianaminocher/ears/HEAD/docs/images/earslogo.png -------------------------------------------------------------------------------- /docs/images/hugo-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rianaminocher/ears/HEAD/docs/images/hugo-logo.png -------------------------------------------------------------------------------- /reproducibility-study-outline.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rianaminocher/ears/HEAD/reproducibility-study-outline.pdf -------------------------------------------------------------------------------- /docs/fonts/lato-v11-latin-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rianaminocher/ears/HEAD/docs/fonts/lato-v11-latin-regular.woff -------------------------------------------------------------------------------- /docs/fonts/lato-v11-latin-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rianaminocher/ears/HEAD/docs/fonts/lato-v11-latin-regular.woff2 -------------------------------------------------------------------------------- /docs/fonts/merriweather-v13-latin-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rianaminocher/ears/HEAD/docs/fonts/merriweather-v13-latin-regular.woff -------------------------------------------------------------------------------- /docs/fonts/merriweather-v13-latin-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rianaminocher/ears/HEAD/docs/fonts/merriweather-v13-latin-regular.woff2 -------------------------------------------------------------------------------- /docs/post/2015-07-23-r-rmarkdown_files/figure-html/pie-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rianaminocher/ears/HEAD/docs/post/2015-07-23-r-rmarkdown_files/figure-html/pie-1.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## ears: evolutionary anthropology reproducibility study 2 | 3 | Our project aims to improve the scientific reproducibility of research within the field of evolutionary anthropology. 4 | 5 | ## About this repository 6 | 7 | This repository contains a [preregistration document](./reproducibility-study-outline.pdf), outlining the proposed research questions for our survey on reproducibility in evolutionary anthropology. 8 | 9 | ## Study results 10 | 11 | The results of this survey are now available online at https://github.com/rianaminocher/reproducibility-analysis and as a preprint at https://psyarxiv.com/4nzc7/ 12 | -------------------------------------------------------------------------------- /docs/tags/index.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Tags on ears: evolutionary anthropology reproducibility study 5 | https://rianaminocher.github.io/ears/tags/ 6 | Recent content in Tags on ears: evolutionary anthropology reproducibility study 7 | Hugo -- gohugo.io 8 | en-us 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.o 8 | *.so 9 | 10 | # Packages # 11 | ############ 12 | # it's better to unpack these files and commit the raw source 13 | # git has its own built in compression methods 14 | *.7z 15 | *.dmg 16 | *.gz 17 | *.iso 18 | *.jar 19 | *.rar 20 | *.tar 21 | *.zip 22 | 23 | # Logs and databases # 24 | ###################### 25 | *.log 26 | *.sql 27 | *.sqlite 28 | 29 | # OS generated files # 30 | ###################### 31 | .DS_Store 32 | .DS_Store? 33 | ._* 34 | .Spotlight-V100 35 | .Trashes 36 | ehthumbs.db 37 | Thumbs.db 38 | 39 | 40 | # R Stuff # 41 | ########### 42 | 43 | .Rhistory 44 | -------------------------------------------------------------------------------- /docs/categories/index.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Categories on ears: evolutionary anthropology reproducibility study 5 | https://rianaminocher.github.io/ears/categories/ 6 | Recent content in Categories on ears: evolutionary anthropology reproducibility study 7 | Hugo -- gohugo.io 8 | en-us 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /docs/js/math-code.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | var i, text, code, codes = document.getElementsByTagName('code'); 3 | for (i = 0; i < codes.length;) { 4 | code = codes[i]; 5 | if (code.parentNode.tagName !== 'PRE' && code.childElementCount === 0) { 6 | text = code.textContent; 7 | if (/^\$[^$]/.test(text) && /[^$]\$$/.test(text)) { 8 | text = text.replace(/^\$/, '\\(').replace(/\$$/, '\\)'); 9 | code.textContent = text; 10 | } 11 | if (/^\\\((.|\s)+\\\)$/.test(text) || /^\\\[(.|\s)+\\\]$/.test(text) || 12 | /^\$(.|\s)+\$$/.test(text) || 13 | /^\\begin\{([^}]+)\}(.|\s)+\\end\{[^}]+\}$/.test(text)) { 14 | code.outerHTML = code.innerHTML; // remove 15 | continue; 16 | } 17 | } 18 | i++; 19 | } 20 | })(); 21 | -------------------------------------------------------------------------------- /docs/css/fonts.css: -------------------------------------------------------------------------------- 1 | @font-face { 2 | font-family: 'Merriweather'; 3 | font-style: normal; 4 | font-weight: 400; 5 | src: local('Merriweather'), local('Merriweather-Regular'), 6 | url('../fonts/merriweather-v13-latin-regular.woff2') format('woff2'), 7 | url('../fonts/merriweather-v13-latin-regular.woff') format('woff'); 8 | } 9 | @font-face { 10 | font-family: 'Lato'; 11 | font-style: normal; 12 | font-weight: 400; 13 | src: local('Lato Regular'), local('Lato-Regular'), 14 | url('../fonts/lato-v11-latin-regular.woff2') format('woff2'), 15 | url('../fonts/lato-v11-latin-regular.woff') format('woff'); 16 | } 17 | body { 18 | font-family: 'Merriweather', serif; 19 | } 20 | h1, h2, h3, h4, h5, h6, .nav, .article-duration, .archive-item-link, .footer { 21 | font-family: 'Lato', sans-serif; 22 | } 23 | -------------------------------------------------------------------------------- /docs/sitemap.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | https://rianaminocher.github.io/ears/2018/07/17/ears-first-post/ 7 | 2018-07-17T00:00:00+00:00 8 | 9 | 10 | 11 | https://rianaminocher.github.io/ears/about/ 12 | 2016-05-05T21:48:51-07:00 13 | 14 | 15 | 16 | https://rianaminocher.github.io/ears/members/ 17 | 2016-05-05T21:48:51-07:00 18 | 19 | 20 | 21 | https://rianaminocher.github.io/ears/categories/ 22 | 0 23 | 24 | 25 | 26 | https://rianaminocher.github.io/ears/post/ 27 | 2018-07-17T00:00:00+00:00 28 | 0 29 | 30 | 31 | 32 | https://rianaminocher.github.io/ears/tags/ 33 | 0 34 | 35 | 36 | 37 | https://rianaminocher.github.io/ears/ 38 | 2018-07-17T00:00:00+00:00 39 | 0 40 | 41 | 42 | -------------------------------------------------------------------------------- /docs/post/index.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Posts on ears: evolutionary anthropology reproducibility study 5 | https://rianaminocher.github.io/ears/post/ 6 | Recent content in Posts on ears: evolutionary anthropology reproducibility study 7 | Hugo -- gohugo.io 8 | en-us 9 | Tue, 17 Jul 2018 00:00:00 +0000 10 | 11 | 12 | 13 | 14 | 15 | ears: overview 16 | https://rianaminocher.github.io/ears/2018/07/17/ears-first-post/ 17 | Tue, 17 Jul 2018 00:00:00 +0000 18 | 19 | https://rianaminocher.github.io/ears/2018/07/17/ears-first-post/ 20 | Evolutionary Anthropology Reproducibility Study 21 | Our project aims to assess and improve the scientific reproducibility of research within the field of evolutionary anthropology. 22 | Reproducible research practices include sharing raw and processed data, analytical code and documenting software and workflows to enable the reproduction of original research results. This is different from replication, which involves establishing findings with new data (e.g. different species, sample, environment). Reproducibility is a criterion for responsible scientific output, but importantly, allows for increased engagement with research, offers opportunities for collaboration and improves productivity. 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /docs/tags/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Tags - ears: evolutionary anthropology reproducibility study 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
32 |
33 | 51 | 52 |
53 | 54 | 55 |
56 | 57 | 58 |
59 | 60 |
61 | 62 | 63 |
64 | 65 | 75 | 76 |
77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /docs/categories/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Categories - ears: evolutionary anthropology reproducibility study 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
32 |
33 | 51 | 52 |
53 | 54 | 55 |
56 | 57 | 58 |
59 | 60 |
61 | 62 | 63 |
64 | 65 | 75 | 76 |
77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /docs/about/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | About - ears: evolutionary anthropology reproducibility study 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 |
30 |
31 | 49 | 50 |
51 | 52 | 53 |
54 | 55 |
56 | 57 | 58 |

About

59 | 60 | 61 | 62 |
63 |

This is a website describing our project on scientific reproducibility in the field of evolutionary anthropology. You can read more about the project and our first study here. You can follow us on GitHub.

64 | 65 |
66 |
67 | 68 | 69 | 70 | 71 |
72 | 73 | 83 | 84 |
85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /docs/post/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Posts - ears: evolutionary anthropology reproducibility study 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
32 |
33 | 51 | 52 |
53 | 54 | 55 |
56 | 57 | 58 |
59 | 60 |

2018

61 | 62 | 68 | 69 | 70 |
71 | 72 | 73 |
74 | 75 | 85 | 86 |
87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | ears: evolutionary anthropology reproducibility study - ears: evolutionary anthropology reproducibility study 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
32 |
33 | 51 | 52 |
53 | 54 | 55 |
56 | 57 | 58 |
59 | 60 |

2018

61 | 62 | 68 | 69 | 70 |
71 | 72 | 73 |
74 | 75 | 85 | 86 |
87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /docs/index.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ears: evolutionary anthropology reproducibility study 5 | https://rianaminocher.github.io/ears/ 6 | Recent content on ears: evolutionary anthropology reproducibility study 7 | Hugo -- gohugo.io 8 | en-us 9 | Tue, 17 Jul 2018 00:00:00 +0000 10 | 11 | 12 | 13 | 14 | 15 | ears: overview 16 | https://rianaminocher.github.io/ears/2018/07/17/ears-first-post/ 17 | Tue, 17 Jul 2018 00:00:00 +0000 18 | 19 | https://rianaminocher.github.io/ears/2018/07/17/ears-first-post/ 20 | Evolutionary Anthropology Reproducibility Study 21 | Our project aims to assess and improve the scientific reproducibility of research within the field of evolutionary anthropology. 22 | Reproducible research practices include sharing raw and processed data, analytical code and documenting software and workflows to enable the reproduction of original research results. This is different from replication, which involves establishing findings with new data (e.g. different species, sample, environment). Reproducibility is a criterion for responsible scientific output, but importantly, allows for increased engagement with research, offers opportunities for collaboration and improves productivity. 23 | 24 | 25 | 26 | About 27 | https://rianaminocher.github.io/ears/about/ 28 | Thu, 05 May 2016 21:48:51 -0700 29 | 30 | https://rianaminocher.github.io/ears/about/ 31 | This is a website describing our project on scientific reproducibility in the field of evolutionary anthropology. You can read more about the project and our first study here. You can follow us on GitHub. 32 | 33 | 34 | 35 | Members 36 | https://rianaminocher.github.io/ears/members/ 37 | Thu, 05 May 2016 21:48:51 -0700 38 | 39 | https://rianaminocher.github.io/ears/members/ 40 | Riana Minocher 41 | riana_minocher@eva.mpg.de 42 | I am a PhD student in the Department for Human Behaviour, Ecology and Culture at the Max-Planck Institute for Evolutionary Anthropology. I am interested in understanding the processes that generate, maintain and affect the patterns of cultural diversity we can observe. During my PhD, I plan to develop theory about the transmission of multiple cultural traits under different social and ecological contexts. I will link this theory to data I collect in the field, and develop (reproducible) research methods to do so. 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /docs/members/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Members - ears: evolutionary anthropology reproducibility study 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 |
30 |
31 | 49 | 50 |
51 | 52 | 53 |
54 | 55 |
56 | 57 | 58 |

Members

59 | 60 | 61 | 62 |
63 |

Riana Minocher

64 | 65 |

riana_minocher@eva.mpg.de

66 | 67 |

I am a PhD student in the Department for Human Behaviour, Ecology and Culture at the Max-Planck Institute for Evolutionary Anthropology. I am interested in understanding the processes that generate, maintain and affect the patterns of cultural diversity we can observe. During my PhD, I plan to develop theory about the transmission of multiple cultural traits under different social and ecological contexts. I will link this theory to data I collect in the field, and develop (reproducible) research methods to do so.

68 | 69 |

I am supervised by Dr. Bret Beheim.

70 | 71 |

We work closely with Dr. Silke Atmaca on this project, along with Claudia Bavero, and help from Kristina Kunze, Anne Büchner and Leonie Ette.

72 | 73 |
74 |
75 | 76 | 77 | 78 | 79 |
80 | 81 | 91 | 92 |
93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /docs/2018/07/17/ears-first-post/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | ears: overview - ears: evolutionary anthropology reproducibility study 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 |
30 |
31 | 49 | 50 |
51 | 52 | 53 |
54 | 55 |
56 | 57 | 2 min read 58 | 59 | 60 |

ears: overview

61 | 62 | 63 | 2018/07/17 64 | 65 | 66 |
67 |

Evolutionary Anthropology Reproducibility Study

68 |

Our project aims to assess and improve the scientific reproducibility of research within the field of evolutionary anthropology.

69 |

Reproducible research practices include sharing raw and processed data, analytical code and documenting software and workflows to enable the reproduction of original research results. This is different from replication, which involves establishing findings with new data (e.g. different species, sample, environment). Reproducibility is a criterion for responsible scientific output, but importantly, allows for increased engagement with research, offers opportunities for collaboration and improves productivity. This is because transparent research methods and available data allow us to improve on existing ways to answer questions, as well as devise new ways to answer questions.

70 |

We are conducting a survey on data availability, author accessibility and scientific reproducibility of published empirical work on a topic relevant to the field of evolutionary anthropology, “social learning”. This is a dense and diverse literature, contributed to by researchers from biology, psychology, sociology, economics, anthropology, philosophy and archaeology. We thus hope our project will have an impact on a wide audience. You can find a current version of the preregistration for our first study here.

71 |

We plan to develop a repository online of reproducible social learning research, to serve as a resource for any interested researcher.

72 |

If you have any thoughts, suggestions or questions about this work, please get in touch: riana_minocher@eva.mpg.de

73 | 74 |
75 |
76 | 77 | 78 | 79 | 80 |
81 | 82 | 92 | 93 |
94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /docs/css/main.css: -------------------------------------------------------------------------------- 1 | html, 2 | body { 3 | margin: 0; 4 | padding: 0; 5 | position: relative; 6 | } 7 | 8 | body { 9 | text-rendering: optimizeLegibility; 10 | -webkit-font-smoothing: antialiased; 11 | -moz-osx-font-smoothing: grayscale; 12 | -moz-font-feature-settings: "liga" on; 13 | } 14 | 15 | h1, 16 | h2, 17 | h3, 18 | h4, 19 | h5, 20 | h6 { 21 | color: #222; 22 | } 23 | 24 | a { 25 | outline: none; 26 | } 27 | 28 | code { 29 | background-color: rgba(0, 0, 0, 0.02); 30 | } 31 | 32 | .wrapper { 33 | overflow: hidden; 34 | position: relative; 35 | } 36 | 37 | .header { 38 | padding: 20px 0; 39 | position: relative; 40 | background: #f5f5f5; 41 | border-bottom: 1px solid #eaeaea; 42 | } 43 | 44 | .nav { 45 | max-width: 800px; 46 | margin: 0 auto; 47 | padding: 0 15px; 48 | text-align: right; 49 | } 50 | 51 | .nav-logo { 52 | float: left; 53 | margin-top: -11px; 54 | transition: transform 150ms ease-out; 55 | } 56 | 57 | .nav-logo:hover { 58 | transform: scale(1.1); 59 | } 60 | 61 | .nav-logo img { 62 | display: block; 63 | max-height: 40px; 64 | width: auto; 65 | border-radius: 50%; 66 | } 67 | 68 | .nav-links { 69 | margin: 0; 70 | padding: 0; 71 | font-size: 14px; 72 | list-style: none; 73 | } 74 | 75 | .nav-links li { 76 | display: inline-block; 77 | margin: 0 0 0 15px; 78 | } 79 | 80 | .nav-links li:first-child { 81 | margin-left: 0; 82 | } 83 | 84 | .nav-links a { 85 | text-decoration: none; 86 | color: #111; 87 | } 88 | 89 | .nav-links a:hover { 90 | color: #000; 91 | } 92 | 93 | .content { 94 | max-width: 700px; 95 | margin: 40px auto 10px; 96 | padding: 0 15px; 97 | font-size: 16px; 98 | line-height: 1.7; 99 | color: #333; 100 | } 101 | 102 | .article-title { 103 | font-size: 34px; 104 | color: #222; 105 | margin: 0; 106 | line-height: 1.3; 107 | } 108 | 109 | .article-date { 110 | color: #ccc; 111 | } 112 | 113 | .article-duration { 114 | margin-top: 12px; 115 | float: right; 116 | font-size: 10px; 117 | padding: 1px 5px; 118 | font-weight: bold; 119 | border-radius: 3px; 120 | background: #bbb; 121 | color: #fff 122 | } 123 | 124 | .article-content p { 125 | margin: 15px 0 25px; 126 | } 127 | 128 | .article-content a { 129 | text-decoration: none; 130 | color: inherit; 131 | border-bottom: 3px solid #CBF8DF; 132 | box-shadow: inset 0 -4px 0 #CBF8DF; 133 | background: transparent; 134 | transition: background 150ms ease; 135 | } 136 | 137 | .article-content a:hover, 138 | .article-content a:focus { 139 | background: #CBF8DF; 140 | } 141 | 142 | .article-content hr { 143 | width: 150px; 144 | margin: 50px auto; 145 | height: 1px; 146 | overflow: hidden; 147 | border: none; 148 | background: #e5e5e5; 149 | } 150 | 151 | .highlight { 152 | border-radius: 3px; 153 | position: relative; 154 | margin: 40px 0; 155 | font-size: 14px; 156 | } 157 | 158 | .highlight pre { 159 | padding: 10px; 160 | } 161 | 162 | .archive { 163 | margin: 0; 164 | padding: 0; 165 | list-style: none; 166 | line-height: 1.3em; 167 | text-align: center; 168 | color: #111; 169 | } 170 | 171 | .archive-title { 172 | font-size: 32px; 173 | margin-top: 60px; 174 | } 175 | 176 | .archive-item { 177 | margin: 5px 0 15px; 178 | padding: 0; 179 | } 180 | 181 | .archive-item-date { 182 | color: #ccc; 183 | font-size: 14px; 184 | padding: 5px 0; 185 | margin-top: 5px; 186 | display: block; 187 | } 188 | 189 | .archive-item-link { 190 | display: inline-block; 191 | text-decoration: none; 192 | font-size: 21px; 193 | font-weight: bold; 194 | color: #222; 195 | padding: 5px 0; 196 | border-bottom: 1px solid #ddd; 197 | } 198 | 199 | .archive-item-link:hover { 200 | border-bottom-color: #222; 201 | } 202 | 203 | .footer { 204 | padding: 25px 0; 205 | text-align: center; 206 | } 207 | 208 | .footer-links { 209 | list-style: none; 210 | margin: 0; 211 | padding: 0; 212 | color: #ccc; 213 | font-size: 14px; 214 | } 215 | 216 | .footer-links li { 217 | margin: 0; 218 | padding: 0; 219 | display: inline-block; 220 | } 221 | 222 | .footer-links a { 223 | color: #888; 224 | text-decoration: none; 225 | transition: color 150ms ease; 226 | margin: 0 15px; 227 | } 228 | 229 | .footer-links a:hover, 230 | .footer-links a:focus { 231 | color: #222; 232 | } 233 | 234 | .footer-links li::before { 235 | content: '/'; 236 | position: relative; 237 | left: -2px; 238 | } 239 | 240 | .footer-links li:first-child::before { 241 | display: none; 242 | } 243 | .footer-links-kudos img { 244 | position: relative; 245 | top: 6px; 246 | margin-left: 2px; 247 | } 248 | 249 | @media (min-width: 600px) { 250 | .header { 251 | padding: 25px 0; 252 | } 253 | 254 | .nav-logo { 255 | margin-top: -13px; 256 | } 257 | 258 | .nav-logo img { 259 | max-height: 50px; 260 | } 261 | 262 | .nav-links { 263 | font-size: 18px; 264 | } 265 | 266 | .nav-links li { 267 | margin: 0 0 0 30px; 268 | } 269 | 270 | .content { 271 | font-size: 19px; 272 | line-height: 1.8; 273 | } 274 | 275 | .highlight { 276 | font-size: 16px; 277 | margin: 40px -20px; 278 | } 279 | 280 | .archive { 281 | text-align: left; 282 | } 283 | 284 | .archive-title { 285 | font-size: 38px; 286 | } 287 | 288 | .archive-item-date { 289 | font-size: 19px; 290 | text-align: right; 291 | float: right; 292 | } 293 | 294 | .archive-item-link { 295 | text-overflow: ellipsis; 296 | max-width: calc(100% - 120px); 297 | white-space: nowrap; 298 | overflow: hidden; 299 | } 300 | 301 | .article-title { 302 | font-size: 42px; 303 | } 304 | 305 | .article-duration { 306 | font-size: 12px; 307 | } 308 | 309 | .footer-links { 310 | font-size: inherit; 311 | } 312 | } 313 | 314 | @media print { 315 | .header, .footer { 316 | display: none; 317 | } 318 | .article-content a { 319 | box-shadow: none; 320 | } 321 | } 322 | -------------------------------------------------------------------------------- /reproducibility-study-outline.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Pre-registration: Data availability, author accessibility and analytical reproducibility of empirical social learning research" 3 | output: 4 | pdf_document: 5 | number_sections: true 6 | bibliography: MPI-Reproducibility.bib 7 | 8 | --- 9 | 10 | ```{r setup, include=FALSE} 11 | # Displays source code 12 | knitr::opts_chunk$set(echo = TRUE) 13 | ``` 14 | 15 | ```{r, echo = FALSE} 16 | # Make code wrap text so it doesn't go off the page when Knitting to PDF 17 | library(knitr) 18 | opts_chunk$set(tidy.opts = list(width.cutoff = 60), tidy = TRUE) 19 | ``` 20 | 21 | *Submitted by:* 22 | Riana Minocher\*, Bret Beheim, Silke Atmaca, Corina Logan, and Richard McElreath 23 | \newline \*riana_minocher@eva.mpg.de 24 | 25 | # Motivation 26 | 27 | Reproducibility, the ability to reproduce results of published research using the data, methods and code of original analyses, is a criterion for responsible scientific output. Reproducibility differs from replication, which involves establishing scientific findings in an independent investigation of the same research question (@ihle2017credible, @patil2016statistical). Encouraging reproducible research practices deters scientific misconduct, increases engagement with research and improves productivity. Importantly, implementing a transparent and reproducible workflow reduces time spent in data organisation and streamlines collaboration, thus improving output within research groups (@lowndes2017open). Reproducible research is published alongside available data and transparent methods, providing the potential for full analysis reproduction with executable analytical code. 28 | 29 | Surveys across scientific disciplines have found reproducibility of original work to be low (30-60%), due to inaccessible authors and data (@vines2014age, @teunis2015responsibility) and improper use of software or methods (@gilbert2012structure). Reproducibility generally improves when data and code are available (@peng2011computational) and when code is executable (@gertler2018norm). We aim to quantify the extent of scientific reproducibility within the field of evolutionary anthropology. Focussing on empirical social learning work, we will integrate these findings to identify, improve and implement reproducible research methods to study social learning. 30 | 31 | # Context 32 | 33 | Social learning, the mechanisms by which social information or behavioural traits, are acquired, used, and diffused among conspecifics, is a topic of long-standing interdisciplinary interest and controversy. Thus, a dense body of empirical research has been contributed to for over 30 years by disciplines including psychology, sociology, economics, anthropology and archaeology. This work has traditionally involved a wide range of study species, from humans and non-human primates, to birds, fish and cetaceans. Selecting a literature that is of interdisciplinary interest is critical, as we aim to demonstrate the state of reproducibility and importance of good scientific practices to a wide and growing audience. 34 | 35 | Previous investigations identify a number of factors associated with low reproducibility. For example, the probability of data availability (@vines2014age) and author accessibility via email (@teunis2015responsibility) decreases with age of publication, while age has no relationship with reproducibility when data is available (@andrew2015dfa). Journal data policies, such as mandated data statements, may increase odds of finding data online (@savage2009sharing). Furthermore, some types of software, analysis pipelines or coding language may be less stable than others (e.g. @andrew2015dfa); methods should be implemented properly and documented clearly (@gilbert2012structure). The degree of information reported in published work is often not sufficient for supplementary work, such as meta-analyses (@manca2018meta). 36 | 37 | The nature of the data and analyses in empirical social learning work lend themselves to unique considerations about data availability and sensitivity: experimental data on human subjects could often be confidential, yet is usually collected for a single purpose with design, and is thus complete, prepared for analysis, easy to share and unlikely to be reused by the same research group. However, we do not expect that our sample should have higher rates of reproducibility than other studies have found. In fact, we are not aware of traditions of collaborative data sharing practices within these disciplines, and consider the possibility that data and analyses may often go undocumented. 38 | 39 | # Research questions 40 | 41 | We aim to identify the state of data availability, author accessibility and scientific reproducibility of empirical research on social learning. We will do this by assessing several questions: 42 | 43 | 1) What proportion of social learning research is published with publicly available data? 44 | 45 | 2) When data is available on request from authors, are authors accessible? 46 | 47 | 3) Are accessible authors able/ willing to share data? 48 | 49 | 3) When data is available online or through request, is it usable, on inspection, for reproducing analyses? 50 | 51 | 4) Is analytical code available? 52 | 53 | 5) When data and analytical code are available, are the original research results reproducible? 54 | 55 | 6) Is there any influence of journal data policy on the availability of data and author willingness to share data? 56 | 57 | 7) Is there any effect of author ID on data availability - i.e. do reproducibility 'norms' exist? 58 | 59 | 8) Does age of research influence data availability and scientific reproducibility? 60 | 61 | 9) Is the methodological approach associated with reproducibility? 62 | 63 | # Methods 64 | 65 | We will quantify the proportions of the sample we find at each stage of our study (Figure 1). During Phase I of this outline, we will be assessing data availability, author accessibility, availability of data through request, data usability and access to analytical code. In Phase II we will attempt to reanalyse research with a set of reproducibility criteria defined by the set of publications we will work with. At this point, we expect to identify the precise research methods that increase scientific reproducibility. We have planned the study in two phases because we are aware Phase II demands a greater investment of time and resources, and must be limited to a subset of publications. We apply a "reasonable researcher" criteria, given that there are always ways to improve reproducibility with more effort, but diminishing returns as effort increases. We will thus document when we succeeded because we persisted beyond the point most researchers would, and we will report results at each stage with mention of the effort required to obtain materials. 66 | 67 | ![study outline](figure1.png){width=70%} 68 | 69 | ### Planned sample 70 | 71 | We focus on the subset of literature within evolutionary anthropology that fits the criteria "observational or experimental social learning". This broadly includes empirical work that involves social learning or diffusion of social information, across any species. We expect to include as much relevant literature in this study as possible for Phase I - we thus have no planned sample size and will continue to collect data until July 2018. Papers assessed during Phase II will be a subsample based on data availability and author accessibility during Phase I. 72 | 73 | ### Data collection 74 | 75 | PHASE I: 76 | Data for the first phase of our study will be collected in three parts: 77 | 78 | 1. Fetching: 79 | 80 | We are collecting information on relevant literature through a variety of sources, including published reviews on social learning, keyword searches on google scholar/ web of science, and personal and public requests to researchers. This information is used to download PDFs of publications. 81 | 82 | 2. Scraping: 83 | 84 | We record article and author metadata, and download bibliographic citations. We will produce and publish a bibliographic database for this literature. 85 | 86 | 3. Digging: 87 | 88 | When supplementary materials are published alongside the research article, we download and store them with metadata and citation. We will search for and record statements about further availability of materials, including raw data, processed data and code for analysis and data processing, in the published article and on the webpage of the publication. If data and/or code are available online, we will record the link and download and store the materials with metadata, citation and supplementary materials. 89 | 90 | We will source contact information for corresponding authors by performing a google search for updated information from the author's own website if existing, or an institutional website, after verifying that the author is currently present there. Alternatively, we will use the email address printed in the most recent paper published by the same author, i.e. disregarding older correspondence addresses. We will email corresponding authors to request data and code. Here, we plan to email authors who have stated data is available on request, as well as authors who have made no materials availability statements, but will report the outcomes of these two investigations separately. 91 | 92 | For each relevant publication, we aim to collect a pdf of the article, a citation, available supplementary materials, data and analysis code. 93 | 94 | PHASE II: 95 | Data for our second phase will be the output of reproducing a subset of studies. We will begin this phase when we have contacted all authors for materials. 96 | 97 | We will attempt to reproduce results of publications in R when original raw data is available, either online or sourced from authors. We expect to divide publications into three categories based on extent of materials available - i) raw or processed data available and methods described, ii) raw data and code in non-R format available, and iii) raw data and R code for processing and analysis available. In all cases, we will compare our reproduction of findings to the claims of the original publication, to make a qualitative assessment about whether the results remain valid. To do this, we will first summarise the key findings of each publication within this sample, before attempting reproduction. We will also record information about the type of analysis, including approach, statistical software used, and degree of statistical information reported. We expect we will need to focus the reproduction on a set of specific criteria for analyses, to maintain comparability across studies. This will depend on the final sample of publications with available materials. We will update our preregistration when we have this sample and before conducting our analyses. 98 | 99 | ### Data exclusion criteria 100 | 101 | We plan to categorise all studies according to author accessibility, data availability, code availability and scientific reproducibility. We will only attempt to reproduce results if data and/ or code are available, and usable. If data are missing or incorrectly annotated, we will not attempt to reproduce results, but record this. 102 | 103 | ### Letter template 104 | 105 | Dear Dr./ Mr./ Ms. AUTHOR NAME, 106 | 107 | My name is YOUR NAME and I am contacting you as part of the Evolutionary Anthropology Reproducibility Study (EARS), based at the Max Planck Institute for Evolutionary Anthropology. 108 | 109 | We are assessing the reproducibility of scientific work in evolutionary anthropology, measuring how often published results can be reproduced using their original data, code, and methods. A working pre-registration of our first study is available at https://rianaminocher.github.io/ears/. 110 | 111 | For our first investigation, we are focusing on empirical work in social learning, both observational and experimental. As such, your YEAR study with CO-AUTHORS (FULL NAMES), TITLE (CITATION KEY), published in JOURNAL, has been included in our starting sample. 112 | 113 | We have looked online, but have not been able to find data or analysis code to accompany this paper. We would very much appreciate if you could share your data with us for this reproducibility study. If you also have code or scripting used for your analysis, this will help us in our attempt to reproduce your work. 114 | 115 | All these materials and correspondences will remain private among members of EARS project, but we are planning to develop a redacted database from our collected materials and will contact you at a later date for your permission. 116 | 117 | If you are unable to share the data and code with us, please let us know why, e.g. data is not readily available, or is lost, or is confidential. We understand that data in such studies can be confidential, and if this is the case, would like to know more about the nature of your ethical obligations. As part of our project, we are working on developing ways that sensitive data could be shared to aid scientific reproducibility, while protecting participants' information. 118 | 119 | Thank you for your time, and please let us know if you have any questions. We're all EARS! (We're looking forward to your response). 120 | 121 | Sincerely, 122 | 123 | Evolutionary Anthropology Reproducibility Study 124 | 125 | # Planned project output 126 | 127 | 1. Complete bibliography of empirical social learning literature 128 | 2. Database of reproducible social learning work, including data, supplementary materials and analytical code in R 129 | 3. Published report on state of data availability, author accessibility and scientific reproducibility of empirical research on social learning 130 | 131 | # References -------------------------------------------------------------------------------- /MPI-Reproducibility.bib: -------------------------------------------------------------------------------- 1 | Automatically generated by Mendeley Desktop 1.17.13 2 | Any changes to this file will be lost if it is regenerated by Mendeley. 3 | 4 | BibTeX export options can be customized via Preferences -> BibTeX in Mendeley Desktop 5 | 6 | @article{steegen2016multiverse, 7 | abstract = {Empirical research inevitably includes constructing a dataset by processing raw data into a form ready for statistical analysis. Data processing often involves choices among several reasonable options for excluding, transforming and coding data. We suggest that instead of performing only one analysis, researchers could perform a multiverse analysis, which involves performing all analyses across the whole set of alternatively processed data sets corresponding to a large set of reasonable scenarios. Using a worked example focusing on the effect of fertility on religiosity and political attitudes, we show that analyzing a single data set can be misleading, and propose a multiverse analysis as an alternative practice. A multiverse analysis offers an idea of how much the conclusions change because of arbitrary choices in data construction, and gives pointers as to which choices are most consequential in the fragility of the result.}, 8 | annote = {a multiverse analysis 9 | 10 | performing all analyses across the whole set of alternatively processed data sets corresponding to 11 | a large set of reasonable scenarios. 12 | 13 | 14 | raw data do not uniquely give rise to a single data set for 15 | analysis but rather to multiple alternatively processed 16 | data sets, depending on the specific combination of 17 | choices—a many worlds or multiverse of data sets. 18 | 19 | 20 | multiverse analysis: 21 | 22 | enhances transparency by providing a detailed picture of the robustness or fragility of statistical results, and it helps 23 | identifying the key choices that conclusions hinge on 24 | 25 | 26 | involves listing the different reasonable choices during 27 | each step of data processing. 28 | 29 | the analysis of interest 30 | (in this case, an ANOVA or a logistic regression) is per-formed across all the alternatively constructed data sets. 31 | 32 | alternative options for data construction 33 | requires judgment about which options can be consid-ered reasonable and will typically depend on the experi-mental design, the research question, and the researchers 34 | performing the research. 35 | 36 | 37 | multiverse analysis is 38 | valuable, regardless of the inferential framework (fre-quentist or Bayesian), and regardless of the specific way 39 | uncertainty is quantified: a p value, an effect size, a con-fidence (Cumming, 2013) or credibility (Kruschke, 2010) 40 | interval, or a Bayes factor (Morey {\&} Rouder, 2011). 41 | 42 | MODEL MULTIVERSE 43 | 44 | If the choice for 45 | a single model specification out of the model multiverse 46 | cannot be justified, a model multiverse analysis can be 47 | performed to reveal the effect of this arbitrary choice on 48 | the statistical result. 49 | 50 | Patel, Burford, and Ioannidis (2015), focusing 51 | on the choices in deciding which predictors and covari-ates to include. Such a model multiverse analysis is 52 | related to perturbation analysis (Geisser, 1993) and to 53 | sensitivity analysis in economics (e.g., Leamer, 1985) and 54 | in Bayesian statistics (e.g., Kass {\&} Raftery, 1995), 55 | 56 | more complete analysis, the multiverse of data sets 57 | could be crossed with the multiverse of models to further 58 | reveal the multiverse of statistical results.}, 59 | author = {Steegen, Sara and Tuerlinckx, Francis and Gelman, Andrew and Vanpaemel, Wolf}, 60 | doi = {10.1177/1745691616658637}, 61 | file = {:Users/rianaminocher/Documents/reading/Steegen{\_}et{\_}al{\_}2016{\_}Multiverse analysis.pdf:pdf}, 62 | isbn = {1745-6924 (Electronic)$\backslash$r1745-6916 (Linking)}, 63 | issn = {17456924}, 64 | journal = {Perspectives on Psychological Science}, 65 | keywords = {arbitrary choices,data processing,good research practices,multiverse analysis,selective reporting,transparency}, 66 | number = {5}, 67 | pages = {702--712}, 68 | pmid = {27694465}, 69 | title = {{Increasing Transparency Through a Multiverse Analysis}}, 70 | volume = {11}, 71 | year = {2016} 72 | } 73 | @article{wicherts2011willingness, 74 | abstract = {BACKGROUND: The widespread reluctance to share published research data is often hypothesized to be due to the authors' fear that reanalysis may expose errors in their work or may produce conclusions that contradict their own. However, these hypotheses have not previously been studied systematically.$\backslash$n$\backslash$nMETHODS AND FINDINGS: We related the reluctance to share research data for reanalysis to 1148 statistically significant results reported in 49 papers published in two major psychology journals. We found the reluctance to share data to be associated with weaker evidence (against the null hypothesis of no effect) and a higher prevalence of apparent errors in the reporting of statistical results. The unwillingness to share data was particularly clear when reporting errors had a bearing on statistical significance.$\backslash$n$\backslash$nCONCLUSIONS: Our findings on the basis of psychological papers suggest that statistical results are particularly hard to verify when reanalysis is more likely to lead to contrasting conclusions. This highlights the importance of establishing mandatory data archiving policies.}, 75 | author = {Wicherts, Jelte M. and Bakker, Marjan and Molenaar, Dylan}, 76 | doi = {10.1371/journal.pone.0026828}, 77 | file = {:Users/rianaminocher/Documents/reading/Wicherts{\_}et{\_}al{\_}2011{\_}Willingness to share data.PDF:PDF}, 78 | isbn = {1932-6203 (Electronic)$\backslash$n1932-6203 (Linking)}, 79 | issn = {19326203}, 80 | journal = {PLoS ONE}, 81 | number = {11}, 82 | pages = {1--7}, 83 | pmid = {22073203}, 84 | title = {{Willingness to share research data is related to the strength of the evidence and the quality of reporting of statistical results}}, 85 | volume = {6}, 86 | year = {2011} 87 | } 88 | @article{yang2015value, 89 | abstract = {The study protocol, publications, full study report detailing all analyses, and participant-level dataset constitute the main documentation of methods and results for health research. However, journal publications are available for only half of all studies and are plagued by selective reporting of methods and results. The protocol, full study report, and participant-level dataset are rarely available. The quality of information provided in study protocols and reports is variable and often incomplete. Inaccessibility of full information for the vast majority of studies wastes billions of dollars, introduces bias, and has a detrimental impact on patient care and research. To help improve this situation at a systemic level, three main actions are warranted. Firstly, it is importantthat academic institutions and funders reward investigators who fully disseminate their research protocols, reports, and participant-level datasets. Secondly, standards for the content of protocols, full study reports, and data sharing practices should be rigorously developed and adopted for all types of health research. Finally, journals, funders, sponsors, research ethics committees, regulators, and legislators should implement and enforce policies supporting study registration and availability of journal publications, full study reports, and participant-level datasets.}, 90 | archivePrefix = {arXiv}, 91 | arxivId = {15334406}, 92 | author = {Chan, An-Wen and Song, Fujian and Vickers, Andrew and Jefferson, Tom and Dickersin, Kay and G{\o}tzsche, Peter C. and Krumholz, Harlan M. and Ghersi, Davina and van der Worp, H. Bart}, 93 | doi = {10.1126/science.1249098.Sleep}, 94 | eprint = {15334406}, 95 | file = {:Users/rianaminocher/Documents/reading/Chan{\_}et{\_}al{\_}2014{\_}Reducing research waste.pdf:pdf}, 96 | isbn = {0000000000000}, 97 | issn = {1527-5418}, 98 | journal = {The Lancet}, 99 | number = {9913}, 100 | pages = {257--266}, 101 | pmid = {24655651}, 102 | title = {{Increasing value and reducing waste: addressing inaccessible research}}, 103 | volume = {383}, 104 | year = {2014} 105 | } 106 | @article{stodden2018empirical, 107 | author = {Stodden, Victoria and Seiler, Jennifer and Ma, Zhaokun}, 108 | doi = {10.1073/pnas.1708290115}, 109 | file = {:Users/rianaminocher/Documents/reading/Stodden{\_}et{\_}al{\_}2018{\_}Empirical computational reproducibility.pdf:pdf}, 110 | issn = {0027-8424}, 111 | journal = {Proceedings of the National Academy of Sciences}, 112 | number = {11}, 113 | pages = {2584--2589}, 114 | pmid = {29531050}, 115 | title = {{An empirical analysis of journal policy effectiveness for computational reproducibility}}, 116 | url = {http://www.pnas.org/lookup/doi/10.1073/pnas.1708290115}, 117 | volume = {115}, 118 | year = {2018} 119 | } 120 | @article{lowndes2017open, 121 | abstract = {Reproducibility starts with having a transparent and streamlined workflow. Here, the authors describe how they achieved this using open data tools for the collaborative Ocean Health Index project.}, 122 | author = {Lowndes, Julia S.Stewart and Best, Benjamin D. and Scarborough, Courtney and Afflerbach, Jamie C. and Frazier, Melanie R. and O'Hara, Casey C. and Jiang, Ning and Halpern, Benjamin S.}, 123 | doi = {10.1038/s41559-017-0160}, 124 | file = {:Users/rianaminocher/Documents/reading/Lowndes{\_}et{\_}al{\_}2017{\_}Path to better science OHI.pdf:pdf}, 125 | isbn = {4690786631}, 126 | issn = {2397334X}, 127 | journal = {Nature Ecology and Evolution}, 128 | number = {6}, 129 | pmid = {28812630}, 130 | title = {{Our path to better science in less time using open data science tools}}, 131 | volume = {1}, 132 | year = {2017} 133 | } 134 | @article{gertler2018norm, 135 | abstract = {Replication is essential for building confidence in research studies1, yet it is still the exception rather than the rule2,3. That is not necessarily because funding is unavailable — it is because the current system makes original authors and replicators antagonists. Focusing on the fields of economics, political science, sociology and psychology, in which ready access to raw data and software code are crucial to replication efforts, we survey deficiencies in the current system. We propose reforms that can both encourage and reinforce better behaviour — a system in which authors feel that replication of software code is both probable and fair, and in which less time and effort is required for replication. Current incentives for replication attempts reward those efforts that overturn the original results. In fact, in the 11 top-tier economics journals we surveyed, we could find only 11 replication studies — in this case, defined as reanalyses using the same data sets — published since 2011. All claimed to refute the original results. We also surveyed 88 editors and co-editors from these 11 journals. All editors who replied (35 in total, including at least one from each journal) said they would, in principle, publish a replication study that overturned the results of an original study. Only nine of the respondents said that they would consider publishing a replication study that confirmed the original results. We also personally experienced antagonism between replicators and authors in a programme sponsored by the International Initiative for Impact Evaluation (3ie), a non-governmental organization that actively funds software-code replication. We participated as authors of original studies (P.G. and S.G.) and as the chair of 3ie's board of directors (P.G.). In our experience, the programme worked liked this: 3ie selected influential papers to be replicated and then held an open competition, awarding approximately US{\$}25,000 for the replication of each study4. The organization also offered the original authors the opportunity to review and comment on the replications. Of 27 studies commissioned, 20 were completed, and 7 (35{\%}) reported that they were unable to fully replicate the results in the original article. The only replication published in a peer-reviewed journal5 claimed to refute the results of the original paper. Despite 3ie's best efforts, adversarial relationships developed between original and replication researchers. Original authors of five of the seven non-replicated studies wrote in public comments that the replications actively sought to refute their results and were nitpicking. One group stated that the incentives of replicators to publish “could lead to overstatement of the magnitude of criticism” (see go.nature.com/2gecz3b). Others made similar points. Although one effort replicated all the results in the original paper, the originating authors wrote, “we disagree with the unnecessarily aggressive tone of some statements in the replication report (particularly in the abstract)” (see go.nature.com/2esdjkr). Another group felt that “the statement that our original conclusions were robust was buried”(see go.nature.com/2siufet). The 3ie-sponsored replication5 of one highly cited paper6 resulted in an acrimonious debate that became known as the Worm Wars. Several independent scholars speculated that assumptions made by the replicators had more to do with overturning results than with any scientific justification. Replication costs A first step to getting more replications is making them easier. This could be done by requiring authors to publicly post the data and code used to produce the results in their studies — although this in itself would not redress the incentive to gain a publication by overturning results. Ready access to data and code would ease replication attempts. In our survey of journal websites, the mid-tier economics journals (as ranked by impact factor) and those in sociology and psychology rarely asked for these resources to be made available. By contrast, almost all of the top-tier journals in economics have policies that require software code and data to be made available to editors before publication. This is also true of most of the political-science journals we assessed, and all three of the broad science journals in our survey (see Supplementary information). In addition, many of the top-tier economics journals explicitly ask authors to post raw data as well as estimation data — the final data set used to produce the results after data clean-up and manipulation of variables. These are usually placed on a publicly accessible website that is designated or maintained by the journal. There seems to be no such norm in psychology and sociology journals (see ‘Data checked?').  Source: P. Gertler, S. Galiani {\&} M. Romero (unpublished data)}, 136 | author = {Gertler, Paul and Galiani, Sebastian and Romero, Mauricio}, 137 | doi = {10.1038/d41586-018-02108-9}, 138 | file = {:Users/rianaminocher/Documents/reading/Gertler{\_}et{\_}al{\_}2018{\_}Replication economics.pdf:pdf}, 139 | journal = {Nature}, 140 | pages = {417--419}, 141 | title = {{How to make replication the norm}}, 142 | url = {https://www.nature.com/articles/d41586-018-02108-9}, 143 | volume = {554}, 144 | year = {2018} 145 | } 146 | @article{manca2018meta, 147 | author = {Manca, Andrea and Cugusi, Lucia and Dvir, Zeevi and Deriu, Franca}, 148 | doi = {10.1016/j.jclinepi.2018.01.009}, 149 | file = {:Users/rianaminocher/Documents/reading/Manca{\_}et{\_}al{\_}2018{\_}Non-correspondence meta-analysis.pdf:pdf}, 150 | issn = {1878-5921}, 151 | journal = {Journal of Clinical Epidemiology}, 152 | keywords = {Correspondence,Data extraction,Data sharing,Meta-analysis}, 153 | number = {0}, 154 | pmid = {29408344}, 155 | publisher = {Elsevier Inc}, 156 | title = {{Non-corresponding authors in the era of meta-analyses.}}, 157 | url = {http://www.ncbi.nlm.nih.gov/pubmed/29408344}, 158 | volume = {0}, 159 | year = {2018} 160 | } 161 | @article{patil2016statistical, 162 | author = {Patil, Prasad and Peng, Roger D and Leek, Jeffrey T}, 163 | doi = {10.1101/066803}, 164 | file = {:Users/rianaminocher/Documents/reading/Patil{\_}2016{\_}Statistical definition for reproducibility and replicability.pdf:pdf}, 165 | journal = {BioRxiV}, 166 | pages = {8--13}, 167 | title = {{A statistical definition for reproducibility and replicability}}, 168 | year = {2016} 169 | } 170 | @article{ihle2017credible, 171 | abstract = {Science is meant to be the systematic and objective study of the world but evidence suggests that scientific practices are sometimes falling short of this expectation. In this invited idea, we argue that any failure to conduct research according to a documented plan (lack of reliability) and/or any failure to ensure that reconducting the same project would provide the same finding (lack of reproducibility), will result in a low probability of independent studies reaching the same outcome (lack of replicability). After outlining the challenges facing behavioral ecology and science more broadly and incorporating advice from international organizations such as the Center for Open Science (COS), we present clear guidelines and tutorials on what we think open practices represent for behavioral ecologists. In addition, we indicate some of the currently most appropriate and freely available tools for adopting these practices. Finally, we suggest that all journals in our field, such as Behavioral Ecology, give additional weight to transparent studies and therefore provide greater incentives to align our scientific practices to our scientific values. Overall, we argue that producing demonstrably credible science is now fully achievable for the benefit of each researcher individually and for our community as a whole.}, 172 | author = {Ihle, Malika and Winney, Isabel S. and Krystalli, Anna and Croucher, Michael}, 173 | doi = {10.1093/beheco/arx003}, 174 | file = {:Users/rianaminocher/Documents/reading/Ihle{\_}2017{\_}Striving for transparent and credible research.pdf:pdf}, 175 | isbn = {1045-2249}, 176 | issn = {14657279}, 177 | journal = {Behavioral Ecology}, 178 | keywords = {Acknowledging Open Practices,Badges,Integrity,Open science initiative,Software,TOP guidelines,Toolkit}, 179 | number = {2}, 180 | pages = {348--354}, 181 | title = {{Striving for transparent and credible research: Practical guidelines for behavioral ecologists}}, 182 | volume = {28}, 183 | year = {2017} 184 | } 185 | @article{Wacker2017, 186 | abstract = {Within multiple fields alarming reproducibility problems are now obvious to most: The majority of the reported effects are either false positives or the population effect size is much smaller than expected based on the initial studies (e.g., Ioannidis, 2005; Button et al., 2013; Open Science Collaboration, 2015; Baker, 2016; Nichols et al., 2017). Assuming that neither outright scientific fraud (Fanelli, 2009) nor severe deficits in methodological training are the norm, likely reasons for this inacceptable status quo include the following: (A) a high prevalence of severely underpowered studies (e.g., Button et al., 2013), (B) hypothesizing after results are known (HARKing; Kerr, 1998), (C) intentionally or unintentionally exploiting researcher degrees of freedom (Simmons et al., 2011) in data processing and analysis and thereby pushing the p-value of statistical tests below the conventional significance level without being transparent concerning all the variables and approaches that have been tried out (P-HACKING), and (D) selective reporting of research findings and publication bias. Several options for pre-registration of hypotheses are now readily available providing the opportunity to effectively prevent HARKing (e.g., OSF.io, AsPredicted.org). However, suggestions to address the other three issues have so far met with the following challenges:}, 187 | author = {Wacker, Jan}, 188 | doi = {10.3389/fpsyg.2017.01332}, 189 | file = {:Users/rianaminocher/Documents/reading/Wacker{\_}2017{\_}Forking path analysis.pdf:pdf}, 190 | issn = {16641078}, 191 | journal = {Frontiers in Psychology}, 192 | keywords = {Cooperative forking path analysis,False positives,Replication,Reproducibility,Researcher degrees of freedom,Scientific practice,Statistical power}, 193 | number = {AUG}, 194 | pages = {8--11}, 195 | title = {{Increasing the reproducibility of science through close cooperation and forking path analysis}}, 196 | volume = {8}, 197 | year = {2017} 198 | } 199 | @article{peng2011computational, 200 | abstract = {Computational science has led to exciting new developments, but the nature of the work has exposed limitations in our ability to evaluate published findings. Reproducibility has the potential to serve as a minimum standard for judging scientific claims when full independent replication of a study is not possible.}, 201 | archivePrefix = {arXiv}, 202 | arxivId = {0901.4552}, 203 | author = {Peng, Roger D.}, 204 | doi = {10.1126/science.1213847}, 205 | eprint = {0901.4552}, 206 | file = {:Users/rianaminocher/Documents/reading/Peng{\_}2011{\_}Reproducible research.pdf:pdf}, 207 | isbn = {1095-9203 (Electronic)$\backslash$r0036-8075 (Linking)}, 208 | issn = {10959203}, 209 | journal = {Science}, 210 | number = {6060}, 211 | pages = {1226--1227}, 212 | pmid = {22144613}, 213 | title = {{Reproducible research in computational science}}, 214 | volume = {334}, 215 | year = {2011} 216 | } 217 | @article{teunis2015responsibility, 218 | abstract = {BACKGROUND: Publication of a manuscript does not end an author's responsibilities. Reasons to contact an author after publication include clarification, access to raw data, and collaboration. However, legitimate questions have been raised regarding whether these responsibilities generally are being met by corresponding authors of biomedical publications.$\backslash$n$\backslash$nQUESTIONS/PURPOSES: This study aims to establish (1) what proportion of corresponding authors accept the responsibility of correspondence; (2) identify characteristics of responders; and (3) assess email address decay with time. We hypothesize that the response rate is unrelated to journal impact factor.$\backslash$n$\backslash$nMETHODS: We contacted 450 corresponding authors throughout various fields of biomedical research regarding the availability of additional data from their study, under the pretense of needing these data for a related review article. Authors were randomly selected from 45 journals whose impact factors ranged from 52 to 0; the source articles were published between May 2003 and May 2013. The proportion of corresponding authors who replied, along with author characteristics were recorded, as was the proportion of emails that were returned for inactive addresses; 446 authors were available for final analysis.$\backslash$n$\backslash$nRESULTS: Fifty-three percent (190/357) of the authors with working email addresses responded to our request. Clinical researchers were more likely to reply than basic/translational scientists (51{\%} [114/225] versus 34{\%} [76/221]; p{\textless}0.001). Impact factor and other author characteristics did not differ. Logistic regression analysis showed that the odds of replying decreased by 15{\%} per year (odds ratio [OR], 0.85; 95{\%} CI, 0.79-0.91; p{\textless}0.001), and showed a positive relationship between clinical research and response (OR, 2.0; 95{\%} CI, 1.3-2.9; p=0.001). In 2013 all email addresses (45/45) were reachable, but within 10 years, 49{\%} (21/43) had become invalid.$\backslash$n$\backslash$nCONCLUSIONS: Our results suggest that contacting corresponding authors is problematic throughout the field of biomedical research. Defining the responsibilities of corresponding authors by journals more explicitly-particularly after publication of their manuscript-may increase the response rate on data requests. Possible other ways to improve communication after research publication are: (1) listing more than one email address per corresponding author, eg, an institutional and personal address; (2) specifying all authors' email addresses; (3) when an author leaves an institution, send an automated reply offering alternative ways to get in touch; and (4) linking published manuscripts to research platforms.}, 219 | author = {Teunis, Teun and Nota, Sjoerd P F T and Schwab, Joseph H.}, 220 | doi = {10.1007/s11999-014-3868-3}, 221 | file = {:Users/rianaminocher/Documents/reading/Teunis{\_}et{\_}al{\_}2015{\_}Author responsibility.pdf:pdf}, 222 | issn = {15281132}, 223 | journal = {Clinical Orthopaedics and Related Research}, 224 | number = {2}, 225 | pages = {729--735}, 226 | pmid = {25123243}, 227 | title = {{Do Corresponding Authors Take Responsibility for Their Work? A Covert Survey}}, 228 | volume = {473}, 229 | year = {2015} 230 | } 231 | @article{savage2009sharing, 232 | abstract = {BACKGROUND: Many journals now require authors share their data with other investigators, either by depositing the data in a public repository or making it freely available upon request. These policies are explicit, but remain largely untested. We sought to determine how well authors comply with such policies by requesting data from authors who had published in one of two journals with clear data sharing policies. METHODS AND FINDINGS: We requested data from ten investigators who had published in either PLoS Medicine or PLoS Clinical Trials. All responses were carefully documented. In the event that we were refused data, we reminded authors of the journal's data sharing guidelines. If we did not receive a response to our initial request, a second request was made. Following the ten requests for raw data, three investigators did not respond, four authors responded and refused to share their data, two email addresses were no longer valid, and one author requested further details. A reminder of PLoS's explicit requirement that authors share data did not change the reply from the four authors who initially refused. Only one author sent an original data set. CONCLUSIONS: We received only one of ten raw data sets requested. This suggests that journal policies requiring data sharing do not lead to authors making their data sets available to independent investigators.}, 233 | author = {Savage, Caroline J. and Vickers, Andrew J.}, 234 | doi = {10.1371/journal.pone.0007078}, 235 | file = {:Users/rianaminocher/Documents/reading/Savage{\_}2009{\_}Empirical.PDF:PDF}, 236 | isbn = {1932-6203}, 237 | issn = {19326203}, 238 | journal = {PLoS ONE}, 239 | number = {9}, 240 | pages = {9--11}, 241 | pmid = {19763261}, 242 | title = {{Empirical study of data sharing by authors publishing in PLoS journals}}, 243 | volume = {4}, 244 | year = {2009} 245 | } 246 | @article{gilbert2012structure, 247 | abstract = {Reproducibility is the benchmark for results and conclusions drawn from scientific studies, but systematic studies on the reproducibility of scientific results are surprisingly rare. Moreover, many modern statistical methods make use of 'random walk' model fitting procedures, and these are inherently stochastic in their output. Does the combination of these statistical procedures and current standards of data archiving and method reporting permit the reproduction of the authors' results? To test this, we reanalysed data sets gathered from papers using the software package STRUCTURE to identify genetically similar clusters of individuals. We find that reproducing structure results can be difficult despite the straightforward requirements of the program. Our results indicate that 30{\%} of analyses were unable to reproduce the same number of population clusters. To improve this, we make recommendations for future use of the software and for reporting STRUCTURE analyses and results in published works. {\{}{\textcopyright}{\}} 2012 Blackwell Publishing Ltd.}, 248 | author = {Gilbert, Kimberly J. and Andrew, Rose L. and Bock, Dan G. and Franklin, Michelle T. and Kane, Nolan C. and Moore, Jean S{\'{e}}bastien and Moyers, Brook T. and Renaut, S{\'{e}}bastien and Rennison, Diana J. and Veen, Thor and Vines, Timothy H.}, 249 | doi = {10.1111/j.1365-294X.2012.05754.x}, 250 | file = {:Users/rianaminocher/Documents/reading/Gilbert{\_}et{\_}al{\_}2012{\_}STRUCTURE reproducibility.pdf:pdf}, 251 | isbn = {1365-294X}, 252 | issn = {09621083}, 253 | journal = {Molecular Ecology}, 254 | keywords = {population clustering,population genetics,reproducibility,structure}, 255 | number = {20}, 256 | pages = {4925--4930}, 257 | pmid = {22998190}, 258 | title = {{Recommendations for utilizing and reporting population genetic analyses: The reproducibility of genetic clustering using the program structure}}, 259 | volume = {21}, 260 | year = {2012} 261 | } 262 | @article{Roche2017, 263 | author = {Roche, Dominique G}, 264 | doi = {10.1126/science.aan8158}, 265 | file = {:Users/rianaminocher/Documents/reading/Roche{\_}2017{\_}Science open data.pdf:pdf}, 266 | journal = {Science}, 267 | number = {6352}, 268 | pages = {654}, 269 | title = {{Evaluating Science's open-data policy}}, 270 | volume = {357}, 271 | year = {2017} 272 | } 273 | @article{andrew2015dfa, 274 | abstract = {Data are the foundation of empirical research, yet all too often the datasets underlying published papers are unavailable, incorrect, or poorly curated. This is a serious issue, because future researchers are then unable to validate published results or reuse data to explore new ideas and hypotheses. Even if data files are securely stored and accessible, they must also be accompanied by accurate labels and identifiers. To assess how often problems with metadata or data curation affect the reproducibility of published results, we attempted to reproduce Discriminant Function Analyses (DFAs) from the field of organismal biology. DFA is a commonly used statistical analysis that has changed little since its inception almost eight decades ago, and therefore provides an opportunity to test reproducibility among datasets of varying ages. Out of 100 papers we initially surveyed, fourteen were excluded because they did not present the common types of quantitative result from their DFA or gave insufficient details of their DFA. Of the remaining 86 datasets, there were 15 cases for which we were unable to confidently relate the dataset we received to the one used in the published analysis. The reasons ranged from incomprehensible or absent variable labels, the DFA being performed on an unspecified subset of the data, or the dataset we received being incomplete. We focused on reproducing three common summary statistics from DFAs: the percent variance explained, the percentage correctly assigned and the largest discriminant function coefficient. The reproducibility of the first two was fairly high (20 of 26, and 44 of 60 datasets, respectively), whereas our success rate with the discriminant function coefficients was lower (15 of 26 datasets). When considering all three summary statistics, we were able to completely reproduce 46 (65{\%}) of 71 datasets. While our results show that a majority of studies are reproducible, they highlight the fact that many studies still are not the carefully curated research that the scientific community and public expects.}, 275 | author = {Andrew, Rose L. and Albert, Arianne Y.K. and Renaut, Sebastien and Rennison, Diana J. and Bock, Dan G. and Vines, Tim}, 276 | doi = {10.7717/peerj.1137}, 277 | file = {:Users/rianaminocher/Documents/reading/Andrew{\_}et{\_}al{\_}2015{\_}DFA reproducibility.pdf:pdf}, 278 | issn = {2167-8359}, 279 | journal = {PeerJ}, 280 | pages = {e1137}, 281 | title = {{Assessing the reproducibility of discriminant function analyses}}, 282 | url = {https://peerj.com/articles/1137}, 283 | volume = {3}, 284 | year = {2015} 285 | } 286 | @article{vines2014age, 287 | abstract = {Policies ensuring that research data are available on public archives are increasingly being implemented at the government [1], funding agency [2-4], and journal [5, 6] level. These policies are predicated on the idea that authors are poor stewards of their data, particularly over the long term [7], and indeed many studies have found that authors are often unable or unwilling to share their data [8-11]. However, there are no systematic estimates of how the availability of research data changes with time since publication. We therefore requested data sets from a relatively homogenous set of 516 articles published between 2 and 22 years ago, and found that availability of the data was strongly affected by article age. For papers where the authors gave the status of their data, the odds of a data set being extant fell by 17{\%} per year. In addition, the odds that we could find a working e-mail address for the first, last, or corresponding author fell by 7{\%} per year. Our results reinforce the notion that, in the long term, research data cannot be reliably preserved by individual researchers, and further demonstrate the urgent need for policies mandating data sharing via public archives. {\textcopyright} 2014 Elsevier Ltd.}, 288 | annote = {possible step - 289 | meta analyse the reproducibility literature: how does ev anth compare (in short) to other fields 'assessed' --{\textgreater} this may be an incentive for better practices / understanding where the issue lies}, 290 | archivePrefix = {arXiv}, 291 | arxivId = {1312.5670}, 292 | author = {Vines, Timothy H. and Albert, Arianne Y K and Andrew, Rose L. and D{\'{e}}barre, Florence and Bock, Dan G. and Franklin, Michelle T. and Gilbert, Kimberly J. and Moore, Jean S{\'{e}}bastien and Renaut, S{\'{e}}bastien and Rennison, Diana J.}, 293 | doi = {10.1016/j.cub.2013.11.014}, 294 | eprint = {1312.5670}, 295 | file = {:Users/rianaminocher/Documents/reading/Vines{\_} 2014{\_}Avaliability of research data declines rapidly with article age.pdf:pdf}, 296 | isbn = {0960-9822}, 297 | issn = {09609822}, 298 | journal = {Current Biology}, 299 | number = {1}, 300 | pages = {94--97}, 301 | pmid = {24361065}, 302 | title = {{The availability of research data declines rapidly with article age}}, 303 | volume = {24}, 304 | year = {2014} 305 | } 306 | @article{Culina2018, 307 | abstract = {Open access to data is revolutionizing the sciences. To allow ecologists and evolutionary biologists to confidently find and use the existing data, we provide an overview of the landscape of online data infrastructures, and highlight the key points to consider when using open data. We introduce an online collaborative platform to keep a community-driven, updated list of the best sources that enable search for data in one interface. In doing so, our aim is to lower the barrier to accessing open data, and encourage its use by researchers hoping to increase the scope, reliability and value of their findings.}, 308 | author = {Culina, Antica and Baglioni, Miriam and Crowther, Tom W. and Visser, Marcel E. and Woutersen-Windhouwer, Saskia and Manghi, Paolo}, 309 | doi = {10.1038/s41559-017-0458-2}, 310 | file = {:Users/rianaminocher/Documents/reading/Culina{\_}2018{\_}Navigating unfolding open data landscape in eco evo.pdf:pdf}, 311 | issn = {2397-334X}, 312 | journal = {Nature Ecology {\&} Evolution}, 313 | number = {3}, 314 | pages = {420--426}, 315 | publisher = {Springer US}, 316 | title = {{Navigating the unfolding open data landscape in ecology and evolution}}, 317 | url = {http://www.nature.com/articles/s41559-017-0458-2}, 318 | volume = {2}, 319 | year = {2018} 320 | } 321 | --------------------------------------------------------------------------------