├── .gitignore
├── LICENSE
├── README.md
├── content
├── _legal
│ └── imprint.md
├── _scripts
│ └── credits.js
├── _styles
│ └── extra.css
├── _theme
│ ├── branding
│ │ ├── apple-touch-icon.png
│ │ ├── favicon-192.png
│ │ ├── favicon-256.png
│ │ ├── favicon-512.png
│ │ ├── social-m.png
│ │ └── social-s.png
│ ├── main.html
│ ├── navigation
│ │ └── rainbow.png
│ └── partials
│ │ ├── footer.html
│ │ ├── footer_homepage.html
│ │ ├── header.html
│ │ ├── hero_homepage.html
│ │ ├── hero_moving.html
│ │ ├── hero_rainbow.html
│ │ ├── imprint.html
│ │ └── social.html
├── admin
│ ├── config.yml
│ └── index.html
├── analyze
│ ├── mistakes.md
│ └── reproducibility.md
├── change
│ ├── community
│ │ ├── collaboration.md
│ │ ├── contact.md
│ │ └── contribute.md
│ ├── educate
│ │ ├── OSC_Flyer_15x15_Englisch.png
│ │ ├── ask.md
│ │ ├── first-steps.md
│ │ └── literature.md
│ ├── institutions
│ │ └── hiring.md
│ ├── recognize
│ │ └── badges.md
│ └── teaching
│ │ └── resources.md
├── collect
│ └── tools.md
├── disseminate
│ ├── articles
│ │ ├── oa-outlets.md
│ │ ├── open-access.md
│ │ ├── review.md
│ │ └── sharing.md
│ └── preprints
│ │ ├── engage.md
│ │ ├── motivation.md
│ │ └── servers.md
├── index.md
├── plan
│ ├── preregistration
│ │ ├── change.md
│ │ ├── how.md
│ │ ├── scope.md
│ │ ├── teaching.md
│ │ └── why.md
│ └── registered-reports.md
├── share
│ ├── licenses.md
│ ├── open-data
│ │ ├── best-practices.md
│ │ ├── definition.md
│ │ ├── privacy.md
│ │ └── repositories.md
│ └── open-protocols.md
├── site.webmanifest
└── theorize
│ └── synthesis.md
├── mkdocs.yml
├── requirements.txt
├── runtime.txt
└── scripts
└── find_orphans.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # Ignore build output
2 | site/
3 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Attribution-ShareAlike 4.0 International
2 |
3 | =======================================================================
4 |
5 | Creative Commons Corporation ("Creative Commons") is not a law firm and
6 | does not provide legal services or legal advice. Distribution of
7 | Creative Commons public licenses does not create a lawyer-client or
8 | other relationship. Creative Commons makes its licenses and related
9 | information available on an "as-is" basis. Creative Commons gives no
10 | warranties regarding its licenses, any material licensed under their
11 | terms and conditions, or any related information. Creative Commons
12 | disclaims all liability for damages resulting from their use to the
13 | fullest extent possible.
14 |
15 | Using Creative Commons Public Licenses
16 |
17 | Creative Commons public licenses provide a standard set of terms and
18 | conditions that creators and other rights holders may use to share
19 | original works of authorship and other material subject to copyright
20 | and certain other rights specified in the public license below. The
21 | following considerations are for informational purposes only, are not
22 | exhaustive, and do not form part of our licenses.
23 |
24 | Considerations for licensors: Our public licenses are
25 | intended for use by those authorized to give the public
26 | permission to use material in ways otherwise restricted by
27 | copyright and certain other rights. Our licenses are
28 | irrevocable. Licensors should read and understand the terms
29 | and conditions of the license they choose before applying it.
30 | Licensors should also secure all rights necessary before
31 | applying our licenses so that the public can reuse the
32 | material as expected. Licensors should clearly mark any
33 | material not subject to the license. This includes other CC-
34 | licensed material, or material used under an exception or
35 | limitation to copyright. More considerations for licensors:
36 | wiki.creativecommons.org/Considerations_for_licensors
37 |
38 | Considerations for the public: By using one of our public
39 | licenses, a licensor grants the public permission to use the
40 | licensed material under specified terms and conditions. If
41 | the licensor's permission is not necessary for any reason--for
42 | example, because of any applicable exception or limitation to
43 | copyright--then that use is not regulated by the license. Our
44 | licenses grant only permissions under copyright and certain
45 | other rights that a licensor has authority to grant. Use of
46 | the licensed material may still be restricted for other
47 | reasons, including because others have copyright or other
48 | rights in the material. A licensor may make special requests,
49 | such as asking that all changes be marked or described.
50 | Although not required by our licenses, you are encouraged to
51 | respect those requests where reasonable. More_considerations
52 | for the public:
53 | wiki.creativecommons.org/Considerations_for_licensees
54 |
55 | =======================================================================
56 |
57 | Creative Commons Attribution-ShareAlike 4.0 International Public
58 | License
59 |
60 | By exercising the Licensed Rights (defined below), You accept and agree
61 | to be bound by the terms and conditions of this Creative Commons
62 | Attribution-ShareAlike 4.0 International Public License ("Public
63 | License"). To the extent this Public License may be interpreted as a
64 | contract, You are granted the Licensed Rights in consideration of Your
65 | acceptance of these terms and conditions, and the Licensor grants You
66 | such rights in consideration of benefits the Licensor receives from
67 | making the Licensed Material available under these terms and
68 | conditions.
69 |
70 |
71 | Section 1 -- Definitions.
72 |
73 | a. Adapted Material means material subject to Copyright and Similar
74 | Rights that is derived from or based upon the Licensed Material
75 | and in which the Licensed Material is translated, altered,
76 | arranged, transformed, or otherwise modified in a manner requiring
77 | permission under the Copyright and Similar Rights held by the
78 | Licensor. For purposes of this Public License, where the Licensed
79 | Material is a musical work, performance, or sound recording,
80 | Adapted Material is always produced where the Licensed Material is
81 | synched in timed relation with a moving image.
82 |
83 | b. Adapter's License means the license You apply to Your Copyright
84 | and Similar Rights in Your contributions to Adapted Material in
85 | accordance with the terms and conditions of this Public License.
86 |
87 | c. BY-SA Compatible License means a license listed at
88 | creativecommons.org/compatiblelicenses, approved by Creative
89 | Commons as essentially the equivalent of this Public License.
90 |
91 | d. Copyright and Similar Rights means copyright and/or similar rights
92 | closely related to copyright including, without limitation,
93 | performance, broadcast, sound recording, and Sui Generis Database
94 | Rights, without regard to how the rights are labeled or
95 | categorized. For purposes of this Public License, the rights
96 | specified in Section 2(b)(1)-(2) are not Copyright and Similar
97 | Rights.
98 |
99 | e. Effective Technological Measures means those measures that, in the
100 | absence of proper authority, may not be circumvented under laws
101 | fulfilling obligations under Article 11 of the WIPO Copyright
102 | Treaty adopted on December 20, 1996, and/or similar international
103 | agreements.
104 |
105 | f. Exceptions and Limitations means fair use, fair dealing, and/or
106 | any other exception or limitation to Copyright and Similar Rights
107 | that applies to Your use of the Licensed Material.
108 |
109 | g. License Elements means the license attributes listed in the name
110 | of a Creative Commons Public License. The License Elements of this
111 | Public License are Attribution and ShareAlike.
112 |
113 | h. Licensed Material means the artistic or literary work, database,
114 | or other material to which the Licensor applied this Public
115 | License.
116 |
117 | i. Licensed Rights means the rights granted to You subject to the
118 | terms and conditions of this Public License, which are limited to
119 | all Copyright and Similar Rights that apply to Your use of the
120 | Licensed Material and that the Licensor has authority to license.
121 |
122 | j. Licensor means the individual(s) or entity(ies) granting rights
123 | under this Public License.
124 |
125 | k. Share means to provide material to the public by any means or
126 | process that requires permission under the Licensed Rights, such
127 | as reproduction, public display, public performance, distribution,
128 | dissemination, communication, or importation, and to make material
129 | available to the public including in ways that members of the
130 | public may access the material from a place and at a time
131 | individually chosen by them.
132 |
133 | l. Sui Generis Database Rights means rights other than copyright
134 | resulting from Directive 96/9/EC of the European Parliament and of
135 | the Council of 11 March 1996 on the legal protection of databases,
136 | as amended and/or succeeded, as well as other essentially
137 | equivalent rights anywhere in the world.
138 |
139 | m. You means the individual or entity exercising the Licensed Rights
140 | under this Public License. Your has a corresponding meaning.
141 |
142 |
143 | Section 2 -- Scope.
144 |
145 | a. License grant.
146 |
147 | 1. Subject to the terms and conditions of this Public License,
148 | the Licensor hereby grants You a worldwide, royalty-free,
149 | non-sublicensable, non-exclusive, irrevocable license to
150 | exercise the Licensed Rights in the Licensed Material to:
151 |
152 | a. reproduce and Share the Licensed Material, in whole or
153 | in part; and
154 |
155 | b. produce, reproduce, and Share Adapted Material.
156 |
157 | 2. Exceptions and Limitations. For the avoidance of doubt, where
158 | Exceptions and Limitations apply to Your use, this Public
159 | License does not apply, and You do not need to comply with
160 | its terms and conditions.
161 |
162 | 3. Term. The term of this Public License is specified in Section
163 | 6(a).
164 |
165 | 4. Media and formats; technical modifications allowed. The
166 | Licensor authorizes You to exercise the Licensed Rights in
167 | all media and formats whether now known or hereafter created,
168 | and to make technical modifications necessary to do so. The
169 | Licensor waives and/or agrees not to assert any right or
170 | authority to forbid You from making technical modifications
171 | necessary to exercise the Licensed Rights, including
172 | technical modifications necessary to circumvent Effective
173 | Technological Measures. For purposes of this Public License,
174 | simply making modifications authorized by this Section 2(a)
175 | (4) never produces Adapted Material.
176 |
177 | 5. Downstream recipients.
178 |
179 | a. Offer from the Licensor -- Licensed Material. Every
180 | recipient of the Licensed Material automatically
181 | receives an offer from the Licensor to exercise the
182 | Licensed Rights under the terms and conditions of this
183 | Public License.
184 |
185 | b. Additional offer from the Licensor -- Adapted Material.
186 | Every recipient of Adapted Material from You
187 | automatically receives an offer from the Licensor to
188 | exercise the Licensed Rights in the Adapted Material
189 | under the conditions of the Adapter's License You apply.
190 |
191 | c. No downstream restrictions. You may not offer or impose
192 | any additional or different terms or conditions on, or
193 | apply any Effective Technological Measures to, the
194 | Licensed Material if doing so restricts exercise of the
195 | Licensed Rights by any recipient of the Licensed
196 | Material.
197 |
198 | 6. No endorsement. Nothing in this Public License constitutes or
199 | may be construed as permission to assert or imply that You
200 | are, or that Your use of the Licensed Material is, connected
201 | with, or sponsored, endorsed, or granted official status by,
202 | the Licensor or others designated to receive attribution as
203 | provided in Section 3(a)(1)(A)(i).
204 |
205 | b. Other rights.
206 |
207 | 1. Moral rights, such as the right of integrity, are not
208 | licensed under this Public License, nor are publicity,
209 | privacy, and/or other similar personality rights; however, to
210 | the extent possible, the Licensor waives and/or agrees not to
211 | assert any such rights held by the Licensor to the limited
212 | extent necessary to allow You to exercise the Licensed
213 | Rights, but not otherwise.
214 |
215 | 2. Patent and trademark rights are not licensed under this
216 | Public License.
217 |
218 | 3. To the extent possible, the Licensor waives any right to
219 | collect royalties from You for the exercise of the Licensed
220 | Rights, whether directly or through a collecting society
221 | under any voluntary or waivable statutory or compulsory
222 | licensing scheme. In all other cases the Licensor expressly
223 | reserves any right to collect such royalties.
224 |
225 |
226 | Section 3 -- License Conditions.
227 |
228 | Your exercise of the Licensed Rights is expressly made subject to the
229 | following conditions.
230 |
231 | a. Attribution.
232 |
233 | 1. If You Share the Licensed Material (including in modified
234 | form), You must:
235 |
236 | a. retain the following if it is supplied by the Licensor
237 | with the Licensed Material:
238 |
239 | i. identification of the creator(s) of the Licensed
240 | Material and any others designated to receive
241 | attribution, in any reasonable manner requested by
242 | the Licensor (including by pseudonym if
243 | designated);
244 |
245 | ii. a copyright notice;
246 |
247 | iii. a notice that refers to this Public License;
248 |
249 | iv. a notice that refers to the disclaimer of
250 | warranties;
251 |
252 | v. a URI or hyperlink to the Licensed Material to the
253 | extent reasonably practicable;
254 |
255 | b. indicate if You modified the Licensed Material and
256 | retain an indication of any previous modifications; and
257 |
258 | c. indicate the Licensed Material is licensed under this
259 | Public License, and include the text of, or the URI or
260 | hyperlink to, this Public License.
261 |
262 | 2. You may satisfy the conditions in Section 3(a)(1) in any
263 | reasonable manner based on the medium, means, and context in
264 | which You Share the Licensed Material. For example, it may be
265 | reasonable to satisfy the conditions by providing a URI or
266 | hyperlink to a resource that includes the required
267 | information.
268 |
269 | 3. If requested by the Licensor, You must remove any of the
270 | information required by Section 3(a)(1)(A) to the extent
271 | reasonably practicable.
272 |
273 | b. ShareAlike.
274 |
275 | In addition to the conditions in Section 3(a), if You Share
276 | Adapted Material You produce, the following conditions also apply.
277 |
278 | 1. The Adapter's License You apply must be a Creative Commons
279 | license with the same License Elements, this version or
280 | later, or a BY-SA Compatible License.
281 |
282 | 2. You must include the text of, or the URI or hyperlink to, the
283 | Adapter's License You apply. You may satisfy this condition
284 | in any reasonable manner based on the medium, means, and
285 | context in which You Share Adapted Material.
286 |
287 | 3. You may not offer or impose any additional or different terms
288 | or conditions on, or apply any Effective Technological
289 | Measures to, Adapted Material that restrict exercise of the
290 | rights granted under the Adapter's License You apply.
291 |
292 |
293 | Section 4 -- Sui Generis Database Rights.
294 |
295 | Where the Licensed Rights include Sui Generis Database Rights that
296 | apply to Your use of the Licensed Material:
297 |
298 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right
299 | to extract, reuse, reproduce, and Share all or a substantial
300 | portion of the contents of the database;
301 |
302 | b. if You include all or a substantial portion of the database
303 | contents in a database in which You have Sui Generis Database
304 | Rights, then the database in which You have Sui Generis Database
305 | Rights (but not its individual contents) is Adapted Material,
306 |
307 | including for purposes of Section 3(b); and
308 | c. You must comply with the conditions in Section 3(a) if You Share
309 | all or a substantial portion of the contents of the database.
310 |
311 | For the avoidance of doubt, this Section 4 supplements and does not
312 | replace Your obligations under this Public License where the Licensed
313 | Rights include other Copyright and Similar Rights.
314 |
315 |
316 | Section 5 -- Disclaimer of Warranties and Limitation of Liability.
317 |
318 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
319 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
320 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
321 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
322 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
323 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
324 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
325 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
326 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
327 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
328 |
329 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
330 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
331 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
332 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
333 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
334 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
335 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
336 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
337 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
338 |
339 | c. The disclaimer of warranties and limitation of liability provided
340 | above shall be interpreted in a manner that, to the extent
341 | possible, most closely approximates an absolute disclaimer and
342 | waiver of all liability.
343 |
344 |
345 | Section 6 -- Term and Termination.
346 |
347 | a. This Public License applies for the term of the Copyright and
348 | Similar Rights licensed here. However, if You fail to comply with
349 | this Public License, then Your rights under this Public License
350 | terminate automatically.
351 |
352 | b. Where Your right to use the Licensed Material has terminated under
353 | Section 6(a), it reinstates:
354 |
355 | 1. automatically as of the date the violation is cured, provided
356 | it is cured within 30 days of Your discovery of the
357 | violation; or
358 |
359 | 2. upon express reinstatement by the Licensor.
360 |
361 | For the avoidance of doubt, this Section 6(b) does not affect any
362 | right the Licensor may have to seek remedies for Your violations
363 | of this Public License.
364 |
365 | c. For the avoidance of doubt, the Licensor may also offer the
366 | Licensed Material under separate terms or conditions or stop
367 | distributing the Licensed Material at any time; however, doing so
368 | will not terminate this Public License.
369 |
370 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
371 | License.
372 |
373 |
374 | Section 7 -- Other Terms and Conditions.
375 |
376 | a. The Licensor shall not be bound by any additional or different
377 | terms or conditions communicated by You unless expressly agreed.
378 |
379 | b. Any arrangements, understandings, or agreements regarding the
380 | Licensed Material not stated herein are separate from and
381 | independent of the terms and conditions of this Public License.
382 |
383 |
384 | Section 8 -- Interpretation.
385 |
386 | a. For the avoidance of doubt, this Public License does not, and
387 | shall not be interpreted to, reduce, limit, restrict, or impose
388 | conditions on any use of the Licensed Material that could lawfully
389 | be made without permission under this Public License.
390 |
391 | b. To the extent possible, if any provision of this Public License is
392 | deemed unenforceable, it shall be automatically reformed to the
393 | minimum extent necessary to make it enforceable. If the provision
394 | cannot be reformed, it shall be severed from this Public License
395 | without affecting the enforceability of the remaining terms and
396 | conditions.
397 |
398 | c. No term or condition of this Public License will be waived and no
399 | failure to comply consented to unless expressly agreed to by the
400 | Licensor.
401 |
402 | d. Nothing in this Public License constitutes or may be interpreted
403 | as a limitation upon, or waiver of, any privileges and immunities
404 | that apply to the Licensor or You, including from the legal
405 | processes of any jurisdiction or authority.
406 |
407 |
408 | =======================================================================
409 |
410 | Creative Commons is not a party to its public
411 | licenses. Notwithstanding, Creative Commons may elect to apply one of
412 | its public licenses to material it publishes and in those instances
413 | will be considered the “Licensor.” The text of the Creative Commons
414 | public licenses is dedicated to the public domain under the CC0 Public
415 | Domain Dedication. Except for the limited purpose of indicating that
416 | material is shared under a Creative Commons public license or as
417 | otherwise permitted by the Creative Commons policies published at
418 | creativecommons.org/policies, Creative Commons does not authorize the
419 | use of the trademark "Creative Commons" or any other trademark or logo
420 | of Creative Commons without its prior written consent including,
421 | without limitation, in connection with any unauthorized modifications
422 | to any of its public licenses or any other arrangements,
423 | understandings, or agreements concerning use of licensed material. For
424 | the avoidance of doubt, this paragraph does not form part of the
425 | public licenses.
426 |
427 | Creative Commons may be contacted at creativecommons.org.
428 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | Welcome to the **open science knowledge base**! This is an annotated collection of practical resources for researchers interested in using open practices in their work. [Additions and revisions](content/change/community/contribute.md) are very welcome!
2 |
3 |
--------------------------------------------------------------------------------
/content/_legal/imprint.md:
--------------------------------------------------------------------------------
1 | # Imprint / Impressum
2 |
3 | **This page contains the legal information required by German law, and is therefore kept in German. Please contact us if you have further questions.**
4 |
5 | ----
6 |
7 | ### Betreiber und Kontakt gem. § 5 TMG
8 |
9 |
10 |
11 |
12 |
13 | Felix Henninger
14 | Cognition Lab
15 | University of Koblenz-Landau
16 | Gebäude K
17 | Fortstraße 7
18 | 76829 Landau, Germany
19 |
29 |
30 | ----
31 |
32 | ### Haftung für Inhalte
33 |
34 | Als Diensteanbieter sind wir gemäß § 7 Abs.1 TMG für eigene Inhalte auf diesen Seiten nach den allgemeinen Gesetzen verantwortlich. Nach §§ 8 bis 10 TMG sind wir als Diensteanbieter jedoch nicht verpflichtet, übermittelte oder gespeicherte fremde Informationen zu überwachen oder nach Umständen zu forschen, die auf eine rechtswidrige Tätigkeit hinweisen.
35 |
36 | Verpflichtungen zur Entfernung oder Sperrung der Nutzung von Informationen nach den allgemeinen Gesetzen bleiben hiervon unberührt. Eine diesbezügliche Haftung ist jedoch erst ab dem Zeitpunkt der Kenntnis einer konkreten Rechtsverletzung möglich. Bei Bekanntwerden von entsprechenden Rechtsverletzungen werden wir diese Inhalte umgehend entfernen.
37 |
38 | ### Haftung für Links
39 |
40 | Unser Angebot enthält Links zu externen Websites Dritter, auf deren Inhalte wir keinen Einfluss haben. Deshalb können wir für diese fremden Inhalte auch keine Gewähr übernehmen. Für die Inhalte der verlinkten Seiten ist stets der jeweilige Anbieter oder Betreiber der Seiten verantwortlich. Die verlinkten Seiten wurden zum Zeitpunkt der Verlinkung auf mögliche Rechtsverstöße überprüft. Rechtswidrige Inhalte waren zum Zeitpunkt der Verlinkung nicht erkennbar.
41 |
42 | Eine permanente inhaltliche Kontrolle der verlinkten Seiten ist jedoch ohne konkrete Anhaltspunkte einer Rechtsverletzung nicht zumutbar. Bei Bekanntwerden von Rechtsverletzungen werden wir derartige Links umgehend entfernen.
43 |
44 | ### Urheberrecht
45 |
46 | Die durch die Seitenbetreiber erstellten Inhalte und Werke auf diesen Seiten unterliegen dem deutschen Urheberrecht. Die Vervielfältigung, Bearbeitung, Verbreitung und jede Art der Verwertung außerhalb der Grenzen des Urheberrechtes bedürfen der schriftlichen Zustimmung des jeweiligen Autors bzw. Erstellers. Downloads und Kopien dieser Seite sind nur für den privaten, nicht kommerziellen Gebrauch gestattet.
47 |
48 | Soweit die Inhalte auf dieser Seite nicht vom Betreiber erstellt wurden, werden die Urheberrechte Dritter beachtet. Insbesondere werden Inhalte Dritter als solche gekennzeichnet. Sollten Sie trotzdem auf eine Urheberrechtsverletzung aufmerksam werden, bitten wir um einen entsprechenden Hinweis. Bei Bekanntwerden von Rechtsverletzungen werden wir derartige Inhalte umgehend entfernen.
49 |
50 | Quellenangabe: e-recht24.de
51 |
--------------------------------------------------------------------------------
/content/_scripts/credits.js:
--------------------------------------------------------------------------------
1 | {
2 | const el = document.querySelector('[data-md-component="credits"]')
3 | const repo = 'felixhenninger/open-science-knowledge-base'
4 | const creditsPrepend = 'Curated by
11 | Welcome to the open science knowledge base!
12 |
13 |
14 | This is an annotated collection
15 | of practical resources
16 | for researchers interested in using open practices.
17 | Additions and revisions are very welcome!
18 |
5 |
6 |
--------------------------------------------------------------------------------
/content/_theme/partials/imprint.html:
--------------------------------------------------------------------------------
1 |
9 |
--------------------------------------------------------------------------------
/content/_theme/partials/social.html:
--------------------------------------------------------------------------------
1 |
10 |
--------------------------------------------------------------------------------
/content/admin/config.yml:
--------------------------------------------------------------------------------
1 | backend:
2 | name: git-gateway
3 | branch: master # Branch to update (optional; defaults to master)
4 | squash_merges: true
5 |
6 | publish_mode: editorial_workflow
7 |
8 | # Article media
9 | media_folder: "content/_media/uploads"
10 | public_folder: "/_media/uploads"
11 |
12 | # Collection setup
13 | collections:
14 | - name: "page"
15 | label: "Page"
16 | folder: "content"
17 | create: true
18 | slug: "{{slug}}"
19 | fields:
20 | - {label: "Title", name: "title", widget: "string"}
21 | - {label: "Body", name: "body", widget: "markdown"}
22 | - {label: "Publish Date", name: "date", widget: "string"}
23 |
--------------------------------------------------------------------------------
/content/admin/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | Content Manager
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/content/analyze/mistakes.md:
--------------------------------------------------------------------------------
1 | # How can I avoid mistakes during analysis?
2 |
3 | [**Everyone makes mistakes during data analysis.**](https://twitter.com/mcxfrank/status/1010254884938047488) Careful checks of the literature consistently that [exact results are difficult to reproduce](https://osf.io/preprints/bitss/39cfb/), and [reporting errors are fairly common](https://doi.org/10.3758/s13428-011-0089-5) (see also [Nuijten et al., 2016](https://dx.doi.org/10.3758/s13428-015-0664-2), below). All of us have very likely made mistakes, and are likely to make more in the future. Open practices help others spot these and correct them; we can work to reduce the incidence of errors.
4 |
5 | ## Tools
6 |
7 | * **Statcheck** automatically checks the consistency of APA-formatted test statistics and _p_ values in manuscripts. It can process `HTML`, `PDF` and `DOC` files directly, but can also check individual test statistics. The [online version](http://statcheck.io/) works without installation; an [`R` package](http://cran.r-project.org/package=statcheck) is also available for local use. Using statcheck, [Nuijten et al. (2016)](https://dx.doi.org/10.3758/s13428-015-0664-2) re-checked over a quarter-million _p_ values spanning almost 30 years of psychological research, and found inconsistent reported results in one in two articles, and qualitative divergences from the reported conclusions in one in eight articles.
8 |
9 | !!! info "See also"
10 | In general, **any practice that increases the transparency of your results is also likely to help you spot room for improvement**. Many more ideas are listed on the [reproducible analysis](./reproducibility.md) page.
11 |
--------------------------------------------------------------------------------
/content/analyze/reproducibility.md:
--------------------------------------------------------------------------------
1 | # What makes an analysis reproducible?
2 |
3 | **The goal of reproducible data analysis is to document and communicate your analysis so that other researchers can easily follow your procedure and replicate it results.**
4 |
5 | The list of demands and best practices can seem overwhelming at times, but as [Klein et al. (2018)](https://psyarxiv.com/rtygm/) note, even small steps are helpful, and often reduce effort on part of the original analyst:
6 | > [the adoption of reproducible workflows] can be piecemeal -- each incremental step towards complete transparency adds positive value.
7 |
8 | ----
9 |
10 | ## Introductions
11 |
12 | * A great introduction to this topic with a focus on psychology is the [**practical guide for transparency in psychological science**](https://psyarxiv.com/rtygm/) by Klein et al. (2018), who also provide extensive [supplemental resources](http://psych-transparency-guide.uni-koeln.de/) with practical tips for reproducible analyses, and propose an exemplary [folder structure](http://psych-transparency-guide.uni-koeln.de/folder-structure.html) for data in [online repositories](../share/open-data/repositories.md).
13 | * Daniel Lakens has written an excellent and very comprehensive step-by-step [**tutorial on computational reproducibility**](https://docs.google.com/document/d/1WvApy4ayQcZaLRpD6bvAqhWncUaPmmRimT016-PrLBk/edit) using `R` and Markdown.
14 |
15 | ----
16 |
17 | ## Open-source tools
18 |
19 | **The tools we use for analysis should allow colleagues to see what we have done, and re-run or adapt our steps.** Freely available tools that can be picked up by any colleague are often more helpful than proprietary, commercial tools, but either will do. Thankfully, there are several sets of tools to choose from:
20 |
21 | ### Analysis frameworks
22 |
23 | * [JASP](https://jasp-stats.org/) and [Jamovi](https://www.jamovi.org/) are easy-to-use, graphical interfaces for statistical analysis. Both make possible fully reproducible analyses without needing to write code.
24 | * The [`R`](https://cran.rstudio.com/) programming language and the corresponding [RStudio](https://www.rstudio.com/) interface are probably the most common analysis tools in the social sciences.
25 | * [Python](https://www.python.org/) and [Jupyter](https://jupyter.org/) provide an alternative approach.
26 | * [Julia](https://julialang.org/) is a programming language focussed on high-performance numerical computation.
27 |
28 | !!! tip "Resources for learning `R`"
29 | If you are new to `R`, there are several fantastic resources to help you get started:
30 |
31 | * RStudio maintains a [repository of **`R` Tutorials**](https://www.rstudio.com/online-learning/), and Lorne Campbell has put together another list of [resources for learning to use `R`](https://osf.io/ny5bq/).
32 | * The Software Carpentry offers an [**introduction to `R`** for non-programmers](http://swcarpentry.github.io/r-novice-gapminder/).
33 | * RStudio's [**`R` Cheatsheets**](https://www.rstudio.com/resources/cheatsheets/) are a fantastic resource for finding `R` commands for any particular purpose.
34 | * [`R` for Data Science](http://r4ds.had.co.nz/) by Grolemund & Wickham is a more comprehensive, and more advanced, tutorial that covers all aspects of data analysis.
35 |
36 | ----
37 |
38 | ## Open and documented data formats
39 |
40 | **Like the tools used for analysis, data is most useful if it can be reused by anyone.**
41 |
42 | * Text-based data formats such as [comma-seperated values](https://en.wikipedia.org/wiki/Comma-separated_values) are probably the most widely understood format for tabular data. If in doubt, the Library of Congress provides some guidelines around [data formats for long-term archival](https://www.loc.gov/preservation/resources/rfs/data.html).
43 | * Wickham (2014) provides guidelines and examples for creating [**tidy datasets**](https://www.jstatsoft.org/article/view/v059i10).
44 | * It's often helpful to create a **codebook** with further information about your data. The [`codebook` package for `R`](https://rubenarslan.github.io/codebook/) can help you create an overview of you data automatically, and is especially helpful for survey data.
45 |
46 | ----
47 |
48 | ## Public code
49 |
50 | **As with the above steps, publishing code can seem daunting, but Barnes (2010) makes a compelling argument to [publish your computer code: It is good enough](https://dx.doi.org/10.1038/467753a).** The code that produced them is often a vital part of the results you report, as it documents the precise steps in the analysis. For added transparency, syntax files can be shared alongside data in a [public online repository](../share/open-data/repositories.md).
51 |
52 | ----
53 |
54 | ## Further steps
55 |
56 | Beyond sharing your code, you can take several additional steps further increase the ease with with fellow researchers (and yourself) can follow and reproduce your analysis:
57 |
58 | * Give your files (data, analysis scripts, etc.) [self-explanatory names](http://kbroman.org/dataorg/pages/names.html) or follow a [standardized folder structure](http://psych-transparency-guide.uni-koeln.de/folder-structure.html).
59 | * [**Self-contained projects**](https://www.tidyverse.org/articles/2017/12/workflow-vs-script/) reduce the degree to which a programme is tied to any specific computer, ensuring that analyses can be re-run by others.
60 | * [**Literate programming**](https://en.wikipedia.org/wiki/Literate_programming) is the practice of combining code interwoven with the narrative of the analysis such as explanations and discussions of the intermediate steps. For example, [Notebooks in RStudio](https://rmarkdown.rstudio.com/r_notebooks.html) or [Jupyter](https://jupyter.org) contain not only code, but also the output of every analysis step, and can also contain your commentary. They also produce `HTML` and `PDF` reports that make it possible to inspect all results without re-running the analysis. **Comments** in scripts also facilitate understanding.
61 | * **Consistency** can also help add clarity to code. For added standardization, you might consider adopting [naming conventions in `R`](https://www.r-bloggers.com/consistent-naming-conventions-in-r/), follow a styleguide such as the one proposed in [Advanced `R`](http://adv-r.had.co.nz/Style.html), or the [Google Styleguide](https://google.github.io/styleguide/Rguide.xml), or let a package like [FormatR](https://yihui.name/formatr/) do the work for you.
62 |
63 | ### Version control
64 |
65 | **Version control** systems track changes to code (and other files) over time, documenting the history of a project, and providing backups of earlier versions to fall back on.
66 |
67 | !!! info "Version control resources"
68 | * Bryan (2017) provides an excellent introduction to the most popular such system, `Git`, in the context of research data and `R`, if you have [a moment to talk about version control](https://dx.doi.org/10.1080/00031305.2017.1399928). The author also shares her extensive step-by-step guide to [Happy Git and GitHub for the useR](http://happygitwithr.com/).
69 | * Vuorre & Curley (2018) provide a tutorial on [curating research assets](https://doi.org/10.1177%2F2515245918754826) with `Git`, tailored specifically to psychology.
70 |
71 | ### Dependency management
72 |
73 | **Your results may depend on the specific versions of the software you used in your analysis** -- both the analysis framework, and the plugins and packages you installed on your computer. Documenting these dependencies, such as the package versions you relied on, helps others to recreate the exact environment that you conducted the analysis in. This helps avoid [works-on-my-machine errors](http://psych-transparency-guide.uni-koeln.de/analytic-reproducibility.html#avoid-works-on-my-machine-errors) that can be extremely difficult to pin down.
74 |
75 | !!! info "Dependency management resources"
76 | **Approaches**
77 |
78 | * In **`R`**, the [packrat package](https://rstudio.github.io/packrat/) stores a snapshot of your package library and allows others to reproduce the exact same state. The [Checkpoint package](https://github.com/RevolutionAnalytics/checkpoint) will reinstall packages available on a specific date. The built-in `sessionInfo()` command lists the used versions of every active package.
79 | * In **Python**, the [`virtualenv`](https://virtualenv.pypa.io) helps manage environments on a per-project basis. [Conda](https://conda.io) aims to manage dependencies in any language, but is most common in the Python world.
80 | * **Containers** are the latest addition to the dependency management toolkit. Instead of just recreating the set of installed packages, _containers capture an entire system_, and often contain instructions for automatically setting up the system from scratch. This ensures that every part of the analysis environment can be reproduced exactly, and safely transferred between computers if desired. Thus, an entire analyis can be packaged and (re-)run on almost any computer, including external online services. Containers are probably most useful where dependencies go beyond a single analysis framework and associated packages, for example in the case of complex toolchains.
81 |
82 | **Containers**
83 |
84 | Containers are frequently used in software engineering, and are a very stable and dependable technology that is slowly finding its way into scientific practice. The most common software for managing containers is [Docker](https://docs.docker.com/get-started/).
85 |
86 | * [Green & Clyburne-Sherin (2018)](https://psyarxiv.com/mf82t/) motivate and review containers in the context of reproducible analyses in the social sciences. Their manuscript introduces the commercial [CodeOcean](https://codeocean.com/) service, which can run analysis containers over the internet.
87 | * The [Singularity](https://singularityware.github.io/) project is building a container system designed specifically to meet the needs of researchers and academic institutions. The project's motivation is to increase the ["mobility of compute"](http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0177459) around high-performance computing systems, and provide a central [hub for sharing containers](http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0188511).
88 | * The [Scientific Filesystem](https://sci-f.github.io/) aims to make containers more accessible and transparent.
89 |
--------------------------------------------------------------------------------
/content/change/community/collaboration.md:
--------------------------------------------------------------------------------
1 | # Which projects is the community working on?
2 |
3 | * The [**Psychological Science Accelerator**](https://psysciacc.org/) organizes distributed large-scale data collection projects worldwide. It is open to everyone, and its members are positively prolific!
4 | * [**Study Swap**](https://osf.io/9aj5g/) is an online platform to support collaborative research (with a focus on psychology). It is designed to facilitate any type of exchange, such as requesting or sharing resources and finding collaborators. The project provides a brief [step-by-step guide](https://twitter.com/Study_Swap/status/1016501689959047168) and a [tutorial paper](https://psyarxiv.com/wqhbj/) that describe the process.
5 | * The [**ManyLab Project**](https://osf.io/89vqh/) is another of the first collaborative research projects. Its goals are to specifically investigate questions that require a grand scale of inter-lab collaboration:
6 | * [Heterogenity of replication results](https://osf.io/ebmf8/) across samples and settings (there is also a similar [follow-up project](https://osf.io/8cd4r/) that is not yet published)
7 | * [Data quality in participant pools across the semester](https://osf.io/csygd/)
8 | * [Effects of expert-led vs in-house study design](https://osf.io/8ccnw/) on replication success
9 | * [Effects of pre-data-collection peer-review](https://osf.io/7a6rd/) on replicability
10 |
11 | !!! question
12 | **There's probably a lot we are missing!** Can you help provide further information?
13 |
--------------------------------------------------------------------------------
/content/change/community/contact.md:
--------------------------------------------------------------------------------
1 | title: Establishing contact
2 |
3 | # How can I find out what's going on in the open science world?
4 |
5 | **The Open Science community is a vibrant, friendly and collaborative group** of people working together to improve psychological science. Many discussions take place in the open, and there is always a lot going on. How can we keep up with the latest developments?
6 |
7 | ## Social media
8 |
9 | Much of the day-to-day chatter can be found on social media:
10 |
11 | * [**Twitter**](https://twitter.com/) provides a never-ending firehose of discussions. The [@OpenScience](https://twitter.com/openscience) account often collects news and articles regarding Open Science across disciplines; the organizations below are also represented. If you follow some voices you find interesting, you'll likely soon get a feeling for ongoing discussions.
12 | * [**Facebook**](https://facebook.com/) also hosts some forums related to open science, though less so. An active group in the realm of methodology and statistics is [PsychMAP](https://www.facebook.com/groups/psychmap/about/).
13 | * **Blogs** reflect individual opinions in a longer format. You might enjoy the [100% CI](http://www.the100.ci/), Simine Vazire's [Sometimes I'm Wrong](http://sometimesimwrong.typepad.com/) or [Felix Schönbrodt's blog](http://www.nicebread.de/)?
14 | * **Podcasts** can be great for casual listening to discussions. Examples you might find interesting are the [Open Science Radio](http://www.openscienceradio.org/category/english-episodes/) or [The Black Goat](http://www.theblackgoatpodcast.com). Sau-Chin Chen put together an [overview of Psychological Science Podcasts](http://scchen.com/en/post/ps-podcasts/), which are mostly hosted by Open Science Promoters.
15 |
16 | ??? seealso "Being an academic on social media"
17 | Academic social media is great at giving voice to a wide and diverse group of people, but dealing with the deluge of information can be exhausting. It's hard if not impossible to keep up, and everybody seems to be working so hard and achieving so much! As much as the banter can be uplifting, it can at times be depressing.
18 |
19 | * The members of the podcast [The Black Goat](http://www.theblackgoatpodcast.com), themselves veteran prolific tweeps, discuss the [**blooming, buzzing confusion**](http://www.theblackgoatpodcast.com/posts/a-blooming-buzzing-confusion/) that is social media.
20 | * Dorothy Bishop lays out [**a simple rule to reduce stress**](https://deevybee.blogspot.com/2018/05/how-to-survive-on-twitter-simple-rule.html) caused by unpleasant and negative messages.
21 |
22 | ## Organizations
23 |
24 | There are several organizations that have made it their mission to foster open science, especially in the field of psychology:
25 |
26 | * The [**Society for the Improvement of Psychological Science**](http://improvingpsych.org/) \(SIPS\) is a grass-roots initiative to improve research practices on both an individual \(through training and outreach\) and institutional level. SIPS organize an [annual meeting](http://improvingpsych.org/meetings/) which focusses on collaboration in the service of open science.
27 | * The [**Center for Open Science**](https://cos.io/) runs the [Open Science Framework](https://osf.io/), and provides several related services around [preprints](https://cos.io/our-products/osf-preprints/), [open conference materials](https://cos.io/our-products/osf-meetings/), and similar efforts. The COS is represented through a worldwide network of [ambassadors](https://cos.io/our-communities/become-ambassador/).
28 |
29 | ??? seealso "Local initiatives"
30 | Besides larger organisations, there are several local, university-level initiatives. The German network of open-science initiatives maintains a [list of local working groups](https://osf.io/tbkzh/wiki/home/) in Germany and beyond.
31 |
--------------------------------------------------------------------------------
/content/change/community/contribute.md:
--------------------------------------------------------------------------------
1 | title: Contributing
2 |
3 | # How can I contribute to this resource?
4 |
5 | Wow, thanks for considering a contribution to this knowledge base! You're awesome, and your help is much appreciated.
6 |
7 | ----
8 |
9 | ## Suggest a new page
10 |
11 | We're always on the lookout for new topics that are useful for researchers trying to improve the reproducibility of their own research and that of our field in general, and we'd be glad to add content.
12 |
13 | If you have an idea for a new entry, please reach out to [Felix Henninger](http://felixhenninger.com), ideally by email, but postcards are also welcome. To save you some effort, please be invited to copy-and-paste the following template:
14 |
15 | Hi Felix, dear team,
16 |
17 | I really enjoyed the Open Science Knowledge Base, and I think you
18 | could make it even better by adding a page on [ topic ]. This would
19 | extend the section [ section ], and a good question header would be
20 | [ question? ]. Would you consider adding a page like this?
21 |
22 | I've done some searching, and I think [ ] would make good resources
23 | that are relatively stable and cater to the practical needs of our
24 | fellow researchers. [ I'm extra-awesome and have summarized their
25 | main contributions, which are ... ]
26 |
27 | I understand that we're all super-busy and working hard, so feel
28 | free to take some time if you need to, but I'd really appreciate to
29 | be able to make a contribution, and I think my colleagues would
30 | find this information useful, too.
31 |
32 | Thanks for your help, and kind regards. Yours ever awesome,
33 |
34 |
35 | -A friendly and smart contributor
36 |
37 |
38 | ## How can I develop content locally?
39 | These instructions are intended for those users on Linux or Unix flavored systems
40 | with a command line. If you are using Windows, the easier option might be to
41 | [open an issue](https://github.com/FelixHenninger/open-science-knowledge-base/issues) or use the contact above to request a change.
42 |
43 |
44 | ### 1. Fork and Clone the Repository on Github
45 | If you want to dig in and contribute directly, you can clone the repository,
46 | check out a new branch, and then develop locally. FIrst, you should fork the
47 | repository to your Github account. If your username is `waffles` after you
48 | fork, you would clone like this:
49 |
50 | ```bash
51 | $ git clone https://www.github.com/waffles/open-science-knowledge-base
52 | $ cd open-science-knowledge-base
53 | ```
54 |
55 | ### 2. Create a New Branch
56 |
57 | At this point, you are on the `master` branch, which is the main branch of the
58 | repository that should be kept in line with the upstream (the primary repository).
59 | In order to cleanly separate your changes, you should first checkout a new branch:
60 |
61 | ```bash
62 | $ git checkout -b add/my-addition master
63 | ```
64 |
65 | ### 3. Make Changes
66 | Then, make changes to your heart's content! The project uses [mkdocs](https://www.mkdocs.org)
67 | so you should first install it:
68 |
69 | ```bash
70 | $ pip install mkdocs
71 | ```
72 |
73 | The theme that we use is [Material](https://squidfunk.github.io/mkdocs-material/) so
74 | you need to install that too.
75 |
76 | ```bash
77 | $ pip install mkdocs-material
78 | ```
79 |
80 | and then start a local server to see your changes in action!
81 |
82 | ```bash
83 | $ mkdocs serve
84 | INFO - Building documentation...
85 | INFO - Cleaning site directory
86 | [I 180628 10:57:20 server:292] Serving on http://127.0.0.1:8000
87 | [I 180628 10:57:20 handlers:59] Start watching changes
88 | [I 180628 10:57:20 handlers:61] Start detecting changes
89 | ```
90 |
91 | And open up [http://127.0.0.1:8000](http://127.0.0.1:8000) to see the site. As you
92 | edit the local text files, you will see the server rebuild automatically!
93 | If you need more detail or instruction, please see the link provided above.
94 |
95 | ### 4. Create a Pull Request
96 |
97 | When you are ready you can opena [pull request](https://help.github.com/articles/about-pull-requests/) to the primary repository master branch, and discuss your changes with the maintainers. Please be sure to include details about your contribution.
98 |
--------------------------------------------------------------------------------
/content/change/educate/OSC_Flyer_15x15_Englisch.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/how-to-open-science/knowledge-base/3fa049b144c78ad7f4656729e369c20c203e7e0b/content/change/educate/OSC_Flyer_15x15_Englisch.png
--------------------------------------------------------------------------------
/content/change/educate/ask.md:
--------------------------------------------------------------------------------
1 | title: Asking questions
2 |
3 | # Where can I ask further questions?
4 |
5 | As the standards in our field of science evolve, it's natural that all kinds of additional questions pop up.
6 |
7 | * [**Ask Open Science**](https://ask-open-science.org/) is an online Q&A site specifically for Open Science practices (across all subjects)
8 | * [**Online Communities**](../community/contact.md) are often happy to answer questions
9 | * If you have a question that could be valuable for other researchers, please consider [contributing to this collection](../community/contribute.md)!
10 |
--------------------------------------------------------------------------------
/content/change/educate/first-steps.md:
--------------------------------------------------------------------------------
1 | # What are first steps I can take?
2 |
3 | So you're looking to **get started right now**? The [Open Science Flyer (`PDF`)](https://osf.io/hktmf/) by the German National Open Science Initiative provides some concrete steps you can take to get your feet wet.
4 |
5 | 
6 |
--------------------------------------------------------------------------------
/content/change/educate/literature.md:
--------------------------------------------------------------------------------
1 |
2 | # Where can I learn more about the issues?
3 |
4 | After more than half a decade of discussion, the literature on reproducibility is substantial. Thankfully, brave authors have provided summaries and reviews:
5 |
6 | * [Spellman, Gilbert, & Corker \(2017\)](https://psyarxiv.com/ak6jr) provide practical, actionable advice for **Open Science: What, Why, and How**
7 | * [Nelson, J. Simmons, & Simonsohn \(2017\)](https://dx.doi.org/10.1146/annurev-psych-122216-011836) have written an excellent overview of **Psychology's Renaissance**, with a focus on questionable research practices and suggested mitigation strategies.
8 | * The [Berkeley Initiative for Transparency in the Social Sciences (BITSS)
9 | ](https://www.bitss.org/) provides a five-week online course on [**Transparent and Open Social Science Research**](https://www.futurelearn.com/courses/open-social-science-research) that covers every step of the research process. Similarly, the [**Open Science MOOC**](https://opensciencemooc.eu/) is developing [course material](https://docs.google.com/document/d/1KuTSECSYHXZmZX15GDjyD65pJ90eRMhHVEZ-1trsw30/edit#) for learning about Open Science, and has collected many interesting links for different topics that often go beyond scholarly articles.
10 | * The [**Open Science Training Handbook**](https://book.fosteropenscience.eu) is a fantastic and comprehensive resource that covers many topics around open science. It's built for people who would like to create open science training, and very in-depth.
11 |
12 | !!! seealso "see also"
13 | Crüwell et al. (2018) have put together [**8 Easy Steps to Open Science**](https://psyarxiv.com/cfzyx), an annotated reading list.
14 |
15 | Brent Roberts and Dan Simmons have compiled an extensive [**Reproducibility and Replicability Reading List**](https://docs.google.com/document/d/14lBD0aZDPij2Z6AOpAharOAtmt6ZBI0EuF3_tu8m66I/) which lists further literature by topic.
16 |
17 | The [**Open Research Glossary**](https://docs.google.com/document/d/1uXZzyXPHNcjCPiR5qkzEuB5u2PUIYQzq0mrG9BtD-Qo) by Jon Tennant and Ross Mounce defines key terms used throughout the Open Science movement.
18 |
--------------------------------------------------------------------------------
/content/change/institutions/hiring.md:
--------------------------------------------------------------------------------
1 | # How can we acknowledge open practices in hiring decisions?
2 |
3 | * Felix Schönbrodt is collecting [**academic job offers that mentioned open science**](https://osf.io/7jbnt/). His department has instituted a hiring policy that [asks for plans and previous experience around transparent research](http://www.nicebread.de/open-science-hiring-policy-lmu/).
4 | * There is an ongoing initiative to [**define and certify open research practices**](https://docs.google.com/document/d/1ty43Syw0Flkh8ncjW8MZArIkvYe8hLwwhLlIwbtSk_Y/edit) so that the individual or institutional level of experience with these practices can be assessed.
5 |
6 | !!! danger
7 | **We're still collecting resources here.** Do you know things we should add?
8 |
--------------------------------------------------------------------------------
/content/change/recognize/badges.md:
--------------------------------------------------------------------------------
1 | # How do the Open Science Badges work?
2 |
3 | **The [Open Science Badges](https://osf.io/tvyxz/) are simple visual indicators that a publication or project followed open practices.** [Kidwell et al. (2016)](https://doi.org/10.1371%2Fjournal.pbio.1002456) describe the badges and their underlying motivation, and show that the rate of openly archived data increased dramatically after their introduction at _Psychological Science_, as did the proportion of openly available materials.
4 |
5 | The badges indicate [**preregistration**](../../plan/preregistration/why.md) of a project's design and analysis, the presence of [**open data**](../../share/open-data/definition.md) and the **open availability of materials**. There is also a [**protected access open data**](https://osf.io/tvyxz/wiki/8.%20Approved%20Protected%20Access%20Repositories/) badge for critical data that is placed in a repository, yet are only accessible following a standardized authorization procedure. [**Preregistration plus**](https://osf.io/6q7nm/) indicates the presence of an analysis plan that corresponds directly to the reported analysis.
6 |
7 |
8 | !!! info "Further resources"
9 | * Every badge comes with a [set of **criteria**](https://osf.io/tvyxz/wiki/1.%20View%20the%20Badges/) that govern whether it can be conferred.
10 | * The Open Science Framework maintains a [list of **journals that offer badges**](https://osf.io/tvyxz/wiki/5.%20Adoptions%20and%20Endorsements/).
11 |
--------------------------------------------------------------------------------
/content/change/teaching/resources.md:
--------------------------------------------------------------------------------
1 | # Where can I find resources for teaching open science?
2 |
3 | Open Science is here to stay. Therefore, students deserve to be prepared for a future where reproducable research practices are standard. We have found that students intuitively understand the necessity for changes to the scientific practice, and enjoy applying state-of-the-art methods in their projects.
4 |
5 | As with scientific resources, members of the Open Science community have created and share freely their teaching materials. Here are a few resources:
6 |
7 | * Many researchers have made their [**course syllabi for open and reproducible methods**](https://osf.io/vkhbt/) openly available.
8 | * At the SIPS Meeting 2018, a Hackathon collected a whole bunch of [teaching resources](https://osf.io/bts5d/wiki/home/), including syllabi, videos, podcasts, journal articles and much more.
9 | * The [Framework for Open and Reproducible Research Training](https://forrt.netlify.com) presents recommendations to improve the adoption of open science teaching practices by modifying the academic incentive structure.
10 | * The [Society for the Improvement of Psychological Science](http://improvingpsych.org/) has collected [**teaching and training materials**](https://osf.io/ehpt4/) in general, as well as a collection of [**modular course materials for research methods**](https://osf.io/zbwr4/wiki/home/). For example, ...
11 | * The [Collaborative Replications and Education Project](https://osf.io/wfc6u/wiki/home/) provides teaching materials around a set of five studies that undergraduate students can replicate, with the aim of publishing the resulting data when enough has been gathered.
12 | * The [Project Teaching Integrity in Empirical Research](https://www.projecttier.org/tier-classroom) (TIER) develops methods and tools for enhancing research transparency through teaching. These can be used by faculty who teach quantitative methods or supervise student research or by students who want to adopt transparent and replicable research practices independetly.
13 |
14 | * If you teach a lab course, you might be interested in these [(German) slides provided by the LMU Munich](https://osf.io/mgwk8/).
15 |
--------------------------------------------------------------------------------
/content/collect/tools.md:
--------------------------------------------------------------------------------
1 | Which open tools can I use for data collection?
2 | ===============================================
3 |
4 | Over the past decades, a wealth of open, freely available tools for data collection has been made available to the scientific community. Because their source code is available and their internals can be examined and extended, these tools tend to have a rich set of features and an active user community. [Gezelter (2015)](https://doi.org/10.1021%2Facs.jpclett.5b00285) and [Ince et al. (2012)](https://doi.org/10.1038%2Fnature10836) provide further reasons for favoring open-source software in all areas of research.
5 |
6 | In the following, we list a few of the available tools for different types of research. All of these are freely available as open-source software.
7 |
8 | ----
9 |
10 | Laboratory-based data collection
11 | --------------------------------
12 |
13 | * [**OpenSesame**](http://osdoc.cogsci.nl) is a powerful, free and open experimental builder with a graphical interface for building studies.
14 | * [**PsychoPy**](http://www.psychopy.org/) has a more basic interface, but makes available many powerful features through additional code.
15 | * Several further options are available for researchers looking to program studies from scratch, for example [PyEPL](http://pyepl.sourceforge.net/) and [Expyriment](http://www.expyriment.org/) for Python, and the [Psychtoolbox](http://psychtoolbox.org/) for [Matlab](http://www.mathworks.com/) and its open analogue [Octave](http://www.octave.org/).
16 |
17 | ----
18 |
19 | Online / browser-based data collection
20 | --------------------------------------
21 |
22 | ### Questionnaires
23 |
24 | * [**LimeSurvey**](https://www.limesurvey.org/) is a very versatile tool with a graphical interface for building online questionnaires and surveys.
25 | * [**formr**](https://formr.org/) uses the [R programming language](http://r-project.org/) to enable very flexible and powerful logic within questionnaires, for example immediate graphical feedback, and complex designs with multiple waves and reminders.
26 |
27 | ### Experiments
28 |
29 | * [**lab.js**](https://lab.js.org) offers an easy-to-use graphical interface for constructing studies, similar to laboratory-based experimental software.
30 | * [**jsPsych**](http://jspsych.org/) is a framework for programming experiments in JavaScript.
31 |
32 | ### Study hosting
33 | * [**JATOS**](http://www.jatos.org) and [**Tatool**](http://tatool.ch/) help coordinate data collection, and allow researchers to construct batteries from multiple tasks. They also offer additional features such as a powerful integration with [Amazon Mechanical Turk](https://www.mturk.com/), and (in the case of JATOS) capabilities for interaction between participants through chat and economic games.
34 | * [**Pavlovia**](https://pavlovia.org/) is a repository for sharing and hosting online studies. It provides full version history for all files, automated data collection, and a public issue tracker.
35 |
--------------------------------------------------------------------------------
/content/disseminate/articles/oa-outlets.md:
--------------------------------------------------------------------------------
1 | # Where can I publish with Open Access?
2 |
3 | The [**Directory of Open Access Journals**](https://doaj.org/) is a comprehensive, searchable database of publication outlets that support OA.
4 |
--------------------------------------------------------------------------------
/content/disseminate/articles/open-access.md:
--------------------------------------------------------------------------------
1 | # What is Open Access?
2 |
3 | Open Access ensures that scientific articles are available [free of charge and other barriers](https://open-access.net/DE-EN/information-on-open-access/open-access-journals). This ensures that fellow researchers as well as the public are able to access scientific ideas and results.
4 |
5 | !!! seealso "See also"
6 | * [**open-access.net**](https://open-access.net) is a treasure trove of information regarding open access, compiled by several German university libraries. We link extensively to their resources below.
7 | * The article [Ten **Hot Topics around Scholarly Publishing**](https://www.mdpi.com/2304-6775/7/2/34) tackles common concerns and questions around open access from the perspective of a researcher.
8 |
9 | ----
10 |
11 | ## **Paths** to Open Access
12 |
13 | There are two ways or [**strategies to ensure the public availablility of publications**](https://open-access.net/en/information-on-open-access/open-access-strategies/):
14 |
15 | * **Gold Open Access** denotes that the published version of an article or book is immediately made openly accessible as part of the publication process.
16 | * For **Green Open Access**, scientific texts are available freely outside of the publication outlets' platform by the author. This is also known as _self-archiving_ or _self-publishing_.
17 |
18 | ### **Gold** Open Access
19 |
20 | There are several [**business models**](https://open-access.net/en/information-on-open-access/business-models#c831) for providing Gold OA:
21 |
22 | * Publishers can levy one-time **Article Processing Charges** (APCs) for providing open access to articles.
23 | * Some journals provide **No-fee Open Access** which is free for authors (sometimes also known as _diamond OA_), largely through institutional funding of journals.
24 |
25 | !!! warning "Hybrid models: Per-article OA"
26 |
27 | The gold approach assumes that Open Access is the default for an entire journal rather than an exception on a per-article basis. However, some commercial publishers provide **open access to individual articles**, collecting _both_ APCs and subscription revenue.
28 |
29 | This practice of charging twice is known as **double-dipping**, and while it meets the above definitions in principle, it is controversial because it costs institutions doubly.
30 |
31 | ----
32 |
33 | ### **Green** Open Access
34 |
35 | For Green OA approaches, the first consideration is _whether_, _when_, and _in which format_, public archival can take place. The main options are:
36 |
37 | * A **postprint**, which is identical in content, but not in formatting, to the **published version**, which is typeset by the publisher.
38 | * **Immediate** publishing or archival following an **embargo** period.
39 |
40 | The second question is _where_ to archive papers. Again, there are several possibilities:
41 |
42 | * **Domain- or field-specific repositories** that gather paper for a certain field.
43 | * **Institutional repositories** that collect material for all scientists at a certain institution.
44 | * A common, though discouraged, practice is to archive papers on **personal homepages**. The difficulty with this approach is that
45 |
46 | !!! seealso "More resources"
47 |
48 | * [Am I allowed to **share my papers** openly after publication?](../sharing)
49 |
--------------------------------------------------------------------------------
/content/disseminate/articles/review.md:
--------------------------------------------------------------------------------
1 | title: Reviewing papers
2 |
3 | # What can I do as a reviewer to encourage open science?
4 |
5 | Reviewers provide quality control and a seal of approval for the published scientific literature. They have considerable influence over what gets published, and can use this leverage to encourage more open practices.
6 |
7 | * If you would like to commit to requiring open practices in the papers you review, you could consider joining the [**Peer Reviewers' Openness Initiative**](https://opennessinitiative.org/). Members of the PRO pledge to require open data and materials, or an explanation why either is not possible, before reviewing a paper. [Morey et al. (2016)](http://rsos.royalsocietypublishing.org/content/3/1/150547) present and discuss this initiative in detail.
8 |
--------------------------------------------------------------------------------
/content/disseminate/articles/sharing.md:
--------------------------------------------------------------------------------
1 | title: Sharing papers
2 |
3 | # Can I share my papers openly after publication?
4 |
5 | The degree to and the format in which you can (legally) share your papers openly, for example by uploading `pdf` files to your personal homepage or any public repository, depends on your agreement with the publisher.
6 |
7 | The [Sherpa/Romeo database](http://www.sherpa.ac.uk/romeo/index.php) catalogues policies by journal for many publishers.
8 |
9 | If a publisher asks for a transfer of copyrights during the publication process, you may be able to negotiate a more favorable agreement. The [SPARC Author Addendum](https://sparcopen.org/our-work/author-rights) provides a template that (among other things) allows the author to disseminate a typeset version of the article for non-commercial purposes.
10 |
--------------------------------------------------------------------------------
/content/disseminate/preprints/engage.md:
--------------------------------------------------------------------------------
1 | # How can I engage with preprints?
2 |
3 | **Preprints not only serve to disseminate research more quickly, they can also help improve the quality of the published literature before it's printed and (almost) unalterable.** Imagine if you could exchange your thoughts with the authors and incorporate changes instead of being stuck with a final version!
4 |
5 | * The [**PreReview**](https://www.prereview.org/) project aims to make preprints and reviews thereof commonplace. They outline their [motivation](https://www.prereview.org/users/8850/articles/198235-welcome-to-prereview) in detail, and provide a [step-by-step guide](https://prereview.org/users/8850/articles/199648-how-to-post-with-prereview-org) for preprint reviews on the [Authorea](https://authorea.com/) platform. They also suggest a [prereview journal club](https://www.prereview.org/users/164141/articles/205430-how-to-start-a-prereview-journal-club-tips-email-templates), and provide a starter kit for interested researchers.
6 | * There are several **platforms that collect commentary** on papers and preprints alike, such as [PubPeer](https://pubpeer.com/), which works across journals and preprints, and [hypothes.is](https://web.hypothes.is), which is offered specifically by some publishers. Some publishers also allow annotations on papers directly, for example [eLife sciences](https://elifesciences.org).
7 |
--------------------------------------------------------------------------------
/content/disseminate/preprints/motivation.md:
--------------------------------------------------------------------------------
1 | title: Motivation
2 |
3 | # What is a preprint, and why should I publish one?
4 |
5 | According to Wikipedia, a [preprint](https://en.wikipedia.org/wiki/Preprint) is
6 |
7 | > **[...] a version of a scholarly or scientific paper that precedes publication in a peer-reviewed scholarly or scientific journal.** The preprint may be available, often as a non-typeset version available free, before and/or after a paper is published in a journal.
8 |
9 | A preprint makes a manuscript widely available independently of its formal publication. This provides greater and earlier exposure compared to the traditional route, which has advantages and downsides.
10 |
11 | * Nikolaus Kriegskorte lists [**selfish reasons for preprint posting**](https://thewinnower.com/papers/3674-the-selfish-scientist-s-guide-to-preprint-posting) and discusses the importance of timing.
12 | * In fields with a long tradition of preprints like physics, availability of preprints appears to result in a [fairly large](http://akademiai.com/doi/abs/10.1007/s11192-007-1661-8) [boost](https://link.springer.com/article/10.1007%2Fs11207-006-0262-7) [in citations](https://arxiv.org/abs/astro-ph/0411275v1), though it is difficult to establish causality and results might not generalize to other subjects. More careful randomized controlled trials suggest that [open-access publishing certainly increases exposure, but does not translate to added citations](http://www.bmj.com/content/337/bmj.a568.full).
13 |
--------------------------------------------------------------------------------
/content/disseminate/preprints/servers.md:
--------------------------------------------------------------------------------
1 | title: Preprint servers
2 |
3 | # Where can I find or publish preprints?
4 |
5 | Though a preprint could be circulated through any medium in principle, in practice, fields often establish central repositories that disseminate manuscripts further, for example by notifying interested community members, and screening uploaded content. There are several such repositories or _preprint servers_.
6 |
7 | To benefit best from the additional exposure a preprint gains, you'll probably want to choose a location based on where your manuscript will find the largest number of interested readers. For psychology, the [Society for the Improvement of Psychological Science](http://improvingpsych.org/) has established [**PsyArXiv**](https://psyarxiv.com/), which has since been named the [APA's preferred preprint service](http://blog.psyarxiv.com/2017/08/14/apa-names-psyarxiv-preferred-preprint-service/).
8 |
9 | Other fields have a long tradition of preprint publication and established central repositories, especially [arXiv](https://arxiv.org/) for physics, mathematics, and computer science. Because preprint submission is so common in these areas, some journals allow for submissions directly through arXiv. The [Open Science Framework]() provides a [domain-general service for preprint repositories](https://osf.io/preprints), which powers a number of preprint repositories.
10 |
--------------------------------------------------------------------------------
/content/index.md:
--------------------------------------------------------------------------------
1 | # Changing scientific practice
2 |
3 | **Science is rapidly moving towards more open, transparent practice**, posing new questions and challenges. This knowledge base is designed to serve as a practical, hands-on introduction as well as a reference for budding and veteran open scientists.
4 |
5 | ----
6 |
7 | ## Bringing about **change**
8 |
9 | * [What are some **steps I can take right now**?](change/educate/first-steps.md)
10 |
11 | ### Educating ourselves and others
12 |
13 | * [Where can I **learn more** about the issues?](change/educate/literature.md)
14 | * [Where can I **ask further questions**?](change/educate/ask.md)
15 |
16 | ### The Open Science Community
17 |
18 | * [How can I find out **what's going on** in the open science world?](change/community/contact.md)
19 | * [Which **projects** is the community working on?](change/community/collaboration.md)
20 | * [How can I **contribute** to this resource?](change/community/contribute.md)
21 |
22 | ### Recognizing open practices
23 |
24 | * [How do the **Open Science Badges** work?](change/recognize/badges.md)
25 | * [How can we acknowledge open practices in **hiring decisions**?](change/institutions/hiring.md)
26 |
27 | ----
28 |
29 | ## **Theorizing**
30 |
31 | ### Knowledge **synthesis**
32 |
33 | * [How can I **keep track of the reliability of results**?](theorize/synthesis.md)
34 |
35 | ----
36 |
37 | ## **Planning** research
38 |
39 | ### Preregistration
40 |
41 | * [**Why** preregister research?](plan/preregistration/why.md)
42 | * [**How** can I preregister a study?](plan/preregistration/how.md)
43 | * [Which **level of detail** is appropriate in a preregistration?](plan/preregistration/scope.md)
44 | * [Can I **change my plans** after preregistration?](plan/preregistration/change.md)
45 | * [How can I use preregistration **in class**?](plan/preregistration/teaching.md)
46 |
47 | ### Registered reports
48 | * [How do **registered reports** work?](plan/registered-reports.md)
49 |
50 | ----
51 |
52 | ## **Collecting** data
53 |
54 | * [Which open **tools** can I use?](collect/tools.md)
55 |
56 | ----
57 |
58 | ## **Analyzing** data
59 |
60 | * [What defines a **reproducible analysis**?](analyze/reproducibility.md)
61 | * [How can I **avoid mistakes** in the analysis?](analyze/mistakes.md)
62 |
63 | ----
64 |
65 | ## **Sharing** scientific resources
66 |
67 | * [How can I set conditions for the use of my materials? How do **licenses** work?](share/licenses.md)
68 |
69 | ### Open protocols
70 | * [How can I share my **research process**?](share/open-protocols.md)
71 |
72 | ### Open data
73 |
74 | * [What are the **best practices** around open data?](share/open-data/best-practices.md)
75 | * [Where can I **upload data and materials**?](share/open-data/repositories.md)
76 | * [How can I preserve participants' **privacy**?](share/open-data/privacy.md)
77 |
78 | ----
79 |
80 | ## **Disseminating** research
81 |
82 | ### Preprints
83 |
84 | * [**What** is a preprint, and why should I publish one?](disseminate/preprints/motivation.md)
85 | * [**Where** can I find or publish preprints?](disseminate/preprints/servers.md)
86 | * [How can I **engage with preprints**?](disseminate/preprints/engage.md)
87 |
88 |
89 | ### Publication
90 |
91 | * [What is **open access**?](disseminate/articles/open-access.md)
92 | * [Which **outlets** support open access?](disseminate/articles/oa-outlets.md)
93 | * [Am I allowed to **share my papers** openly after publication?](disseminate/articles/sharing.md)
94 | * [What can I do as a reviewer to **encourage open science**?](disseminate/articles/review.md)
95 |
96 | ### Registered reports
97 | * [How do **registered reports** work?](plan/registered-reports.md)
98 |
99 | ### Teaching
100 |
101 | * [Where can I find **resources for teaching** open science?](change/teaching/resources.md)
102 | * [How can I use **preregistration in class**?](plan/preregistration/teaching.md)
103 |
--------------------------------------------------------------------------------
/content/plan/preregistration/change.md:
--------------------------------------------------------------------------------
1 | title: Changing plans
2 |
3 | # Can I change my plans after preregistration?
4 |
5 | **Yes!** It's not realistic to demand that researchers anticipate every little decision upfront. However, we should be transparent about the changes we make (that's what the preregistration is about). As an article on the Open Science Framework notes, preregistration is [a plan, not a prison](https://cos.io/blog/preregistration-plan-not-prison/); it also provides practical guidance for different stages of research.
6 |
--------------------------------------------------------------------------------
/content/plan/preregistration/how.md:
--------------------------------------------------------------------------------
1 | title: Methods
2 |
3 | # How can I preregister a study?
4 |
5 | **In a preregistration, you record your intentions regarding data collection and analysis.** This clearly documents which ideas were specified in advance. When you choose to preregister, your choices come down to the format of the preregistration \(especially the amount of detail and whether you use a pre-defined template\) and the place where it is archived.
6 |
7 | ## Format
8 |
9 | In principle, the format of your preregistration is arbitrary as long as it states your plans clearly. Similarly, it need not be long, but any additional detail will help you prove that you followed a confirmatory research strategy. In practice, templates with pre-specified questions and prompts \(as opposed to an open-ended, narrative preregistration\) can help guide you through the process and make sure all critical questions are addressed.
10 |
11 | The Open Science Framework has collected [pre-made documents for each of these templates](https://osf.io/zab38/wiki/home/).
12 |
13 | * **AsPredicted** boils the preregistration down to [eight questions](https://aspredicted.org/nfj4s.pdf), and is probably the shortest and most straight-forward format available.
14 | * **Brandt et al. \(2013\)** provide a [recipe specifically for preregistering replications](https://osf.io/4jd46/)
15 | * **van 't Veer and Gina-Sorolla** \(2016\) suggest a [template geared toward experimental research](https://osf.io/56g8e/), and helpfully distinguish between essential and optional, but recommended, elements.
16 | * The **OSF Prereg** provides arguably the most [comprehensive template](https://osf.io/jea94/).
17 | * For **(re)analyses of existing data**, there's a [secondary data preregistration template](https://osf.io/djgvw/) by Sara J. Weston and Marjan Bakker
18 |
19 | ## Archive
20 |
21 | As with the format, where you put your preregistration is up to you. A piece of paper in your desk drawer on which you sketched your hypotheses can be helpful, however, your preregistration becomes more convincing if a third party can vouch for the specific date on which you fixed your pre-registration ([Corker, 2017, discusses different degrees of public commitment](https://scienceofpsych.wordpress.com/2016/02/05/so-you-want-to-pre-register-a-study/)).
22 |
23 | If you are using a service that archives files and records changes such as [FigShare](https://figshare.com/) or [Zenodo](https://zenodo.org/), you can pre-register by uploading your document to a public or private repository. Some services additionally simplify the process by providing a form and generating a record from your responses:
24 |
25 | * [**AsPredicted**](https://aspredicted.org/) provides an online version of its preregistration form, and generates an [anonymous PDF file](https://aspredicted.org/nfj4s.pdf) from the responses, which is accessible only using a secret link provided to the registrant. The link is never made public, so it is up to the authors to share it with reviewers and include it in manuscripts.
26 | * The [**Open Science Framework**](http://osf.io/) allows for every project to contain registrations, which contain a snapshot of all files in the project as well any of the above templates, each of which the OSF provides in form of an online questionnaire. Registrations can be public directly, or embargoed to a maximum of four years. The OSF has provided a [step-by-step written guide to preregistration](http://help.osf.io/m/registrations/l/524205-register-your-project) as well as an extensive [video tutorial](https://www.youtube.com/watch?v=EnKkGO3OM9c).
27 | * Further **domain-specific archives** also exist for a variety of subjects, such as [ClinicalTrials.gov](https://clinicaltrials.gov), run by the US National Library of Medicine, and other international clinical trials registry platforms as listed by the [WHO](http://www.who.int/ictrp/network/primary/en/), the registry for randomized controlled trials of the [American Economic Association](https://www.socialscienceregistry.org/), [Evidence in Governance and Politics](https://egap.org/register-your-design) or the [Registry for International Development Impact Evaluations](http://ridie.3ieimpact.org/).
28 |
--------------------------------------------------------------------------------
/content/plan/preregistration/scope.md:
--------------------------------------------------------------------------------
1 | title: Level of detail
2 |
3 | # Which level of detail is appropriate in a preregistration?
4 |
5 | This is a question that has no definite answer yet (as far as we know). However, there are some rough recommendations available:
6 |
7 | * [Nelson, Simmons, & Simonsohn](http://datacolada.org/64) discuss this issue and offer some general guidelines.
8 |
--------------------------------------------------------------------------------
/content/plan/preregistration/teaching.md:
--------------------------------------------------------------------------------
1 | title: Preregistration in class
2 |
3 | # How can I use preregistration in class?
4 |
5 | ## Why?
6 |
7 | Preregistering a student project can have several **practical advantages** (alongside [theoretical benefits](./why.md))
8 |
9 | * Going through a preregistration questionnaire can help **identify open questions and underspecified parts of theory or analysis**, because students are forced to think through the entire data collection process. This is especially true in comparison to unstructured exposé-style documents that often focus on the issues which are most straightforward or discussed in the most detail.
10 | * Once completed, the document helps **remind students (and supervisors) of previously made plans and considerations**, reducing duplicative discussions later in the project and providing a structure for students to work independently.
11 | * A preregistration clearly **defines the scope of an empirical project**, guiding a student through their report. Clearly defined hypothesis alleviate the pressure to find 'interesting' results at any cost, and help assure students of the value of their work regardless of outcome.
12 | * Students often quickly grasp the aim of preregistrations, and find it motivating to participate in **state-of-the-art transparent science**.
13 |
14 |
15 |
16 |
--------------------------------------------------------------------------------
/content/plan/preregistration/why.md:
--------------------------------------------------------------------------------
1 | title: Motivation
2 |
3 | # Why preregister research?
4 |
5 | **By preregistering research, you create a public record of your hypotheses and planned analyses before you conduct them**, so that you can later demonstrate that you did not change hypotheses after learning your results \(HARKing; [Kerr, 1998](https://dx.doi.org/10.1207/s15327957pspr0203_4)\), nor that you changed the analysis to create a specific result. This increases the credibility of your findings, and is [increasingly becoming the standard in our field](https://www.psychologicalscience.org/observer/preregistration-becoming-the-norm-in-psychological-science).
6 |
7 | ## Questionable research practices
8 |
9 | Preregistration is all about proving that we don't engage in questionable research practices \(QRPs\).
10 |
11 | * [Nuzzo \(2015\)](https://dx.doi.org/10.1038/526182a) highlights the potential for self-deception in research, and summarizes countermeasures \(preregistration being one of them\)
12 | * [Wagenmakers, Wetzels, Borsboom, van der Mass, and Kievit \(2012\)](https://dx.doi.org/10.1177/1745691612463078) list the different biases that can be introduced into an analysis, and argue that only pre-registered research should be labelled 'confirmatory'.
13 | * [Simmons, Nelson, and Simonsohn \(2011\)](https://dx.doi.org/10.1177/0956797611417632) show how flexibility in the analysis inflates the false positive rate by demonstrating that listening to "When I'm sixty-four" by the Beatles rejuvenates participants.
14 | * [John, Loewenstein, and Prelec \(2012\)](https://dx.doi.org/10.1177/0956797611430953) estimate the prevalence of _questionable research practices_ such as the above. [Fiedler and Schwarz \(2016\)](https://dx.doi.org/10.1177/1948550615612150) argue that their estimates were inflated and that the issue is less widespread.
15 |
16 | ## Preregistration as a solution
17 |
18 | * [van’t Veer and Giner-Sorolla \(2016\)](https://dx.doi.org/10.1016/j.jesp.2016.03.004) provide an excellent overview over the motivation for and history of preregistration in psychology and medicine, discuss advantages, objections, and limitations, and provide practical guidance with a focus on social psychology.
19 |
20 | * In an open letter, Chambers, Munafo and over 80 signatories argue that [trust in science would be improved by study pre-registration](https://www.theguardian.com/science/blog/2013/jun/05/trust-in-science-study-pre-registration) \(2013\).
21 |
22 | !!! seealso "see also"
23 | [**Registered reports**](../registered-reports) are a special case of preregistration, where the study plan is peer-reviewed as part of the publication process. They merit [their own entry](../registered-reports).
24 |
25 | Preregistrations can be augmented or supplanted by **alternative methods** that may be more practical. Srivastava (2018) describes these strategies for [**sound inference in complicated research**](https://psyarxiv.com/bwr48).
26 |
--------------------------------------------------------------------------------
/content/plan/registered-reports.md:
--------------------------------------------------------------------------------
1 | # How do registered reports work?
2 |
3 | **Registered reports** are a form of publication where the preregistration is peer-reviewed before data is collected, and the decision to publish is made regardless of the study's outcome.
4 |
5 | * The [registered reports page at the Center for Open Science](https://cos.io/rr/) provides an excellent collection of resources, including participating journals, practical information for all parties involved, and answers to common questions. Furthermore, they provide [examplary guidelines for authors and reviewers] (https://osf.io/pukzy/).
6 | * [Chambers, Feredoes, Muthukumaraswamy and Etchells \(2014\)](https://doi.org/10.3934/Neuroscience.2014.1.4) introduce registered reports, outline the underlying motivations, and respond to common objections.
7 | * [Greve, Bröder, and Erdfelder \(2013\)](https://dx.doi.org/10.1027/1016-9040/a000144) suggest a similar idea, _result-blind peer review_.
8 |
--------------------------------------------------------------------------------
/content/share/licenses.md:
--------------------------------------------------------------------------------
1 | # Licensing content
2 |
3 | As the creator of an original work, you are typically granted far-reaching rights that govern and limit redistribution and -use of your creation by default. **By applying a license to your work, you make clear what others can do with the things you're sharing, and also the conditions under which you're providing them.** You can also require others who copy your work to do things in return.
4 |
5 | You're free to draft your own terms, but **it's common to apply a pre-made license template**. There are several options to choose from that vary in their terms, and in their suitability for data and materials as opposed to source code.
6 |
7 | !!! tip "Getting started"
8 |
9 | * The [**public license selector**](https://ufal.github.io/public-license-selector/) helps you narrow down the applicable licenses through a series of questions.
10 | * The [**Scientific Licensing Portal**](http://forschungslizenzen.de/) (German) provides an excellent review of the legal landscape around copyright and license options, with a focus on German law.
11 | * A FAQ by the [Leibniz Information Centre for Science and Technology](https://www.tib.eu) addresses [**legal aspects of dealing with scientific data**](https://zenodo.org/record/1173546#.WyEIVKlCRXh) (German). A formal legal [expertise around the **current German legal situation**](https://www.ianus-fdz.de/projects/ap6-geschaeftsmodell/wiki/Expertise_AP6_2) (German) summarizes issues especially around digital archeology.
12 |
13 | ----
14 |
15 | ## General-purpose licenses
16 |
17 | For all material except source code, [**creative commons licenses**](https://creativecommons.org/share-your-work/) are the most common choice. They always require attribution to the original source, and optionally disallow commercial use and derivative works. You can also require others to share modifications under the same terms. The creative commons website will walk you through [choosing a license](https://creativecommons.org/choose/).
18 |
19 | The [**CC0**](https://creativecommons.org/share-your-work/public-domain/cc0/) license helps you waive your rights as the originator of a creative work to the greatest possible extent, placing your material in the public domain.
20 |
21 | !!! note "Licensing materials and papers"
22 |
23 | * [**Licensing your work on PsyArXiv**](http://blog.psyarxiv.com/2018/05/14/licensing-work-psyarxiv/) by Hannah Moshontz discusses licenses in the context of **preprints**.
24 | * You can [**set a license for any OSF project**](http://help.osf.io/m/bestpractices/l/611430-licensing).
25 | * The DataWiz knowledge base has a more detailed entry on [licenses for sharing data](https://datawizkb.leibniz-psychology.org/index.php/after-collection/what-should-i-know-about-licenses/).
26 |
27 | ----
28 |
29 | ## Licenses for program code
30 |
31 | Open licenses have been used for decades in computer science and software engineering, where sharing code has a long tradition. Therefore, a wider range of licenses specific to software code is available.
32 |
33 | * [**Choose a license**](https://choosealicense.com/) by [GitHub](https://github.com/) provides a succinct [summary of popular licenses](https://choosealicense.com/licenses/).
34 | * The [**Open Source Initiative**](https://opensource.org) approves open-source licenses and maintains a more [complete list](https://opensource.org/licenses).
35 |
36 |
37 | !!! note "Licensing software"
38 |
39 | * The Software Sustainability Institute provides an in-depth discussion of [**choosing an open-source license**](https://www.software.ac.uk/choosing-open-source-licence) focussed on scientific software.
40 | * [TLDR legal](https://tldrlegal.com/) summarizes most open source licenses, and lists the rights and demands they confer.
41 |
--------------------------------------------------------------------------------
/content/share/open-data/best-practices.md:
--------------------------------------------------------------------------------
1 | # What are the best practices around open data?
2 |
3 | * [Goodman et al. (2014)](journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1003542) outline _Ten Simple Rules for the Care and Feeding of Scientific Data_
4 | * The [Harvard Biomedical Data Management](https://datamanagement.hms.harvard.edu/) group summarize best practices around [security and access](https://datamanagement.hms.harvard.edu/security-access), [file naming](https://datamanagement.hms.harvard.edu/file-naming-conventions) as well as [versioning](https://datamanagement.hms.harvard.edu/versioning-1), with a clear focus on biomedical data.
5 | * [Born-open data](https://doi.org/10.3758%2Fs13428-015-0630-z) is archived publicly directly or very soon after its collection, ideally in an automated way. This practice maximises transparency by obviating the need for on-demand data sharing from the start.
6 | * The German Psychological Society (DGPs) provides [Guidelines for Data Management in Psychological Science](http://www.dgps.de/fileadmin/documents/Empfehlungen/Data_Management_eng_9.11.16.pdf).
7 |
8 | !!! seealso "See also"
9 | The [**DataWiz Knowledge Base**](https://datawizkb.leibniz-psychology.org) holds much more extensive information regarding all aspects of data handling. Don't miss this excellent resource!
10 |
--------------------------------------------------------------------------------
/content/share/open-data/definition.md:
--------------------------------------------------------------------------------
1 | title: Definition
2 |
3 | # What makes open data open?
4 |
5 | According to the [**Open Definition**](https://opendefinition.org/od/2.1/en/),
6 | > Knowledge is open if anyone is free to access, use, modify, and share it — subject, at most, to measures that preserve provenance and openness.
7 |
8 | The [**FAIR principles**](https://www.go-fair.org/fair-principles/) provide guidelines to make data more **f**indable, **a**ccessible, **i**nteroperable, and **r**eusable.
9 |
--------------------------------------------------------------------------------
/content/share/open-data/privacy.md:
--------------------------------------------------------------------------------
1 | title: Preserving privacy
2 |
3 | # How can I preserve participants' privacy?
4 |
5 | **Ensuring the anonymity of published data can be a tricky task. We owe it to our participants to guarantee their privacy, while being as transparent about our research as possible.**
6 |
7 | * [Meyer (2018)](http://journals.sagepub.com/doi/full/10.1177/2515245917747656) provides _Practical Tips for Ethical Data Sharing_
8 | * [Fraser & Willison (2009)](http://www.ehealthinformation.ca/wp-content/uploads/2014/08/2009-Tools-for-De-Identification-of-Personal-Health.pdf) present various strategies and _Tools for De-Identification of Personal Health Information_. As examples of more recent implementations of these approaches, [anonymizer](https://github.com/paulhendricks/anonymizer) for `R` obfuscates potentially identifying information, the online tool [Amnesia](https://amnesia.openaire.eu/index.html) shortens identifiers (e.g. postcodes) until they are no longer unique; [Faker](http://faker.readthedocs.io/en/master/) for Python can [generate random, but plausible substitute data](https://www.districtdatalabs.com/a-practical-guide-to-anonymizing-datasets-with-python-faker/).
9 | * The Open Science Framework has defined criteria for a [Protected Access Open Data Badge](https://osf.io/tvyxz/wiki/8. Approved Protected Access Repositories/) and has collected a list of repositories where sensitive data can be stored safely.
10 | * For highly sensitive data, analyses can be made reproducable by publishing raw descriptive data alongside correlation or covariance matrices. For example, [Fried et al. (2017)](https://osf.io/mjd2h/) report a fully open and reproducible analysis based on a publicly available aggregate version of their data.
11 |
12 | !!! seealso "See also"
13 | The [**DataWiz Knowledge Base**](https://datawizkb.leibniz-psychology.org) provides much more information about [informed consent](https://datawizkb.leibniz-psychology.org/index.php/before-my-project-starts/informed-consent/) and [participants privacy](https://datawizkb.leibniz-psychology.org/index.php/before-my-project-starts/what-should-i-know-about-privacy/), among many other issues. Don't miss this excellent resource!
14 |
--------------------------------------------------------------------------------
/content/share/open-data/repositories.md:
--------------------------------------------------------------------------------
1 | title: Repositories
2 |
3 | # Where can I upload data and materials?
4 |
5 | You can upload your data and materials in a number of places, for example:
6 |
7 | * The [**Open Science Framework**](https://osf.io/), run by the [Center for Open Science](https://cos.io) is probably the most comprehensive platform for open data in the social sciences. Its scope extends beyond the publication of data alone, as it aims to facilitate [collaboration between multiple users](http://help.osf.io/m/projects/c/167765) even before a project is completed. [Projects can be kept private](http://help.osf.io/m/projects/l/524048-control-your-privacy-settings), and [secret, read-only links](http://help.osf.io/s/support/m/links_forks/l/524049-create-a-view-only-link-for-a-project) can be generated for reviewers to access materials before a manuscript is made public. Snapshots of projects can also be created for [preregistration](../../plan/preregistration/how.md), and [DOIs can be generated for public data](http://help.osf.io/m/sharing/l/524208-create-dois-and-arks). [Soderberg (2018)](http://journals.sagepub.com/doi/full/10.1177/2515245918757689) provides a step-by-step guide to using the OSF for sharing data.
8 | * [**Zenodo**](https://zenodo.org) is a data repository run by [CERN](https://home.cern/) in collaboration with the European Union's [OpenAIRE](https://www.openaire.eu/) project. It is mostly a data storage service, with few collaboration features. Projects can be fully open, limited-access (upon request or opened after an embargo date) or closed-access. DOIs are generated automatically for projects.
9 | * [**Databrary**](https://databrary.org) is a data sharing service designed around sensitive data, particularly audio and video material of participants. It limits access to projects to researchers who have signed an agreement regarding the use of data. The project is described in detail by [Gilmore, Kennedy, & Adolph (2018)](http://journals.sagepub.com/doi/abs/10.1177/2515245917746500).
10 | * [**Figshare**](https://figshare.com/) is a commercial data repository run by a subsidiary of the [Holtzbrinck Publishing Group](https://en.wikipedia.org/wiki/Holtzbrinck_Publishing_Group). Like Zenodo, it primarily provides storage space, alongside an online viewer for documents and presentations. DOIs are generated automatically for repositories.
11 |
12 |
13 | ### Repository overviews
14 |
15 | The above list is non-exhaustive, and there are more repositories and evaluation criteria. These have been summarized in other places:
16 |
17 | * The [Harvard Biomedical Data Management](https://datamanagement.hms.harvard.edu/) group provides a much more [**in-depth comparison of available repositories**](https://datamanagement.hms.harvard.edu/overview-data-repositories) and provides recommendations for [choosing a repository](https://datamanagement.hms.harvard.edu/data-deposit-storage).
18 | * [Nature scientific data](https://www.nature.com/sdata/) maintains a list of [**recommended data repositories**](https://www.nature.com/sdata/policies/repositories#social).
19 | * The [Registry of Research Data Repositories](https://www.re3data.org/) provides an extensive list of data repositories across domains.
20 | * [Klein et al. (2018)](https://psyarxiv.com/rtygm) include a table of of repositories at the end of their supplementary material.
21 |
--------------------------------------------------------------------------------
/content/share/open-protocols.md:
--------------------------------------------------------------------------------
1 | # How can I document my methods so that others can reproduce my work more easily?
2 |
3 | **The goal of open protocols is to document and communicate your research methods unambiguously, so that other researchers can easily replicate your exact procedures.**
4 |
5 | The term *protocol* refers to the precise method for carrying out or reproducing a given experiment. Depending on your field of study, this can include everything from your study setup to the exact procedure you followed when analyzing your data. As with the [data](../open-data/definition), this documentation is best made openly accessible and linked to from your manuscript.
6 |
7 | !!! seealso "See also"
8 | By including the [**21-word statement**](https://dx.doi.org/10.2139/ssrn.2160588) in your methods section, you can make clear that you have reported how the sample size was determined, and all variables and conditions that were included in your study.
9 |
10 | ----
11 |
12 | ## Approaches
13 |
14 | Starting from screenshots or a video recording of your study, there are several **informal** methods of documenting your exact procedures.
15 |
16 | Beyond these, several institutions have created **formal standards** that provide precise guidance around what should be reported, and how.
17 |
18 | * The TIER project has developed the [**DRESS Protocol**](https://www.projecttier.org/tier-protocol/dress-protocol/). It is a set of *standards for documenting empirical research in the social sciences* that specify in detail the content and organization a study protocol.
19 |
20 | !!! seealso "Other fields"
21 | **Biomedical** researchers in particular have defined a wide range of protocols and standards for different types of research, ranging from human or animal studies to systematic reviews. BMJOpen provides an [overview](https://bmjopen.bmj.com/pages/authors/#study_protocols) of items that should be included in a study protocol, and lists [reporting guidelines](https://bmjopen.bmj.com/pages/authors/#reporting_guidelines) for various purposes. Similarly, the [Equator Network](http://www.equator-network.org/) collects different reporting guidelines, and provides guidance for [selecting the appropriate one](http://www.equator-network.org/toolkits/selecting-the-appropriate-reporting-guideline/).
22 |
23 | For **benchtop experiments**, the website [protocols.io](https://www.protocols.io/) provides a framework to specify exactly how to perform a procedure, and to share and collaboratively extend available protocols.
24 |
--------------------------------------------------------------------------------
/content/site.webmanifest:
--------------------------------------------------------------------------------
1 | {
2 | "name": "",
3 | "short_name": "",
4 | "icons": [
5 | {
6 | "src": "/_theme/branding/favicon-192.png",
7 | "sizes": "192x192",
8 | "type": "image/png"
9 | },
10 | {
11 | "src": "/_theme/branding/favicon-256.png",
12 | "sizes": "256x256",
13 | "type": "image/png"
14 | },
15 | {
16 | "src": "/_theme/branding/favicon-512.png",
17 | "sizes": "512x512",
18 | "type": "image/png"
19 | }
20 | ],
21 | "theme_color": "#ffffff",
22 | "background_color": "#ffffff",
23 | "display": "standalone"
24 | }
25 |
--------------------------------------------------------------------------------
/content/theorize/synthesis.md:
--------------------------------------------------------------------------------
1 | # How can I keep track of the reliability of results?
2 |
3 | As the replication crisis casts doubt on the reliability of research findings, and replications become more common, **we might need to reevaluate our confidence in published results more often** -- in either direction. **How can we keep up-to-date?**
4 |
5 | ## Summarizing replication efforts
6 |
7 | * [**Curate science**](http://curatescience.org/) collects replications and registered reports and matches them to the original articles. The service aggregates and visualizes effect sizes of the original study and its replications, and computes a meta-analytic average.
8 | * [**PsychFileDrawer**](http://psychfiledrawer.org) archives attempted replications of specific studies and whether the replication was achieved, making it easier to estimate the robustness of an effect or finding resources for meta-analyses.
9 |
10 | ## Collecting errata and retractions
11 |
12 | * [**Open Retractions**](http://openretractions.com/) tracks the status of every article indexed on PubMed and CrossRef: Whether it was retracted, and whether errata are available. The [retractcheck](https://github.com/libscie/retractcheck) package for `R` automates this process.
13 | * The blog [**Retraction Watch**](https://retractionwatch.com/) maintains an ongoing collection and commentary on retracted papers, with a (non-exclusive) focus on biomedical research.
14 |
--------------------------------------------------------------------------------
/mkdocs.yml:
--------------------------------------------------------------------------------
1 | site_name: Open Science Knowledge Base
2 | site_description: An annotated collection of open science resources
3 | site_url: https://how-to-open.science
4 | docs_dir: 'content'
5 | theme:
6 | name: 'material'
7 | custom_dir: 'content/_theme'
8 | extra_css:
9 | - '_styles/extra.css'
10 | extra_javascript:
11 | - '_scripts/credits.js'
12 | copyright: 'Built by scientists like you.'
13 | markdown_extensions:
14 | - admonition
15 | - meta
16 | - pymdownx.details
17 | - pymdownx.tilde
18 | repo_name: 'felixhenninger/open-science-knowledge-base'
19 | repo_url: 'https://github.com/felixhenninger/open-science-knowledge-base'
20 | edit_uri: 'edit/master/content/'
21 | strict: true
22 | nav:
23 | - Overview: 'index.md'
24 | - Changing scientific practice:
25 | - First steps: 'change/educate/first-steps.md'
26 | - Learning about issues:
27 | - Introduction to reproducibility: 'change/educate/literature.md'
28 | - Where to ask questions: 'change/educate/ask.md'
29 | - Teaching resources: 'change/teaching/resources.md'
30 | - Open Science community:
31 | - Establishing contact: 'change/community/contact.md'
32 | - Ongoing projects: 'change/community/collaboration.md'
33 | - Contributing: 'change/community/contribute.md'
34 | - Recognizing open science:
35 | - Badges: 'change/recognize/badges.md'
36 | - Changing institutions:
37 | - Hiring practices: 'change/institutions/hiring.md'
38 | - Theorizing:
39 | - Knowledge synthesis: 'theorize/synthesis.md'
40 | - Planning research:
41 | - Preregistration:
42 | - Motivation: 'plan/preregistration/why.md'
43 | - Methods: 'plan/preregistration/how.md'
44 | - Scope: 'plan/preregistration/scope.md'
45 | - Binding character: 'plan/preregistration/change.md'
46 | - Teaching: 'plan/preregistration/teaching.md'
47 | - Registered reports: 'plan/registered-reports.md'
48 | - Collecting data:
49 | - Tools: 'collect/tools.md'
50 | - Analysis:
51 | - Reproducibility: 'analyze/reproducibility.md'
52 | - Avoiding mistakes: 'analyze/mistakes.md'
53 | - Sharing:
54 | - Open Data:
55 | - Definition: 'share/open-data/definition.md'
56 | - Best practices: 'share/open-data/best-practices.md'
57 | - Repositories: 'share/open-data/repositories.md'
58 | - Ensuring privacy: 'share/open-data/privacy.md'
59 | - Licenses: 'share/licenses.md'
60 | - Open Protocols: 'share/open-protocols.md'
61 | - Dissemination:
62 | - Preprints:
63 | - Motivation: 'disseminate/preprints/motivation.md'
64 | - Preprint servers: 'disseminate/preprints/servers.md'
65 | - Engaging with preprints: 'disseminate/preprints/engage.md'
66 | - Articles:
67 | - Sharing papers: 'disseminate/articles/sharing.md'
68 | - Open access: 'disseminate/articles/open-access.md'
69 | - OA outlets: 'disseminate/articles/oa-outlets.md'
70 | - Peer review: 'disseminate/articles/review.md'
71 | - Registered reports: 'plan/registered-reports.md'
72 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | mkdocs>=1
2 | mkdocs-material
3 |
--------------------------------------------------------------------------------
/runtime.txt:
--------------------------------------------------------------------------------
1 | 3.6
2 |
--------------------------------------------------------------------------------
/scripts/find_orphans.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | import os, yaml
4 |
5 | # Collect markdown files included in sitemap -----------------------------------
6 |
7 | def leaf_nodes(site_tree):
8 | output = []
9 |
10 | for subtree in site_tree:
11 | for k, v in subtree.items():
12 | if type(v) == str:
13 | output.append(v)
14 | elif type(v) == list:
15 | output += leaf_nodes(v)
16 | else:
17 | print('encountered weird data type', type(v))
18 |
19 | return output
20 |
21 | included_files = []
22 |
23 | with open('../mkdocs.yml', 'r') as stream:
24 | try:
25 | site_tree = yaml.safe_load(stream)['pages']
26 | included_files = leaf_nodes(site_tree)
27 | except yaml.YAMLError as exc:
28 | print(exc)
29 |
30 | # Collect actually present markdown files --------------------------------------
31 |
32 | available_files = []
33 |
34 | for root, directory, files in os.walk('../content'):
35 | filtered_files = filter(lambda f: f.endswith('.md'), files)
36 | file_paths = map(
37 | lambda f: os.path.relpath(os.path.join(root, f), '../content'), filtered_files
38 | )
39 | available_files += file_paths
40 |
41 | included_files = set(included_files)
42 | available_files = set(available_files)
43 |
44 | if len(included_files - available_files) is not 0:
45 | print(
46 | 'In sitemap but not directory',
47 | included_files - available_files
48 | )
49 |
50 | if len(available_files - included_files) is not 0:
51 | print(
52 | 'In directory but not sitemap',
53 | available_files - included_files
54 | )
55 |
--------------------------------------------------------------------------------
4 |