├── .github ├── FUNDING.yml └── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ ├── improvement.md │ └── venue-addition-to-the-commitment-list.md ├── .gitignore ├── .idea ├── .gitignore ├── aclrollingreview.iml ├── inspectionProfiles │ └── profiles_settings.xml ├── misc.xml ├── modules.xml └── vcs.xml ├── .ruby-version ├── 404.md ├── CNAME ├── Gemfile ├── LICENSE ├── README.md ├── _config.yml ├── _includes ├── admonition.html ├── analytics.html ├── analytics_head.html ├── bluesky-icon.html ├── disqus.html ├── meta.html ├── pullquote.html ├── svg-icons.html └── toc.html ├── _layouts ├── default.html ├── page.html ├── page_toc.html └── post.html ├── _posts ├── 2021-03-18-here-we-are.md ├── 2021-04-05-initialCFP.md ├── 2021-04-07-reviewerinterest.md ├── 2021-04-12-submissions.md ├── 2021-04-13-reviewforms.md ├── 2021-04-14-venues.md ├── 2021-04-27-welcome-new-reviewers.md ├── 2021-04-30-prlead.md ├── 2021-04-30-some-important-dates.md ├── 2021-05-19-tech-ad.md ├── 2021-08-26-communicationsteam.md ├── 2021-10-12-status-report.md ├── 2021-10-20-emergency-reviewers.md ├── 2021-10-26-november-submissions.md ├── 2021-11-03-new-reviewer-tutorial.md ├── 2021-11-17-modified-timeline-November.md ├── 2021-12-02-ARR-changes-December.md ├── 2022-01-06-Modified-timeline-January.md ├── 2022-01-11-choices.md ├── 2022-01-13-resubmissions.md ├── 2022-01-17-modified-timeline-December.md ├── 2022-01-28-declaring-a-conflict-of-interest.md ├── 2022-02-04-review-data-collection.md ├── 2022-02-17-stats.md ├── 2022-03-03-recognition.md ├── 2022-04-07-six-week-cycles.md ├── 2022-04-13-in-cycle-author-response.md ├── 2022-07-05-max-cycle-load-openreview.md ├── 2022-08-31-arr-eic-changes.md ├── 2022-08-31-changes-based-on-the-ACL-reviewing-survey.md ├── 2022-10-11-key-changes-in-the-october-cycle.md ├── 2022-10-24-newly-formed-arr-board.md ├── 2022-12-9-Updates-on-December-Cycle.md ├── 2023-02-15-new-ethics-chair.md ├── 2023-05-12-ACL-Rolling-Review-update.md ├── 2023-05-19-update-on-data-collection copy.md ├── 2023-06-15-ethics-chair.md ├── 2023-09-06-ACL-2023-session.md ├── 2023-09-21-submission-dates-for-2024.md ├── 2023-10-26-reviewing-update.md ├── 2023-10-31-reviewing-update2.md ├── 2024-01-15-anonymity.md ├── 2024-02-13-great-oct23-reviewers.md ├── 2024-02-14-ethics-flagging-guidelines.md ├── 2024-03-19-new-eic.md ├── 2024-03-31-april-cycle.md ├── 2024-04-01-reviewing-workload-requirement.md ├── 2024-04-30-peer-review-data-collection.md ├── 2024-06-28-ARR-seeks-Editors-in-Chief.md ├── 2024-09-18-editorial-assistant.md ├── 2024-09-18-workflow-manager.md ├── 2025-02-10-reviewing-workload-adjustment.md └── 2025-05-05-incentives.md ├── _sass ├── _bluesky-icon.scss ├── _darcula.scss ├── _highlights.scss ├── _reset.scss ├── _svg-icons.scss ├── _variables.scss └── admonition.scss ├── about.md ├── acguidelines.md ├── actioneditorreviewform.md ├── aes.md ├── areas.md ├── assets ├── simple-jekyll-search.min.js └── style.scss ├── authorchecklist.md ├── authors.md ├── cfp.md ├── changelog.md ├── commitments.json ├── datacollection.md ├── dates.md ├── ethicsreviewertutorial.md ├── favicon.ico ├── fonts ├── EBGaramond-Regular.ttf └── aramisi.ttf ├── future.md ├── images ├── 404.jpg ├── ARR-pre.png ├── ARR.png ├── VenuesOR.PNG ├── arr-paper-cycle.jpg ├── arr_report │ ├── conf-submissions.png │ ├── days-from-submission-to-meta-review.png │ ├── days-from-submission-to-review.png │ ├── num-submissions.png │ └── reviewer-pool.png ├── categories.png ├── datacollection.png ├── favicon-32x32.png ├── galaxy.jpg ├── guidelines │ ├── 1-AC-console.png │ ├── 10-AC-messaging-reviewers-part2.png │ ├── 11-AC-chat.png │ ├── 12-AC-submit-meta-part1.png │ ├── 13-AC-submit-meta-part2.png │ ├── 14-AC-request-recognition.png │ ├── 15-AC-assign-emergency-part1.png │ ├── 16-AC-assign-emergency-part2.png │ ├── 16-AC-assign-emergency-part2_new.png │ ├── 17-AC-assign-emergency-part3.png │ ├── 17-AC-assign-emergency-part3_new.png │ ├── 18-AC-assign-emergency-part4.png │ ├── 19-AC-reviewer-email.png │ ├── 2-AC-resubmission-link.png │ ├── 3-AC-overall-assessment.png │ ├── 4-AC-OR-login.png │ ├── 5-AC-console-access.png │ ├── 6-AC-reviewer-in-pool.png │ ├── 7-AC-reviewer-not-in-pool.png │ ├── 8-AC-official-comment-button.png │ ├── 9-AC-messaging-reviewers-part1.png │ ├── RI_form1.png │ ├── RI_form2.png │ ├── RI_form3.png │ ├── emergency_declaration.png │ ├── metareview_issue_report.png │ ├── review_issue_report.png │ └── reviewer_assignment_interface.png ├── icons │ ├── align-left-solid.svg │ ├── bluesky.svg.png │ ├── bolt-solid.svg │ ├── bug-solid.svg │ ├── circle-check-solid.svg │ ├── circle-info-solid.svg │ ├── circle-question-solid.svg │ ├── fire-solid.svg │ ├── list-ol-solid.svg │ ├── list-solid.svg │ ├── pen-solid.svg │ ├── quote-right-solid.svg │ ├── times-circle-solid.svg │ └── triangle-exclamation-solid.svg ├── mobile-demo.png ├── people │ ├── anna_rogers.jpg │ ├── atsuki_yamaguchi.jpg │ ├── dhruv_naik.jpg │ ├── dongqi_liu.jpg │ ├── freda_shi.jpg │ ├── gabriella_skitalinskaya.jpg │ ├── goggi_sara.jpg │ ├── himanshu_beniwal.jpg │ ├── holy_lovenia.jpeg │ ├── isabelle_augenstein.jpg │ ├── jing_jiang.jpg │ ├── jonathan_kummerfeld.jpg │ ├── julia_hockenmaier.jpg │ ├── jun_suzuki.jpg │ ├── malihe_alikhani.jpg │ ├── manuel_mager.png │ ├── margot_mieskes.jpeg │ ├── mausam.jpg │ ├── michael_white.jpg │ ├── ning_shi.jpg │ ├── placeholder.jpg │ ├── ruohao_guo.jpg │ ├── santosh.jpg │ ├── sarvnaz_karimi.jpeg │ ├── shabnam_behzad.jpg │ ├── shirley.jpg │ ├── shubham_mittal.jpg │ ├── shubhra_kanti.jpg │ ├── smaranda_muresan.jpg │ ├── sonakshi_chauhan.png │ ├── sudipta_kar.jpg │ ├── thamar_solorio.jpg │ ├── vincent_ng.jpg │ ├── vinodkumar_prabhakaran.jpg │ ├── viviane_moreira.jpg │ ├── vlilja_ovrelid.jpg │ ├── xiaodan_zhu.jpg │ └── yoshitomo_matsubara.jpg ├── reverie-demo.png ├── reverie-text.png ├── reverie.png └── search.png ├── index.html ├── mentorship.md ├── organization.md ├── organizers.md ├── others.md ├── people.md ├── policies.md ├── responsibleNLPresearch.md ├── reviewerfaq.md ├── reviewerguidelines.md ├── reviewers.md ├── reviewform.md ├── reviewing.md ├── saes.md ├── search.html ├── search.json ├── static ├── responsibleNLPresearch.pdf ├── responsibleNLPresearch.tex └── service_history-api_v1_v2_all.json ├── submissionform.md └── twitter-bot ├── README.md ├── requirements.txt └── twitter_bot.py /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: amitmerchant1990 4 | custom: paypal.me/AmitMerchant 5 | patreon: amitmerchant 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve the ARR website aclrollingreview.org 4 | title: '' 5 | labels: bug 6 | assignees: jkkummerfeld 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug in the [ARR website](https://aclrollingreview.org/) is. 12 | 13 | Note: this is **not** for bugs related OpenReview. Please email [the tech team](https://aclrollingreview.org/people) directly for that. 14 | 15 | **Suggested fix** 16 | [Optional] A suggestion for what we should do to address the issue. 17 | 18 | **Additional context** 19 | Add any other context about the problem here. 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Other (e.g., OpenReview issues) 4 | url: https://aclrollingreview.org/people 5 | about: Issues in this repository relate to the ARR website (https://aclrollingreview.org/) only. Please send issues and suggestions related to OpenReview directly to the tech team. 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/improvement.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Improvement 3 | about: Suggest an idea for for the ARR website aclrollingreview.org 4 | title: '' 5 | labels: enhancement 6 | assignees: jkkummerfeld 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | Note: this is **not** for ideas or bugs related OpenReview. Please email [the tech team](https://aclrollingreview.org/people) directly for that. 14 | 15 | **Describe the solution you'd like** 16 | A clear and concise description of what you want to happen. 17 | 18 | **Describe alternatives you've considered** 19 | A clear and concise description of any alternative solutions or features you've considered. 20 | 21 | **Additional context** 22 | Add any other context or screenshots about the feature request here. 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/venue-addition-to-the-commitment-list.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Venue addition to the commitment list 3 | about: Request to have your venue added to the list on https://aclrollingreview.org/dates 4 | title: '' 5 | labels: '' 6 | assignees: jkkummerfeld 7 | 8 | --- 9 | 10 | We prefer to receive Pull Requests for this (faster for us to merge). However, if you have any issues with that process, please provide: 11 | 12 | Venue name: TODO 13 | Commitment Date: TODO 14 | Website / CFP URL: TODO 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | _site 2 | .DS_Store 3 | .jekyll 4 | .jekyll-metadata 5 | .bundle 6 | .sass-cache 7 | Gemfile.lock 8 | node_modules 9 | package.json 10 | *.gem 11 | *.gemspec 12 | .jekyll-cache 13 | *.swp 14 | secret.txt 15 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /.idea/aclrollingreview.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 3.1.4 2 | -------------------------------------------------------------------------------- /404.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: 404 - Page not found 4 | permalink: /404 5 | --- 6 | 7 | Sorry, we can't find that page that you're looking for. You can try again by going [back to the homepage]({{ site.baseurl }}/). 8 | 9 | [Constructocat by https://github.com/jasoncostello]({{ site.baseurl }}/) 10 | -------------------------------------------------------------------------------- /CNAME: -------------------------------------------------------------------------------- 1 | aclrollingreview.org -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | git_source(:github) {|repo_name| "https://github.com/#{repo_name}" } 6 | 7 | gem "github-pages", group: :jekyll_plugins 8 | 9 | # If you want to use Jekyll native, uncomment the line below. 10 | # To upgrade, run `bundle update`. 11 | 12 | # gem "jekyll" 13 | 14 | gem "wdm", "~> 0.1.0" if Gem.win_platform? 15 | gem 'bootstrap', '~> 5.1.0' 16 | 17 | # If you have any plugins, put them here! 18 | group :jekyll_plugins do 19 | # gem "jekyll-archives" 20 | gem "jekyll-feed" 21 | gem 'jekyll-sitemap' 22 | gem 'hawkins' 23 | end 24 | 25 | gem "webrick", "~> 1.8" 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Amit Merchant 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "About" 4 | permalink: /about 5 | --- 6 | 7 | Homepage for [ACL Rolling Review](https://aclrollingreview.org) 8 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | # 2 | # This file contains configuration flags to customize your site 3 | # 4 | 5 | # Name of your site (displayed in the header) 6 | name: ACL Rolling Review 7 | 8 | # Name of the author 9 | #author: ACL Rolling Review 10 | 11 | # Short bio or description (displayed in the header) 12 | description: A peer review platform for the Association for Computational Linguistics 13 | 14 | # URL of your avatar or profile pic (you could use your GitHub profile pic) 15 | avatar: /images/ARR.png 16 | 17 | # 18 | # Flags below are optional 19 | # 20 | 21 | # Includes an icon in the footer for each username you enter 22 | footer-links: 23 | dribbble: 24 | email: support@aclrollingreview.org 25 | facebook: 26 | flickr: 27 | github: 28 | instagram: 29 | linkedin: 30 | pinterest: 31 | rss: rss 32 | twitter: ReviewAcl 33 | bluesky: aclrollingreview.bsky.social 34 | stackoverflow: 35 | youtube: # channel/ or user/ 36 | googleplus: # anything in your profile username that comes after plus.google.com/ 37 | playconsole: 38 | 39 | # Enter your Disqus shortname (not your username) to enable commenting on posts 40 | # You can find your shortname on the Settings page of your Disqus account 41 | disqus: 42 | 43 | # Enter your Google Analytics web tracking code (e.g. UA-2110908-2) to activate tracking 44 | #google_analytics: UA-43339302-11 45 | 46 | # For newer "GA4" analytics, use the following instead of the "UA" entry above 47 | google_analytics_ga4: G-NQ9259P3ZZ 48 | 49 | # Your website URL (e.g. http://amitmerchant1990.github.io or http://www.amitmerchant.com) 50 | # Used for Sitemap.xml and your RSS feed 51 | url: http://aclrollingreview.org 52 | enforce_ssl: https://aclrollingreview.org 53 | 54 | # If you're hosting your site at a Project repository on GitHub pages 55 | # (http://yourusername.github.io/repository-name) 56 | # and NOT your User repository (http://yourusername.github.io) 57 | # then add in the baseurl here, like this: "/repository-name" 58 | # baseurl: "aclrollingreview.org" 59 | 60 | # 61 | # !! You don't need to change any of the configuration flags below !! 62 | # 63 | 64 | permalink: /:title/ 65 | 66 | # The release of Jekyll Now that you're using 67 | version: v1.2.0 68 | 69 | # Jekyll 3 now only supports Kramdown for Markdown 70 | kramdown: 71 | # Use GitHub flavored markdown, including triple backtick fenced code blocks 72 | input: GFM 73 | gfm_quirks: 74 | - checkbox_support 75 | # Jekyll 3 and GitHub Pages now only support rouge for syntax highlighting 76 | syntax_highlighter: none #rouge 77 | syntax_highlighter_opts: 78 | # Use existing pygments syntax highlighting css 79 | #css_class: 'highlight' 80 | disable: true 81 | parse_block_html: false #true 82 | 83 | # Set the Sass partials directory, as we're using @imports 84 | sass: 85 | style: :expanded # You might prefer to minify using :compressed 86 | 87 | # Use the following plug-ins 88 | plugins: 89 | - jekyll-sitemap # Create a sitemap using the official Jekyll sitemap gem 90 | - jekyll-feed # Create an Atom feed using the official Jekyll feed gem 91 | - jekyll-seo-tag 92 | - jekyll-paginate 93 | - jekyll-redirect-from 94 | 95 | include: ['_pages'] 96 | 97 | paginate: 6 98 | paginate_path: /page:num/ 99 | 100 | # Exclude these files from your production _site 101 | exclude: 102 | - Gemfile 103 | - Gemfile.lock 104 | - LICENSE 105 | - README.md 106 | - CNAME 107 | 108 | -------------------------------------------------------------------------------- /_includes/admonition.html: -------------------------------------------------------------------------------- 1 | {% if include.type.size > 0 and include.title.size > 0 and include.body.size > 0 %} 2 | {% assign types = "note, abstract, info, tip, success, question, warning, failure, danger, bug, example, quote" | split: ", " %} 3 | {% if types contains include.type %} 4 |
5 |

{{ include.title }}

6 |

7 | {{ include.body }} 8 |

9 |
10 | {% endif %} 11 | {% endif %} -------------------------------------------------------------------------------- /_includes/analytics.html: -------------------------------------------------------------------------------- 1 | {% if site.google_analytics %} 2 | 3 | 15 | 16 | {% endif %} 17 | -------------------------------------------------------------------------------- /_includes/analytics_head.html: -------------------------------------------------------------------------------- 1 | {% if site.google_analytics_ga4 %} 2 | 3 | 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /_includes/bluesky-icon.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /_includes/disqus.html: -------------------------------------------------------------------------------- 1 | {% if site.disqus %} 2 |
3 |
4 | 15 | 16 |
17 | {% endif %} -------------------------------------------------------------------------------- /_includes/meta.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {% if page.excerpt %} 6 | 7 | {% elsif page.description %} 8 | 9 | {% else %} 10 | 11 | {% endif %} 12 | 13 | 14 | {% if page.title %} 15 | 16 | 17 | {% endif %} 18 | 19 | {% if page.image %} 20 | 21 | 22 | {% else %} 23 | 24 | 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /_includes/pullquote.html: -------------------------------------------------------------------------------- 1 | {{ include.quote }} -------------------------------------------------------------------------------- /_includes/svg-icons.html: -------------------------------------------------------------------------------- 1 | {% if site.footer-links.dribbble %}{% endif %} 2 | {% if site.footer-links.email %}{% endif %} 3 | {% if site.footer-links.facebook %}{% endif %} 4 | {% if site.footer-links.flickr %}{% endif %} 5 | {% if site.footer-links.github %}{% endif %} 6 | {% if site.footer-links.instagram %}{% endif %} 7 | {% if site.footer-links.linkedin %}{% endif %} 8 | {% if site.footer-links.pinterest %}{% endif %} 9 | {% if site.footer-links.rss %}{% endif %} 10 | {% if site.footer-links.twitter %}{% endif %} 11 | {% if site.footer-links.bluesky %}{% endif %} 12 | {% if site.footer-links.stackoverflow %}{% endif %} 13 | {% if site.footer-links.youtube %}{% endif %} 14 | {% if site.footer-links.googleplus %}{% endif %} 15 | {% if site.footer-links.playconsole %}{% endif %} 16 | -------------------------------------------------------------------------------- /_layouts/page.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | --- 4 | 5 |
6 | 7 |

{{ page.title }}

8 | 9 |
10 | {{ content }} 11 |
12 |
13 | -------------------------------------------------------------------------------- /_layouts/page_toc.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | --- 4 | 5 |
6 | 7 |

{{ page.title }}

8 | 9 | {% include toc.html html=content %} 10 | 11 |
12 | {{ content }} 13 |
14 | 15 | 16 |
+
17 |
18 |
19 | Table of Contents 20 | 21 |
22 |
23 |
24 | 25 |
26 | 27 | 82 | 83 | 203 | -------------------------------------------------------------------------------- /_layouts/post.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | --- 4 | 5 |
6 |

{{ page.title }}

7 | 8 |
9 |

{{site.author}} · {{ page.date | date: "%B %e, %Y" }}

10 | {% if page.last_modified_at %} 11 |

(Updated: {{ page.last_modified_at | date: "%b %-d, %Y" }})

12 | {% endif %} 13 |
14 | {% if post %} 15 | {% assign categories = post.categories %} 16 | {% else %} 17 | {% assign categories = page.categories %} 18 | {% endif %} 19 | {% for category in categories %} 20 | {{category}} 21 | {% unless forloop.last %} {% endunless %} 22 | {% endfor %} 23 |
24 |
25 | 26 |
27 | {{ content }} 28 |
29 | 30 |
31 |

Twitter, Facebook, Bluesky

32 |
33 | 34 | 37 | 38 | {% include disqus.html %} 39 |
40 | -------------------------------------------------------------------------------- /_posts/2021-03-18-here-we-are.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Here we are 4 | categories: [Admin] 5 | --- 6 | 7 | ACL Rolling Review is here! We'll be running a first trial with [EMNLP 2021](https://2021.emnlp.org/) (thank you very very much to the organizers!). 8 | 9 | We have some great technical staff, Nils Dycke and Kushal Arora. 10 | 11 | We will be asking for your submissions and your participation as action editors and reviewers very, very soon. 12 | -------------------------------------------------------------------------------- /_posts/2021-04-05-initialCFP.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Call for Papers 4 | categories: [Admin] 5 | --- 6 | 7 | The ACL Rolling Review [call for papers](/cfp) is up. The first submission deadline will be 15 May 2021, which is also the last submission deadline to be considered for the EMNLP 2021 main conference. 8 | -------------------------------------------------------------------------------- /_posts/2021-04-07-reviewerinterest.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Reviewer Interest Survey 4 | categories: [Admin] 5 | --- 6 | 7 | We invite you to express interest in becoming a reviewer or action editor for ACL Rolling Review via our [reviewer interest survey](/reviewers). 8 | -------------------------------------------------------------------------------- /_posts/2021-04-12-submissions.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Submission Site 4 | categories: [Admin] 5 | --- 6 | 7 | ACL Rolling Review is now accepting submissions! Please go to [the Authors page](/authors) to preview the fields in the submission form and find the link to the submission site. 8 | -------------------------------------------------------------------------------- /_posts/2021-04-13-reviewforms.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Review Forms 4 | categories: [Admin] 5 | --- 6 | 7 | The draft review forms are up at [here](/reviewers)! 8 | And we still welcome volunteer reviewers/action editors! 9 | -------------------------------------------------------------------------------- /_posts/2021-04-14-venues.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: ACL Conferences and Workshops and ARR 4 | categories: [Admin] 5 | --- 6 | 7 | if you are running an ACL-affiliated conference or workshop, you can use ARR to accept submissions! Please contact the ARR editors (editors@aclrollingreview.org) and we will be happy to set it up for you. 8 | -------------------------------------------------------------------------------- /_posts/2021-04-27-welcome-new-reviewers.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Action Editors and Reviewers 4 | categories: [Admin] 5 | --- 6 | 7 | We are sending invitations out to action editors and reviewers. And we have *submissions*!! Thank you for volunteering, responding to invites, submitting! 8 | Please create / update your [Open Review](https://openreview.net) profile so we can do good conflict of interest handling and paper matching. 9 | -------------------------------------------------------------------------------- /_posts/2021-04-30-prlead.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Opening for a PR Lead 4 | categories: [Admin] 5 | --- 6 | 7 | We have an opening for a PR lead. Responsibilities: maintain website; manage social media presence; liaise with ACL publicity chairs. To apply, contact editors at aclrollingreview.org. 8 | 9 | Update: this opening has been filled! See the [people](/people) page for details. 10 | -------------------------------------------------------------------------------- /_posts/2021-04-30-some-important-dates.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Dates 4 | categories: [Admin] 5 | --- 6 | 7 | We have added a [dates](/dates) page. 8 | * Authors, refer to this page for a list of ARR participating venues and important dates. 9 | * Reviewers and action editors, refer to this page for relevant dates each month. 10 | * Conference and workshop organizers, refer to this page to figure out what the last ARR submission date should be for submissions to your venue. Contact us to have your venue added. 11 | 12 | -------------------------------------------------------------------------------- /_posts/2021-05-19-tech-ad.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Opening for technical staff 4 | categories: [Admin] 5 | --- 6 | 7 | We are looking for a few extra members of the technical staff, who implement the OpenReview interface, automated systems for reviewer management, paper-reviewer matching, etc. Please contact [cto@aclrollingreview.org](mailto:cto@aclrollingreview.org) with a quick self-introduction if interested! 8 | 9 | This is a volunteer position for the ACL academic society, and it will require at most a few hours per week 10 | 11 | Contributions can be made in various ways. Even just basic Python coding skills would be useful, but there are also some areas where the ability to write database code or machine learning models would be additionally helpful. 12 | -------------------------------------------------------------------------------- /_posts/2021-08-26-communicationsteam.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Opening for Communications Team members 4 | categories: [Admin] 5 | --- 6 | 7 | We have openings for members of a new “communications team.” This role (alongside the current communications lead) would encompass website updates, social media presence, and triaging/responding to support email. To apply, contact `editors at aclrollingreview.org`. 8 | -------------------------------------------------------------------------------- /_posts/2021-10-12-status-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: ARR Status Report 4 | categories: [Admin] 5 | --- 6 | 7 | The ACL Rolling Review (ARR) officially kicked off in early 2021, with the first call for papers in April, and the first deadline in May. This initiative was started with the [promise](https://www.aclweb.org/adminwiki/index.php?title=ACL_Rolling_Review_Proposal) of a unified reviewer pool, more consistent reviewing, lower average reviewer load, and faster submission to publication timelines. 8 | 9 | Now, 5 months and 5 deadlines into the new system, we wanted to give an update on the ARR process. 10 | 11 | ## ARR Statistics 12 | Let’s start with some statistics. 13 | 14 | 15 | ### Submissions 16 | The number of submissions each month is shown in the bar chart below. One third to one half of the submissions were opted in to be anonymous preprints hosted on OpenReview, with links tweeted by the [ARRPreprints](https://twitter.com/ARRPreprints) twitter bot. So far, this bot has 343 followers, significantly less than [similar](https://twitter.com/arxiv_cscl) [accounts](https://twitter.com/arXiv_Daily). Most tweets get about 100-200 impressions, although a few have reached up to 15K impressions. 17 | 18 | We expect there to be dramatically more submissions in the October and November deadlines. 19 | 20 | ![Number of Submissions](/images/arr_report/num-submissions.png) 21 | 22 | 23 | ### Reviewer Pool 24 | The review process involves both reviewers and Action Editors (commonly called Area Chairs in most conferences). The number of reviewers in the pool (shown in the figure below) has hovered around 1200, with about 100 to 150 action editors. In contrast, [ACL 2021](https://aclanthology.org/2021.acl-long.pdf) had 60 SACs, 323 ACs, and 3,685 primary reviewers, to deal with 3,350 full paper submissions. 25 | 26 | Our reviewer pool is dramatically smaller than the pool in ACL 2021. 27 | 28 | ![Reviewer Pool](/images/arr_report/reviewer-pool.png) 29 | 30 | ### Timing 31 | 32 | The original goal of ARR was to have all reviews and meta-reviews completed within 35 days. The process requires that each paper has 3 reviews, and once those are complete, a meta-review. This is a pretty tight turnaround. By comparison, in [ACL 2021](https://2021.aclweb.org/calls/papers/), the time from submission to discussion period (which does not include meta-reviews) was 51 days, almost twice the length of ARR review+meta-review period. The time from submission to notification amounted to 92 days, nearly 3 times what ARR is aiming for. 33 | 34 | In the graph below, each data point is the time taken for an individual review after the submission deadline. A handful of rapid reviewers got their reviews in before 20 days (thank you!), many reviews are completed +/- 2 day from the deadline, and a non-trivial number (about 25%) are finished after the deadline, with one or two stretching even as far as 90+ days later. 35 | 36 | The graph also shows relevant deadlines from ACL 2021 as dashed vertical lines. While ARR is not yet hitting all promised reviewing target deadlines, notice that the timeline is compressed compared to ACL. 37 | 38 | (This graph doesn’t include the 24 reviews that are not yet finished, most of which are from the August deadline.) 39 | 40 | ![Days from Submission to Review](/images/arr_report/days-from-submission-to-review.png) 41 | 42 | The following graph is analogous, but looking at time from submission to meta-review. 43 | 44 | ![Days from Submission to Meta-Review](/images/arr_report/days-from-submission-to-meta-review.png) 45 | 46 | 47 | ### Conference Submissions 48 | ARR provides no decisions, only reviews, which can be submitted to a conference or workshop. As of writing, a total of 26 papers with ARR reviews have been submitted to conferences. The breakdown is shown in the bar chart below. 49 | 50 | ![Conference Submissions](/images/arr_report/conf-submissions.png) 51 | 52 | ## Major Challenges 53 | 54 | ### Review Time 55 | The major challenge ARR has faced is getting reviews back in time. As of writing (Sept 2021), there is 1 paper from June that has still not received all 3 reviews, and there are a handful in each subsequent deadline as well. These delays have happened for several reasons, including technical issues with OpenReview, mistakes we made (e.g. failing to send reminders), but overall overwhelmingly because of unresponsive reviewers and meta-reviewers. 56 | 57 | We are keenly aware of the issues with late reviews, and the ripple effects on conference submission deadlines. We are working to improve the process in several ways: 58 | 59 | - Improving the assignment of action editors and reviewers, and automating the process of reminding reviewers and action editors. 60 | - Recruiting emergency reviewers and action editors who can step in when a reviewer/action editor doesn't complete their job. 61 | 62 | _We cannot succeed unless reviewers and action editors complete their work on time_. ARR is a peer-reviewing process: authors depend on timely delivery of reviews, and reviewers and action editors are also authors. 63 | 64 | ### Reviewers Declining 65 | 66 | One of the biggest challenges that action editors have faced is reviewers declining to review, leading to late reviews. This is happening for a variety of reasons, including reviewers being assigned papers that they don’t feel qualified to review, or papers assigned with very tight deadlines. 67 | 68 | We are working on ways of addressing this. One partial remedy is to provide reviewers with no option to decline (especially not on a per-assignment basis). Instead, **reviewers will have to request** to be exempt prior to the submission deadline, and will be asked to acknowledge when they have received a review request. 69 | 70 | Other (smaller) remedies will be more technical in nature -- such as simple interface changes, for example, including the name of the action editor requesting the review in the email request (there is no action editor <> reviewer anonymity) to facilitate the communication between reviewers and action editors. 71 | 72 | ### Technical Issues 73 | 74 | Another hurdle has been in stretching the OpenReview system to a new use case. OpenReview has been successfully used for several years in a number of large conferences, including [NeurIPS](https://openreview.net/group?id=NeurIPS.cc) and [ICML](https://openreview.net/group?id=ICML.cc). But in each of these cases, there is a single deadline with a single reviewer pool. ARR has a deadline every month, and OpenReview creates an entirely new conference each month. We have to copy over reviewers and action editors from the prior month. This makes many editorial duties significantly more difficult. For example, it is not possible to track all missing reviews and meta-reviews in one place; similarly, it is at the moment not possible to track and balance overall reviewing load across months. 75 | 76 | We are improving the process with each deadline, and also working closely with developers at OpenReview. We are hopeful that in time many of the problems will be ironed out, leading to a smoother and easier process. 77 | 78 | ## Looking Ahead 79 | We are anticipating a heavier load in the following few months since both [ACL 2022](https://www.2022.aclweb.org/) and [NAACL 2022](https://2022.naacl.org/) will adopt ARR as their sole reviewing system. We ask for patience and understanding from authors, and strict adherence to timelines from reviewers and meta-reviewers. We are also working together with ACL and NAACL 2022 chairs to make sure that every manuscript submitted to the respective ARR deadlines is processed in time and can ultimately (if so desired by the authors) commit to those conferences. 80 | 81 | We are also excited about a number of new initiatives, including training for reviewers and action editors, [mentoring](/mentorship) for new reviewers, automated review quality assessment, and a new reviews corpus. In the prior conference reviewing system, many of these initiatives would have been organized on a conference-by-conference basis. Having a centralized reviewing system makes these tasks much simpler. 82 | 83 | As it becomes more widely adopted, and as the process becomes smoother with time, we hope that ARR will prove to be a valuable reviewing system that ensures reviews of consistently high quality with a sustainable amount of effort from the *ACL community. 84 | 85 | 86 | 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /_posts/2021-10-20-emergency-reviewers.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Emergency Reviewers and Action Editors 4 | categories: [Admin] 5 | search_index: false 6 | --- 7 | 8 | We are recruiting volunteer emergency reviewers and emergency action editors. To volunteer, fill out (*Update: These forms are no longer in use. Please contact the editors to volunteer*): 9 | * Emergency [*action editor* form]() 10 | * Emergency [*reviewer* form]() 11 | 12 | **Responsibilities**: 13 | * As an emergency *action editor*, you will step in at the request of the editors in chief when the action editor assigned to paper is unable to or does not complete their responsibility for a paper. You will chase down reviewers (or find emergency reviewers) and complete a meta review so that authors get their reviews in a timely fashion. 14 | * As an emergency *reviewer*, you will step in at the request of an action editor when the reviewer assigned to paper is unable to or does not complete their responsibility for a paper. You will read a paper and provide a review, probably within a week. 15 | 16 | **Availability needed**: You should be available to swing into action between the 10th and 20th of a month. As with other ARR roles, you can indicate your unavailability for any given month(s). 17 | 18 | **Level of experience**: You should have all the experience of a regular action editor (publishing regularly in *ACL venues and a demonstrated track record of reviewing for *ACL venues), as well as the ability to focus under time pressure. 19 | 20 | In recognition of this extra level of service, ACL Rolling Review will be happy to provide you with a letter of recognition that you can show your employer, as well as displaying your name on our website and acknowledging you at conferences. 21 | -------------------------------------------------------------------------------- /_posts/2021-10-26-november-submissions.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: November 2021 Submmissions 4 | categories: [Admin] 5 | --- 6 | 7 | November submissions are open! Please submit your papers [here](https://openreview.net/group?id=aclweb.org/ACL/ARR/2021/November). -------------------------------------------------------------------------------- /_posts/2021-11-03-new-reviewer-tutorial.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: We have a new Reviewer Tutorial 4 | categories: [Admin] 5 | --- 6 | 7 | We have a new [reviewer tutorial](/reviewertutorial)! Thanks to Anna Rogers and Isabelle Augenstein for creating it. 8 | -------------------------------------------------------------------------------- /_posts/2021-11-17-modified-timeline-November.md: -------------------------------------------------------------------------------- 1 | For November, working with the ACL 2022 senior area chairs and program chairs, we will have a slightly modified ARR timeline. 2 | * Nov 19-22 - SACs check AE assignments 3 | * Nov 25 - finalized AE assignments 4 | * November 27 - AEs notified of assignments 5 | * Nov 27-29 - desk rejects and manual check of reviewer assignment by AEs 6 | * Nov 29 - Reviewers notified of assignments 7 | * Nov 29 - Dec 27: reviewing 8 | * Dec 28-29 - AEs find emergency reviewers and lead discussions 9 | * Dec 28-31 - Emergency reviewing 10 | * Dec 28-Jan 7 - AEs prepare meta reviews 11 | * Jan 10, 2022 - ARR reviews for a Nov 15 submission sent to the authors 12 | * Jan 15, 2022 - ACL 2022 commitment deadline 13 | 14 | 15 | -------------------------------------------------------------------------------- /_posts/2021-12-02-ARR-changes-December.md: -------------------------------------------------------------------------------- 1 | 2 | We are introducing several changes to ACL Rolling Review starting with the December 2021 deadline. 3 | 4 | ## Changes to the timeline 5 | 6 | In November, we received many submissions! In December, we will go back to our [usual timeline](/dates). There’s a shared calendar there you can import to your own calendar. 7 | 8 | ## Changes to the submission process 9 | 10 | ### Responsible NLP Research 11 | 12 | In collaboration with the NAACL 2022 program chairs and Jesse Dodge, and following input from the NAACL reproducibility chairs, Anna Rogers, Margot Mieskes, and the ACL Ethics Committee, we are rolling out a new responsible NLP research checklist. This checklist is intended to inform and educate. 13 | Before December, ARR pointed authors to separate ethics and reproducibility checklists. Completion of these checklists was strongly encouraged but not mandatory. Nothing was uploaded as part of submitting to ARR. 14 | Starting in December, authors will be required to complete the new responsible NLP research checklist. It can be found [here](/responsibleNLPresearch/). There will be a field in the submission form for uploading the completed checklist. 15 | 16 | ### Submission Type 17 | 18 | We have added a question about submission type (long paper/short paper). This information will be used to balance load across reviewers, and by program chairs at publication venues. 19 | 20 | ### Willing to Review 21 | 22 | Finally, we are making permanent the last question in the submission form, which requires authors submitting to ARR to agree to review if qualified and invited. 23 | 24 | The fields in the submission form are here: [Submission Form Fields](/submissionform). 25 | 26 | As always, authors may refer to the [Authors’ Guide to OpenReview](https://docs.google.com/presentation/d/1kJeoAfwbnFapUN0ySLSoOm11-2odz48DGS1DEzNs03k/edit?usp=sharing). 27 | 28 | ## Changes to the review process 29 | 30 | We have modified the review form also. 31 | 32 | ### Overall Assessment 33 | 34 | We have modified the overall assessment question, with very valuable work from Isabelle Augenstein and Anna Rogers and input from Jesse Dodge. 35 | 36 | ### Responsible NLP Research 37 | 38 | Instead of a single ethics question, there are now three. This enables us to better assess whether to send a paper for ethics review. 39 | 40 | The revised review form can be found here: [Review Form](/reviewform). 41 | 42 | -------------------------------------------------------------------------------- /_posts/2022-01-06-Modified-timeline-January.md: -------------------------------------------------------------------------------- 1 | As in November, ARR will have a slightly modified timeline in January (ntoe that all deadlines are 11:59pm AOE): 2 | * January 15th: Submission deadline 3 | * January 16th-24th: Reviewer/AE assignment 4 | * January 24th-27th: SAC/AE paper check and reassignment 5 | * January 28th-February 15th: Reviewing 6 | * February 16th-26th: Meta-reviews and reviewer chasing 7 | * February 27th: Review release 8 | -------------------------------------------------------------------------------- /_posts/2022-01-11-choices.md: -------------------------------------------------------------------------------- 1 | **There is no need to withdraw any papers from ARR after reviewing is complete** (in fact if you do withdraw your paper from an ARR cycle, we won't be able to transfer that paper + reviews to any venue you "commit" to). Once you have your reviews, your paper is considered to no longer be under review to ARR. 2 | 3 | For those of you who have received reviews for any previous ARR cycle and are wondering what your options are, you and your co-authors have a (non-exclusive) choice: 4 | * "commit" your paper + reviews to [ACL 2022](https://openreview.net/group?id=aclweb.org/ACL/2022/Conference) by 15 January 2022 and see if it is accepted (if it is accepted, you could modify the paper before camera ready) 5 | * "commit" your paper + reviews to [NAACL 2022](https://2022.naacl.org/) by 4 March 2022 and see if it is accepted 6 | * *revise your paper* and resubmit it to [ARR](https://openreview.net/group?id=aclweb.org/ACL/ARR/2022) (including to the January 15th 2022 deadline, although that is probably too soon for a revision, practically speaking) to get new reviews. Please see the [call for papers](/cfp) for the requirements and options for resubmissions. 7 | * a combination of these is also possible. 8 | 9 | If you are interested in "committing" to another venue, please refer to [this page](/dates). 10 | 11 | 12 | -------------------------------------------------------------------------------- /_posts/2022-01-13-resubmissions.md: -------------------------------------------------------------------------------- 1 | We have received several questions about resubmitting to ARR. We have now modified the [call for papers](/cfp) to address these questions, which have to do with adding/removing authors, providing responses to reviewers, and requesting new reviewers. 2 | -------------------------------------------------------------------------------- /_posts/2022-01-17-modified-timeline-December.md: -------------------------------------------------------------------------------- 1 | For the December cycle, ARR will have a slightly modified timeline. Reviews will be released on January 28th given that reviewing started a bit later due to the end of the semester, the overlapping heavy load from November and the holidays. We also wanted to give a bit more time to AEs to be mindful of weekend days. 2 | * December 15th: Submission deadline 3 | * December 16th-24th: Reviewer/AE assignment 4 | * December 24th-28th: SAC/AE paper check and reassignment 5 | * December 29th-January 18th: Reviewing 6 | * January 19th-27th: Meta-reviews and reviewer chasing 7 | * January 28th: Review and meta-review release 8 | -------------------------------------------------------------------------------- /_posts/2022-01-28-declaring-a-conflict-of-interest.md: -------------------------------------------------------------------------------- 1 | As an author, reviewer or action editor for OpenReview, when and how should you declare a conflict of interest? 2 | 3 | The ACL has a [conflict of interest policy](https://aclweb.org/adminwiki/index.php?title=ACL_Conference_Conflict-of-interest_policy). It says: 4 | "A person has a conflict of interest with a submitted paper if that person: 5 | 1. is a co-author of the paper, or 6 | 2. has been a student or supervisor of one of the authors in the previous five years, or 7 | 3. has co-authored a paper or collaborated with one of the authors in the previous five years, or 8 | 4. is employed at the same company or institution as an author, or 9 | 5. has any other circumstances that could cause a bias in evaluating the paper." 10 | 11 | Everyone 12 | -------- 13 | 14 | We try to automatically identify conflicts of interest using the OpenReview profiles of authors on a submission and of each potential reviewer or action editor. We check: 15 | * submission co-authorship by looking at the submission 16 | * student / supervisor relationships via your declared relationships in your OpenReview profile 17 | * historical co-authorship by using your publications from your DBLP and Semantic Scholar entries in your OpenReview profile 18 | * employment via your domains / employment in your OpenReview profile, backing off to your email addresses if necessary 19 | * other circumstances using your declared conflicts of interest in your OpenReview profile 20 | 21 | This means that it is critical to *have a complete OpenReview profile*. Now, if you don't regularly publish, or are a first-time author, you may not have Semantic Scholar or DBLP entries. But as soon as you get that first publication in a computer science discipline (congratulations!) please add those profiles. 22 | 23 | These helpful guides show you how to set up your OpenReview profile and so much more: 24 | * [Authors' Guide to OpenReview](https://docs.google.com/presentation/d/1kJeoAfwbnFapUN0ySLSoOm11-2odz48DGS1DEzNs03k/edit#slide=id.p) 25 | * [Reviewers' Guide to OpenReview](https://docs.google.com/presentation/d/1CkfR94WxEPEZEyCN--ydC7K3wY4g-5ZiFd2HM8LRSXg/edit#slide=id.p) 26 | 27 | Reviewers and Action Editors 28 | ---------------------------- 29 | If you take a look at a submission and think, oh, I might have a conflict of interest here, you should report it *immediately*. You should not review the submission. This is why it's a good idea to take a look at your assignments as soon as you receive them. 30 | Reviewers should report COIs to their action editor; action editors to their senior action editor. If you are not removed from the submission within 24 hours, email editors@aclrollingreview.org. 31 | 32 | Some Frequently Asked Questions 33 | ------------------------------- 34 | 35 | * Isn't it a lot of work to set up a profile? - It takes about 10 minutes to set up a profile; and [you'd have to have a profile](https://acl2020.org/blog/conflict-of-interest/) whether we used OpenReview or START. 36 | 37 | * Can't the system be gamed by providing incomplete or incorrect information? - Yes, malicious people could do this. This would be a clear violation of professional behavior, and a matter for the ACL ethics committee. 38 | 39 | * Can't you just desk reject every submission where the authors don't have complete profiles? - First-time authors won't yet have Semantic Scholar or DBLP entries, so we can't just desk reject because a profile doesn't contain this information. 40 | 41 | * My co-author has never published before or is not in our field. Do *they* need an OpenReview profile? It's a lot of overhead for making a submission. - This is the flip side of the coin from the previous question. If you have a co-author who couldn't make a profile for some reason, please add an Offical Comment to the editors (program chairs) on your submission in OpenReview. 42 | 43 | * Can't AEs just check manually? - Our field is really large; an AE is unlikely to know all or even any of the potential conflicts of interest between authors and reviewers, even if they knew the identities of the authors, which they don't. 44 | 45 | * The ACL policy is overly broad; I work at a big tech company and shouldn't have conflicts with 500 other people I've never met. - That would be a question for the ACL exec. 46 | 47 | * In the presence of "noise" (e.g. incomplete profiles) how can we minimize the assignment of submissions to reviewers with conflicts of interest? - We do not assign reviewers for whom we don't have basic information (so if you are wondering why you have never received an assignment, check your OpenReview profile). Beyond this, we *could* minimize potential conflicts of interest by assigning reviewers based on the inverse of their expertise match to a submission - the opposite of what we do now. However, this would lead to bad reviews from people with no expertise in the subject matter. A more nuanced policy (second best match?) is also possible. However, this shows that great conflict of interest handling and good reviewer assignment are highly related, and the best way to get both is to _complete your OpenReview profile_. 48 | -------------------------------------------------------------------------------- /_posts/2022-02-04-review-data-collection.md: -------------------------------------------------------------------------------- 1 | One of the initiatives that ARR supports is the ongoing collection of data about peer-review in NLP. Nils Dycke, Ilia Kuznetsov and Iryna Gurevych have been leading an [ACL-approved initiative](https://www.aclweb.org/adminwiki/index.php?title=Review_Data_Collection_at_*ACL) to collect review data on an opt-in basis. They provide a status update and invite discussion [here](https://openreview.net/forum?id=28n-0nBPTch). 2 | -------------------------------------------------------------------------------- /_posts/2022-02-17-stats.md: -------------------------------------------------------------------------------- 1 | We have [**data**](http://stats.aclrollingreview.org). Thanks to great work from Sebastin Santy on our tech team, we now have a regularly updated statistics dashboard for ACL Rolling Review. When you refer to it, please note the last updated date as data for the current cycles may be somewhat delayed. As always, feedback welcome. 2 | -------------------------------------------------------------------------------- /_posts/2022-03-03-recognition.md: -------------------------------------------------------------------------------- 1 | Action editors and reviewers may now request letters of recognition for their ARR service! Instructions can be found in the: 2 | * [Action editors' guide to OpenReview](https://docs.google.com/presentation/d/1OynnI8TxlBcSt3J1YCJbxdLsMT91D2Hr1zVtorLkeao/edit#slide=id.p) 3 | * [Reviewers' guide to OpenReview](https://docs.google.com/presentation/d/1CkfR94WxEPEZEyCN--ydC7K3wY4g-5ZiFd2HM8LRSXg/edit#slide=id.p) 4 | 5 | The letter of recognition will be sent to your contact email address as an email attachment at the end of the latest cycle. It will be a PDF, on ARR letterhead, using your preferred name and acknowledging your cumulative number of reviews / metareviews. 6 | 7 | Suggestions for improvements to the wording in the letter of recognition are welcome. 8 | 9 | Thanks to Nils Dycke for implementing this new functionality *inside OpenReview*. 10 | -------------------------------------------------------------------------------- /_posts/2022-04-07-six-week-cycles.md: -------------------------------------------------------------------------------- 1 | Effective the 15 April 2022 deadline, with permission of the ACL executive and the ACL committee on reviewing, ARR will be moving to a six-week cycle with deadlines every six weeks. 2 | 3 | Why? 4 | ---- 5 | 6 | The original proposal for ACL Rolling Review was for a [five-week cycle with monthly deadlines](https://www.aclweb.org/adminwiki/index.php?title=ACL_Rolling_Review_Proposal). However: 7 | * It takes more time than anticipated to check submissions and resubmissions and handle (re)assignments and desk rejects each cycle, especially in months with more than 300 submissions. Just running the code for reviewer assignment takes several hours, and then action editors and reviewers propose modifications which are made by hand. 8 | * Chasing after reviewers, and doing emergency reviewing and meta reviewing, takes time. 9 | * It is confusing for action editors, reviewers and the tech team when cycles overlap. 10 | 11 | What Will Be The New Deadlines? 12 | ------------------------------- 13 | 14 | There will be nine per year: 15 | * 1 March 16 | * 15 April 17 | * 1 June 18 | * 15 July 19 | * 1 September 20 | * 15 October 21 | * 1 December 22 | * 15 January 23 | 24 | This set of deadlines avoids the dreaded "holiday deadline" that was 15 December 2021. 25 | 26 | How Long Will I Have to Revise and Resubmit? 27 | -------------------------------------------- 28 | 29 | With this cycle, authors will typically have six weeks to revise their submission if they want to resubmit it to the next cycle; slightly less if their submission requires emergency (meta)reviews or ethics reviewing. Authors are always welcome to resubmit to a later cycle, as long as their paper has not been accepted to a publication venue and otherwise meets the conditions outlined in the [Call For Papers](/cfp). 30 | 31 | Now Can We Have Author Response? 32 | -------------------------------- 33 | 34 | A six-week cycle allows for a *very brief* period for in-cycle author response to reviews, to correct errors of fact or alert action editors to serious reviewer confusion. The primary means for authors to respond to reviewers will still be revise and resubmit. There will be more on this in the next blog post! 35 | 36 | -------------------------------------------------------------------------------- /_posts/2022-04-13-in-cycle-author-response.md: -------------------------------------------------------------------------------- 1 | Starting with the April 15 deadline, there will be a brief period for author response within the ARR review cycle. 2 | * for 5 days after the deadline for regular reviews, authors can make comments on the reviews, using the discussion forum within OpenReview (as is done, for example, with ICLR) 3 | * the comments should be limited to clear factual errors or serious misunderstandings 4 | * reviewers may respond to authors' comments or update their reviews, but are not obligated to do so 5 | * action editors will take these comments into account when writing their metareview 6 | 7 | This is made possible by the slightly longer cycle announced in the previous blog post. 8 | 9 | The *primary route* for responding to reviews will remain revise and resubmit. In-cycle responses to reviews should not include, for example, new experimental results or data analyses requested by reviewers. 10 | -------------------------------------------------------------------------------- /_posts/2022-07-05-max-cycle-load-openreview.md: -------------------------------------------------------------------------------- 1 | Starting from the July 2022 ARR cycle, the Reviewers and Action Editors can specify their unavailability and maximum load directly in OpenReview. Load specification is designed as a reviewer/AE task in OpenReview, to be completed before the submission deadline for the respective cycle (for the July 22 cycle, the deadline for specifying the max. load is July 15). A detailed guide on how to declare your max. load / unavailability can be found [here](https://docs.google.com/document/d/1_UEnoQVl27vg-IGyHZlt09nfShZxSofef2GDXo2pipI/edit?usp=sharing). 2 | -------------------------------------------------------------------------------- /_posts/2022-08-31-arr-eic-changes.md: -------------------------------------------------------------------------------- 1 | Two new EICs have started their service (and several new EiCs are expected to start soon): 2 | * Mausam 3 | * Thamar Solorio 4 | 5 | Three current EICs have / will soon be stepping down: 6 | * Amanda Stent (moving to the tech team in October) 7 | * Goran Glavaš (steps down in October) 8 | * Pascale Fung 9 | 10 | Thank you Pascale, Goran, and Amanda for your massive efforts! And thank you to Mausam and Thamar for stepping into these roles. 11 | 12 | -------------------------------------------------------------------------------- /_posts/2022-08-31-changes-based-on-the-ACL-reviewing-survey.md: -------------------------------------------------------------------------------- 1 | In May and June, the ACL reviewing committee conducted a survey on the future of reviewing in the ACL community. Follow [this link for results and analysis of the survey](https://www.cis.lmu.de/~hs/acl22/panel/survey.html), including both the multiple choice questions and the free-text responses. 2 | 3 | Based on those responses, the panel discussion at ACL, feedback from the ACL and NAACL Program Chairs (PCs), and discussion with the ACL Executive, we have a range of updates about ARR: 4 | 5 | Changes that are **already in effect**: 6 | * An ARR board is being created to oversee ARR, including approving new ARR initiatives, approving new ARR Editor in Chiefs (EICs), and managing collaboration between ARR and conferences. The board will have five members: an ARR EIC, an ACL Executive member, two past PCs, and one other member of the community. 7 | * ACL is providing funding to OpenReview (OR) to get the ongoing support of an OR employee for ARR activities as well as all other ACL activities that are carried out through OR. 8 | * Additional EICs are being recruited to spread the load of running ARR. As part of recruitment, we are working to ensure continued diversity in the EIC team. The plan is to grow the team to seven EIC in the coming weeks. 9 | 10 | Plan for future changes and ARR activities (note, the goal is to complete items by the end of the month they are listed in) 11 | 12 | #### August 13 | * A monthly blog post to share updates on ARR. 14 | 15 | #### September 16 | * Make requesting new reviewers and/or a new AE for a resubmission easier and have the request be accepted by default. 17 | * Adding the ability for authors to identify relevant track(s) for their paper, for AEs and reviews to indicate relevant track(s) for themselves, and for that information to be considered in AE and reviewer assignments. This corresponds to the ‘soft tracks’ option from the survey. 18 | * Adjust reviewer assignment matching to consider demographics to ensure there is at least one senior reviewer and that no two reviewers for a paper are from the same research group. 19 | * Implement author evaluation of reviews. 20 | * Adjusting the review form to support [the new ACL Awards Policy](https://www.aclweb.org/adminwiki/index.php/ACL_Conference_Awards_Policy). 21 | 22 | #### October 23 | * Release a paper on arXiv about findings from the first year of ARR. 24 | * Reconsidering the wording and structure of the review form to make it be explicitly focused on the main ACL conferences (ACL, AACL, EACL, EMNLP, NAACL). 25 | 26 | #### November 27 | * Refine guidelines for meta reviewers and consider running a tutorial session or mentoring process. 28 | 29 | #### December 30 | * Switch to an **8 week cycle** and **guarantee all reviews** will be ready by the end of the cycle. 31 | * Explore aligning cycles with conferences. 32 | 33 | #### February 2023 34 | * Develop a mentoring system for new reviewers (either new to reviewing or new to the field). 35 | 36 | #### May 2023 37 | * Conduct another survey to get feedback from the community. 38 | 39 | #### July 2023 40 | * Present results and analysis of the survey. 41 | 42 | 43 | #### Under discussion / Exploration 44 | * Adding support for subreviewers / secondary reviewers. 45 | * Allow papers that are missing reviews or a meta-review to be committed to conference deadlines. 46 | * Improvements to the user interface and reviewing process to encourage more reviewer discussion. 47 | 48 | Thank you to all of the community members who provided input! 49 | 50 | -------------------------------------------------------------------------------- /_posts/2022-10-11-key-changes-in-the-october-cycle.md: -------------------------------------------------------------------------------- 1 | At ARR, we are slowly and steadily making changes to reflect the ACL reviewing survey results. In its October 15th cycle, we bring two key changes. 2 | 3 | 1) Starting this cycle, ARR will be a two-monthly cycle instead of a 1.5 monthly. Accompanying this change will be a (near)-guarantee (barring exceptional circumstances) that all papers will get their reviews before the start of the next cycle, in this case 15th December. We recognize that a two monthly submissions are less than ideal for the goal of encouraging rapid review cycles. And we might as well revisit this once ARR processes get stabilized. However, this is a price we ought to pay right now to ensure that all papers receive their reviews in time – authors can plan better, and reviewers start to be more accountable. 4 | 5 | 2) Based on popular demand (a.k.a. survey results), we have introduced “tracks” in the review mechanism. As a first step, tracks will provide some agency to the reviewers and action editors: paper-reviewer matching will highly encourage matches where the paper’s track is common with one of the reviewer’s tracks. In the long run, tracks will become a vehicle for scaling up ARR, as we will associate senior action editors with a track, who will oversee papers in their track. This latter functionality is yet to be implemented. 6 | 7 | While we're at it, please also welcome two new editors in chief, who joined us recently: 8 | - Lilja Øvrelid, Professor at University of Oslo, Norway 9 | - Viviane P. Moreira, Professor at Universidade Federal do Rio Grande do Sul, Brazil 10 | 11 | As always for any comments/suggestions, please write to editors@aclrollingreview.org 12 | -------------------------------------------------------------------------------- /_posts/2022-10-24-newly-formed-arr-board.md: -------------------------------------------------------------------------------- 1 | 2 | As promised in our blog entry for August, ACL has established an ARR board that will oversee the 3 | activities of ARR. We suggest readers go back to [that blog entry](/changes-based-on-the-ACL-reviewing-survey/) 4 | for more information on the responsibilities of the board. In this update to the community we would 5 | like to formally welcome the founding members of the ARR Board: 6 | 7 | - Julia Hockenmaier, University of Illinois at Urbana-Champaign, USA 8 | - Yusuke Miyao, The University of Tokyo, Japan 9 | - Smaranda Muresan, Columbia University, USA 10 | - Thamar Solorio, University of Houston, USA 11 | 12 | The board is just getting settled into their new roles so please be patient while this happens and be on 13 | the lookout for more information from them in the near future. We expect their roles to also adjust with 14 | time as we form a better understanding on how this board can support the needs of the ACL community. For 15 | now, let’s just welcome them and thank them for their willingness to volunteer in this capacity. 16 | -------------------------------------------------------------------------------- /_posts/2022-12-9-Updates-on-December-Cycle.md: -------------------------------------------------------------------------------- 1 | The major changes happening for the December cycle are the introduction of Senior Action Editors (SAEs), and a new reviewing assignment process to include at least one non-student reviewer per paper. These changes are also a response to community feedback. SAEs will help us during key phases of the review process. They will provide another layer of review quality control for reviewers and AEs. The team of SAEs is almost complete and will be added to the website pretty soon. 2 | 3 | In an effort to streamline our processes, we are also trying to integrate all tasks inside the Openeview infrastructure. Expect some small changes in how you complete the tasks in upcoming cycles. 4 | 5 | Lastly, given that the December cycle has a winter holiday in between, we prepared a timeline that hopefully is as friendly to everyone as feasible so we can have three full weeks for reviewers to work on their assigned papers, not including the week from 12/23-01/01. Here’s the plan for the december timeline: 6 | 7 | - 12/1-12/15/2022: Submission window 8 | - 12/18/2022: Assignments to SAEs and AEs 9 | - 12/19-12/28/2022: Paper and reviewer assignment checks by SAE and AEs 10 | - 01/02-01/23/2023 Reviewing period 11 | - 01/24/2023: AEs check review quality 12 | - 01/30-02/03/2023: Author rebuttal period 13 | - 02/08/2023: Meta-reviews due 14 | - 02/10/2023: Deadline to complete the review process for all papers 15 | 16 | Useful reminders to everyone: 17 | - Check your spam folders for missed emails from OpenReview regularly during the cycle. This is still a source of delays in the process. We are using a new email address: [aclrollingreview@openreview.org]() in the hope that this email will be more easily identified by you as something that needs attention. 18 | - We will continue to ask reviewers and action editors to indicate their availability and max load, this is already happening now. This is our earliest check point to prevent reviewing delays every cycle. We thank you for your understanding and prompt reply to this! 19 | - Authors who submit papers to the December cycle are automatically added as reviewers. Read the bullet point above for what to do if for some reason you can’t help on this cycle (hint: availability and max load tasks). 20 | -------------------------------------------------------------------------------- /_posts/2023-02-15-new-ethics-chair.md: -------------------------------------------------------------------------------- 1 | We’re delighted to share the news that Vinodkumar Prabhakaran is joining the ACL RR team as Ethics Chair. Vinod is a senior research scientist at Google's Responsible AI org, working on issues at the intersection of AI and society. 2 | -------------------------------------------------------------------------------- /_posts/2023-05-12-ACL-Rolling-Review-update.md: -------------------------------------------------------------------------------- 1 | See the latest updates, Stats, and FAQs from ACL Rolling Reviews that were presented in EACL 2023. You can find the presentation slides [here](https://drive.google.com/file/d/1gZRHAAWyIqbvk5xs5TyNDDEMNm_YfpLx/view?usp=sharing). -------------------------------------------------------------------------------- /_posts/2023-05-19-update-on-data-collection copy.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Update on Data Collection 4 | categories: [Admin] 5 | --- 6 | 7 | We have added an FAQ on the on-going [peer review data collection](/datacollection)! Check it out for more information on the data and how you can contribute. 8 | -------------------------------------------------------------------------------- /_posts/2023-06-15-ethics-chair.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Malihe Alikhani joins ARR as Ethics co-chair 4 | categories: [Admin] 5 | --- 6 | 7 | We're pleased to announce that Malihe Alikhani has accepted to take on the role of Ethics co-chair for ARR. Malihe works towards designing inclusive and equitable language technologies. She develops systems that can communicate and collaborate with diverse populations, especially those from underserved communities. She studies how learning models might become biased and how we can mitigate them. She is currently an assistant professor of computer science at the University of Pittsburgh. Beginning in the Fall of 2023, she will move to Northeastern University. 8 | 9 | We're super excited to have you on board, Malihe; welcome! 10 | 11 | -------------------------------------------------------------------------------- /_posts/2023-09-21-submission-dates-for-2024.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Submission dates and process for 2024 Conferences 4 | categories: [Admin] 5 | --- 6 | 7 | The use of an ARR-only submission process has been agreed by Program Chairs (PCs), and General Chairs of the upcoming conferences in 2024 (EACL, NAACL and ACL, referred to in this document as \*CL conferences), with feedback from the ACL Rolling Review (ARR) Initiative, the ARR board, and the ACL Exec. There will be no direct submission option. This decision was announced during the ACL 2023 business meeting. 8 | The purpose of this communication is to update those that were not present, and to provide more details about the coordinated plan. 9 | In this document, submission deadline refers to the paper submission to the ARR reviewing portal in Open Review. Commitment deadline refers to the deadline for authors to submit their fully ARR-reviewed submission to the conference portal for consideration at that conference. Decisions available refer to the deadline by which conference PCs will send notifications to paper authors. 10 | 11 | The dates and cycles relevant for this coordinated submission process are as follows (Table 1 below shows key dates by conference and month): 12 | - 15 Oct 2023: October ARR Cycle - EACL submission deadline 13 | - 15 Dec 2023: ARR reviews & meta-reviews available to authors of October cycle 14 | - 15 Dec 2023: December ARR Cycle - NAACL submission deadline 15 | - 20 Dec 2023: EACL commitment deadline 16 | - 15 Jan 2024: EACL decisions available 17 | - 15 Feb 2024: ARR reviews & meta-reviews available to authors of December cycle 18 | - 15 Feb 2024: February ARR Cycle - ACL submission deadline 19 | - 20 Feb 2024: NAACL commitment deadline 20 | - 15 Mar 2024: NAACL decisions available 21 | - 15 Apr 2024: ARR reviews & meta-reviews available to authors of February cycle 22 | - 20 Apr 2024: ACL commitment deadline 23 | - 15 May 2024: ACL decisions available 24 | 25 | Each cycle will include its corresponding author response period before the meta-review stage. Exact time will be announced in the conference CFPs. Relevant aspects of the plan above are discussed below. 26 | 27 | **Anonymity period** 28 | Unless the ACL enacts a change in policy, each conference will follow the current ACL wide anonymity embargo policy where authors cannot release a deanonymized preprint during the review cycle or between commitment and decisions, plus one month before submission and commitment. More information here: [ARR Call For Papers](/cfp#anonymity-period) and [ARR Author instructions](/authors). 29 | 30 | **Acceptance rates** 31 | To calculate each conference’s acceptance rate, we will record, at the time of submission to ARR, authors’ intention to commit to one of the three conferences. This doesn’t imply a hard commitment. Authors may choose to commit to a different venue after receiving their reviews. 32 | 33 | Month | EACL 2024 | NAACL 2024 | ACL 2024 34 | ----- | --------- | ---------- | -------- 35 | Oct 2023 | Last ARR submission deadline: 10/15 | | 36 | Dec 2023 | ARR reviews available: 12/15
Commitment deadline: 12/20 | Last ARR submission deadline: 12/15 | 37 | Jan 2024 | Decision notification: 01/15 | 38 | Feb 2024 | | ARR reviews available: 02/15
Commitment deadline: 02/20 | Last ARR submission deadline: 02/15 | 39 | Mar 2024 | | Decision notification: 03/15 | 40 | Apr 2024 | | | ARR reviews available: 04/15
Commitment deadline: 04/20 | 41 | May 2024 | | | Decision notification: 05/15 | 42 | 43 | **Table 1. Timeline of submission and reviewing activities for the upcoming conferences** 44 | 45 | **Program Committee** 46 | Each set of PCs will recruit their own team of Senior Area Chairs, these SACs will be the same group of people in ARR and the conference for the corresponding submission cycle. So for example, for the upcoming October cycle, EACL PCs will select their team of SACs. They will serve as SACs in ARR for that cycle, and then also support PCs during the final assessment of papers committed to EACL. This is the team that will help make final accept/reject decisions after commitment of papers. PCs will encourage, and facilitate, discussions among reviewers/ACs and SACs after the commitment of papers, to further improve the final decision process. However, reviewers, and area chairs will be recruited by all teams, although this may happen in stages. So please be prepared to receive an invite to join the reviewing process soon! 47 | 48 | **Resubmissions to ARR and Commitments to Conferences** 49 | Once a paper has been fully reviewed, authors have the following (mutually exclusive) options as next steps for their manuscript: 50 | - Commit their paper and reviews to be considered for publication by a future conference venue accepting ARR reviewed versions. Authors will have the ability to include a response to their meta-review during commitment. If the paper is not accepted for publication by the authors’ chosen venue, it remains in ARR as before, available either for commitment to another venue or revision and resubmission. 51 | - Revise their paper and resubmit to ARR for another round of reviewing. Authors can request a change of reviewers and/or meta-reviewer. We will do our best to keep the same reviewers unless otherwise requested by authors, but it depends on reviewer availability. New reviewers will not see the old reviews until after they submit their review. 52 | - Take their paper to a different venue that does not take ARR-reviewed papers. 53 | 54 | Figure 1 below shows a diagram of the process for an ARR reviewed paper: 55 | 56 | ![ARR Paper Cycle](/images/arr-paper-cycle.jpg){:class="img-responsive"} 57 | 58 | **Figure 1. Wrofklow diagram of the author choices once a submission has been fully reviewed by ARR.** 59 | 60 | More detailed information and FAQ about the process can be found on [the Authors page](/authors) 61 | 62 | There will be more information coming out soon, but reach out to any of the PCs teams or to ARR in case you have questions. 63 | 64 | Email contacts: 65 | - ARR EiCs: editors@aclrollingreview.org 66 | - EACL PCs: eacl2024pcs@googlegroups.com 67 | - NAACL PCs: naacl2024-programchairs@googlegroups.com 68 | - ACL PCs: acl2024-programchairs@googlegroups.com 69 | 70 | -------------------------------------------------------------------------------- /_posts/2023-10-26-reviewing-update.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: ARR October Cycle Update 4 | categories: [Admin] 5 | --- 6 | 7 | We have been busy with housekeeping tasks after the submission period ended. In brief, at this time we have: 8 | - Added authors of October submissions to the reviewing pool 9 | - Assigned papers to Action Editors (AEs), Senior Action Editors (SAEs) and reviewers 10 | - Sent instructions to AEs and SAEs re their ToDo tasks 11 | 12 | AEs and SAEs are now reviewing their assignments and making minor tweaks to reviewing assignments where required. We will start the review process on Monday October 30th. We will be sending out instructions to reviewers soon. 13 | 14 | October cycle by the numbers: 15 | * We received a total of 1,271 submissions 16 | * We have assigned papers to 1,437 reviewers: 17 | * Max reviewing load is 6 and average load is 2.67 18 | * We have assigned papers to 266 AEs: 19 | * Max load for AEs is 12 papers and average load is 4.75 20 | * A total of 55 Senior Action Editors are overseeing the review process 21 | 22 | More soon! 23 | -------------------------------------------------------------------------------- /_posts/2023-10-31-reviewing-update2.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: ARR October Cycle - Reviewing Has Started 4 | categories: [Admin] 5 | --- 6 | 7 | We have now completed the paper assignment to reviewers, and the review process has started. 8 | 9 | Reviews are due Nov 20th. A detailed guide has been shared with reviewers and the support team is regularly checking for and responding to questions. 10 | 11 | Many thanks for all the support from reviewers in this process! 12 | 13 | -------------------------------------------------------------------------------- /_posts/2024-01-15-anonymity.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Update to Anonymity Policy 4 | categories: [Admin] 5 | --- 6 | 7 | The ACL has adopted a new [anonymity policy](https://www.aclweb.org/adminwiki/index.php/ACL_Anonymity_Policy) effective for all future submissions, including to ARR. This new policy replaces the old policy that prohibited authors from posting or advertising non-anonymous preprints during a period starting one month before the submission or commitment deadline, and continuing while the submission is under review. 8 | 9 | The [new policy](https://www.aclweb.org/adminwiki/index.php/ACL_Policies_for_Review_and_Citation) takes effect in ARR beginning with the Feb. 15 submission and commitment cycles. Under the new policy, submissions will remain anonymous during peer review, but __authors are free to post and discuss non-anonymous preprints at any time__. To protect anonymity during peer review, ARR will take measures to prioritize reviews by reviewers who are not aware of the author identities. Authors are reminded that widely sharing the work will make it harder to recruit reviewers. (Venues will also [institute awards](https://www.aclweb.org/adminwiki/index.php/ACL_Conference_Awards_Policy#Additional_Award_Slots_for_Unpublicized_Work) for unpublicized work. These awards are decided by program committees, not ARR.) 10 | 11 | The new policy is now reflected in the ARR [call for papers](/cfp), [author page](/authors), [reviewing page](/reviewing), [review form](/reviewform), [reviewer tutorial](/reviewertutorial), and [guidelines for commitment venues](/organizers). 12 | -------------------------------------------------------------------------------- /_posts/2024-02-13-great-oct23-reviewers.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Thanking Great Reviewers from October 2023 4 | categories: [Admin] 5 | --- 6 | 7 | Thank you to all of the reviewers from the October 2023 cycle! Your effort was critical to getting high quality reviews back to authors on time. 8 | 9 | During the cycle, we asked Action Editors to indicate reviewers who did a particularly great job. We'd like to publicly acknowledge you all - thank you! 10 | 11 | Milind Agarwal, 12 | Sumeet Agarwal, 13 | Nader Akoury, 14 | Miguel A. Alonso, 15 | Ion Androutsopoulos, 16 | Rico Angell, 17 | Diego Antognini, 18 | Ekaterina Artemova, 19 | Ioana Baldini, 20 | Yu Bao, 21 | Lisa Beinborn, 22 | Gábor Berend, 23 | Aditya Bhargava, 24 | Laura Biester, 25 | Terra Blevins, 26 | Emanuela Boros, 27 | Ana Brassard, 28 | Sky CH-Wang, 29 | Pere-Lluís Huguet Cabot, 30 | Pengshan Cai, 31 | Rui Cao, 32 | Yixin Cao, 33 | Yekun Chai, 34 | Chung-Chi Chen, 35 | Guanyi Chen, 36 | Yi Chen, 37 | Cheng-Han Chiang, 38 | Patricia Chiril, 39 | Hyundong Justin Cho, 40 | Fenia Christopoulou, 41 | Iulia Maria Comsa, 42 | Sam Davidson, 43 | Björn Deiseroth, 44 | Daryna Dementieva, 45 | Yang Deng, 46 | Ritam Dutt, 47 | Mohamed Elgaar, 48 | Guy Emerson, 49 | Nils Feldhus, 50 | Besnik Fetahu, 51 | Anette Frank, 52 | Dayne Freitag, 53 | William Gantt, 54 | Vagrant Gautam, 55 | Ariel Gera, 56 | Thamme Gowda, 57 | Jialiang Guo, 58 | Zhen Guo, 59 | Richard Hahnloser, 60 | Chikara Hashimoto, 61 | Shreya Havaldar, 62 | Timothy J. Hazen, 63 | Zhengqi He, 64 | Maartje Ter Hoeve, 65 | Ari Holtzman, 66 | Seokhee Hong, 67 | Mark Hopkins, 68 | Tom Hosking, 69 | David M Howcroft, 70 | Kung-Hsiang Huang, 71 | Taichi Iki, 72 | Md Saiful Islam, 73 | Eugene Jang, 74 | Harsh Jhamtani, 75 | Pei Ke, 76 | Katherine A. Keith, 77 | Douwe Kiela, 78 | Tracy Holloway King, 79 | Svetlana Kiritchenko, 80 | Venelin Kovatchev, 81 | Saurabh Kulshreshtha, 82 | Jenny Kunz, 83 | Tatsuki Kuribayashi, 84 | Shuhei Kurita, 85 | Shuaiqi LIU, 86 | Tsz Kin Lam, 87 | Dongkyu Lee, 88 | Ji-Ung Lee, 89 | Colin Leong, 90 | Bryan Li, 91 | Dongyuan Li, 92 | Lucy H. Lin, 93 | Xudong Lin, 94 | Tal Linzen, 95 | Enrico Liscio, 96 | Aiwei Liu, 97 | Fenglin Liu, 98 | Lu Liu, 99 | Qian Liu, 100 | Kyle Lo, 101 | Yi Lu, 102 | Ziqiao Ma, 103 | Craig MacDonald, 104 | Sumit Madan, 105 | Mounica Maddela, 106 | Rahmad Mahendra, 107 | Mieradilijiang Maimaiti, 108 | Sandeep Mathias, 109 | R. Thomas McCoy, 110 | Niloofar Mireshghallah, 111 | Syrielle Montariol, 112 | Hyeonseok Moon, 113 | Luca Moroni, 114 | Maximilian Mozes, 115 | Diane Napolitano, 116 | Aurélie Névéol, 117 | Ansong Ni, 118 | Garrett Nicolai, 119 | Dmitry Nikolaev, 120 | Shu Okabe, 121 | Eda Okur, 122 | Peyman Passban, 123 | Sachin Pawar, 124 | Pavel Pecina, 125 | Massimo Piccardi, 126 | Francesco Piccinno, 127 | Tiago Pimentel, 128 | Joan Plepi, 129 | Linlu Qiu, 130 | Leonardo Ranaldi, 131 | Yanghui Rao, 132 | Vipul Kumar Rathore, 133 | Agha Ali Raza, 134 | Siyu Ren, 135 | Darcey Riley, 136 | Wiem Ben Rim, 137 | Anthony Rios, 138 | Rudolf Rosa, 139 | Domenic Rosati, 140 | Mukund Rungta, 141 | Yves Scherrer, 142 | David Schlangen, 143 | Sebastian Schuster, 144 | Amit Seker, 145 | Rico Sennrich, 146 | Ori Shapira, 147 | Qiang Sheng, 148 | Anthony Sicilia, 149 | Damien Sileo, 150 | Mohammad Golam Sohrab, 151 | Yixiao Song, 152 | Felix Stahlberg, 153 | Julius Steen, 154 | Ian Stewart, 155 | Shane Storks, 156 | Phillip Benjamin Ströbel, 157 | Hanbo Sun, 158 | Rui Sun, 159 | Makoto Takamoto, 160 | Ece Takmaz, 161 | Chris Thomas, 162 | Julien Tourille, 163 | Martin Tutek, 164 | Asahi Ushio, 165 | Sowmya Vajjala, 166 | Vincent Vandeghinste, 167 | Vijay Viswanathan, 168 | MinhDuc Vo, 169 | Henning Wachsmuth, 170 | Liang Wang, 171 | Ruoyao Wang, 172 | Zhaowei Wang, 173 | Leon Weber-Genzel, 174 | Orion Weller, 175 | Rodrigo Wilkens, 176 | Steven R. Wilson, 177 | Hongqiu Wu, 178 | Amelie Wuehrl, 179 | Kaige Xie, 180 | Pengyu Xu, 181 | Shuntaro Yada, 182 | Jing Nathan Yan, 183 | Chenghao Yang, 184 | Torsten Zesch, 185 | Chen Zhang, 186 | Haoran Ranran Zhang, 187 | Zixuan Zhang, 188 | Jie Zhao, 189 | Xiaoyan Zhao, 190 | Mingyang Zhou, 191 | Shengqi Zhu, 192 | Wanrong Zhu, 193 | Caleb Ziems 194 | 195 | -------------------------------------------------------------------------------- /_posts/2024-02-14-ethics-flagging-guidelines.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: ACL ARR Ethics Review Flagging Guidelines 4 | categories: [Admin] 5 | --- 6 | By Malihe Alikhani and Vinodkumar Prabhakaran 7 | 8 | Ethics reviews are a cornerstone of maintaining integrity and responsibility in academic research. As we navigate through a multitude of submissions for our upcoming conference proceedings, it's vital to ensure that our papers adhere to ethical standards, rules, and regulations. This guide aims to assist reviewers and action editors (AEs) in detecting papers that may need an in-depth ethics review, as well as preparing the reviewers to conduct effective and efficient ethics reviews, ensuring a high standard of compliance throughout our conference proceedings. 9 | 10 | # The Importance of Ethics Review 11 | Ethics reviews are not just a formality; they are critical to the integrity of academic discourse. In particular, the ethics reviews are meant to assess whether the submitted research exhibits any substantial ethical issues, that is, issues that pose an increased risk of harm outside the current norms of NLP or CL research. It is especially critical to uphold these standards as we engage in research that deals with sensitive data and/or controversial topics. As ethics reviewers, you play a pivotal role in safeguarding these standards. 12 | 13 | Before diving into the review process, familiarize yourself with the specific guidelines for ethics reviewing that ARR has drafted. Remember, an ethics review is not just about ticking boxes, nor is it a technical review; it's about engaging critically with the material and its implications. You can find more information about this and about the process in our previous blog posts: 14 | 15 | Guidelines for Ethics Reviewing: 16 | 17 | ACL code of conduct: 18 | 19 | # Guidelines for Flagging Papers for Ethics Review 20 | 21 | One of the critical steps in the ARR pipeline is the flagging of papers for ethics reviewing. This is done by the action editors during the early stages of the reviewing cycle, or by the reviewers during the deeper technical review they perform. They are required to provide a justification along with flagging any papers they deem to require an in-depth ethics review. 22 | 23 | This is an important step that routes papers to the ethics chairs which are then assigned for full in-depth ethics reviews after an initial assessment. While ethics reviews are essential in maintaining the integrity of research, it’s vital to discern when they are truly needed. Over-flagging not only burdens the review process but might also hinder valuable research from moving forward. 24 | 25 | Based on the past few cycles of ARR ethics review operationalization, we have identified a few patterns that result in false flags. We surface these recurring patterns here as we believe it will be helpful for reviewers and action editors to correctly identify the papers that should be flagged for deeper ethics reviewing, while also eliminating false flags stemming from misconceptions about the ethics reviewing. 26 | 27 | ## What to Avoid? 28 | 29 | 1. **No justification:** A huge majority of papers get flagged without the justification field being filled in, or left as “N/A”, “None”, “No ethical concerns” etc. This unnecessarily adds to the workload of the ethics chairs and reviewers. If you are flagging a paper for full ethics review, please try to give a clear and succinct justification so that ethics chairs can appropriately assign ethics reviewers for the paper. 30 | 2. **Flagging for Missing Section(s):** Another substantial number of papers get incorrectly flagged because of missing ethical considerations section, or missing limitations section, or incompletely/incorrectly filled Responsible AI checklist. While these issues may need to be flagged to the AE’s attention, and in some cases may justify desk rejection (in case of Limitations section for certain conferences), they do not justify a full ethics review. Adding an ethical considerations section is not mandatory; however if you believe there are specific ethical issues that warrant a full review, please flag the paper. 31 | 3. **Flagging for Copyright, Consent, Transparency:** While it is important to ensure that ARR process correctly flags submissions that may be in violation of copyright policies of datasets used, inadequate informed consent process, or transparency of presented artifacts, these issues in and of themselves do not justify a fuller ethics review. Instead, since you have already identified the issue, you can call out the authors in your review or meta review itself to address these issues. However, if there are aspects that require a deeper look, you can of course flag it for a full ethics review, outlining the concern. 32 | 33 | ## Common Misconceptions 34 | 35 | 1. **All Data-centric Papers require Ethics Review:** A common misstep is assuming that any paper discussing data usage needs a full ethics review. This is not the case. Instead, focus should be on potential misuse or unethical handling of data. A mere description of data usage does not automatically warrant an in-depth review. 36 | 2. **Use of Human Annotators require Ethics Review:** The involvement of human annotators in a study does not on its own justify an ethics review. However, if there are specific concerns about the human annotation step, such as potential exploitation or ethical lapses, these may warrant an in-depth ethics review. 37 | 3. **All Datasets on Sensitive Topics require Ethics Review:** Datasets that feature critical content, such as misinformation detection, might raise eyebrows due to their potential dual use. But again, the mere presence of such content does not automatically demand a full ethics review. If it involves engaging with specific marginalized communities (e.g., linguistic minorities) or if the content of the annotation involves potentially traumatizing data (e.g., involving hate speech or other gory concepts), then it may justify an in-depth ethics review to assess whether adequate safeguards were taken. In such cases, explain your reasons as justification while flagging for ethics review. 38 | 4. **All Papers on High-stakes Domains require Ethics Review:** While all of NLP research can arguably have downstream human impact, data/methodological innovations on high-stakes domains may have an immediate impact on users or communities. These include educational use cases such as automated grading scenarios, or medical use cases such as in psychotherapy, or legal use cases such as criminality prediction. These may also include challenges in de-identification in high-stakes domains such as healthcare. However, just because a paper focuses on such a high-stakes domain doesn’t by default necessitate a full ethics review. What matters is the context and application of the research. If regulations have been followed and necessary precautions taken, and these are detailed in the paper, then sensitive research areas can be explored without automatically qualifying for a full ethics review. 39 | 5. **All Papers on NLP ethics require Ethics Review:** A paper engaging with questions on NLP ethics and fairness in and of itself does not justify an in-depth ethics review. However, many of these papers do tend to engage with topics that are potentially sensitive and may sometimes need an in-depth review. But this determination should be based on such additional contexts, rather than merely for the topic being NLP ethics. 40 | 41 | -------------------------------------------------------------------------------- /_posts/2024-03-19-new-eic.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Anna Rogers joins the ARR as a new Editor-in-Chief! 4 | --- 5 | 6 | Anna Rogers is an Associate Professor in the Computer Science Department at the IT University of Copenhagen. 7 | Her main research area is Natural Language Processing, in particular analysis and evaluation of pre-trained language models. 8 | She is a veteran program chair of the 61st Annual Meeting of the Association for Computational Linguistics (ACL 2023), the lead author of the current ARR guidelines for reviewers, and contributor to the Responsible NLP Checklist. 9 | 10 | Welcome to the team Anna! 11 | -------------------------------------------------------------------------------- /_posts/2024-03-31-april-cycle.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: April 2024 Cycle Opens April 4th 4 | --- 5 | 6 | Submissions will open on April 4th for the April ARR Cycle. 7 | The site is available [here](https://openreview.net/group?id=aclweb.org/ACL/ARR/2024/April). 8 | 9 | As usual, the deadline is April 15th. 10 | We will be releasing further information about changes in this cycle shortly. 11 | -------------------------------------------------------------------------------- /_posts/2024-04-01-reviewing-workload-requirement.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Upcoming changes from April cycle - reviewing workload requirement 4 | --- 5 | 6 | As we continue to add incremental changes to our review process, and OpenReview (OR) infrastructure, we want to highlight an important change that will be implemented for the upcoming April cycle: a reviewing workload requirement. 7 | 8 | We are very happy to see the larger number of submissions coming due to our major conferences adopting ARR as their default reviewing mode. We hope this continues and would like to clarify that we’re always open to receiving submissions from anyone, especially newcomers to the field! 9 | 10 | However, with increasingly larger numbers of submissions, we face the need to implement a more robust approach to covering reviewing needs. Therefore, with the support of the ARR Board, we have decided to adopt a reviewing workload requirement for papers. 11 | This requirement has to be satisfied at the time of submission. 12 | 13 | What does this mean for authors? 14 | 15 | It means that during submission time, authors have to specify which person, among the coauthors in the current submission, will be covering reviewing for at least three papers and to certify that whoever has been specified as reviewer meets the following conditions: 16 | 17 | a) Has an updated OR profile[^1] (the person has provided their semantic scholar and an updated email address where they can receive OR messages). 18 | 19 | b) Has published at least three papers in our \*CL conferences[^2] in the last 5 years (note exceptions below). 20 | 21 | c) Once added as a reviewer, the person promptly completes their reviewer registration form for the cycle. 22 | 23 | Understandably, there will be several exceptions to this requirement, and we will be more than happy to accommodate papers falling under reasonable exceptions, including: 24 | 1. A paper whose authors are new to our community. 25 | 2. A paper whose authors have not met requirement **b)**. 26 | 3. A paper whose authors are already involved in the reviewing process in some capacity (as Area Chairs, as Senior Area Chairs, etc). 27 | 28 | Authors can indicate another reason to request an exception that will then be reviewed by the EiCs. 29 | 30 | Unless the paper falls under categories 1-3 above, or there is another valid reason to request an exception, the paper will be subject to desk rejection if authors fail to comply with the reviewing workload. This means we will verify that we have a complete OR profile and that those authors identified as reviewers have fulfilled their max load and expertise task. 31 | In the past, as per our CFP, all authors qualifying to review were automatically added to the reviewing pool. But we faced challenges with many of these authors, they either: (1) did not have a complete OR profile, and thus we were unable to assign them papers, or (2) never answered our email requests to specify max load/expertise. 32 | 33 | This new policy may seem too harsh to some. 34 | We would like to point out that helping out to cover reviewing for three papers is a low bar that doesn’t really cover the workload of having a paper complete a full review cycle. 35 | A full review cycle includes effort from the Senior Area Chair to check that the reviewing assignments are sensible, an Area Chair (AC) that will help monitor reviewers' progress, as well as help to engage them during author rebuttal. Then, the AC needs to review the paper, digest the reviews, the authors' response, and the reviewers' reply, if any, in order to then write a meta-review. Finally, there is another set of people who may engage with the paper during commitment. Our goal is that the review process is a bit less stressful by securing some fraction of the reviewing capacity needed during the submission process. Furthermore, the more people are in the reviewing pool, the better paper-reviewer matches can be secured, which means a better experience for reviewers and higher-quality feedback for the authors. 36 | 37 | As always, we welcome your feedback, send an email to editors@aclrollingreview.org. 38 | We will continue to review our processes with the goal of improving the overall reviewing experience for everyone, and increasing trust in the community. 39 | 40 | Footnotes: 41 | 42 | [^1]: Regarding the first requirement. To understand what an up to date OR profile looks like, see this example: Complete OR ProfileIn contrast, this example profile is missing all the key fields:Incomplete OR Profile 43 | 44 | [^2]: The term \*CL conferences is used loosely here. We include AACL/IJCNLP, ACL, CL, COLING, CONLL, EACL, EMNLP, NAACL, LREC, and TACL. We don't distinguish between short and long papers, and we do count Findings papers. 45 | -------------------------------------------------------------------------------- /_posts/2024-04-30-peer-review-data-collection.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: New Streamlined and More Comprehensive Peer-review Data Collection Protocol 4 | --- 5 | 6 | **\[tldr:\]** ARR is expanding its data collection to further insights. 7 | Authors can now consent to donate their paper reviews and meta data during paper submission, reviewers and ACs can consent to donate their data for the cycle. 8 | Consider donating your peer-review data! You can always revisit this decision as needed. 9 | 10 | As the number of submissions increases, our publication model is changing. 11 | However, we still do not know much about the path papers take, the decision processes of reviewers, etc. 12 | To do just that, we will start collecting data to better understand the submission cycle, gain new insights into review quality and reviewer behavior, and more. 13 | This new data will provide ample research opportunities. 14 | 15 | With the approval of the ACL Ethics Committee, ACL RR has launched a new peer-review data collection protocol during the April submission cycle. 16 | This new protocol builds on the initiative by [Gurevych, Kuznetsov and Dycke (2021)](https://arr-data.aclweb.org/resources/) and is a community-driven effort. 17 | Read more on this: [https://arr-data.aclweb.org/](https://arr-data.aclweb.org/ ) 18 | 19 | 20 | **Background** 21 | 22 | Frequent users of the ARR might remember our previous protocol for requesting data collection from authors and reviewers. 23 | That protocol approached authors to donate their data only after accept/reject decisions have been made. 24 | The details can be found here: https://aclrollingreview.org/datacollection. 25 | As has been documented in the publications using this data \[1,2\], the resulting data is skewed towards papers that received a positive outcome during the peer review process. 26 | Recognizing the need to motivate more research into the design of NLP to support peer-review, we propose to streamline the process of data collection by asking authors to consent to sharing their data during paper submission. 27 | In addition, we expand the content being collected to now request metadata donation. 28 | 29 | 30 | **What is new in this Protocol?** 31 | 32 | Authors now decide on data donation during submission and can revise this decision throughout the reviewing process. 33 | Reviewers will now also be requested to donate their metadata (i.e. scores), in addition to the content of the reviews, and have the ability to specify which content they want to donate. 34 | Meta-reviewers will be asked to sign a license agreement to donate their meta-reviews of a cycle. The collection of meta-reviews is a new addition to the protocol. 35 | For additional information on the data collection protocol, as well as the changes made in comparison to the earlier protocol, see the [Protocol page](https://arr-data.aclweb.org/protocol). 36 | 37 | \[1\] Yes-Yes-Yes: Proactive Data Collection for ACL Rolling Review and Beyond 38 | Nils Dycke, Ilia Kuznetsov, Iryna Gurevych (2022). Findings of EMNLP. 39 | 40 | \[2\] NLPeer: A Unified Resource for the Computational Study of Peer Review 41 | Nils Dycke, Ilia Kuznetsov, Iryna Gurevych (2023). ACL-2023. 42 | -------------------------------------------------------------------------------- /_posts/2024-06-28-ARR-seeks-Editors-in-Chief.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: ACL Rolling Review is inviting nominations for several Editors in Chief positions 4 | --- 5 | 6 | The ACL Rolling Review (ARR) initiative is seeking nominations (self-nominations are permitted and encouraged), for several co-Editors-in-Chief (henceforth “EiCs”, with the “co-” understood) positions, serving a 2.5-year term[^1]. Successful candidates should be able to start at the end of summer or as early as possible. 7 | 8 | ARR is increasingly used as the main submission channel for *CL conferences. In the last four cycles alone, ARR has received and managed, close to 10,000 submissions within 2 month cycles. To continue supporting these large cycles, there is a pressing need to scale the EiCs team managing the bi-monthly cycles. EiCs rotate to lead reviewing cycles. However, the entire team coordinates and oversees the following activities: 9 | 10 | * Recruit and manage (senior) area chairs ((S)ACs) and reviewers 11 | * Coordinate with conference program chairs, in principle becoming guest PCs during cycles tied to *CL conferences 12 | * Oversee review forms and CFPs 13 | * Review and update submission tracks 14 | * Make final desk reject decisions on the advice of SACs 15 | * Provide reports to the ACL Exec and ARR board 16 | * General work on the ARR system hosted on OpenReview 17 | * Coordinate with the rest of the ARR team, including support, communications, etc. 18 | * Assess and propose reviewing policies that are submitted for consideration to the ARR board and/or the ACL exec 19 | 20 | The ideal candidate would have a strong publication record across more than one NLP research area and a solid record as an outstanding member of the*CL community. Experience serving in ARR as SAC and/or AC is highly valued, as is having served as a past PC in one of our top *CL conferences and/or as an EiC for one of our *CL journals. At a minimum, candidates should have at least some experience in managing submission and reviewing processes, for example, as workshop co-organizers. 21 | 22 | Being an ARR Editor-in-Chief is a major, ongoing time commitment requiring, on average, 2-3 hours a week but absorbing a significant amount of time during peak cycle phases. When an EiC is leading a cycle, it is expected that they can dedicate a large portion of their time during these peak critical times. However, contributing to our community this way is also a very rewarding experience. We are aware that the ARR process needs to continue to improve; we are looking for candidates with ideas on improving what we do who are also eager to dedicate their energy and time to help us implement better reviewing practices and workflows. We welcome disrupters! 23 | 24 | ARR is a diverse team, and we are determined to keep it that way. This includes researchers at different career stages, although we recognize that due to the requirements we specified, most junior researchers might not yet be eligible. 25 | 26 | **Nomination procedure** 27 | 28 | Nominations should be submitted via email to ARR board and EiC member, Thamar Solorio, [thamar.solorio@mbzuai.ac.ae](mailto:thamar.solorio@mbzuai.ac.ae), with a Cc: to Mausam, [mausam@cs.washington.edu](mailto:mausam@cs.washington.edu) by July 31st, 2024, subject line “Nomination for ARR co-EiC”, and should contain the following information: 29 | 30 | * Name, email address, and website of the nominee 31 | 32 | If not a self-nomination, confirmation that the nominee is willing to serve (a dated excerpt of an email from the nominee suffices); if a self-nomination, a statement affirming willingness to serve 33 | 34 | Statement of nominee’s goals/vision for ARR and/or relevant prior experience and/or reasons for interest in the position. The length of the statement is up to the nominator, but 1 paragraph may suffice, and more than 4 paragraphs are not anticipated to be necessary. 35 | 36 | Nominations will be acknowledged by reply email; should acknowledgment not be received within 3 days, please resend your email. 37 | 38 | The ARR board will evaluate the nominations and make decisions in coordination with the existing ARR EiC team, after which the search committee chair will contact the selected nominee via email. ARR board members are: Yuki Arase, Julia Hockenmeier, Smaranda Muersan, and EiC Thamar Solorio. 39 | 40 | [^1]:The last 6 months are not considered as full on EiCs but more of a transitions period to support training of new EiCs. 41 | 42 | -------------------------------------------------------------------------------- /_posts/2024-09-18-editorial-assistant.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: ARR Job Ads -- Editorial Assistant 4 | --- 5 | 6 | The Association for Computational Linguistics Rolling Review (ARR) is seeking an editorial assistant to support the regular operations of ARR bi-monthly reviewing cycles. This position sits at the heart of the peer-review team that handles our major ACL conferences. The goal of this role is to support the editors-in-chief by managing some of the standard tasks involved in the peer-review process, such as supporting the communication between EiCs, the Senior Area Chairs, and the rest of the research community. 7 | 8 | POSITION SUMMARY 9 | 10 | The editorial assistant will: 11 | 12 | * Support the regular activities of the peer-review process in collaboration with the entire ARR team and conference PCs. 13 | * Support the communication of the ARR with the rest of the community. This includes reviewing and improving existing information about the review process (website and template emails/answers). 14 | * Handle standard tasks such as recognition letters, awards, and straightforward desk rejections. 15 | * Support standard tasks of the review process, such as sending assignments and reminder emails. 16 | * Provide support to handle reviewing situations that need special attention. 17 | * Work closely with the ARR team and conference PCs to provide a smooth transition between the review and commitment stages. 18 | 19 | REQUIRED SKILLS AND KNOWLEDGE 20 | 21 | Candidates are required to have: 22 | 23 | * Strong organizational and time-management skills 24 | * Demonstrated attention to detail 25 | * Strong oral and written communication skills 26 | * Knowledge and experience interacting with different software and APIs 27 | * Some awareness of the peer-review process in scientific communities 28 | * Salary range: $12,000-$15,000/year 29 | 30 | The work is done remotely; interaction with the ARR team and conference PC will be via online meetings. 31 | 32 | ADDITIONAL INFORMATION 33 | 34 | The workflow manager works under the direct supervision of the ARR Editors-in-Chief. 35 | 36 | The start date of this position is November 1, 2024. 37 | 38 | The workload varies somewhat over the course of the bi-monthly cycles but averages approximately 20 hours weekly. 39 | 40 | To apply, submit a cover letter, CV/resume, and three references. 41 | 42 | If you have questions about this position or to submit an application, contact us via email at: editors@aclrollingreview.org [1] and thamar.solorio@mbzuai.ac.ae [2]. 43 | -------------------------------------------------------------------------------- /_posts/2024-09-18-workflow-manager.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: ARR Job Ads -- Workflow Manager 4 | --- 5 | 6 | The Association for Computational Linguistics Rolling Review (ARR) seeks a workflow manager to support the regular operations of ARR bi-monthly reviewing cycles. This position sits at the heart of the peer-review infrastructure that handles our major ACL conferences. The goal of this role is to support the customization of the infrastructure used by ARR, which currently is OpenReview. 7 | 8 | POSITION SUMMARY 9 | 10 | The workflow manager will: 11 | 12 | * Lead the implementation of data manipulation from OpenReview required to fit the needs of the ARR peer-review process. 13 | * Collaborate with the ARR team (editors in chief, CTO, and tech support) on identifying, designing, and implementing software solutions to streamline the review process in ARR. 14 | * Work closely with the ARR team and conference PCs to provide a smooth transition between the review and commitment stages. 15 | * Collaborate with the ARR team and support the implementation of data-driven approaches to improve the review process at ARR. 16 | * Support the creation of varied types of reports about the review process for the ARR team and conference PCs. 17 | * Provide troubleshooting and tech support regarding the OpenReview infrastructure to the ARR team. 18 | * Salary range: $30,000-$40,000/year 19 | 20 | REQUIRED SKILLS AND KNOWLEDGE 21 | 22 | Candidates are required to have: 23 | 24 | * Solid software engineering training 25 | * Experience integrating solutions into legacy software 26 | * Solid programming skills in Python, javascript, HTML 27 | * Strong organizational and time-management skills 28 | * Demonstrated attention to detail 29 | * Strong communication skills 30 | * Knowledge and experience with software versioning systems 31 | 32 | The work is done remotely; interaction with the ARR team and conference PC will be via online meetings. 33 | 34 | ADDITIONAL INFORMATION 35 | 36 | The workflow manager works under the direct supervision of the ARR Editors-in-Chief. 37 | 38 | The desired start date of this position is November 1, 2024. 39 | 40 | The workload varies somewhat throughout the bi-monthly cycles but averages approximately 20 hours weekly. 41 | 42 | To apply, submit a cover letter, CV/resume, and three references. 43 | 44 | If you have questions about this position or to apply, contact us via 45 | 46 | email at: editors@aclrollingreview.org [1] and thamar.solorio@mbzuai.ac.ae [2]. 47 | -------------------------------------------------------------------------------- /_posts/2025-02-10-reviewing-workload-adjustment.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: post 3 | title: Changes to reviewing workload requirement starting from February cycle 4 | --- 5 | 6 | We want to highlight two important changes we are implementing for the February cycle: (1) a minimum review load requirement for those volunteer reviewers who are nominated for multiple submissions, and (2) the possibility of nominating an experienced colleague as a reviewing volunteer for a submission. 7 | 8 | We introduced a reviewing workload requirement for submissions in April 2024 ([link](https://aclrollingreview.org/reviewing-workload-requirement/)). This requirement has helped to significantly enlarge the reviewer pool, which is needed for major cycles that receive a large number of submissions. We thank all of you who have adhered to this requirement. 9 | 10 | As we continue to fine-tune our review process, we would like to implement the following two important changes: 11 | 12 | (1) If a person is nominated as the reviewing volunteer for multiple submissions, we now require this volunteer reviewer to provide **4 reviews per submission**, with a cap of 16 reviews. For example, if a volunteer reviewer is nominated for 3 submissions, then we expect this reviewer to provide a minimum of 12 reviews; if a volunteer reviewer is nominated for 4 or more submissions, then the reviewer is expected to provide 16 reviews. We hope that by introducing this change, authors submitting multiple papers can share a larger reviewing load, which we deem a fair practice for all authors. 13 | 14 | **Clarification:** If the same senior author has more than 4 submissions, then authors must coordinate so that the team overall contributes 4 reviews for each of their papers, but no one reviewer is responsible for more than 4 papers. If a qualified author already serves as an ARR AC in this cycle, this accounts for up to 4 papers they submit as authors. Serving as a SAC accounts for up to 6 papers as an author, and serving as a PC - up to 10. 15 | 16 | (2) In the past there have been submissions with no authors qualified as reviewers based on our criteria. We are happy to announce that we will now relax the requirement to allow authors to nominate an experienced colleague who is not an author of the submission to be the reviewing volunteer of that submission. This non-author reviewing volunteer is also expected to provide 4 reviews per submission for which the volunteer is nominated for. 17 | 18 | The above changes can also be found in the OpenReview submission form, as shown below. Please also note that we reserve the right to desk reject papers that do not contribute to reviewing effort without sufficient justification. 19 | 20 | ![img](https://lh7-rt.googleusercontent.com/docsz/AD_4nXc1VPAhSSO3VD3iG5NTthFOCNxTE-yQpuPRx7O-CNZRnii1tv8Gdjlryy6EwTulk93NWS-TLoHD38fHjCW-vtFA-fxEIQ9aPoGVwqiqwUccy59Kd1FgJmoxd11amS_N_dwD__Gh3g?key=5nFmQJ_ZS-2l9rXmQOYTre6Z) 21 | 22 | As always, we welcome your feedback. Please send an email to editors@aclrollingreview.org with your comments and suggestions. We will continue to review our processes with the goal of improving the overall reviewing experience and increasing trust in the community. 23 | -------------------------------------------------------------------------------- /_sass/_bluesky-icon.scss: -------------------------------------------------------------------------------- 1 | // Bluesky icon styles 2 | .svg-icon.bluesky { 3 | background-image: url('/images/icons/bluesky.svg.png'); 4 | background-size: 32px 32px; 5 | background-position: center; 6 | background-repeat: no-repeat; 7 | opacity: 0.9; 8 | transition: opacity 0.3s ease; 9 | border-radius: 50%; 10 | 11 | &:hover { 12 | opacity: 1; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /_sass/_darcula.scss: -------------------------------------------------------------------------------- 1 | .highlight { 2 | background-color: #1e1c3f; 3 | padding: 7px 7px 7px 10px; 4 | overflow: auto; 5 | font-size: 95%; 6 | } 7 | 8 | code { 9 | font-family:'Bitstream Vera Sans Mono','Courier', monospace; 10 | color: #d8d8d8; 11 | } 12 | 13 | .highlighter-rouge { 14 | background-color: #FAFAFA; 15 | color: #FF554A; 16 | font-size: 16px; 17 | } 18 | 19 | .highlight .hll { background-color: #f1fa8c } 20 | .highlight { background: #282a36; color: #f8f8f2 } 21 | .highlight .c { color: #6272a4 } /* Comment */ 22 | .highlight .err { color: #f8f8f2 } /* Error */ 23 | .highlight .g { color: #f8f8f2 } /* Generic */ 24 | .highlight .k { color: #ff79c6 } /* Keyword */ 25 | .highlight .l { color: #f8f8f2 } /* Literal */ 26 | .highlight .n { color: #f8f8f2 } /* Name */ 27 | .highlight .o { color: #ff79c6 } /* Operator */ 28 | .highlight .x { color: #f8f8f2 } /* Other */ 29 | .highlight .p { color: #f8f8f2 } /* Punctuation */ 30 | .highlight .ch { color: #6272a4 } /* Comment.Hashbang */ 31 | .highlight .cm { color: #6272a4 } /* Comment.Multiline */ 32 | .highlight .cp { color: #ff79c6 } /* Comment.Preproc */ 33 | .highlight .cpf { color: #6272a4 } /* Comment.PreprocFile */ 34 | .highlight .c1 { color: #6272a4 } /* Comment.Single */ 35 | .highlight .cs { color: #6272a4 } /* Comment.Special */ 36 | .highlight .gd { color: #8b080b } /* Generic.Deleted */ 37 | .highlight .ge { color: #f8f8f2; text-decoration: underline } /* Generic.Emph */ 38 | .highlight .gr { color: #f8f8f2 } /* Generic.Error */ 39 | .highlight .gh { color: #f8f8f2; font-weight: bold } /* Generic.Heading */ 40 | .highlight .gi { color: #f8f8f2; font-weight: bold } /* Generic.Inserted */ 41 | .highlight .go { color: #44475a } /* Generic.Output */ 42 | .highlight .gp { color: #f8f8f2 } /* Generic.Prompt */ 43 | .highlight .gs { color: #f8f8f2 } /* Generic.Strong */ 44 | .highlight .gu { color: #f8f8f2; font-weight: bold } /* Generic.Subheading */ 45 | .highlight .gt { color: #f8f8f2 } /* Generic.Traceback */ 46 | .highlight .kc { color: #ff79c6 } /* Keyword.Constant */ 47 | .highlight .kd { color: #8be9fd; font-style: italic } /* Keyword.Declaration */ 48 | .highlight .kn { color: #ff79c6 } /* Keyword.Namespace */ 49 | .highlight .kp { color: #ff79c6 } /* Keyword.Pseudo */ 50 | .highlight .kr { color: #ff79c6 } /* Keyword.Reserved */ 51 | .highlight .kt { color: #8be9fd } /* Keyword.Type */ 52 | .highlight .ld { color: #f8f8f2 } /* Literal.Date */ 53 | .highlight .m { color: #bd93f9 } /* Literal.Number */ 54 | .highlight .s { color: #f1fa8c } /* Literal.String */ 55 | .highlight .na { color: #50fa7b } /* Name.Attribute */ 56 | .highlight .nb { color: #8be9fd; font-style: italic } /* Name.Builtin */ 57 | .highlight .nc { color: #50fa7b } /* Name.Class */ 58 | .highlight .no { color: #f8f8f2 } /* Name.Constant */ 59 | .highlight .nd { color: #f8f8f2 } /* Name.Decorator */ 60 | .highlight .ni { color: #f8f8f2 } /* Name.Entity */ 61 | .highlight .ne { color: #f8f8f2 } /* Name.Exception */ 62 | .highlight .nf { color: #50fa7b } /* Name.Function */ 63 | .highlight .nl { color: #8be9fd; font-style: italic } /* Name.Label */ 64 | .highlight .nn { color: #f8f8f2 } /* Name.Namespace */ 65 | .highlight .nx { color: #f8f8f2 } /* Name.Other */ 66 | .highlight .py { color: #f8f8f2 } /* Name.Property */ 67 | .highlight .nt { color: #ff79c6 } /* Name.Tag */ 68 | .highlight .nv { color: #8be9fd; font-style: italic } /* Name.Variable */ 69 | .highlight .ow { color: #ff79c6 } /* Operator.Word */ 70 | .highlight .w { color: #f8f8f2 } /* Text.Whitespace */ 71 | .highlight .mb { color: #bd93f9 } /* Literal.Number.Bin */ 72 | .highlight .mf { color: #bd93f9 } /* Literal.Number.Float */ 73 | .highlight .mh { color: #bd93f9 } /* Literal.Number.Hex */ 74 | .highlight .mi { color: #bd93f9 } /* Literal.Number.Integer */ 75 | .highlight .mo { color: #bd93f9 } /* Literal.Number.Oct */ 76 | .highlight .sa { color: #f1fa8c } /* Literal.String.Affix */ 77 | .highlight .sb { color: #f1fa8c } /* Literal.String.Backtick */ 78 | .highlight .sc { color: #f1fa8c } /* Literal.String.Char */ 79 | .highlight .dl { color: #f1fa8c } /* Literal.String.Delimiter */ 80 | .highlight .sd { color: #f1fa8c } /* Literal.String.Doc */ 81 | .highlight .s2 { color: #f1fa8c } /* Literal.String.Double */ 82 | .highlight .se { color: #f1fa8c } /* Literal.String.Escape */ 83 | .highlight .sh { color: #f1fa8c } /* Literal.String.Heredoc */ 84 | .highlight .si { color: #f1fa8c } /* Literal.String.Interpol */ 85 | .highlight .sx { color: #f1fa8c } /* Literal.String.Other */ 86 | .highlight .sr { color: #f1fa8c } /* Literal.String.Regex */ 87 | .highlight .s1 { color: #f1fa8c } /* Literal.String.Single */ 88 | .highlight .ss { color: #f1fa8c } /* Literal.String.Symbol */ 89 | .highlight .bp { color: #f8f8f2; font-style: italic } /* Name.Builtin.Pseudo */ 90 | .highlight .fm { color: #50fa7b } /* Name.Function.Magic */ 91 | .highlight .vc { color: #8be9fd; font-style: italic } /* Name.Variable.Class */ 92 | .highlight .vg { color: #8be9fd; font-style: italic } /* Name.Variable.Global */ 93 | .highlight .vi { color: #8be9fd; font-style: italic } /* Name.Variable.Instance */ 94 | .highlight .vm { color: #8be9fd; font-style: italic } /* Name.Variable.Magic */ 95 | .highlight .il { color: #bd93f9 } /* Literal.Number.Integer.Long */ 96 | -------------------------------------------------------------------------------- /_sass/_highlights.scss: -------------------------------------------------------------------------------- 1 | 2 | .highlight { 3 | background-color: #1e1c3f; 4 | padding: 7px 7px 7px 10px; 5 | overflow: auto; 6 | font-size: 90%; 7 | } 8 | 9 | code { 10 | font-family:'Bitstream Vera Sans Mono','Courier', monospace; 11 | color: #d8d8d8; 12 | } 13 | 14 | .highlighter-rouge { 15 | background-color: #FAFAFA; 16 | color: #FF554A; 17 | } 18 | 19 | .highlight .c { color: #586E75 } /* Comment */ 20 | .highlight .err { color: #93A1A1 } /* Error */ 21 | .highlight .g { color: #93A1A1 } /* Generic */ 22 | .highlight .k { color: #859900 } /* Keyword */ 23 | .highlight .l { color: #93A1A1 } /* Literal */ 24 | .highlight .n { color: #93A1A1 } /* Name */ 25 | .highlight .o { color: #859900 } /* Operator */ 26 | .highlight .x { color: #CB4B16 } /* Other */ 27 | .highlight .p { color: #93A1A1 } /* Punctuation */ 28 | .highlight .cm { color: #586E75 } /* Comment.Multiline */ 29 | .highlight .cp { color: #859900 } /* Comment.Preproc */ 30 | .highlight .c1 { color: #586E75 } /* Comment.Single */ 31 | .highlight .cs { color: #859900 } /* Comment.Special */ 32 | .highlight .gd { color: #2AA198 } /* Generic.Deleted */ 33 | .highlight .ge { color: #93A1A1; font-style: italic } /* Generic.Emph */ 34 | .highlight .gr { color: #DC322F } /* Generic.Error */ 35 | .highlight .gh { color: #CB4B16 } /* Generic.Heading */ 36 | .highlight .gi { color: #859900 } /* Generic.Inserted */ 37 | .highlight .go { color: #93A1A1 } /* Generic.Output */ 38 | .highlight .gp { color: #93A1A1 } /* Generic.Prompt */ 39 | .highlight .gs { color: #93A1A1; font-weight: bold } /* Generic.Strong */ 40 | .highlight .gu { color: #CB4B16 } /* Generic.Subheading */ 41 | .highlight .gt { color: #93A1A1 } /* Generic.Traceback */ 42 | .highlight .kc { color: #CB4B16 } /* Keyword.Constant */ 43 | .highlight .kd { color: #268BD2 } /* Keyword.Declaration */ 44 | .highlight .kn { color: #859900 } /* Keyword.Namespace */ 45 | .highlight .kp { color: #859900 } /* Keyword.Pseudo */ 46 | .highlight .kr { color: #268BD2 } /* Keyword.Reserved */ 47 | .highlight .kt { color: #DC322F } /* Keyword.Type */ 48 | .highlight .ld { color: #93A1A1 } /* Literal.Date */ 49 | .highlight .m { color: #2AA198 } /* Literal.Number */ 50 | .highlight .s { color: #2AA198 } /* Literal.String */ 51 | .highlight .na { color: #93A1A1 } /* Name.Attribute */ 52 | .highlight .nb { color: #E91E63 } /* Name.Builtin */ 53 | .highlight .nc { color: #268BD2 } /* Name.Class */ 54 | .highlight .no { color: #CB4B16 } /* Name.Constant */ 55 | .highlight .nd { color: #268BD2 } /* Name.Decorator */ 56 | .highlight .ni { color: #CB4B16 } /* Name.Entity */ 57 | .highlight .ne { color: #CB4B16 } /* Name.Exception */ 58 | .highlight .nf { color: #268BD2 } /* Name.Function */ 59 | .highlight .nl { color: #93A1A1 } /* Name.Label */ 60 | .highlight .nn { color: #93A1A1 } /* Name.Namespace */ 61 | .highlight .nx { color: #B39DDB } /* Name.Other */ 62 | .highlight .py { color: #93A1A1 } /* Name.Property */ 63 | .highlight .nt { color: #268BD2 } /* Name.Tag */ 64 | .highlight .nv { color: #268BD2 } /* Name.Variable */ 65 | .highlight .ow { color: #859900 } /* Operator.Word */ 66 | .highlight .w { color: #93A1A1 } /* Text.Whitespace */ 67 | .highlight .mf { color: #2AA198 } /* Literal.Number.Float */ 68 | .highlight .mh { color: #2AA198 } /* Literal.Number.Hex */ 69 | .highlight .mi { color: #2AA198 } /* Literal.Number.Integer */ 70 | .highlight .mo { color: #2AA198 } /* Literal.Number.Oct */ 71 | .highlight .sb { color: #586E75 } /* Literal.String.Backtick */ 72 | .highlight .sc { color: #2AA198 } /* Literal.String.Char */ 73 | .highlight .sd { color: #93A1A1 } /* Literal.String.Doc */ 74 | .highlight .s2 { color: #2AA198 } /* Literal.String.Double */ 75 | .highlight .se { color: #CB4B16 } /* Literal.String.Escape */ 76 | .highlight .sh { color: #93A1A1 } /* Literal.String.Heredoc */ 77 | .highlight .si { color: #2AA198 } /* Literal.String.Interpol */ 78 | .highlight .sx { color: #2AA198 } /* Literal.String.Other */ 79 | .highlight .sr { color: #DC322F } /* Literal.String.Regex */ 80 | .highlight .s1 { color: #2AA198 } /* Literal.String.Single */ 81 | .highlight .ss { color: #2AA198 } /* Literal.String.Symbol */ 82 | .highlight .bp { color: #268BD2 } /* Name.Builtin.Pseudo */ 83 | .highlight .vc { color: #268BD2 } /* Name.Variable.Class */ 84 | .highlight .vg { color: #268BD2 } /* Name.Variable.Global */ 85 | .highlight .vi { color: #268BD2 } /* Name.Variable.Instance */ 86 | .highlight .il { color: #2AA198 } /* Literal.Number.Integer.Long */ 87 | -------------------------------------------------------------------------------- /_sass/_reset.scss: -------------------------------------------------------------------------------- 1 | 2 | /***************/ 3 | /* MEYER RESET */ 4 | /***************/ 5 | 6 | html, body, div, span, applet, object, iframe, 7 | h1, h2, h3, h4, h5, h6, p, blockquote, pre, 8 | a, abbr, acronym, address, big, cite, code, 9 | del, dfn, em, img, ins, kbd, q, s, samp, 10 | small, strike, strong, sub, sup, tt, var, 11 | b, u, i, center, 12 | dl, dt, dd, ol, ul, li, 13 | fieldset, form, label, legend, 14 | table, caption, tbody, tfoot, thead, tr, th, td, 15 | article, aside, canvas, details, embed, 16 | figure, figcaption, footer, header, hgroup, 17 | menu, nav, output, ruby, section, summary, 18 | time, mark, audio, video { 19 | margin: 0; 20 | padding: 0; 21 | border: 0; 22 | font-size: 100%; 23 | font: inherit; 24 | vertical-align: baseline; 25 | } 26 | // HTML5 display-role reset for older browsers 27 | article, aside, details, figcaption, figure, 28 | footer, header, hgroup, menu, nav, section { 29 | display: block; 30 | } 31 | body { 32 | line-height: 1; 33 | } 34 | ol, ul { 35 | list-style: none; 36 | } 37 | blockquote, q { 38 | quotes: none; 39 | } 40 | blockquote:before, blockquote:after, 41 | q:before, q:after { 42 | content: ''; 43 | content: none; 44 | } 45 | table { 46 | border-collapse: collapse; 47 | border-spacing: 0; 48 | } 49 | // Apply a natural box layout model to all elements 50 | // from: http://www.paulirish.com/2012/box-sizing-border-box-ftw/ 51 | *, *:before, *:after { 52 | -moz-box-sizing: border-box; -webkit-box-sizing: border-box; box-sizing: border-box; 53 | } 54 | -------------------------------------------------------------------------------- /_sass/_variables.scss: -------------------------------------------------------------------------------- 1 | 2 | // 3 | // VARIABLES 4 | // 5 | 6 | // Colors 7 | $blue: #4183C4; 8 | 9 | // Grays 10 | $black: #000; 11 | $darkerGray: #222; 12 | $darkGray: #333; 13 | $gray: #666; 14 | $lightGray: #eee; 15 | $white: #fff; 16 | 17 | // Font stacks 18 | $helvetica: Helvetica, Arial, sans-serif; 19 | $helveticaNeue: "Helvetica Neue", Helvetica, Arial, sans-serif; 20 | $georgia: Georgia, serif; 21 | 22 | // Mobile breakpoints 23 | @mixin mobile { 24 | @media screen and (max-width: 640px) { 25 | @content; 26 | } 27 | } -------------------------------------------------------------------------------- /_sass/admonition.scss: -------------------------------------------------------------------------------- 1 | // source for this and liquid template: https://www.adamsdesk.com/posts/admonitions-jekyll/ 2 | $primary-color: #fc0; 3 | $primary-bgcolor: rgba(55.59%, 44.41%, 0%, .4); 4 | $admonitions: 5 | //class (type), icon filename, icon/border color, title bg color 6 | ('note', 'pen-solid.svg') 7 | ('abstract', 'align-left-solid.svg') 8 | ('info', 'circle-info-solid.svg', '#00b0ff', '#293d52') 9 | ('tip', 'fire-solid.svg', '#ff9100', '#433a38') 10 | ('success', 'check-circle-solid.svg', '#00c953', '#294040') 11 | ('question', 'circle-question-solid.svg', '#00b8d4', '#293e4e') 12 | ('warning', 'triangle-exclamation-solid.svg', '#ff9100', '#564b3c') 13 | ('failure', 'times-circle-solid.svg', '#ff5252', '#564444') 14 | ('danger', 'bolt-solid.svg', '#ff1744', '#563e43') 15 | ('bug', 'bug-solid.svg', '#f50057', '#553c45') 16 | ('example', 'list-solid.svg', '#9e9e9e', '#4c4c4c') 17 | ('quote', 'quote-right-solid.svg', '#9e9e9e', '#4c4c4c') 18 | ; 19 | .admonition { 20 | margin: 1.5625em 0; 21 | overflow: hidden; 22 | color: #808080; 23 | page-break-inside: avoid; 24 | background-color: #cecaca; 25 | border-left: .3rem solid $primary-color; 26 | border-radius: .1rem; 27 | } 28 | .admonition p { 29 | padding: 0 1rem; 30 | } 31 | .admonition .admonition-title { 32 | color: #ddd; 33 | background-color: $primary-bgcolor; 34 | font-weight: 700; 35 | line-height: 3rem; 36 | } 37 | .admonition-title::before { 38 | margin-right: .5rem; 39 | width: 1.2rem; 40 | height: 1.2rem; 41 | display: inline-block; 42 | content: ''; 43 | -webkit-mask-size: cover; 44 | mask-size: cover; 45 | background-color: $primary-color; 46 | vertical-align: text-bottom; 47 | } 48 | @each $name, $icon, $icon-color, $title-color in $admonitions { 49 | @if $icon-color { 50 | .admonition.#{$name} { 51 | border-left-color: #{$icon-color}; 52 | } 53 | } 54 | @if $title-color { 55 | .admonition.#{$name} .admonition-title { 56 | background-color: #{$title-color}; 57 | } 58 | } 59 | .admonition.#{$name} .admonition-title::before { 60 | -webkit-mask: url("/images/icons/#{$icon}") no-repeat 50% 50%; 61 | mask: url("/images/icons/#{$icon}") no-repeat 50% 50%; 62 | @if $icon-color { 63 | background-color: #{$icon-color}; 64 | } 65 | } 66 | } -------------------------------------------------------------------------------- /about.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "About" 4 | permalink: /about 5 | search_index: true 6 | --- 7 | 8 | A peer review platform for the [Association for Computational Linguistics](https://aclweb.org), originally created on the basis of this [proposal](https://www.aclweb.org/adminwiki/index.php?title=ACL_Rolling_Review_Proposal). 9 | 10 | Follow us on [Twitter](https://twitter.com/ReviewAcl) and [Facebook](https://www.facebook.com/aclrollingreview). 11 | 12 | 13 | Acknowledgments 14 | 15 | Template from [Amit Merchant](https://github.com/amitmerchant1990/reverie) 16 | 17 | Logo from [Ozan Irsoy](https://github.com/oir) 18 | 19 | -------------------------------------------------------------------------------- /actioneditorreviewform.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Action Editor Form" 4 | permalink: /actioneditorform 5 | --- 6 | 7 | ### Paper Summary 8 | 9 | Describe what this paper is about. This should help SACs at publication venues understand the sessions into which the paper might fit. 10 | 11 | ### Summary of Strengths 12 | 13 | What are the major strengths of this paper? This should help SACs at publication venues understand why they might want to accept the paper. 14 | 15 | ### Summary of Weaknesses 16 | 17 | What revisions could the authors make to the research and the paper that would improve it? This should help authors understand the reviews in context, and help them plan any future resubmission. 18 | 19 | ### Overall Assessment 20 | 21 | What is your overall assessment of this paper? 22 | 23 | * 5, The paper is largely complete and there are no clear points of revision 24 | * 4, There are minor points that may be revised 25 | * 3, There are major points that may be revised 26 | * 2, The paper would need significant revisions to reach a publishable state 27 | * 1, Even after revisions, the paper is not likely to be publishable at an *ACL venue 28 | 29 | ### Suggested Venue(s) 30 | 31 | You are encouraged to suggest conferences or workshops that would be suitable for this paper. 32 | 33 | ### Best Paper 34 | 35 | Could the camera-ready version of this paper merit consideration for an 'outstanding paper' award (up to 2.5% of accepted papers at *ACL conferences will be recognized in this way)? Outstanding papers should be either fascinating, controversial, surprising, impressive, or potentially field-changing. Awards will be decided based on the camera-ready version of the paper. 36 | 37 | * Yes 38 | * Maybe 39 | * No 40 | 41 | If the answer is Yes or Maybe, please justify your decision: 42 | 43 | ### Ethical Concerns 44 | 45 | Independent of your judgement of the quality of the work, please review the ACL code of ethics and list any ethical concerns related to this paper. 46 | 47 | Should this paper be sent for an in-depth ethics review? We have a small ethics committee that can specially review very challenging papers when it comes to ethical issues. If this seems to be such a paper, then please explain why here, and we will try to ensure that it receives a separate review. 48 | 49 | ## Review Feednack 50 | 51 | Please list the ids of all reviewers who went beyond expectations in terms of providing informative and constructive reviews and discussion. For example: jAxb, zZac 52 | 53 | Please list the ids of all reviewers whose reviews did not meet expectations. For example: jAxb, zZac 54 | -------------------------------------------------------------------------------- /assets/simple-jekyll-search.min.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Simple-Jekyll-Search v1.7.2 (https://github.com/christian-fei/Simple-Jekyll-Search) 3 | * Copyright 2015-2018, Christian Fei 4 | * Licensed under the MIT License. 5 | */ 6 | !function(){"use strict";var f={load:function w(t,e){var n=function r(){return window.XMLHttpRequest?new window.XMLHttpRequest:new ActiveXObject("Microsoft.XMLHTTP")}();n.open("GET",t,!0),n.onreadystatechange=function i(e,n){return function(){if(4===e.readyState&&200===e.status)try{n(null,JSON.parse(e.responseText))}catch(t){n(t,null)}}}(n,e),n.send()}};(function y(t){if(!function e(t){return!!t&&"undefined"!=typeof t.required&&t.required instanceof Array}(t))throw new Error("-- OptionsValidator: required options missing");if(!(this instanceof y))return new y(t);var r=t.required;this.getRequiredOptions=function(){return r},this.validate=function(e){var n=[];return r.forEach(function(t){"undefined"==typeof e[t]&&n.push(t)}),n}});var n=function g(t,e){var n=e.length,r=t.length;if(n{title}',templateMiddleware:Function.prototype,sortMiddleware:function(){return 0},noResultsText:"No results found",limit:10,fuzzy:!1,exclude:[]},n=["searchInput","resultsContainer","json"],r=function y(e){if(!function n(t){return!!t&&"undefined"!=typeof t.required&&t.required instanceof Array}(e))throw new Error("-- OptionsValidator: required options missing");if(!(this instanceof y))return new y(e);var r=e.required;this.getRequiredOptions=function(){return r},this.validate=function(e){var n=[];return r.forEach(function(t){"undefined"==typeof e[t]&&n.push(t)}),n}}({required:n});function i(t){o.success(t),l.put(t),function e(){o.searchInput.addEventListener("keyup",function(t){(function e(t){return-1===[13,16,20,37,38,39,40,91].indexOf(t)})(t.which)&&(u(),c(t.target.value))})}()}function u(){o.resultsContainer.innerHTML=""}function a(t){o.resultsContainer.innerHTML+=t}function c(t){(function e(t){return t&&0 29 | 30 | ## Submission to Participating Venues 31 | 32 | Any paper that is not desk rejected will receive reviews. Once authors have reviews they may choose to [revise and resubmit](/authors.md#step4) for another round of reviews, or [commit their paper *and reviews* to a publication venue](/authors.md#step5). 33 | 34 | The upcoming publication venues participating in ARR are listed below. If you represent a publication venue and would like to be added to this list, please see [this page](/organizers). 35 | 36 | | Venue | Final ARR Submission Date | Commitment Date | 37 | | :---: | :-----------------------: | :-------------: | 38 | | [NAACL 2025](https://2025.naacl.org/) | October 15, 2024 | December 16, 2024 | 39 | | [ACL 2025](https://2025.aclweb.org/) | February 15, 2025 | April 20, 2025 | 40 | | [EMNLP 2025](https://2025.emnlp.org/) | May 19, 2025 | July 31, 2025 | 41 | | AACL 2025 | July 28, 2025 | October 5, 2025 | 42 | 43 | ## Past Venues that Accepted ARR Submissions 44 | 45 | In 2022-2023, most *ACL events accepted ARR submissions while also running a direct submission process. Starting from 2024, all main *ACL conferences used ARR exclusively. 46 | 47 | - [EMNLP 2021](https://2021.emnlp.org/) 48 | - [INLG 2021](https://inlg2021.github.io/pages/calls.html) 49 | - [BlackboxNLP 2021](https://blackboxnlp.github.io/) 50 | - [CRAC 2021](https://sites.google.com/view/crac2021/) 51 | - [Eval4NLP 2021](https://eval4nlp.github.io/) 52 | - [FEVER 2021](https://fever.ai/workshop.html) 53 | - [MRL 2021](https://sites.google.com/view/mrl-2021/home?authuser=0) 54 | - [MRQA 2021](https://mrqa.github.io/) 55 | - [NLLP 2021](https://nllpw.org/) 56 | - [SustaiNLP 2021](https://sites.google.com/view/sustainlp2021) 57 | - [WMT 2021](http://statmt.org/wmt21/) 58 | - [NLP Insights 2021](https://insights-workshop.github.io/) 59 | - [NILLI 2021](https://www.cs.mcgill.ca/~pparth2/nilli_workshop/) 60 | - [ACL 2022](https://www.2022.aclweb.org/) 61 | - [ComputeEL-5](https://openreview.net/group?id=aclweb.org/ACL/2022/Workshop/ComputEL) 62 | - [NAACL 2022](https://2022.naacl.org/) 63 | - [MML 2022](https://openreview.net/group?id=aclweb.org/ACL/2022/Workshop/MML) 64 | - [WIT 2022](https://megagon.ai/2nd-workshop-on-deriving-insights-from-user-generated-text-wit/) 65 | - [NLP-Power 2022](https://openreview.net/group?id=aclweb.org/ACL/2022/Workshop/NLP-Power) 66 | - [Repl4NLP 2022](https://sites.google.com/view/repl4nlp2022/) 67 | - [CSSR 2022](https://csrr-workshop.github.io/) 68 | - [Spa-NLP 2022](https://openreview.net/group?id=aclweb.org/ACL/2022/Workshop/Spa-NLP) 69 | - [FEVER 2022](https://fever.ai/) 70 | - [StarSEM2022](https://sites.google.com/view/starsem2022/) 71 | - [Clinical NLP 2022](https://clinical-nlp.github.io/2022/) 72 | - [INLG 2022](https://inlgmeeting.github.io) 73 | - [NLG4Health](https://nlg4health.uvt.nl/) 74 | - [SIGDIAL 2022](https://2022.sigdial.org/) 75 | - [TRAC 2022](https://sites.google.com/view/trac2022/) 76 | - [CoNLL 2022](https://conll.org/) 77 | - AACL-IJCNLP 2022 78 | - [CASE 2022](https://emw.ku.edu.tr/case-2022/) 79 | - [UM-IoS 2022](https://induction-of-structure.github.io/emnlp2022/) 80 | - [BlackBoxNLP 2022](https://blackboxnlp.github.io/) 81 | - [NLP4PI 2022](https://sites.google.com/view/nlp4positiveimpact/) 82 | - [SereTOD 2022](http://seretod.org/) 83 | - [MMNLU 2022](https://mmnlu-22.github.io/) 84 | - [NLLP 2022](https://nllpw.org/) 85 | - [EACL 2023](https://2023.eacl.org/) 86 | - [ACL 2023](https://2023.aclweb.org/calls/main_conference/) 87 | - [MWE 2023](https://multiword.org/mwe2023/) 88 | - [SICon 2023](https://sites.google.com/view/sicon-2023/home) 89 | - [NEJLT volume 9 (1)](https://www.nejlt.org/) 90 | - [LAW 2023](https://sigann.github.io/LAW-XVII-2023) 91 | - [INLG 2023](https://inlg2023.github.io/) 92 | - [SIGDIAL 2023](https://2023.sigdial.org/) 93 | - [EMNLP 2023](https://2023.emnlp.org/) 94 | - [IJCNLP-AACL 2023](http://www.ijcnlp-aacl2023.org/) 95 | - [GenBench 2023](https://genbench.org/workshop) 96 | - [NEJLT volume 9 (2)](https://www.nejlt.org/) 97 | - [EACL 2024 main](https://2024.eacl.org/) 98 | - [EACL 2024 SRW](https://sites.google.com/view/eacl2024srw) 99 | - [LAW 2024](https://sigann.github.io/LAW-XVIII-2024/) 100 | - [WNUT 2024](http://noisy-text.github.io/2024/) 101 | - [MOOMIN 2024](https://moomin-workshop.github.io/) 102 | - [NLP4HR 2024](https://megagon.ai/nlp4hr-2024/) 103 | - [CALD-pseudo 2024](https://mormor-karl.github.io/events/CALD-pseudo/) 104 | - [UncertaiNLP 2024](https://uncertainlp.github.io/) 105 | - [SCI-CHAT 2024](https://sites.google.com/view/dialogue-evaluation/) 106 | - [NAACL 2024](https://2024.naacl.org/) 107 | - [MWE-UD 2024](https://multiword.org/mweud2024/) 108 | - [ACL 2024](https://2024.aclweb.org/) 109 | - [ArabicNLP 2024](https://arabicnlp2024.sigarab.org/) 110 | - [CMCL 2024](https://cmclorg.github.io/) 111 | - [SDProc 2024](https://sdproc.org/2024/index.html) 112 | - [ML4AL 2024](https://www.ml4al.com/) 113 | - [IWSLT 2024](https://iwslt.org) 114 | - [INLG 2024](https://inlg2024.github.io/) 115 | - [EMNLP 2024](https://2024.emnlp.org/) 116 | - [CoNLL 2024](https://conll.org/2024) 117 | - [NLP4DH 2024](https://www.nlp4dh.com/nlp4dh-2024) 118 | - [FEVER 2024](https://fever.ai/workshop.html) 119 | -------------------------------------------------------------------------------- /favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/favicon.ico -------------------------------------------------------------------------------- /fonts/EBGaramond-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/fonts/EBGaramond-Regular.ttf -------------------------------------------------------------------------------- /fonts/aramisi.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/fonts/aramisi.ttf -------------------------------------------------------------------------------- /future.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Future Improvements to ARR" 4 | permalink: /future 5 | --- 6 | 7 | This page describes some ideas that have been proposed by the community, and that we are considering for the future. 8 | 9 | 10 | # Planned Future Work 11 | 12 | * An author feedback mechanism not only for reviewers, but also ACs (for action by SACs) 13 | * Enabling the joint track work by all track SACs 14 | * Improving the workflow to pass reviewed papers to other conferences/workshops 15 | * Improving the integration with proceedings generation 16 | * Reviewer profiles with service history (in development at OpenReview) 17 | * Create a public record of who actually did reviewing, etc for each cycle. 18 | * Using ACL Pub Check to assist PCs with desk reject identification 19 | 20 | # Suggestions 21 | 22 | The following ideas have been suggested and may be considered in future ARR development: 23 | 24 | ## Process and Policy 25 | 26 | * Adding a category of “training” reviewers. Assign them to papers in addition to the normal reviewers. This will help grow the pool of reviewers over time. 27 | * A new type of desk reject, disallowing resubmission (for some rare cases of procedural violations) 28 | 29 | ## OpenReview UI 30 | 31 | * Show AEs and SAEs the meta-review scores in the list of papers. 32 | * Show SAEs, AEs, and reviewers the planned schedule in the OpenReview interface. 33 | * A formal way to indicate secondary reviewer contributions. 34 | * Give authors the ability to download all materials for a paper as a PDF, maintaining anonymisation. 35 | * Mechanism for SAEs to nominate good/bad AEs. 36 | 37 | ## Website 38 | 39 | * On the stats page, show resubmission stats as distributions rather than with box plots (at the moment it is hard to see what the median is). 40 | 41 | ## Other 42 | 43 | * A mentoring system for new reviewers. 44 | * An interactive tutorial for reviewers 45 | * Create a system that detects low-quality review comments at the sentence level. This could either (a) be shared with reviewers, encouraging them to make changes, or (b) shared with AEs, who can then nudge reviewers. 46 | * Encourage authors of resubmissions to mark up the changes. Maybe do it automatically? 47 | * Automatic checks for length violations. 48 | * Integrating the review system of ARR into OrcID so that there is a method for showing reviewing contributions. -------------------------------------------------------------------------------- /images/404.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/404.jpg -------------------------------------------------------------------------------- /images/ARR-pre.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/ARR-pre.png -------------------------------------------------------------------------------- /images/ARR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/ARR.png -------------------------------------------------------------------------------- /images/VenuesOR.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/VenuesOR.PNG -------------------------------------------------------------------------------- /images/arr-paper-cycle.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/arr-paper-cycle.jpg -------------------------------------------------------------------------------- /images/arr_report/conf-submissions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/arr_report/conf-submissions.png -------------------------------------------------------------------------------- /images/arr_report/days-from-submission-to-meta-review.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/arr_report/days-from-submission-to-meta-review.png -------------------------------------------------------------------------------- /images/arr_report/days-from-submission-to-review.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/arr_report/days-from-submission-to-review.png -------------------------------------------------------------------------------- /images/arr_report/num-submissions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/arr_report/num-submissions.png -------------------------------------------------------------------------------- /images/arr_report/reviewer-pool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/arr_report/reviewer-pool.png -------------------------------------------------------------------------------- /images/categories.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/categories.png -------------------------------------------------------------------------------- /images/datacollection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/datacollection.png -------------------------------------------------------------------------------- /images/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/favicon-32x32.png -------------------------------------------------------------------------------- /images/galaxy.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/galaxy.jpg -------------------------------------------------------------------------------- /images/guidelines/1-AC-console.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/1-AC-console.png -------------------------------------------------------------------------------- /images/guidelines/10-AC-messaging-reviewers-part2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/10-AC-messaging-reviewers-part2.png -------------------------------------------------------------------------------- /images/guidelines/11-AC-chat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/11-AC-chat.png -------------------------------------------------------------------------------- /images/guidelines/12-AC-submit-meta-part1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/12-AC-submit-meta-part1.png -------------------------------------------------------------------------------- /images/guidelines/13-AC-submit-meta-part2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/13-AC-submit-meta-part2.png -------------------------------------------------------------------------------- /images/guidelines/14-AC-request-recognition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/14-AC-request-recognition.png -------------------------------------------------------------------------------- /images/guidelines/15-AC-assign-emergency-part1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/15-AC-assign-emergency-part1.png -------------------------------------------------------------------------------- /images/guidelines/16-AC-assign-emergency-part2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/16-AC-assign-emergency-part2.png -------------------------------------------------------------------------------- /images/guidelines/16-AC-assign-emergency-part2_new.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/16-AC-assign-emergency-part2_new.png -------------------------------------------------------------------------------- /images/guidelines/17-AC-assign-emergency-part3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/17-AC-assign-emergency-part3.png -------------------------------------------------------------------------------- /images/guidelines/17-AC-assign-emergency-part3_new.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/17-AC-assign-emergency-part3_new.png -------------------------------------------------------------------------------- /images/guidelines/18-AC-assign-emergency-part4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/18-AC-assign-emergency-part4.png -------------------------------------------------------------------------------- /images/guidelines/19-AC-reviewer-email.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/19-AC-reviewer-email.png -------------------------------------------------------------------------------- /images/guidelines/2-AC-resubmission-link.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/2-AC-resubmission-link.png -------------------------------------------------------------------------------- /images/guidelines/3-AC-overall-assessment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/3-AC-overall-assessment.png -------------------------------------------------------------------------------- /images/guidelines/4-AC-OR-login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/4-AC-OR-login.png -------------------------------------------------------------------------------- /images/guidelines/5-AC-console-access.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/5-AC-console-access.png -------------------------------------------------------------------------------- /images/guidelines/6-AC-reviewer-in-pool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/6-AC-reviewer-in-pool.png -------------------------------------------------------------------------------- /images/guidelines/7-AC-reviewer-not-in-pool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/7-AC-reviewer-not-in-pool.png -------------------------------------------------------------------------------- /images/guidelines/8-AC-official-comment-button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/8-AC-official-comment-button.png -------------------------------------------------------------------------------- /images/guidelines/9-AC-messaging-reviewers-part1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/9-AC-messaging-reviewers-part1.png -------------------------------------------------------------------------------- /images/guidelines/RI_form1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/RI_form1.png -------------------------------------------------------------------------------- /images/guidelines/RI_form2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/RI_form2.png -------------------------------------------------------------------------------- /images/guidelines/RI_form3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/RI_form3.png -------------------------------------------------------------------------------- /images/guidelines/emergency_declaration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/emergency_declaration.png -------------------------------------------------------------------------------- /images/guidelines/metareview_issue_report.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/metareview_issue_report.png -------------------------------------------------------------------------------- /images/guidelines/review_issue_report.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/review_issue_report.png -------------------------------------------------------------------------------- /images/guidelines/reviewer_assignment_interface.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/guidelines/reviewer_assignment_interface.png -------------------------------------------------------------------------------- /images/icons/align-left-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/icons/bluesky.svg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/icons/bluesky.svg.png -------------------------------------------------------------------------------- /images/icons/bolt-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/icons/bug-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/icons/circle-check-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/icons/circle-info-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/icons/circle-question-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/icons/fire-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/icons/list-ol-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/icons/list-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/icons/pen-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/icons/quote-right-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/icons/times-circle-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/icons/triangle-exclamation-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/mobile-demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/mobile-demo.png -------------------------------------------------------------------------------- /images/people/anna_rogers.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/anna_rogers.jpg -------------------------------------------------------------------------------- /images/people/atsuki_yamaguchi.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/atsuki_yamaguchi.jpg -------------------------------------------------------------------------------- /images/people/dhruv_naik.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/dhruv_naik.jpg -------------------------------------------------------------------------------- /images/people/dongqi_liu.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/dongqi_liu.jpg -------------------------------------------------------------------------------- /images/people/freda_shi.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/freda_shi.jpg -------------------------------------------------------------------------------- /images/people/gabriella_skitalinskaya.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/gabriella_skitalinskaya.jpg -------------------------------------------------------------------------------- /images/people/goggi_sara.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/goggi_sara.jpg -------------------------------------------------------------------------------- /images/people/himanshu_beniwal.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/himanshu_beniwal.jpg -------------------------------------------------------------------------------- /images/people/holy_lovenia.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/holy_lovenia.jpeg -------------------------------------------------------------------------------- /images/people/isabelle_augenstein.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/isabelle_augenstein.jpg -------------------------------------------------------------------------------- /images/people/jing_jiang.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/jing_jiang.jpg -------------------------------------------------------------------------------- /images/people/jonathan_kummerfeld.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/jonathan_kummerfeld.jpg -------------------------------------------------------------------------------- /images/people/julia_hockenmaier.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/julia_hockenmaier.jpg -------------------------------------------------------------------------------- /images/people/jun_suzuki.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/jun_suzuki.jpg -------------------------------------------------------------------------------- /images/people/malihe_alikhani.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/malihe_alikhani.jpg -------------------------------------------------------------------------------- /images/people/manuel_mager.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/manuel_mager.png -------------------------------------------------------------------------------- /images/people/margot_mieskes.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/margot_mieskes.jpeg -------------------------------------------------------------------------------- /images/people/mausam.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/mausam.jpg -------------------------------------------------------------------------------- /images/people/michael_white.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/michael_white.jpg -------------------------------------------------------------------------------- /images/people/ning_shi.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/ning_shi.jpg -------------------------------------------------------------------------------- /images/people/placeholder.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/placeholder.jpg -------------------------------------------------------------------------------- /images/people/ruohao_guo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/ruohao_guo.jpg -------------------------------------------------------------------------------- /images/people/santosh.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/santosh.jpg -------------------------------------------------------------------------------- /images/people/sarvnaz_karimi.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/sarvnaz_karimi.jpeg -------------------------------------------------------------------------------- /images/people/shabnam_behzad.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/shabnam_behzad.jpg -------------------------------------------------------------------------------- /images/people/shirley.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/shirley.jpg -------------------------------------------------------------------------------- /images/people/shubham_mittal.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/shubham_mittal.jpg -------------------------------------------------------------------------------- /images/people/shubhra_kanti.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/shubhra_kanti.jpg -------------------------------------------------------------------------------- /images/people/smaranda_muresan.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/smaranda_muresan.jpg -------------------------------------------------------------------------------- /images/people/sonakshi_chauhan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/sonakshi_chauhan.png -------------------------------------------------------------------------------- /images/people/sudipta_kar.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/sudipta_kar.jpg -------------------------------------------------------------------------------- /images/people/thamar_solorio.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/thamar_solorio.jpg -------------------------------------------------------------------------------- /images/people/vincent_ng.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/vincent_ng.jpg -------------------------------------------------------------------------------- /images/people/vinodkumar_prabhakaran.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/vinodkumar_prabhakaran.jpg -------------------------------------------------------------------------------- /images/people/viviane_moreira.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/viviane_moreira.jpg -------------------------------------------------------------------------------- /images/people/vlilja_ovrelid.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/vlilja_ovrelid.jpg -------------------------------------------------------------------------------- /images/people/xiaodan_zhu.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/xiaodan_zhu.jpg -------------------------------------------------------------------------------- /images/people/yoshitomo_matsubara.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/people/yoshitomo_matsubara.jpg -------------------------------------------------------------------------------- /images/reverie-demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/reverie-demo.png -------------------------------------------------------------------------------- /images/reverie-text.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/reverie-text.png -------------------------------------------------------------------------------- /images/reverie.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/reverie.png -------------------------------------------------------------------------------- /images/search.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/images/search.png -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | --- 4 | 5 |
6 |

ACL Rolling Review (ARR) is a centralized reviewing service targeting top-tier conferences 7 | under the Association for Computational Linguistics.

8 |
    9 |
  • Built atop the OpenReview platform, ARR implements the initial stages of conference reviewing in 2 month cycles, 10 | culminating in the release to authors of reviews and a metareview.
    • 11 |
  • Authors may submit a revision to a subsequent ARR cycle and request the same or new reviewers.
    • 12 |
  • Reviews are decoupled from acceptance decisions: as long as a paper has reviews and has not yet been accepted by a venue, 13 | the authors can choose among participating venues, whose program committees will consider the already-reviewed papers for acceptance.
    • 14 |
15 |

For further information about submitting, see the page for Authors.

16 |
17 | 18 |

ARR began with a 2020 proposal. 19 | As of mid-2023, it has evolved with experience and feedback from the community. 20 | In 2024, the ACL, EACL, NAACL, and EMNLP main conferences all used ARR as the sole submission system. The upcoming participating venues are listed at dates and venues. The ARR team continues to welcome input on the design of the review process. See also the ARR changelog and future plans.

21 | 22 |

Follow us on  Twitter,  Facebook and  Bluesky.

23 | 24 |

Updates

25 | 26 |
27 | {% for post in paginator.posts %} 28 | 42 | {% endfor %} 43 | 44 | 45 | {% if paginator.total_pages > 1 %} 46 | 69 | {% endif %} 70 |
71 | -------------------------------------------------------------------------------- /mentorship.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "ARR Mentorship Programme" 4 | permalink: /mentorship 5 | search_index: false 6 | --- 7 | An increasing number of NLP submissions are going through ARR. To make sure that we can handle this growth, and to maintain a diverse and balanced reviewer pool, we would like to set up a reviewer mentorship program in the spirit of previous programs run at ACL. The goal of this program is to help researchers with less reviewing experience to quickly get up to speed with the help of more experienced reviewers. 8 | ## Overview 9 | The program operates as follows: 10 | 1. Mentees and Mentors apply to join the program--see the application forms below. 11 | 2. We match mentors and mentees based on availability, review experience, topical alignment etc. 12 | 3. Mentees and Mentors work through our online material. This will prepare the mentee for their first assignment and remind the mentor what to look for when reviewing for ARR. 13 | 4. Once ready, in the following ARR cycle, we assign a paper to the mentee (provided there is a good match). 14 | 5. In the beginning of the cycle, mentor and mentee meet for an introduction and for high-level advice from mentor to mentee. 15 | * This meeting can also be a place for the mentee to raise open questions from reading the online material below. 16 | 6. The mentee prepares their review in 14 days (so 10 days earlier than normal). 17 | 7. Right after, the mentor and mentee discuss the review and what might need to be improved 18 | * This can involve the action editor too. 19 | * It can happen via video chat or asynchronous communication, down to availability and preferences of mentees and mentors. 20 | 8. The mentee improves the review until stakeholders (mentee, mentor, action editor) decide it has sufficient quality. 21 | 9. After the cycle, mentor and mentee decide if more mentorship is necessary or if the mentee can continue to review independently. 22 | 23 | ## Mentees 24 | 25 | We are looking for researchers in the area of NLP that have experience with NLP paper authoring and/or reviewing (e.g. have an EMNLP or ACL paper, done some reviewing), but don’t yet feel ready to review for ACL conferences and ARR. If you fall into this category and are interested in becoming a review mentee at ARR, stay tuned for updates on our mentoring program. 26 | 27 | Note that we cannot guarantee a place for everyone as the number of mentors is limited. 28 | The form asks for your OpenReview User ID so please create an OpenReview account if you haven’t already done so. 29 | 30 | ## Mentors 31 | We are looking for current ARR reviewers and action editors willing to mentor junior reviewers. If you are interested (and already an ARR reviewer/action editor), please stay tuned for updates on our mentoring program. 32 | 33 | ## Material for Mentors and Mentees 34 | The primary reading material for mentors and mentees is our [ARR tutorial](/reviewertutorial). 35 | 36 | We ask mentees to go through this material (in full). While we expect mentors to have sufficient review experience, we still recommend them to review the material too in order to align with ARR expectations and help with communicating what good and bad reviews are. 37 | 38 | ## Timeline 39 | 40 | We hope that the first mentees and mentors are matched and ready to review by the February 15 cycle. 41 | 42 | ## Contact 43 | 44 | For questions and feedback please email mentorship@aclrollingreview.org. 45 | -------------------------------------------------------------------------------- /organizers.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "For Conference/Workshop Organizers" 4 | permalink: /organizers 5 | --- 6 | 7 | ARR is a standalone centralized reviewing service designed to primarily support conferences and workshops under the umbrella of the Association for Computational Linguistics (ACL). Nonetheless, any other venue may choose to allow ARR-processed manuscripts (i.e., papers with ARR reviews and meta-reviews) to be committed for publication. 8 | 9 | Venues can generally subscribe to ARR in two different configurations: 10 | 11 | 1. Full ARR -- only ARR-reviewed papers are allowed to be committed for publication at the venue (e.g., ACL 2022 and NAACL 2022). 12 | 2. Hybrid -- the venue has a separate reviewing process but also welcomes ARR-reviewed papers to be committed 13 | 14 | "Full ARR" entails a closer collaboration between venue organizers and ARR, as ARR becomes the only path to publication for the venue (e.g., ARR makes an extra effort to make sure that all submissions targeting the venue are processed in time for venue commitment). 15 | 16 | For "Hybrid" venues, ARR is mostly venue-agnostic. We will list such venues on our website and let ARR submissions that target the venue indicate this in the submission form (note that ARR does not need this information itself, we collect it merely to support the venues w.r.t. calculation of their acceptance rates). 17 | 18 | ## Subscribing to ARR (Full or Hybrid) 19 | 20 | Organizers of venues that would like to subscribe to ARR (either in Full or Hybrid setup) should make a pull request to [the ARR venues page](https://github.com/acl-org/aclrollingreview/blob/main/dates.md). They should update this file with the following information: 21 | 22 | - Venue name and website 23 | - Planned commitment deadline 24 | 25 | ## Commitment (fetching ARR Reviews) 26 | 27 | Venue organizers need to decide the exact mechanism by which they will enable authors of ARR-processed papers to commit to their venue. OpenReview will then provide access to the all reviews of the committed papers. In principle, there are the following options: 28 | 29 | 1. The venue uses OpenReview, receiving reviews from ARR only 30 | - In this case, the organizers should [submit a venue request form](https://openreview.net/group?id=OpenReview.net/Support) and inform the OpenReivew staff that they subscribe to ARR. 31 | - Once the commitment venue is deployed, there will be a field present in the submission form named "paper_link" for authors to commit their ARR processed papers. Here authors will simply provide the URL to the OpenReview forum of their ARR submission. This field is validated, so it will check whether the forum link exists in an ARR venue before allowing the author to submit. 32 | - For Full ARR venues, this commitment form is the only submission form to be created. 33 | 34 | 2. The venue uses OpenReview, with its own review process in OpenReview and reviews from ARR 35 | - In this case, submit two venue request forms, one for non-ARR reviewing and another for commitment of ARR-reviews papers. 36 | - OpenReview will provide read access to ARR forums of committed papers to the venue (assigned SACs, PCs) or import ARR reviews into the venue's OpenReview instance depending on the API used for the original ARR submission. Please contact OpenReview () when the submission deadline has passed and you're ready to begin the reviewing process. 37 | 38 | 3. The venue does not use OpenReview 39 | - In this case, the organizers need to figure out how to let the authors of ARR-reviewed papers indicate that they would like to commit to the venue 40 | - The authors need to provide a unique identifier of the ARR submission: the URL of the OpenReview forum of the ARR submission (https://openreview.net/forum?id=XXXXXXXXXXX) 41 | - The organizers send to ARR the list of committed papers (the list of URLs of OpenReview forums of committed papers) 42 | - ARR exports the reviews for committed papers and ships them to the venue organizers 43 | 44 | ## Setting Up an OpenReview Commitment Site 45 | 46 | Once the OpenReview site is set up, the program chairs can navigate to the request form and click on the "Revision" Tab. It is suggested that organizers configure the site as pictured in [picture of OpenReview site with options checked for nonpublic anonymous submissions](/images/VenuesOR.PNG) and add the [additional submission options json](commitments.json). 47 | These additional options include two requests for the paper PDF, one in camera-ready style (ie., with author names) and one that is anonymous, to enable a double-blind process for best paper selection. 48 | 49 | ## Duplicate Submissions 50 | 51 | Venues subscribing to ARR as hybrid are encouraged to adopt the following wording: *A paper may not be simultaneously under review through ARR and YOUR_VENUE. A paper that has or will receive reviews through ARR may not be submitted for review to YOUR_VENUE.* 52 | 53 | ## Informing ARR of Decisions 54 | 55 | ARR provides reviews, it does not make any acceptance or rejection decisions for the venues. ARR, however, forbids revisions to be submitted to ARR while the paper is under consideration (i.e., committed) for publication in a venue. Also, naturally, papers archived in venue's proceedings can no longer be revised and resubmitted to ARR. This is why, once the venue finalizes their program/proceedings, organizers are encouraged to send back to ARR the list of ARR-reviewed submissions that had been committed, with the indication on whether the paper is archived in venue's proceedings (papers accepted for non-archival participation in the venue, are not forbidden from resubmissions to ARR). 56 | 57 | -------------------------------------------------------------------------------- /others.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Others" 4 | permalink: /others 5 | --- 6 | 7 | ## Past Editors in Chief 8 | 9 | - Thamar Solorio, Mohamed Bin Zayed University of Artificial Intelligence 10 | - Aline Villavicencio, University of Sheffield, Federal University of Rio Grande do Sul 11 | - Amanda Stent, Colby College 12 | - Goran Glavaš, University of Würzburg 13 | - Ivan Vladimir Meza Ruiz, Universidad Autonoma de Mexico 14 | - Marie-Catherine de Marneffe, The Ohio State University 15 | - Marine Carpuat, University of Maryland 16 | - Pascale Fung, Hong Kong University of Science & Technology 17 | - Preslav Nakov, Mohamed Bin Zayed University of Artificial Intelligence 18 | - Smaranda Muresan, Columbia University 19 | - Sebastian Riedel, Facebook AI and University College London 20 | 21 | ## Past Contributors to ACL Rolling Review Initiative 22 | 23 | - Graham Neubig, Carnegie Mellon University (Chief Technical Officer, 2021-2023) 24 | - Pengfei Liu, Carnegie Mellon University (Tech Team, 2021-2023) 25 | - Sebastin Santy, University of Washington (Tech Team, 2021-2023) 26 | - Stephen Mayhew, Duolingo (Chief Communication Officer, 2021-2023) 27 | - Nils Dycke, TU Darmstadt (Tech Team, 2021-2024) 28 | 29 | -------------------------------------------------------------------------------- /policies.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Policies" 4 | permalink: /about/policies 5 | search_index: false 6 | --- 7 | 8 | * [ACL Policies for Submission, Review and Citation](https://www.aclweb.org/adminwiki/index.php?title=ACL_Policies_for_Submission,_Review_and_Citation) 9 | * [ACL Privacy Policy](https://www.aclweb.org/adminwiki/index.php?title=ACL_Privacy_Policy) 10 | * [ACL Code of Ethics](https://www.aclweb.org/adminwiki/index.php?title=ACL_Resolutions#March_5.2C_2020) 11 | 12 | -------------------------------------------------------------------------------- /reviewerfaq.md: -------------------------------------------------------------------------------- 1 | --- 2 | permalink: /reviewerfaq 3 | redirect_to: 4 | - /reviewers 5 | search_index: false 6 | --- 7 | -------------------------------------------------------------------------------- /search.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Search 4 | permalink: /search/ 5 | --- 6 | 7 |

Search Results for ""

8 | 9 |
10 |

11 | Note: ARR has been running since March 2021, so there may be older pages with outdated information. In case of conflicts, the sources to be considered primary are the CFP and guidelines for authors, reviewers and area chairs. For authors we also maintain a submission checklist. 12 |

13 |
14 | 15 |
    16 | {% assign all_items = site.pages | concat: site.posts %} 17 | {% for item in all_items %} 18 | {% if item.search_index != false %} 19 |
  • 21 | {{ item.title }} 22 |
    • 23 | {% endif %} 24 | {% endfor %} 25 |
26 | 27 | -------------------------------------------------------------------------------- /search.json: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | [ 4 | {% for post in site.posts %} 5 | { 6 | 7 | "title" : "{% if post.title != "" %}{{ post.title | escape }}{% else %}{{ post.excerpt | strip_html | escape | strip }}{%endif%}", 8 | "url" : "{{ site.baseurl }}{{ post.url }}", 9 | "category" : "{{ post.categories | join: ', '}}", 10 | "date" : "{{ post.date | date: "%B %e, %Y" }}" 11 | } {% unless forloop.last %},{% endunless %} 12 | {% endfor %} 13 | ] -------------------------------------------------------------------------------- /static/responsibleNLPresearch.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acl-org/aclrollingreview/aea1c55db27cd98caec5bbd6c9c6c6ad2ea0b58c/static/responsibleNLPresearch.pdf -------------------------------------------------------------------------------- /submissionform.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Submission Form Fields" 4 | permalink: /submissionform 5 | --- 6 | 7 | | Field | Description | 8 | |-------|--------| 9 | | Title | Title of paper. Add TeX formulas using the following formats: $In-line Formula$ or $$Block Formula$$ | 10 | | Authors | Search author profile by first, middle and last name or email address. If the profile is not found, you can add the author by completing first, middle, and last names as well as author email address. | 11 | | TL;DR | "Too Long; Didn't Read": a short sentence describing your paper | 12 | | Abstract | Abstract of paper. Add TeX formulas using the following formats: $In-line Formula$ or $$Block Formula$$ | 13 | | Pdf | Upload a single PDF containing the paper, references and any appendices | 14 | | Paper Type | See the CFP for the requirements for short and long papers | 15 | | Area | The most relevant research area / track for the paper | 16 | | Previous URL | Provide the URL of your previous submission to ACL Rolling Review if this is a resubmission (this URL will start with https://openreview.net) | 17 | | Previous PDF | Upload a single PDF of your previous submission to ACL Rolling Review if this is a resubmission | 18 | | Response PDF | Upload a single PDF of your responses to the previous reviews to ACL Rolling Review if this is a resubmission | 19 | | Reviewer/Editor Reassignment Request | If this is a resubmission and you think that a reviewer or action editor last time was not qualified to review the paper, please add the reviewer ID (comma-separated for multiple ones) to this box. Add 'AE' for the action editor. This option should be used sparingly, so please only request this in relatively clear circumstances | 20 | | Reviewer/Editor Reassignment Justification | Please specify the reason for any reassignment request. Example reasons include clear lack of expertise in the area or dismissing the work without any concrete comments regarding correctness of the results or argumentation, limited perceived impact of the methods or findings, lack of clarity in exposition, or other valid criticisms. Generally we accept such requests, but the expectation is that papers have been revised based on the earlier reviews and it is up to the discretion of the action editors or editors in chief whether the request is granted. Answers to this question also helps us identify issues in the review process and guide efforts to improve review quality. | 21 | | Software | Each ARR submission can be accompanied by one .tgz or .zip archive containing software (max. 200MB). | 22 | | Data | Each ARR submission can be accompanied by one .tgz or .zip archive containing data (max. 200MB). | 23 | | Preprint | Would the authors like to have a public anonymous pre-print of the submission? | 24 | | Existing Preprints | If there are any publicly available non-anonymous preprints of this paper, please list them here (provide the URLs please) | 25 | | Preferred Venue | If you have a venue that you are hoping to submit this paper to, please enter it here. You must enter the designated acronym from our [list of venues](/dates). Note that entering a preferred venue is not a firm commitment to submit your paper to this venue, but it will help ARR and the venue chairs in planning, so we highly recommend filling in your current intentions. Please enter only your first choice | 26 | | Consent to Share Data | I agree for the anonymized metadata associated with my submission to be included in a publicly available dataset. This dataset WILL include scores, anonymized paper and reviewer IDs that allow grouping the reviews by paper and by reviewer, as well as acceptance decisions and other numerical and categorical metadata. This dataset WILL NOT include any textual or uniquely attributable data like names, submission titles and texts, review texts, author responses, etc. Your decision to opt-in the data does not affect the reviewing of your submission in any way | 27 | | Consent to Review | By submitting this paper, all authors of this paper agree to serve as reviewers for ACL Rolling Review if eligible and invited | 28 | | Responsible NLP Research Questions | A series of questions from the [responsible NLP research checklist](/responsibleNLPresearch) | 29 | -------------------------------------------------------------------------------- /twitter-bot/README.md: -------------------------------------------------------------------------------- 1 | # ACL Rolling Review Preprints Twitter Bot 2 | 3 | This small amount of code will tweet links to anonymous preprints submitted each month. 4 | 5 | Use of a virtual environment is recommended, as well as Python 3.7+ 6 | 7 | ```bash 8 | > python -m venv .pyenv 9 | > source .pyenv/bin/activate 10 | > pip install -r requirements.txt 11 | > 12 | ``` -------------------------------------------------------------------------------- /twitter-bot/requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2020.12.5 2 | chardet==4.0.0 3 | Deprecated==1.2.12 4 | future==0.18.2 5 | idna==2.10 6 | oauthlib==3.1.0 7 | openreview-py==1.0.21 8 | pycryptodome==3.10.1 9 | pylatexenc==2.10 10 | PySocks==1.7.1 11 | requests==2.25.1 12 | requests-oauthlib==1.3.0 13 | six==1.16.0 14 | tld==0.10 15 | tqdm==4.60.0 16 | tweepy==3.10.0 17 | urllib3==1.26.4 18 | wrapt==1.12.1 19 | -------------------------------------------------------------------------------- /twitter-bot/twitter_bot.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import openreview 3 | import tweepy 4 | import os 5 | 6 | # API Key for Arr Preprints Bot 7 | # Contact Stephen Mayhew (twitter: @mayhewsw) for the secrets 8 | CONSUMER_KEY = "wjHFvsbNqcPjM8xh0VphoIfal" 9 | CONSUMER_SECRET = os.environ['TWITTER_CONSUMER_SECRET'] 10 | ACCESS_TOKEN= "1395450619608113156-RxA4yS5IpHjAwEoiV87O8U8JeMmkQi" 11 | ACCESS_TOKEN_SECRET = os.environ['TWITTER_ACCESS_TOKEN_SECRET'] 12 | 13 | def post_links(api: tweepy.API, year: int, month: str, dryrun: bool = False) -> None: 14 | """ 15 | Given a tweepy API, this reads the links from the ACL OpenReview site and 16 | posts them to twitter. 17 | :param api: an authenticated tweepy API 18 | :param year: integer year formatted as YYYY 19 | :param month: string month represented with 3 characters 20 | returns None 21 | """ 22 | guest_client = openreview.Client(baseurl='https://api.openreview.net') 23 | invitation = f"aclweb.org/ACL/ARR/{year}/{month}/-/Blind_Submission" 24 | 25 | blind_submissions = guest_client.get_notes( 26 | invitation=invitation) 27 | 28 | print(f"Retrieved {len(blind_submissions)} submissions from OpenReview for {month} {year}") 29 | 30 | for bs in blind_submissions: 31 | content = bs.content 32 | paper_id = bs.id 33 | 34 | title = content["title"] 35 | paper_link = f"https://openreview.net/pdf?id={paper_id}" 36 | forum_link = f"https://openreview.net/forum?id={paper_id}" 37 | 38 | # the forum link sends to you a landing page for the paper. 39 | tweet = f"{title} {forum_link}" 40 | if dryrun: 41 | print(tweet) 42 | else: 43 | api.update_status(tweet) 44 | 45 | 46 | def oauth_login(): 47 | """Authenticate with twitter using OAuth""" 48 | 49 | # Authenticate to Twitter 50 | auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET) 51 | auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) 52 | 53 | # Create API object 54 | return tweepy.API(auth) 55 | 56 | 57 | def batch_delete(api): 58 | print("You are about to delete all tweets from the account @%s." % api.verify_credentials().screen_name) 59 | print("Does this sound ok? There is no undo! Type yes to carry out this action.") 60 | do_delete = input("> ") 61 | if do_delete.lower() == 'yes': 62 | for status in tweepy.Cursor(api.user_timeline).items(): 63 | try: 64 | api.destroy_status(status.id) 65 | print("Deleted:", status.id) 66 | except Exception: 67 | print("Failed to delete:", status.id) 68 | 69 | 70 | if __name__ == "__main__": 71 | 72 | parser = argparse.ArgumentParser() 73 | parser.add_argument("command", help="Whether to post or delete all tweets", choices=["post", "delete"], type=str) 74 | parser.add_argument("--year", help="This should have 4 digits: 2021", type=int) 75 | parser.add_argument("--month", help="This should be three letters: May, Jun, Jul, etc.", type=str) 76 | parser.add_argument("--dryrun", help="Just print links instead of posting", action="store_true", default=False) 77 | 78 | args = parser.parse_args() 79 | 80 | api = oauth_login() 81 | print("Authenticated as: %s" % api.me().screen_name) 82 | 83 | if args.command == "delete": 84 | print("deleting") 85 | # batch_delete(api) 86 | elif args.command == "post": 87 | if not (args.year and args.month): 88 | print("Must include year and month with posting!") 89 | else: 90 | post_links(api, args.year, args.month, args.dryrun) 91 | pass 92 | else: 93 | print("Shouldn't ever get here...") 94 | --------------------------------------------------------------------------------