├── .DS_Store ├── .devcontainer └── vscode │ └── devcontainer.json ├── .github └── workflows │ ├── preview.yml │ └── publish.yml ├── .gitignore ├── .nojekyll ├── README.md ├── _Documents ├── .DS_Store ├── 20231211_hye_soo_cho.pdf ├── 20231211_paul_schuette.pdf ├── 20240108_R_consortium_JPMA_F.pdf ├── 22008_06_jung.pdf ├── 22008_06_zeballos.pdf ├── FDAInfrastructure.pptx ├── Leng_Update_10-1-21.pptx ├── Letter_to_FDA.Rmd ├── Letter_to_FDA.html ├── OpenSource1c_Paul_S_2023_ASA_biopharma.pdf ├── R Submission Pilot 1 Extension Proposal.pdf ├── R submission 2023 BIOP poster.pdf ├── R submission Pilots to FDA - SDSS presentation.pdf ├── R submission Pilots to FDA - useR presentation.pdf ├── R_Submission_Pilot_3_ADaM_Proposal.pdf ├── R_and_Shiny_in_Regulatory_Submission_to_Swissmedic_2024.pdf ├── R_eCTD_pilot_submission.docx ├── SDSS poster - r submission.pdf ├── SDTCG-SoftwareProgram.docx ├── Summary_R_Pilot2_Submission 27SEP2023.pdf ├── Summary_R_Pilot3_Submission.pdf ├── Summary_R_Pilot_Submission.pdf ├── Summary_R_Pilot_Submission2.pdf ├── Yilong_Zhang_eSub_proposal_12-4-20.pptx ├── pilot_submission_proposal.Rmd ├── pilot_submission_proposal.html ├── pilot_submission_proposal.md ├── pkglite-r-consortium-2021.html ├── project-plan.Rmd └── project-plan.html ├── _Minutes ├── Errors.png ├── Errors2.png ├── R consortium R submission Working Group Updates 9-3-2021.pdf ├── R_Sub_10-07-2022_Minutes.html ├── R_Sub_10-07-2022_Minutes.md ├── R_Sub_10-1-2021_Minutes.Rmd ├── R_Sub_10-1-2021_Minutes.html ├── R_Sub_11-04-22_Minutes.html ├── R_Sub_11-04-22_Minutes.md ├── R_Sub_12-02-2022_Minutes.html ├── R_Sub_12-02-2022_Minutes.md ├── R_Sub_2021-12-03_Minutes.md ├── R_Sub_2022-01-07_Minutes.md ├── R_Sub_2022-02-04_Minutes.md ├── R_Sub_2022-03-04_Minutes.md ├── R_Sub_2022-04-01_Minutes.html ├── R_Sub_2022-04-01_Minutes.md ├── R_Sub_2022-05-06_Minutes.md ├── R_Sub_2022-06-04_Minutes.html ├── R_Sub_2022-06-04_Minutes.md ├── R_Sub_2022-7-1_Minutes.html ├── R_Sub_2022-7-1_Minutes.md ├── R_Sub_6-30-21_Minutes.Rmd ├── R_Sub_6-30-21_Minutes.html ├── R_Sub_8-5-2022_Minutes.html ├── R_Sub_8-5-2022_Minutes.md ├── R_Sub_8-6-2021_Minutes.Rmd ├── R_Sub_8-6-2021_Minutes.html ├── R_Sub_9-02-2022_Minutes.html ├── R_Sub_9-02-2022_Minutes.md ├── R_Sub_9-3-2021_Minutes.Rmd ├── R_Sub_9-3-2021_Minutes.html ├── R_Sub_WG_06-04-21_Minutes.Rmd ├── R_Sub_WG_06-04-21_Minutes.html ├── R_Submissions_WG_01-15-2021_Minutes.Rmd ├── R_Submissions_WG_01-15-2021_Minutes.html ├── R_Submissions_WG_02-05-2021_Minutes.Rmd ├── R_Submissions_WG_02-05-2021_Minutes.html ├── R_Submissions_WG_03-05-2021_Minutes.Rmd ├── R_Submissions_WG_03-05-2021_Minutes.html ├── R_Submissions_WG_04-05-21_Minutes.Rmd ├── R_Submissions_WG_04-05-21_Minutes.html ├── R_Submissions_WG_05-07-21.Rmd ├── R_Submissions_WG_05-07-21.html ├── R_Submissions_WG_12-4-20_Minutes.Rmd ├── R_Submissions_WG_12-4-20_Minutes.html ├── p1.png ├── pilot2_shiny.png └── sec_4.1.2.10.png ├── _freeze ├── about │ └── execute-results │ │ └── html.json ├── minutes_prep │ └── execute-results │ │ └── html.json └── site_libs │ ├── clipboard │ └── clipboard.min.js │ └── quarto-listing │ ├── list.min.js │ └── quarto-listing.js ├── _prototyping └── README.md ├── _quarto.yml ├── _submission-package ├── m1 │ ├── m1toc.pdf │ └── us │ │ └── 12-cov-let │ │ └── cover.pdf ├── m5 │ ├── 53-clin-stud-rep │ │ └── 535-rep-effic-safety-stud │ │ │ └── 5351-stud-rep-contr │ │ │ └── cdiscpilot01 │ │ │ ├── cdiscpilot01.pdf │ │ │ └── narratives.txt │ ├── datasets │ │ ├── cdiscpilot01 │ │ │ ├── analysis │ │ │ │ └── adam │ │ │ │ │ ├── datasets │ │ │ │ │ ├── adae.xpt │ │ │ │ │ ├── adlbc.xpt │ │ │ │ │ ├── adlbh.xpt │ │ │ │ │ ├── adlbhy.xpt │ │ │ │ │ ├── adqsadas.xpt │ │ │ │ │ ├── adqscibc.xpt │ │ │ │ │ ├── adqsnpix.xpt │ │ │ │ │ ├── adsl.xpt │ │ │ │ │ ├── adtte.xpt │ │ │ │ │ ├── advs.xpt │ │ │ │ │ ├── dataguide.pdf │ │ │ │ │ ├── define-v1-updated-html.xsl │ │ │ │ │ ├── define.pdf │ │ │ │ │ └── define.xml │ │ │ │ │ └── programs │ │ │ │ │ ├── adae.sas │ │ │ │ │ └── at14-5-02.sas │ │ │ └── tabulations │ │ │ │ └── sdtm │ │ │ │ ├── ae.xpt │ │ │ │ ├── blankcrf.pdf │ │ │ │ ├── cm.xpt │ │ │ │ ├── define-v1-updated-html.xsl │ │ │ │ ├── define.pdf │ │ │ │ ├── define.xml │ │ │ │ ├── dm.xpt │ │ │ │ ├── ds.xpt │ │ │ │ ├── ex.xpt │ │ │ │ ├── lb.xpt │ │ │ │ ├── mh.xpt │ │ │ │ ├── qs.xpt │ │ │ │ ├── relrec.xpt │ │ │ │ ├── sc.xpt │ │ │ │ ├── se.xpt │ │ │ │ ├── suppae.xpt │ │ │ │ ├── suppdm.xpt │ │ │ │ ├── suppds.xpt │ │ │ │ ├── supplb.xpt │ │ │ │ ├── sv.xpt │ │ │ │ ├── ta.xpt │ │ │ │ ├── te.xpt │ │ │ │ ├── ti.xpt │ │ │ │ ├── ts.xpt │ │ │ │ ├── tv.xpt │ │ │ │ └── vs.xpt │ │ └── datatoc.pdf │ └── m5toc.pdf └── ndatoc.pdf ├── about.qmd ├── data ├── Errors.png ├── Errors2.png └── wg_attendees.xlsx ├── dev_background.qmd ├── faq.qmd ├── index.qmd ├── join.qmd ├── minutes.qmd ├── minutes ├── .DS_Store ├── 2020-12-04 │ └── index.qmd ├── 2021-01-15 │ └── index.qmd ├── 2021-02-05 │ ├── img │ │ └── sec_4.1.2.10.png │ └── index.qmd ├── 2021-03-05 │ └── index.qmd ├── 2021-04-05 │ └── index.qmd ├── 2021-05-07 │ └── index.qmd ├── 2021-06-04 │ └── index.qmd ├── 2021-08-06 │ └── index.qmd ├── 2021-09-03 │ ├── img │ │ └── p1.png │ └── index.qmd ├── 2021-10-01 │ └── index.qmd ├── 2021-11-05 │ ├── img │ │ └── ectd_slide.png │ └── index.qmd ├── 2021-12-03 │ └── index.qmd ├── 2022-01-07 │ └── index.qmd ├── 2022-02-04 │ └── index.qmd ├── 2022-03-05 │ └── index.qmd ├── 2022-04-01 │ └── index.qmd ├── 2022-05-06 │ └── index.qmd ├── 2022-06-04 │ └── index.qmd ├── 2022-07-01 │ └── index.qmd ├── 2022-08-05 │ └── index.qmd ├── 2022-09-02 │ └── index.qmd ├── 2022-10-07 │ └── index.qmd ├── 2022-11-04 │ └── index.qmd ├── 2022-12-02 │ └── index.qmd ├── 2023-01-13 │ └── index.qmd ├── 2023-02-03 │ └── index.qmd ├── 2023-03-03 │ └── index.qmd ├── 2023-04-07 │ ├── index.qmd │ └── repos.png ├── 2023-05-05 │ └── index.qmd ├── 2023-06-02 │ └── index.qmd ├── 2023-07-07 │ └── index.qmd ├── 2023-08-04 │ └── index.qmd ├── 2023-09-01 │ └── index.qmd ├── 2023-10-06 │ └── index.qmd ├── 2023-11-03 │ └── index.qmd ├── 2023-12-01 │ └── index.qmd ├── 2024-01-05 │ ├── index.qmd │ └── output_compare.png ├── 2024-02-02 │ └── index.qmd ├── 2024-03-01 │ ├── KM.png │ ├── index.qmd │ └── pilot4.png ├── 2024-04-05 │ ├── Index.qmd │ └── etcd.png ├── 2024-05-03 │ ├── VM.png │ ├── index.qmd │ └── p3error.png ├── 2024-06-07 │ ├── ADRG.png │ ├── dates.png │ └── index.qmd ├── 2024-08-02 │ └── index.qmd ├── 2024-09-06 │ └── index.qmd ├── 2024-10-04 │ └── index.qmd ├── 2024-11-01 │ └── index.qmd ├── 2024-12-06 │ └── index.qmd ├── 2025-01-10 │ └── index.qmd ├── 2025-02-07 │ └── index.qmd ├── 2025-03-07 │ └── index.qmd ├── 2025-04-04 │ └── index.qmd ├── 2025-05-02 │ └── index.qmd └── _metadata.yml ├── minutes_prep.qmd ├── pilot1.qmd ├── pilot2.qmd ├── pilot3.qmd ├── pilot4.qmd ├── pilot_background.qmd ├── styles.css └── submissions-wg.Rproj /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/.DS_Store -------------------------------------------------------------------------------- /.devcontainer/vscode/devcontainer.json: -------------------------------------------------------------------------------- 1 | // Config options: https://github.com/rocker-org/devcontainer-templates/tree/main/src/r-ver 2 | { 3 | "name": "R submissions WG site", 4 | "image": "ghcr.io/rocker-org/devcontainer/r-ver:4.4", 5 | // Add software 6 | "features": { 7 | // You may wish to switch prerelease to latest for stable development 8 | // Quarto configuration : https://github.com/rocker-org/devcontainer-features/blob/main/src/quarto-cli/README.md 9 | "ghcr.io/rocker-org/devcontainer-features/quarto-cli:1": { 10 | "installTinyTex": false, 11 | "version": "latest" 12 | }, 13 | // Additional R packages 14 | // config: https://github.com/rocker-org/devcontainer-features/tree/main/src/r-packages/README.md 15 | "ghcr.io/rocker-org/devcontainer-features/r-packages:1": { 16 | "packages": "openxlsx2,dplyr", 17 | "installSystemRequirements": true 18 | }, 19 | // Python 20 | // config: https://github.com/devcontainers/features/blob/main/src/python/README.md 21 | "ghcr.io/devcontainers/features/python:1": {} 22 | }, 23 | "customizations": { 24 | "vscode": { 25 | "settings": { 26 | "editor.tabSize": 2, 27 | "r.rterm.linux": "/usr/local/bin/radian", 28 | "r.bracketedPaste": true, 29 | "r.plot.useHttpgd": true, 30 | "[r]": { 31 | "editor.wordSeparators": "`~!@#%$^&*()-=+[{]}\\|;:'\",<>/?" 32 | } 33 | }, 34 | // Enable a development set of extensions for Lua and Quarto 35 | "extensions": ["quarto.quarto", "sumneko.lua", "ionutvmi.path-autocomplete", "hoovercj.vscode-power-mode", "REditorSupport.r", "Posit.shiny", "ms-python-python"] 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /.github/workflows/preview.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_dispatch: 3 | pull_request: 4 | branches: 5 | - main 6 | 7 | name: Preview Site 8 | 9 | jobs: 10 | build-deploy: 11 | runs-on: ubuntu-latest 12 | permissions: 13 | contents: write 14 | deployments: write 15 | steps: 16 | - name: Check out repository 17 | uses: actions/checkout@v4 18 | 19 | - name: Set up R 20 | uses: r-lib/actions/setup-r@v2 21 | with: 22 | use-public-rspm: true 23 | 24 | - name: Install R dependencies 25 | uses: r-lib/actions/setup-r-dependencies@v2 26 | with: 27 | packages: 28 | any::dplyr 29 | any::openxlsx2 30 | any::rmarkdown 31 | 32 | # ============================================ 33 | # Build 34 | # ============================================ 35 | 36 | - name: Set up Quarto 37 | uses: quarto-dev/quarto-actions/setup@v2 38 | 39 | - name: Render Site 40 | uses: quarto-dev/quarto-actions/render@v2 41 | 42 | # ============================================ 43 | # Deploy Preview to Netlify 44 | # ============================================ 45 | 46 | - name: Configure pull release name 47 | if: ${{github.event_name == 'pull_request'}} 48 | env: 49 | PR_NUMBER: ${{ github.event.number }} 50 | run: | 51 | echo "RELEASE_NAME=pr-${PR_NUMBER}" >> $GITHUB_ENV 52 | 53 | - name: Configure branch release name 54 | if: ${{github.event_name != 'pull_request'}} 55 | run: | 56 | # use branch name, but replace slashes. E.g. feat/a -> feat-a 57 | echo "RELEASE_NAME=${GITHUB_REF_NAME/\//-}" >> $GITHUB_ENV 58 | 59 | - name: Create Github "view deployment" button in PR 60 | if: ${{github.event_name == 'pull_request'}} 61 | uses: bobheadxi/deployments@v1 62 | id: deployment 63 | with: 64 | step: start 65 | token: ${{ secrets.GITHUB_TOKEN }} 66 | env: ${{ env.RELEASE_NAME }} 67 | ref: ${{ github.head_ref }} 68 | logs: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' 69 | 70 | - name: Deploy to Netlify 71 | env: 72 | NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} 73 | NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} 74 | REF_NAME: ${{ github.ref_name }} 75 | ALIAS: ${{ steps.deployment.outputs.env }} 76 | run: | 77 | npm install -g netlify-cli 78 | # push main branch to production, others to preview -- 79 | if [ "${REF_NAME}" = "master" ] || [ "${REF_NAME}" = "main" ]; then 80 | netlify deploy --dir=_site/ --prod 81 | else 82 | netlify deploy --dir=_site/ --alias="${ALIAS}" 83 | fi 84 | 85 | - name: Update Github "view deployment" button in PR 86 | uses: bobheadxi/deployments@v1 87 | if: ${{ always() && github.event_name == 'pull_request'}} 88 | with: 89 | step: finish 90 | token: ${{ secrets.GITHUB_TOKEN }} 91 | status: ${{ job.status }} 92 | deployment_id: ${{ steps.deployment.outputs.deployment_id }} 93 | env: ${{ steps.deployment.outputs.env }} 94 | env_url: 'https://${{ steps.deployment.outputs.env }}--${{ secrets.NETLIFY_SITE_NAME}}.netlify.app' 95 | logs: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_dispatch: 3 | push: 4 | branches: 5 | - main 6 | paths-ignore: 7 | - '\.devcontainer/**' 8 | - '\.github/**' 9 | - 'README.md' 10 | 11 | name: Quarto Publish 12 | 13 | jobs: 14 | build-deploy: 15 | runs-on: ubuntu-latest 16 | permissions: 17 | contents: write 18 | steps: 19 | - name: Check out repository 20 | uses: actions/checkout@v3 21 | 22 | - name: Set up Quarto 23 | uses: quarto-dev/quarto-actions/setup@v2 24 | 25 | - name: Set up R 26 | uses: r-lib/actions/setup-r@v2 27 | with: 28 | use-public-rspm: true 29 | 30 | - name: Install dplyr 31 | run: install.packages("dplyr") 32 | shell: Rscript {0} 33 | 34 | - name: Install openxlsx2 35 | run: install.packages("openxlsx2") 36 | shell: Rscript {0} 37 | 38 | - name: Install rmarkdown 39 | run: install.packages("rmarkdown") 40 | shell: Rscript {0} 41 | 42 | - name: Render and Publish 43 | uses: quarto-dev/quarto-actions/publish@v2 44 | with: 45 | target: gh-pages 46 | env: 47 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .Rproj.user 2 | .Rhistory 3 | .RData 4 | .Ruserdata 5 | 6 | /.quarto/ 7 | _site/ 8 | _prototyping/* 9 | !_prototyping/README.md 10 | -------------------------------------------------------------------------------- /.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/.nojekyll -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # submissions-wg 2 | R Submissions Working Group 3 | 4 | Welcome to the **R Consortium "R Submission" Working Group** 5 | 6 | The R submission working group is a cross industry pharma working group 7 | focusing on improving practices of R-based 8 | clinical trial regulatory submission. 9 | 10 | [Website](https://rconsortium.github.io/submissions-wg/) 11 | 12 | # Submission Pilots 13 | 14 | - Pilot 1: R based CDER submission of static outputs (completed) 15 | - Pilot 2: R based CDER submission with a shiny component (completed) 16 | - Pilot 3: R based CDER submission of ADaM dataset (completed) 17 | - Pilot 4: R based CDER submission with a container component (under development) 18 | 19 | 20 | # Related repositories: 21 | 22 | 23 | [Development repo for R submission pilot 1](https://github.com/RConsortium/submissions-pilot1) 24 | 25 | [R submission pilot 1 - Final eCTD submission package to FDA](https://github.com/RConsortium/submissions-pilot1-to-fda) 26 | 27 | [Development repo for R submission pilot 2](https://github.com/RConsortium/submissions-pilot2) 28 | 29 | [R submission pilot 2 - Final eCTD submission package to FDA](https://github.com/RConsortium/submissions-pilot2-to-fda) 30 | 31 | [Development repo for R submission pilot 3](https://github.com/RConsortium/submissions-pilot3-adam) 32 | 33 | [R submission pilot 3 - Final eCTD submission package to FDA](https://github.com/RConsortium/submissions-pilot3-adam-to-fda) 34 | 35 | [Development repo for R submission pilot 4](https://github.com/RConsortium/submissions-pilot4-webR) 36 | 37 | # News 38 | 39 | 40 | [2024/08/08] Pilot 3 (ADaM) FDA response letter received! [link](https://github.com/RConsortium/submissions-wg/blob/main/_Documents/Summary_R_Pilot3_Submission.pdf) 41 | 42 | [2023/09/28] Pilot 2 (shiny) FDA response letter received! [link](https://github.com/RConsortium/submissions-wg/blob/0f1dc5c30985d413f75d196c2b6caa96231b26ee/_Documents/Summary_R_Pilot2_Submission%2027SEP2023.pdf) 43 | 44 | [2023/09/11] [Pilot 3 (ADaM) has been successfully submitted to FDA](https://www.r-consortium.org/announcement/2023/09/11/first-publicly-available-r-based-submission-package-submitted-to-fda-pilot-3) 45 | 46 | [2022/12/07 Successful R-Based Package Submission with Shiny Component to FDA](https://www.r-consortium.org/blog/2022/12/07/update-successful-r-based-package-submission-with-shiny-component-to-fda) 47 | 48 | [2022/03/16 UPDATE: Successful R-based Test Package Submitted to FDA](https://www.r-consortium.org/blog/2022/03/16/update-successful-r-based-test-package-submitted-to-fda) 49 | 50 | [2021/12/08 Successful R-based Test Package Submitted to FDA](https://www.r-consortium.org/blog/2021/12/08/successful-r-based-test-package-submitted-to-fda) 51 | -------------------------------------------------------------------------------- /_Documents/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/.DS_Store -------------------------------------------------------------------------------- /_Documents/20231211_hye_soo_cho.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/20231211_hye_soo_cho.pdf -------------------------------------------------------------------------------- /_Documents/20231211_paul_schuette.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/20231211_paul_schuette.pdf -------------------------------------------------------------------------------- /_Documents/20240108_R_consortium_JPMA_F.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/20240108_R_consortium_JPMA_F.pdf -------------------------------------------------------------------------------- /_Documents/22008_06_jung.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/22008_06_jung.pdf -------------------------------------------------------------------------------- /_Documents/22008_06_zeballos.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/22008_06_zeballos.pdf -------------------------------------------------------------------------------- /_Documents/FDAInfrastructure.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/FDAInfrastructure.pptx -------------------------------------------------------------------------------- /_Documents/Leng_Update_10-1-21.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/Leng_Update_10-1-21.pptx -------------------------------------------------------------------------------- /_Documents/Letter_to_FDA.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Reviewed and Final: Letter to FDA" 3 | author: "Joseph Rickert" 4 | date: "5/13/2021" 5 | output: html_document 6 | --- 7 | 8 | ```{r setup, include=FALSE} 9 | knitr::opts_chunk$set(echo = TRUE) 10 | ``` 11 | 12 | 13 | May 13, 2021 14 | 15 | 16 | Director Ethan Chen 17 | Division of Data Management Services and Solutions 18 | 19 | 20 | Dear Director Chen: 21 | 22 | I am writing to you at the recommendation of Paul Schuette (CDER/OTS/OB) on behalf of the [R Submissions Working Group](https://github.com/RConsortium/submissions-wg) which is organized under the R Consortium$^{1}$. Our working group, primarily composed of representatives from several different pharmaceutical companies, is engaged in an effort to explore and address challenges in adapting our regulatory submission process to accommodate submissions using the R programming language. We are part of a larger effort to facilitate the use of R in the regulatory process that includes the R Consortium working groups: the [R Validation Hub](https://www.pharmar.org/) and [R Tables for Regulatory Submissions](https://github.com/RConsortium/rtrs-wg), and the PhUSE working group [Clinical Statistical Reporting in a Multilingual World](https://advance.phuse.global/display/WEL/Clinical+Statistical+Reporting+in+a+Multilingual+World). This work is also in alignment with the Transcelerate [MSA framework](https://www.transceleratebiopharmainc.com/wp-content/uploads/2021/04/MoA-Initiative_MSAFramework_April-2021.pdf) by pioneering the regulatory delivery of analysis using more rapidly developed, open source tools. 23 | 24 | We are presently working on a pilot submission to demonstrate that it is possible to make all R submissions in a manner that is consistent with the FDA's infrastructure and processes. Because this project is a multi-company, collaborative project being undertaken on behalf of the Pharmaceutical industry, we would like to explore the possibility of registering the R Consortium as a sponsor for the pilot submission. By registering a public and cross-pharma group as a sponsor, we hope to develop pilots in the open, without requiring an individual industry sponsor to manage a pilot submission. 25 | 26 | We appreciate that this would be somewhat out of the ordinary and would like your guidance in this effort. We would very much appreciate it if you, or someone from your office, could attend our next general meeting to discuss the feasibility of having the R Consortium as a sponsor. The meeting is scheduled for 12PM Eastern time on Friday, June 4, 2021. 27 | 28 | Please let me know if this is possible. 29 | 30 | Sincerely, 31 | 32 | 33 | Joseph B. Rickert 34 | Chair, R Consortium Board of Directors 35 | 36 | 37 | $^{1}$ The [R Consortium](https://www.r-consortium.org/) is a non-profit organization with the mission to support the R Language, the R Foundation and the R Community. 38 | 39 | -------------------------------------------------------------------------------- /_Documents/OpenSource1c_Paul_S_2023_ASA_biopharma.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/OpenSource1c_Paul_S_2023_ASA_biopharma.pdf -------------------------------------------------------------------------------- /_Documents/R Submission Pilot 1 Extension Proposal.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/R Submission Pilot 1 Extension Proposal.pdf -------------------------------------------------------------------------------- /_Documents/R submission 2023 BIOP poster.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/R submission 2023 BIOP poster.pdf -------------------------------------------------------------------------------- /_Documents/R submission Pilots to FDA - SDSS presentation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/R submission Pilots to FDA - SDSS presentation.pdf -------------------------------------------------------------------------------- /_Documents/R submission Pilots to FDA - useR presentation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/R submission Pilots to FDA - useR presentation.pdf -------------------------------------------------------------------------------- /_Documents/R_Submission_Pilot_3_ADaM_Proposal.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/R_Submission_Pilot_3_ADaM_Proposal.pdf -------------------------------------------------------------------------------- /_Documents/R_and_Shiny_in_Regulatory_Submission_to_Swissmedic_2024.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/R_and_Shiny_in_Regulatory_Submission_to_Swissmedic_2024.pdf -------------------------------------------------------------------------------- /_Documents/R_eCTD_pilot_submission.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/R_eCTD_pilot_submission.docx -------------------------------------------------------------------------------- /_Documents/SDSS poster - r submission.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/SDSS poster - r submission.pdf -------------------------------------------------------------------------------- /_Documents/SDTCG-SoftwareProgram.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/SDTCG-SoftwareProgram.docx -------------------------------------------------------------------------------- /_Documents/Summary_R_Pilot2_Submission 27SEP2023.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/Summary_R_Pilot2_Submission 27SEP2023.pdf -------------------------------------------------------------------------------- /_Documents/Summary_R_Pilot3_Submission.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/Summary_R_Pilot3_Submission.pdf -------------------------------------------------------------------------------- /_Documents/Summary_R_Pilot_Submission.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/Summary_R_Pilot_Submission.pdf -------------------------------------------------------------------------------- /_Documents/Summary_R_Pilot_Submission2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/Summary_R_Pilot_Submission2.pdf -------------------------------------------------------------------------------- /_Documents/Yilong_Zhang_eSub_proposal_12-4-20.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Documents/Yilong_Zhang_eSub_proposal_12-4-20.pptx -------------------------------------------------------------------------------- /_Documents/pilot_submission_proposal.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "R Pilot Submission for eCTD Compliance" 3 | author: "Mike Stackhouse, Yilong Zhang" 4 | date: "5/6/2021" 5 | output: github_document 6 | --- 7 | 8 | ## Problem Statement 9 | 10 | Interest in R has been growing steadily over the last few years, and several companies have their eyes set of producing a regulatory submission with significant portions of the analysis written in R. While we know that this is acceptable given the Statistical Software Clarifying Statement, there are still several challenges to overcome. Many initiatives across industry are focused on different aspects of this challenge, including the R Validation Hub, PHUSE Clinical Statistical Reporting in a Multilingual World, R Package Validation, the R Consortium R for Regulatory submission, and the R Consortium R Tables for Regulatory Submissions. Most of these initiatives focus on topics like validation, production of expected outputs, and anticipated questions that may arise when interfacing with a regulatory agency, but one question relatively unexplored is the physical delivery of code to the agency. 11 | 12 | The FDA has strict submission guidelines within documents like Technical Conformance Guide, and strict expectations of delivery format following the eCTD. Aspects of these requirements can make the delivery of R code non-trivial, particularly when it comes to R packages. Most R packages exist in open-source, accessible through sources like GitHub or the Comprehensive R Archive Network (CRAN), but it is inevitable that sponsors submitting to the FDA will need a method of delivering a package that cannot be generally accessible to the public. This project will address this challenge in the context of a pilot submission. 13 | 14 | ## Problem Impact 15 | 16 | This project has the potential to give a roadmap to submitting sponsors actively working towards R submissions. Many questions about the process of submitting R code to the FDA will be answered, and there is a large potential to identify questions that must be explored in more detail. 17 | 18 | ## Problem Scope 19 | 20 | Fortunately there is a solid foundation already in existense on which to build this project: 21 | 22 | - [The PHUSE Test Data Factory project upversion of the CDISC Pilot Data](https://github.com/phuse-org/TestDataFactory/tree/main/Updated) 23 | - [The Atorus replication of the CDISC Pilot Project in R](https://github.com/atorus-research/CDISC_pilot_replication) 24 | - [Merck's open-source R package pkglite](https://merck.github.io/pkglite/) 25 | 26 | Atorus' replication of the CDISC pilot gives us a baseline of code to utilize as a starting place. This project replicated the original CDISC pilot's outputs, and thus gives a solid foundation of analysis programs to submit as a test case. 27 | 28 | Merck's R package pkglite was specifically developed to enable the delivery of an R package through the eCTD. This package is able to package the code into a submittable format, and then unpack the code and restore its state to an installable R package, enabling the agency to utilize the delivered code as expected. 29 | 30 | The goal of this project is to conduct a pilot submission using these available tools. The CDISC pilot replication will be submitted, and an R package will be delivered through the eCTD to evaluate if all compliance checks relevant to the programs can be met. 31 | 32 | The scope of this project is limited to evaluating the consumability of the code. Rather than evaluate the entire submission package, we intend to verify that: 33 | - The packaged code is compliant with the eCTD 34 | - A consumer of the eCTD on the agency side is able to unpack and install the delivered R package 35 | - A consumer of the eCTD on the agency side is able to rerun the analaysis package delivered and replicate the results 36 | 37 | Different questions may arise during this process, but those questions will be left for further exploration in follow-ups to this project 38 | 39 | This project is a good candidate for collaboration between PHUSE and the R Consortium. With the R Consortium's wealth of technical expertise, and PHUSE's experience with navigating regulatory agencies (including a recent pilot submission within PHUSE's [Best Practices for Interactive Vizualiations](https://advance.phuse.global/display/WEL/Best+Practices+for+Interactive+Analysis+for+Decision+Making+Submissions) project) and relationship with the FDA. 40 | 41 | ## Project Deliverable(s) and Timeline(s): 42 | 43 | - Submission of pilot ~6 months 44 | - White paper outlining the process and lessons learned ~ 10 months 45 | 46 | ## Tools Required/Planned to be Developed 47 | 48 | Primary tools have been developed. Project requires a pilot submission process through the eCTD, so need collaboration with FDA. 49 | 50 | ## Stakeholders: 51 | 52 | Pharma, CRO, regulators, tech companies 53 | -------------------------------------------------------------------------------- /_Documents/pilot_submission_proposal.md: -------------------------------------------------------------------------------- 1 | R Pilot Submission for eCTD Compliance 2 | ================ 3 | Mike Stackhouse, Yilong Zhang 4 | 5/6/2021 5 | 6 | ## Problem Statement 7 | 8 | Interest in R has been growing steadily over the last few years, and 9 | several companies have their eyes set of producing a regulatory 10 | submission with significant portions of the analysis written in R. While 11 | we know that this is acceptable given the Statistical Software 12 | Clarifying Statement, there are still several challenges to overcome. 13 | Many initiatives across industry are focused on different aspects of 14 | this challenge, including the R Validation Hub, PHUSE Clinical 15 | Statistical Reporting in a Multilingual World, R Package Validation, the 16 | R Consortium R for Regulatory submission, and the R Consortium R Tables 17 | for Regulatory Submissions. Most of these initiatives focus on topics 18 | like validation, production of expected outputs, and anticipated 19 | questions that may arise when interfacing with a regulatory agency, but 20 | one question relatively unexplored is the physical delivery of code to 21 | the agency. 22 | 23 | The FDA has strict submission guidelines within documents like Technical 24 | Conformance Guide, and strict expectations of delivery format following 25 | the eCTD. Aspects of these requirements can make the delivery of R code 26 | non-trivial, particularly when it comes to R packages. Most R packages 27 | exist in open-source, accessible through sources like GitHub or the 28 | Comprehensive R Archive Network (CRAN), but it is inevitable that 29 | sponsors submitting to the FDA will need a method of delivering a 30 | package that cannot be generally accessible to the public. This project 31 | will address this challenge in the context of a pilot submission. 32 | 33 | ## Problem Impact 34 | 35 | This project has the potential to give a roadmap to submitting sponsors 36 | actively working towards R submissions. Many questions about the process 37 | of submitting R code to the FDA will be answered, and there is a large 38 | potential to identify questions that must be explored in more detail. 39 | 40 | ## Problem Scope 41 | 42 | Fortunately there is a solid foundation already in existense on which to 43 | build this project: 44 | 45 | - [The PHUSE Test Data Factory project upversion of the CDISC Pilot 46 | Data](https://github.com/phuse-org/TestDataFactory/tree/main/Updated) 47 | - [The Atorus replication of the CDISC Pilot Project in 48 | R](https://github.com/atorus-research/CDISC_pilot_replication) 49 | - [Merck’s open-source R package 50 | pkglite](https://merck.github.io/pkglite/) 51 | 52 | Atorus’ replication of the CDISC pilot gives us a baseline of code to 53 | utilize as a starting place. This project replicated the original CDISC 54 | pilot’s outputs, and thus gives a solid foundation of analysis programs 55 | to submit as a test case. 56 | 57 | Merck’s R package pkglite was specifically developed to enable the 58 | delivery of an R package through the eCTD. This package is able to 59 | package the code into a submittable format, and then unpack the code and 60 | restore its state to an installable R package, enabling the agency to 61 | utilize the delivered code as expected. 62 | 63 | The goal of this project is to conduct a pilot submission using these 64 | available tools. The CDISC pilot replication will be submitted, and an R 65 | package will be delivered through the eCTD to evaluate if all compliance 66 | checks relevant to the programs can be met. 67 | 68 | The scope of this project is limited to evaluating the consumability of 69 | the code. Rather than evaluate the entire submission package, we intend 70 | to verify that: - The packaged code is compliant with the eCTD - A 71 | consumer of the eCTD on the agency side is able to unpack and install 72 | the delivered R package - A consumer of the eCTD on the agency side is 73 | able to rerun the analaysis package delivered and replicate the results 74 | 75 | Different questions may arise during this process, but those questions 76 | will be left for further exploration in follow-ups to this project 77 | 78 | This project is a good candidate for collaboration between PHUSE and the 79 | R Consortium. With the R Consortium’s wealth of technical expertise, and 80 | PHUSE’s experience with navigating regulatory agencies (including a 81 | recent pilot submission within PHUSE’s [Best Practices for Interactive 82 | Vizualiations](https://advance.phuse.global/display/WEL/Best+Practices+for+Interactive+Analysis+for+Decision+Making+Submissions) 83 | project) and relationship with the FDA. 84 | 85 | ## Project Deliverable(s) and Timeline(s): 86 | 87 | - Submission of pilot \~6 months 88 | - White paper outlining the process and lessons learned \~ 10 months 89 | 90 | ## Tools Required/Planned to be Developed 91 | 92 | Primary tools have been developed. Project requires a pilot submission 93 | process through the eCTD, so need collaboration with FDA. 94 | 95 | ## Stakeholders: 96 | 97 | Pharma, CRO, regulators, tech companies 98 | -------------------------------------------------------------------------------- /_Documents/project-plan.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Project Plan of R Pilot Submission for eCTD Compliance" 3 | output: html_document 4 | --- 5 | 6 | ## Project proposal and vision 7 | 8 | The proposal of the project can be found in [here](https://htmlpreview.github.io/?https://github.com/RConsortium/submissions-wg/blob/main/Documents/pilot_submission_proposal.html) 9 | 10 | Our vision: 11 | 12 | - Make full R submission easy today 13 | - by showing open examples of using current systems 14 | - Make full R submission easy tomorrow 15 | - by influencing future decisions on system/process setup 16 | 17 | ## General Project Milestone: 18 | 19 | This document only cover the first phase with a gold to submit a package with selected analysis. 20 | 21 | There could be extended phase as below: 22 | 23 | - Phase 1: selected common ADaM domain and simple analysis 24 | - pkglite (txt), simple analysis 25 | - Phase 2: extend to commonly used efficacy/safety analysis 26 | - pkglite (txt), complicated analysis; analysis scripts as pkg 27 | - Phase 3: TA specific ADaM domain 28 | - using `.r` as file extension in submission 29 | 30 | The milestone listed below is only for **Phase 1**. 31 | 32 | - Set up R consortium account with FDA. 33 | - Prepare submission program using `pkglite` 34 | - baseline: Table 14-2.01, 35 | - primary analysis: Table 14-3.01, Table 14-3.02 36 | - figure: Figure 14-1 37 | - code: `https://github.com/atorus-research/CDISC_pilot_replication/blob/master/programs` 38 | 39 | > the output can be found in `submission-package/m5/53-clin-stud-rep/535-rep-effic-safety-stud/5351-stud-rep-contr/cdiscpilot01/cdiscpilot01.pdf` 40 | 41 | Note: 42 | 43 | - the format of the table is not necessary to be exactly the same. 44 | - we may consider to use different R packages, e.g. `gt`, `PharmaRTF`, `r2rtf`, etc 45 | 46 | - Update PDF files with details in next section. 47 | - ADRG 48 | - ARM 49 | - `define.xml` and `define.pdf` (if required) 50 | 51 | - Update cover letter 52 | - Including a checklist of items to be reviewed by FDA. 53 | 54 | ## R consortium sponsor account 55 | 56 | - cost of filling from FDA 57 | 58 | ## Expectation 59 | 60 | - cost of filling from FDA 61 | - reconstruct and load submitted R package(s) (non open sourced packages that submitted via eCTD) 62 | - load open sourced packages used in this submission 63 | - reproduce analysis results 64 | - share processes/steps that may be improved 65 | - written response from FDA (A reference but in different purpose https://www.fda.gov/media/99313/download) 66 | 67 | 68 | 69 | ## File Prepration 70 | 71 |
72 | Click to see file structure 73 | 74 | ```{r} 75 | fs::dir_tree("submission-package/") 76 | ``` 77 | 78 |
79 | 80 | - Copied all files in [CDISC pilot submission project](https://bitbucket.cdisc.org/projects/CED/repos/sdtm-adam-pilot-project/browse/updated-pilot-submission-package/900172) 81 | 82 | New program in R 83 | 84 | - ToDo 85 | 86 | Analysis Program: replace `.sas` file to R program with `pkglite`. 87 | 88 | - `submission-package/m5/datasets/cdiscpilot01/analysis/adam/programs/` 89 | 90 | File needs to be updated: 91 | 92 | - `submission-package/ndatoc.pdf` 93 | - `submission-package/m1/us/12-cov-let/cover.pdf` 94 | - `submission-package/m5/m5toc.pdf` 95 | - ADRG: `submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/dataguide.pdf` 96 | - ARM: currently be added in `submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/define.pdf` 97 | - define.xml: `submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/define.xml` 98 | 99 | Files may be used directly: 100 | 101 | - `submission-package/m5/datasets/datatoc.pdf` 102 | - All files in `submission-package/m5/53-clin-stud-rep` 103 | - Contain CSR and narratives. 104 | - All files in `submission-package/m5/datasets/cdiscpilot01/tabulations` 105 | - All files in `submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/*.xpt` 106 | 107 | ## Check list: 108 | 109 | - Hyperlink in PDF and define.xml work properly. 110 | - Confirm SDTM/ADaM data version 111 | - Pass P21 check for all datasets 112 | - Confirm `define.xml` work for both SDTM and ADaM data 113 | - Do we need `submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/define-v1-updated-html.xml` -------------------------------------------------------------------------------- /_Minutes/Errors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Minutes/Errors.png -------------------------------------------------------------------------------- /_Minutes/Errors2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Minutes/Errors2.png -------------------------------------------------------------------------------- /_Minutes/R consortium R submission Working Group Updates 9-3-2021.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Minutes/R consortium R submission Working Group Updates 9-3-2021.pdf -------------------------------------------------------------------------------- /_Minutes/R_Sub_10-07-2022_Minutes.md: -------------------------------------------------------------------------------- 1 | ## Minutes for R Submissions WG 2022-10-07 2 | Compiled by Joseph Rickert (JBR) 3 | 4 | ### Attendees 5 | 6 | * Saghir Bashir -Argenx 7 | * Phil Bowsher - RStudio 8 | * Robert Devine - J&J 9 | * Miriam Fosatti - Merck 10 | * Sean Healey - Pfizer 11 | * Ellis Hughes - FSK 12 | * Ryan Johnson Rstudio 13 | * Ning Leng - Roche 14 | * Tadeusz Lewandoski - Roche 15 | * Qin Li 16 | * Eric Nantz - Lilly 17 | * Thomas Neitmann - Roche 18 | * Sam Parmar - Lilly 19 | * Joseph Rickert - R Consortium / RStudio 20 | * Nan Xiao - Merck 21 | * Renping Zhang - J&J 22 | * Yilong Zhang 23 | * Lei Zhao 24 | * Jizu Zhi - FDA 25 | * Honghong Zhou - Moderna 26 | 27 | 28 | The meeting was recorded and the [video](https://rstudio.zoom.us/rec/share/OfSxaPPUsLuFJtZeUjzsZ6kZoAtCn7GTAj-N1j4zU9gUXOI8NqU5SUXf7rbeG2eu.q4s5o6ZiCBJHId8h?startTime=1665158653000) is available. 29 | 30 | ### Pilot 2 Submission 31 | Ning began her update on the Pilot 2 Submission by thanking everyone who helped with the testing. 32 | 33 | Status: 34 | The submission package is ready. However, the submissions team must address some last minute comments from the FDA. One comment is a request to mask p-values that are automatically reported. The apparent concern is that by default p-values will automatically be updated and reported when users filter the data by subgroups. Because we only have a small test data set this will likely lead to widely varying p-values which could be misinterpreted by general reviewers who are not statisticians. The FDA may request additional outputs to me masked. These comments appear to address the long term use of interactive applications in submissions and are not related to the technical aspects of making the submission. 35 | 36 | Consequently, the meeting scheduled for next Tuesday which was intended for submitting Pilot 2 will now be devoted to addressing FDA comments. The team will reschedule the submission as soon as possible after addressing the comments. We do not anticipate a long delay. There are no technical difficulties with masking additional default outputs. 37 | 38 | In the meantime more testing would be helpful and appreciated. 39 | Follow the instructions in the Appendix for Pilot 2: https://github.com/RConsortium/submissions-pilot2/issues/46 40 | Put issues here: https://github.com/RConsortium/submissions-pilot2/issues/ 41 | 42 | JBR noted that there is now significant material in the GitHub repository for the working group which is indicative of the scope and scale of cooperation of the project. This is itself a success story of some interest. 43 | 44 | ### Pilot 3 - ADaM 45 | 46 | (**13:51min** into the video) Thomas Neitmann presented a proposal by Joel Laxamana and himself to extend Pilot 1 to include ADaM datasets programmed in R. The purpose of the proposal is to: 47 | 48 | 1. Demonstrate the viability of using R for ADaM dataset creation in the context of an FDA submission 49 | 2. Provide the industry with an end-to-end example of an R-based FDA filing 50 | 51 | Thomas' presentation may be found here: https://github.com/RConsortium/submissions-wg/tree/main/Documents 52 | 53 | Yilong suggested taking ODM XML as the source to make it more representative of an end-to-end process. This started a discussion about the value of beginning end-to-end examples from various different starting file formats. 54 | 55 | The present team team includes: 56 | 57 | * Thomas Neitmann - Roche (Co-Lead) 58 | * Joel Laxamana - Roche (Co-Lead) 59 | * Lei Zhao - Roche 60 | * Nicole Jones - Merck 61 | 62 | The team is looking for additional help. Please have a look at the presentation and contact one of the Co-Leads if you would like to get involved. 63 | 64 | The working group agreed to change the name of the project from Pilot 1-ADaM Extension to Pilot 3-ADaM to emphasize that the project will include a significant amount of new work. 65 | 66 | **Action Item** JBR will make Paul Schuette aware of this new project and ask for his feedback. 67 | 68 | 69 | ## DA Japan Talk 70 | 71 | (**31:30** min into video) Tadeusz presented a Shiny application that he developed for a presentation he is going to make at the [DIA Japan conference](https://www.diaglobal.org/en/conference-listing/meetings/2022/10/19th-dia-japan-annual-meeting-2022) on October 9, 2022. The intention of the presentation is to gauge the interest of the Japanese authorities. The session in which Tadeusz will present is called [LS09 Innovative and Dynamic Statistical Analytics for Regulatory Submissions](https://www.diaglobal.org/en/conference-listing/meetings/2022/10/19th-dia-japan-annual-meeting-2022/agenda/09/ls09?ref=LS09) 72 | 73 | ## Subgroups 74 | The working group discussed writing writing some material for the GitHub repository explaining the purpose of the various subgroups, listing the members, and explaining how newcomers can get involved. 75 | 76 | **Action Item** Joe and Eric will meet after the Pilot 2 submission to discuss documenting the subgroups. 77 | 78 | ## Containers 79 | 80 | The working group agreed that the planned project to submit a containerized version of the Pilot 2 submission will require a significant amount of new work and pose several challenges. It should therefore be new, independent pilot submission, perhaps Pilot 4-Containers. 81 | 82 | -------------------------------------------------------------------------------- /_Minutes/R_Sub_10-1-2021_Minutes.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "R_Sub_Minutes_10-1-21" 3 | author: "Joseph Rickert" 4 | date: "10/1/2021" 5 | output: html_document 6 | --- 7 | Minutes 8 | Prepared by Joseph Rickert 9 | 10 | ### Attendees 11 | * Heidi Curinckx 12 | * Bob Engle - BioGen 13 | * Miriam Fossati - Merck 14 | * Christopher Kania - Biogen 15 | * Doug Kelkhoff - Genentech 16 | * Ning Leng - Genentech 17 | * Joseph Rickert - R Consortium 18 | * Michael Rimler - GSK 19 | * Paul Schuette - FDA 20 | * Mike Stackhouse - Atorus 21 | * Ben Staub - GSK 22 | * Adrian Waddell - Roche 23 | * Heng Wang - Genentech 24 | * Peikun Wu - Merck 25 | * Nan Xiao - Merck 26 | * Jiang Xu - FDA 27 | * Renping Zhang - FDA 28 | * Yilong Zhang - Merck 29 | * Jizu Zhi - FDA 30 | 31 | Joseph Rickert brought the meeting to order and Ning Leng presented an up update on the status of the Pilot 1 project using the slide deck Leng_Update_10-1-21.pptx which is available in the [Documents folder](https://github.com/RConsortium/submissions-wg/tree/main/Documents) of submissions-wg GitHub repository. 32 | 33 | Paul Schuette suggested that Ning revise the document to refer to "FDA staff" rather than "FDA Reviewer" which provides more latitude for getting FDA help. Paul also noted that although it might be possible it is not likely that the FDA could accomplish the task requested before the R/Pharma conference which begins on November 2. Those present agreed that it is not necessary to tie the completion of the FDA activities to R/Pharma. 34 | 35 | After Ning's presentation there was a discussion about suggestions for amending the language to section 4.1.2.10 of the Study Data Technical Performance Guide. Doug Kelkhoff noted that in internal discussions within Genentech the idea was floated that, as written the document may already allow passing R code in tar balls through the FDA gateway. If this were indeed true it would greatly simply R submissions since tar balls are a common method of delivering R code. Jian Xu of the FDA suggest that the way to get an answer to this question is to submit the question to the email address provided on the FDA eCTD webpage: [Submit an eCTD or Standardized Data Sample to the FDA](https://www.fda.gov/drugs/electronic-regulatory-submission-and-review/submit-ectd-or-standardized-data-sample-fda). Joe and Doug took the **action item** to write up the question and submit it. 36 | 37 | The group then considered a proposal by Adrian to organize the several GitHub repositories and activities of the Submissions WG using the Teams features of GitHub. The group agreed that this would be a good idea but suggested waiting until after Pilot1 is completed. Adrian will set up a meeting with Joe and Ning to setup the structure. 38 | 39 | The video of today's 10-1-21 meeting is available [here](https://rstudio.zoom.us/rec/share/q3guNFXtWp5aKRYAMB1f7CkW2miueFj_uSBuKnvsAksQi9vYvy9IsEAgBgy5eNje.p4Qupsx4WuBHZgQM) passcode: &1b4VEkR 40 | 41 | The next meeting of the WG will be at 9AM Pacific Time on Friday November 5, 2021. 42 | 43 | 44 | -------------------------------------------------------------------------------- /_Minutes/R_Sub_11-04-22_Minutes.md: -------------------------------------------------------------------------------- 1 | ## Minutes for R Submissions WG 2022-11-04 2 | Compiled by Joseph Rickert (JBR) 3 | 4 | ### Attendees 5 | 6 | * Saghir Bashir -Argenx 7 | * Gregory Chen 8 | * HyeSoo Cho - FDA 9 | * Robert Devine - J&J 10 | * Bob Engle 11 | * Ellis Hughes - FSK 12 | * Joel Laxamana 13 | * Yutong Liu 14 | * Ning Leng - Roche 15 | * Tadeusz Lewandoski - Roche 16 | * Stephanie Lussier 17 | * Eric Nantz - Lilly 18 | * Joseph Rickert - R Consortium / RStudio 19 | * Heng Wang 20 | * Nan Xiao - Merck 21 | * Paul SCHUETTE 22 | * Renping Zhang - J&J 23 | * Yilong Zhang 24 | * Lei Zhao 25 | * Jizu Zhi - FDA 26 | * Honghong Zhou - Moderna 27 | * Lian 28 | * Yiwen 29 | 30 | 31 | The [video](https://www.youtube.com/watch?v=LDVWXgDhzzo) is available on the [R Consortium YouTube Channel](https://www.youtube.com/channel/UC_R5smHVXRYGhZYDJsnXTwg) 32 | 33 | The meeting began with EN summarizing his understanding of the comments made by PS about his preliminary look at the Pilot 2 submission package. 34 | 35 | The following is a summary of Paul's comments which were made in a email exchange with the workgroup members on 10-27-22: 36 | 37 | * Paul has management approval to proceed with Phase 2 of the pilot, with the proviso that the Demographic, Primary and Efficacy Tables are static. 38 | * There is a minor error in the app: the low dose and high dose column settings reversed. 39 | * The efficacy table is slightly misleading as it currently stands. Suggest that the text for the first row read “Study Drug (High Dose)” or similar. 40 | * Suggest an additional table: "Subjects Remaining in the Study" showing how dropouts proceed. (See Paul's email of 10-27-22 for details) This might show how exploratory analysis might add value over the prespecified 41 | * The number of decimal places displayed in the demographic table isn’t always the same as in pilot 1. 42 | * Suggest that the model specification information be included with the efficacy table rather than left to the app information. 43 | * We would like the app to be able to serve as a model for future submissions. 44 | 45 | Eric stated that the updates appear to be relatively minor and would not be difficult to accomplish. 46 | 47 | The submission team agreed to target 11-18-2022 as the target submission date. 48 | 49 | **Action:** JBR to put a 9AM 11-18-2022 on the calendar for the submission. 50 | 51 | JBR asked Paul to describe the boundary between a useful amount and too much interactivity in a shiny submissions app. Paul replied that it a way to think about it would be the difference between exploratory and inferential analysis. The problem with inferential analysis is the usual problem with p-values, subgroups and cherry picking. Inferential analysis is not the way to go. A Kaplan-Meier curve would be considered an exploratory safety analysis. 52 | 53 | When asked if the guidance on interactivity was likely to stand for some time or change with familiarity Paul replied that it was his personal opinion that since it is consistent with the [ASA Statement on p-values](https://www.amstat.org/asa/files/pdfs/p-valuestatement.pdf) it will probably stand. 54 | 55 | Paul also remarked that interactivity facilitating pre-specified analyses would probably be fine. 56 | 57 | The following links may be helpful with understanding potential problems with on subgroup analyses: 58 | 59 | * [Alosh et al. (2015)](https://www.tandfonline.com/doi/full/10.1080/19466315.2015.1077726) 60 | * [Alosh et al. (2016)](https://onlinelibrary.wiley.com/doi/full/10.1002/sim.7167) 61 | 62 | Joel Laxamana gave an update on the Pilot 3 effort: 63 | 64 | * The kickoff meeting for the team was held this past Wednesday 65 | * There are 13 people on the team 66 | * The goal of Pilot 3 is to duplicate Pilot 1 but also generate the ADaM data sets used for the TLGs in Pilot 1 67 | * The idea is to start with the [STDM](https://www.cdisc.org/standards/foundational/sdtm) and show the traceability to [ADaM](https://www.cdisc.org/standards/foundational/adam) 68 | * Pilot 3 will use the `admiral` package along with `tidyverse` packages. 69 | 70 | **Action:** Joel will add this explanatory material to the Pilot 3 GitHub repo. 71 | 72 | **Action:** Joel will send a list of Pilot 3 team members to be added to the invitation list for this meeting. 73 | 74 | **Action:** The Pilot 3 team will submit a written proposal to the FDA describing the process of unpacking the submission and setting up for analysis. The target date is to have this proposal ready for the 12-02-2022 Submissions WG meeting. 75 | 76 | Lei Zhao asked if it would be a problem submitting data in SAS in doing the analysis in R. In general there is no problem, however the FDA suggests using a limited number of tools and using standard tools to do any translations between formats. The FDA prefers using scripted languages as opposed to using a GUI. 77 | 78 | Ning will be presented at the R/Pharma conference on Wednesday, November 9, 2022 at 11AM Eastern Time. 79 | The next meeting of the regular meeting of the group will be at 9AM Pacific Time on Friday, December 2, 2022. 80 | -------------------------------------------------------------------------------- /_Minutes/R_Sub_12-02-2022_Minutes.md: -------------------------------------------------------------------------------- 1 | ## Minutes for R Submissions WG 2022-11-04 2 | Compiled by Joseph Rickert (JBR) 3 | 4 | ### Attendees 5 | 6 | * Phil Bowsher 7 | * HyeSoo Cho - FDA 8 | * Heidi Curinckx 9 | * Robert Devine - J&J 10 | * Bob Engle 11 | * Miriam Fossatti - Merck 12 | * Declan Hodges 13 | * Ellis Hughes - FSK 14 | * Ryan Johnson 15 | * Chris Kania 16 | * Joel Laxamana 17 | * Ning Leng - Roche 18 | * Stephanie Lussier 19 | * Eric Nantz - Lilly 20 | * Thomas Neitmann 21 | * Sam Parmar 22 | * Xin Qiu 23 | * Joseph Rickert - R Consortium / RStudio 24 | * Nan Xiao - Merck 25 | * Paul SCHUETTE 26 | * Brian Sucher 27 | * Dadong Zhang 28 | * Renping Zhang - J&J 29 | * Yilong Zhang 30 | * Jizu Zhi - FDA 31 | 32 | ### Pilot 2 33 | * Ning Leng and Eric Nantz reported that Pilot 2 was submitted on 2022-11-18, upholding the working groups tradition of submitting right before Thanksgiving, and that confirmation was received from the the FDA. 34 | * Paul Schuette explained that since Pilot 2 was not a production submission it was necessary to make a manual request to receive it. Consequently, his group did not receive it until earlier this wee. 35 | * Heysoo Cho said that she will begin working on the submission next week. 36 | * Eric Nantz offered to be available to help if the FDA had any questions. 37 | 38 | ### Pilot 3 39 | Joel Laxamana briefly reviewed the preparations his team is making for Pilot 3 and said that the proposal is ready for transmission to the FDA. Paul suggested that the proposal should be sent to everyone in the working group and not just the FDA. Joel agreed to put the proposal in the Submissions WG GitHub repository and send the link to everyone in the working group. 40 | 41 | Joel noted that many of the Pilot 3 concepts were taken from the ADRG for Pilot 2 and mentioned that they are checking against the CDISC versions as they build the package. 42 | 43 | ### Pilot 4 44 | Eric said that he hopes to ramp up Pilot 4 work after the Pilot 2 submission. The simplest plan will be just to containerize the Pilot 2 submission. [Podman](https://podman.io/) is the initial choice for a the container technology. 45 | 46 | The group also expressed about aligning the Pilot 4 project with the Validation Hub to stand up a repo of validated packages. This would open up the possibility of hosting the container on the validation repo. The Validation Hub repo for this project is: https://github.com/pharmaR/regulatory-r-repo-wg 47 | 48 | 49 | 50 | ### Other Business 51 | 52 | JBR asked Ning and Joel for summary paragraphs for Pilot 2 and Pilot 3 for inclusion in end of the year reports on working group status. Ning is working on a blog post about Pilot 2 for inclusion in the R Consortium Blog. 53 | 54 | There were no updates available about efforts to make pilot submissions to other regulatory agencies. 55 | 56 | The following events were identified as possibilities for delivering presentations: 57 | 58 | * The Appsilon Shiny conference: [ShinyConf 2023](https://shinyconf.appsilon.com/) March 15-17 - Ning and Eric are drafting an abstract 59 | * R/Medicine June 8-9 60 | * R/Pharma 61 | * useR! 2023 62 | * Posit::conf 63 | * ASA BioPharma SWE Workshop in September 64 | 65 | #### Other future plans: 66 | 67 | * Ning noted that she and Yilong have been thinking about a journal submission and have begun writing a draft. 68 | * The group also agreed that they would like to do another R - Adoption Webinar. 69 | * Finally, Eric noted that he would like to rework the Submissions repository site and build it on a Quarto framework. 70 | * **The next meeting of the Submissions WG will be at 10AM PST, Friday 13, 2023.** This is a one week delay to move the meeting beyond the holiday season. 71 | 72 | -------------------------------------------------------------------------------- /_Minutes/R_Sub_2022-06-04_Minutes.md: -------------------------------------------------------------------------------- 1 | ## Minutes for R Submissions WG 2022-06-04 2 | Compiled by Joseph Rickert 3 | 4 | ### Attendees 5 | 6 | * Gabriel Becker 7 | * Michael Blanks - Beigene 8 | * Phil Bowsher - RStudio 9 | * Gregory Chen - MSD 10 | * Robert Devine - J&J 11 | * Bob Engle - Biogen 12 | * Steven Haesendonckx - J&J 13 | * Sean Healey - Pfizer 14 | * Ellis Hughs - GSK 15 | * Ryan Johnson - RStudio 16 | * Tadeusz Lewandowski - Roche 17 | * Xin Qiu J&J 18 | * Joseph Rickert - R Consortium / RStudio 19 | * Kui Shen 20 | * Phanikumar Tata 21 | * Nan Xiao - Merck 22 | * jizu Zhi - FDA 23 | 24 | Unfortunately, the video of the meeting is not available. 25 | 26 | JBR began the meeting with this agenda: 27 | 28 | * Pilot 2 Status Update 29 | * R Adoption Webinar 30 | * Pilot 3 Status? 31 | * Japan Submission Status? 32 | * China Submission Status? 33 | * Review project Board 34 | * Open Discussion 35 | 36 | ### Pilot 2 Status 37 | 38 | The following status was provided in an email to JBR by Ning Leng 39 | 40 | * The Pilot 2 team is addressing feedback from this working group 41 | * The Roche “NEST” software, which includes the `teal` package, is scheduled to be open sourced in mid June. 42 | * After `teal` open source: Pilot 2 team will package up the `shiny` app code using the `golem` package. 43 | * Pilot 2 is on track for July Submission 44 | 45 | JBR asked the following questions: 46 | 47 | * Will teal be put on CRAN? 48 | * Is the gateway ready? 49 | * Is the [cover letter](https://github.com/RConsortium/submissions-wg/issues/80) for Pilot 2 fine? 50 | 51 | No one present was sure whether the `teal` package would be submitted to CRAN before the submission or whether the Pilot 2 submission would be made with this package on GitHub. Tad L. noted `teal` will have an Apache 2.0 license. 52 | 53 | Tad Lewandowski noted that the Pilot 2 execution team received several helpful comments on the `shiny` app. The team plans to keep the initial app as simple as possible and will consider the ideas proposed in the comments as extensions for the future. He also noted hat the Roche team has built additional `shiny` apps that have been made public and are available for use in future submissions that collectively form a "clinical studies results viewer". These include: 54 | 55 | * [Efficacy](https://genentech.shinyapps.io/teal_efficacy/) 56 | * [Exploratory](https://genentech.shinyapps.io/teal_exploratory/) 57 | * [Safety](https://genentech.shinyapps.io/teal_safety/) 58 | 59 | 60 | The next meeting of the execution team will be on 6-16-2022 61 | 62 | ### R Adowtion Series Webinar 63 | The next R Consortium R Adoption series webinar will feature the work of the Submissions Working group 64 | The tentative agenda for the 2022 July R Adoption webinar is: 65 | 66 | * Tentative date: July 14 67 | * Meeting length: 1.5 hr 68 | * Opening (5min) - Coline, R consortium/Roche 69 | * Presentation: Achieving Regulatory Approval Using R (20min) Tae Hyun Jung, FDA 70 | * Presentation: Review experience of the R consortium R submission pilot 1 (20min) Hye Soo Cho, FDA 71 | * Panel discussion (45min) 72 | * Moderator: Ning Leng, R consortium/Roche 73 | * Panelists: Paul Schuette, Hye Soo Cho, and Tae Hyun Jung 74 | 75 | ### Pilot 3 76 | JBR asked the group to begin thinking about Pilot 3. He noted that we had been thinking about using container technology in a submission, but the WG has not completed its investigation into container technology and the FDA does not appear to be ready for such a submission. 77 | 78 | Tad suggested that questions about the content of a Pilot 3 submission would be appropriate for the panel discussion of the upcoming webinar. 79 | 80 | ### Japan Submission 81 | 82 | Tad and his colleagues have had a paper accepted for the upcoming [DIA Conference.](https://www.diaglobal.org/en/conference-listing/meetings/2022/10/19th-dia-japan-annual-meeting-2022) 83 | 84 | ### China Submission 85 | 86 | No update. 87 | 88 | 89 | 90 | ### General Discussion 91 | 92 | The group then updated the project [kanban board](https://github.com/RConsortium/submissions-wg/projects/2) moving the `shiny` documentation to the "In Progress" column. 93 | 94 | The discussion then turned towards considering the topic of HTA, [Health Technology Assessment](https://www.nlm.nih.gov/nichsr/hta101/ta10103.html) submissions that was considered in our previous meeting. We are only at the beginning stages and need discuss which European agency we would like to target and what we could bring to the HTA process. As was noted at our previous meeting, [NICE](https://www.nice.org.uk/) in the U.K. and [IQWIG](https://www.iqwig.de/en/about-us/methods/results/hta-reports/) in Germany are the primary candidates. 95 | 96 | **Action** HTA submissions will be a topic for the July Meeting. Tad Lewandowski will ask his colleague Christian Buehrer to speak about Roche's efforts in this area. 97 | 98 | Gregory Chen noted that several members of the WG have expressed in interest in HTA submissions. He also suggested that it would be very helpful to involve experts in this area to assist us. 99 | 100 | **Action** Gregory agreed to take the lead on compiling a list of experts whose participation could help the working group. 101 | 102 | **Action** Once the list is available, JBR agreed to contact people on the list to seek their participation. 103 | 104 | The next meeting of the WG will be at 9AM Pacific Time, on Friday July 1, 2022. 105 | -------------------------------------------------------------------------------- /_Minutes/R_Sub_6-30-21_Minutes.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "R_Submissions_WG_06-30-21_Minutes" 3 | author: "Joseph Rickert" 4 | date: "6/30/2021" 5 | output: html_document 6 | --- 7 | 8 | Minutes 9 | Prepared by Joseph Rickert 10 | 11 | ### Attendees 12 | 13 | * Heather Crandall - FDA 14 | * Bob Engle - Biogen 15 | * Steven Hasendinckx - J&J 16 | * Chris Kania - Biogen 17 | * Doug Kelkhoff - Genentech 18 | * Ning Leng - Genentech 19 | * Nicholas Masel 20 | * Eli Miller - Atorus 21 | * Joseph Rickert - R Consortium 22 | * Paul Schuette - FDA 23 | * Jiang Xu - FDA 24 | * Yilong Zhang - Merck 25 | 26 | 27 | ### Recording 28 | The meeting was recorded and the video is available at the link given below. 29 | 30 | ### Minutes 31 | 32 | Joseph Rickert opened the meeting and suggested that the group pick up where we left off last time and discuss the feasibility of having the R Consortium sponsor the pilot trial. The group briefly considered whether it would be easier for a pharma company to sponsor the submission. However after a brief discussion, those presently decided that it would be far better for the R Consortium to be the sponsor, because it would make it clear that this is an industry effort and that all work would be carried out in the open. 33 | 34 | There was consensus that Director Chen and his Office at the FDA were generally supportive of the effort and that there did not appear to be regulatory or other impediments with the RC being the sponsor. There is some uncertainty as to whether there would be a fee associated with the submission and how much that fee might be. 35 | 36 | There was a discussion about what should be included in first pilot submission and what the next steps should be. The members present concluded that the first pilot submission should be simple, but not trivial, that it should be something more than a "Hello World". All agreed that it would be optimal to plan the content of sequential submissions in order to make continuous progress with small steps in the beginning and perhaps more ambitious efforts as the project progressed. 37 | 38 | There was also some discussion about the form of the submission and the documentation that should be included. Paul Schuette suggested that the submission package should include a cover letter to set expectations. The letter should describe the submission at a high level, and describe how the objectives will be accomplished. The documentation for the submission package should include: 39 | * A cover letter 40 | * A detailed description of the submission package 41 | * Some suggestions how the FDA reviewers could evaluate the submission 42 | * A description of the kind of feedback that would be helpful 43 | * Perhaps a written response from the FDA describing their evaluation 44 | 45 | 46 | ### Action Items 47 | The members present agreed to divide the work as follows. 48 | 49 | ####Sponsor Application 50 | Joe Rickert, Eli Miller and Doug Kelkhoff will look into identifying all of the steps in the application process and begin the process.. 51 | 52 | #### Prepare the Submission Package 53 | Yilong Zhang volunteered to take the lead in designing and assembling the submission package. Those participating in this task will use the GitHub issues to communicate tasks. Yilong also offered to look into setting up a GitHub project. 54 | 55 | #### FDA Participation 56 | Paul agreed to see if his group could be the primary FDA contact. 57 | 58 | ### Video Recording 59 | The video recording of the meeting is available at: https://zoom.us/rec/share/u-mjqMhw-1SKYBIRM-xqrmGDWCQQBHxYqwhesCIe8Galpmzwup2QvKDmVBmXpvBI.Qb-Ez1OX75zdgoyF Passcode: ax2L&E71 60 | 61 | ### Next Meeting 62 | The next meeting will be held on Friday, August 6, 2021 at 9AM Pacific Time. 63 | -------------------------------------------------------------------------------- /_Minutes/R_Submissions_WG_02-05-2021_Minutes.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "R_Submissions_WG_02-05-2021_Minutes" 3 | author: "Joseph Rickert" 4 | date: "2/5/2021" 5 | output: html_document 6 | --- 7 | 8 | The meeting was held on zoom at 9AM Pacific Time on Friday, February 4th. The minutes taken by Joseph Rickert 9 | 10 | ### Attendees 11 | 12 | Attendees included: 13 | 14 | * Joseph Rickert - R Consortium 15 | * Adrian Waddell - Roche 16 | * Paul Schuette - FDA 17 | * Bella Feng - 18 | * Yilong Zhang - Merck 19 | * Eric Nantz - Lilly 20 | * Doug Kelkhoff - Genentech 21 | * Mike Stackhouse - Atorus Research 22 | * Nan Xiao - Merck 23 | * Ning Leng - Genentech 24 | * Sascha Ahrweiler - Bayer 25 | * Steven Hasendinckx J&J 26 | * Phil Bowsher - RStudio 27 | * Bob Engle - Biogen 28 | 29 | 30 | ### Minutes 31 | 32 | Joseph Rickert brought the meeting to order, reminded everyone of the [GitHub repo](https://github.com/RConsortium/submissions-wg) for the working group where minutes and other documents are kept, and summarized the goal articulated in the previous meeting to develop a pilot environment to study approaches for creating R submissions. 33 | 34 | Mike Stackhouse noted that the group should focus on creating an environment that was "consumable" by the FDA. Paul Schuette agreed with this. He noted that the experiment he did with Eric Nantz two years ago (discussed in a previous meeting) was the closest he has come so far. 35 | 36 | Next followed a discussion about the FDA computing environment, a discussion thread that developed over the course of the meeting. Some key points are: 37 | * FDA reviewers and researchers work primarily on desktop machines that mostly run Windows 10. 38 | * There are two classes of desktop computers 1) regulatory computers and 2) scientific computers. 39 | * The scientific computers are not as rigorously controlled as the scientific computers. 40 | * Paul has installed Ubuntu on a few of the scientific computers 41 | * Desktop computers run SAS version 9.4, but the build may differ 42 | * A perceived advantage of SAS is that it generally runs older software 43 | * There is a pilot project going on at the FDA to run SAS on a server, which would ensure a common build. 44 | * When asked: If the FDA decided to move to Linux, how long would it take to convert all of the desktops from Windows to Linux, Paul speculated that it would take a minimum of two years. The implication is that Windows is not going away anytime soon. 45 | * When asked about the installation of R packages, Paul indicated that installation was not a problem. 46 | * Obtaining R packages is more of an issue. In theory, the FDA could use MRAN or some other repository which enables downloading specific versions of packages. 47 | 48 | The discussion turned to considering section 4.1.2.10 of the Study Data Technical Performance Guide: 49 | 50 | 51 | 52 | 53 | Yilong Zhang stated that his company has focused on the sentence: "The Specific Software utilized should be specified in the ADRG." 54 | Mike Stackhouse explained that the PHUSE working group is is leading is also focused on interpreting this sentence. 55 | 56 | Paul noted that companies are obligated to use the SAS export file format, but are not obligated to use SAS. Bella Feng noted that a recent FDA oncology pilot explicitly specifies SAS. 57 | 58 | Paul noted that many FDA analyses are hybrid using both SAS and R. The group acknowledge that all R submissions were the ultimate goal, we should be planning to work with mixed submissions for some time. 59 | 60 | Mike Stackhouse stated that, although there are many corner cases, we should focus on building a reference platform along the lines of least resistance and go for what is easily achievable. The group generally agreed with this. 61 | 62 | Towards the end of the meeting Paul mentioned that although the that section 4.1.2.10 specifies that programs should be submitted as text files, it does not mean that .txt extensions must be used. This surprised everyone. When asked if the FDA document could be revised to make this more clear, Paul stated that it was possible. He said that the document is generally revised once or twice a year, the most recent revision is dated Nov 20, 2020. 63 | 64 | ** Action Item:** Paul stated that if the group wanted to suggest the wording to revise the document, he would submit it to the responsible FDA committee. He noted that to make the change it would need to be approved by the committee and go through the normal approval process. 65 | 66 | Joseph Rickert asked the group to use the repo to submit their suggested changes to the wording about using text, and reminded everyone that the next meeting was set for Friday, March 5, 2021. 67 | 68 | Joseph Rickert, Mike Stackhouse and Sascha Ahrweiler agreed to meet before the next Submissions group meeting to try and map out the various PHUSE, PSI and R Consortium working groups that are doing similar or complementary work. 69 | 70 | The link to the video of the meeting is: 71 | https://zoom.us/rec/share/gFQancacysPFHm0Czrw3sAYOCrt35BgTbcodBdY2LGld4fkykDypkL99PX5o0i-c.JRCWwmZfO9Ui8G6f 72 | 73 | Passcode: +GKR9&i6 74 | 75 | 76 | -------------------------------------------------------------------------------- /_Minutes/R_Submissions_WG_03-05-2021_Minutes.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "R_Submissions_WG_03_05-2021_Minutes" 3 | author: "Joseph Rickert" 4 | date: "3/5/2021" 5 | output: html_document 6 | --- 7 | 8 | ```{r setup, include=FALSE} 9 | knitr::opts_chunk$set(echo = TRUE) 10 | ``` 11 | 12 | 13 | ### Attendees 14 | 15 | * Joseph Rickert - R Consortium 16 | * Adrian Waddell - Roche 17 | * Paul Schuette - FDA 18 | * Bella Feng - EQRX 19 | * Yilong Zhang - Merck 20 | * Doug Kelkhoff - Genentech 21 | * Mike Stackhouse - Atorus Research 22 | * Nan Xiao - Merck 23 | * Ning Leng - Genentech 24 | * Steven Hasendinckx J&J 25 | * Bryant Chen - FDA 26 | * Ojesh Upadhyay - GSK 27 | 28 | ### Minutes 29 | Joseph Rickert opened the meeting at 9AM Pacific Time and pointed to the agenda in the Discussion Section of the GitHub repo. 30 | 31 | Agenda: 32 | 9:00: Doug Kelkhoff - R Validation Hub infrastructure 33 | 9:05: Nan Xiao the R package `pkglite` which packs/unpacks multiple R packages into txt file is going to be used in an upcoming FDA pilot eCTD submission 34 | 9:30: Open discussion 35 | We will continue the discussion initiated by Paul Schuette last time about modifying the software section of the [study data technical conformance guide](https://www.fda.gov/industry/fda-resources-data-standards/study-data-standards-resources) 36 | 37 | Doug provided an overview of the R Validation Hub protect, pointed out areas of overlap with this working group and welcomed those interested to participate in the Validation Hub effort. Detailed information on the R Validation Hub project which is also organized as an R Consortium working group can be found [here](https://www.pharmar.org/). 38 | 39 | Next, Nan Xiao of Merck gave an overview and demo of the R package `pkglite` which is expected to be up on CRAN soon. 40 | 41 | The specific aims of the package are: 42 | 43 | * Aim 1: To provide a tool for packing and restoring R packages as plaintext assets that are easy to store, transfer, and review. 44 | * Aim 2: To provide a grammar for specifying the file packing scope that is functional, precise, and extendable. 45 | * Aim 3: To provide a standard for exchanging the packed asset that is unambiguous, human-friendly, and machine-readable. 46 | 47 | Nan's presentation can be found in the Document's section of the R Consortium/submissions-wg GitHub repository. 48 | 49 | Yilong stated that he hoped that our working group could collaborate on a pilot submission to the FDA. He prepared the following outline to be included in these minutes. 50 | 51 | Goal: prepare a open source dummy submission and submit to eCTD portal 52 | 53 | The information below is to track potential high level action items. Individual issues can be created to track the progress later. 54 | 55 | Data source: [CDISC pilot submission](https://bitbucket.cdisc.org/projects/CED/repos/sdtm-adam-pilot-project/browse) 56 | 57 | R program for table listing and figure (TLFs): [program](program: https://github.com/atorus-research/CDISC_pilot_replication) 58 | 59 | General action items: 60 | 61 | 1. Make sure this use case is following CDISC terms of use 62 | 2. Identify TLFs in-scope for submission 63 | 64 | Action items for dummy submission package preparation: 65 | 66 | 1. Identify point of contact to initiate dummy submission request 67 | 2. Prepare analysis results metadata 68 | 3. Prepare step-by-step instruction to re-run analysis based on code in program folder and put in ADRG: link 69 | 4. Wrap-up submission package 70 | 71 | Those present all seemed to be enthusiastic about cooperating on the generating a dummy submission. 72 | 73 | Next, followed a lengthy discussion on the required process. Mike Stackhouse suggested that we work with PHUSE to suggest people who may be knowledgeable about past CDISC submissions. Paul Schuette recommended Zak Skrivanek. 74 | 75 | Yilong Zhang mentioned that Merck intends to use this package as part of a pilot oncology submission in the near future. Bella mentioned that the FDA Oncology group had specified SAS. 76 | 77 | Ning Leng of mentioned that Roche is preparing a rare disease submission for 2022. 78 | 79 | Joe Rickert asked the group: assuming the `pkglite` dummy submission is successful, how far the this would advance us towards the ultimate goal of an all R submission. 80 | 81 | While everyone was very excited about the `pkglite` all agreed with Doug Kelkhoff's sober assessment that the industry would ultimately want a more "R idiomatic" solution that took advantage of R naming conventions and did not introduce special requirements for R package maintenance. 82 | 83 | Mike Stackhouse noted that a successful `pkglite` submission would break down a significant barrier to R submissions, but that there were other hurdles. Nevertheless, he asserted that it would be a significant advance to be able to say here is "how" you can do an R submission. 84 | 85 | It the few minutes that remained, Joe Rickert brought the groups attention to the copy of the word document containing the text of section 4.1.2.10 of the FDA document pertaining to software and asked everyone mark up the document to suggest ideal wording from the industry's point of view. 86 | 87 | The group agreed to met next on Friday April 1, at 9AM Pacific time. 88 | 89 | The zoom recording of this meeting is available [here](https://zoom.us/rec/share/o-E_jt-wuDDbUCcEhQFwdsE2vZhPlAdcoqo0mgLNmKgqAe3DCDBkxTH_A7eR_MEa.MoCkXvxa5JzE9F2d) (Passcode: d9N%+JWY). 90 | -------------------------------------------------------------------------------- /_Minutes/R_Submissions_WG_04-05-21_Minutes.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "R_Submissions_WG_04-05-21_Minutes" 3 | author: "Joseph Rickert" 4 | date: "4/6/2021" 5 | output: html_document 6 | --- 7 | 8 | These Minutes were compiled by Joseph Rickert 9 | 10 | ### Attendees 11 | 12 | * Joseph Rickert - R Consortium 13 | * Paul Schuette - FDA 14 | * Yilong Zhang - Merck 15 | * Doug Kelkhoff - Genentech 16 | * Nan Xiao - Merck 17 | * Ning Leng - Genentech 18 | * Bob Engle - Biogen 19 | * Eric Nantz - Lilly 20 | * Phil Bowsher - RStudio 21 | * Sean Lopp - RStudio 22 | 23 | Joe Rickert brought the meeting to order and reminded attendees about the action item from the previous meeting to consider the SDTCG_SoftwareProgram document and comment on changes to the language that the group would like the FDA to consider. 24 | 25 | Next, Yilong Zhang shared a proposal that Mike Stackhouse submitted to PhHUSE outlining a pilot submission to the FDA prepared for PhUSE for consideration by the group. This document R_eCTD_pilot_submissions.docx. 26 | 27 | For the remainder of the meeting the attendees examined the content of the document and discuss how the project should proceed. The prevailing sentiment was that the pilot submission should be an R Consortium Submissions working group project. 28 | 29 | The video recording for the meeting is available here: 30 | https://zoom.us/rec/share/0uljlGZRgZGx9zZuRgloMOIH92qvtScRq1YtUZhxnC0GF2KxkMXsycDYykqHCIXl.pz5myyWa0kJ5JdnO 31 | Passcode: =0yt$%V9 32 | 33 | It was agreed that the group would like to go back to meeting on Fridays. The next meeting is scheduled for Friday, May 7, 2021. Thereafter, the group will continue to meet on the first Friday of the month. 34 | 35 | -------------------------------------------------------------------------------- /_Minutes/R_Submissions_WG_12-4-20_Minutes.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "R_Submissions_WG_12-4-20_Minutes" 3 | author: "Joseph Rickert" 4 | date: "12/8/2020" 5 | output: html_document 6 | --- 7 | 8 | ```{r setup, include=FALSE} 9 | knitr::opts_chunk$set(echo = TRUE) 10 | ``` 11 | 12 | 13 | ## R Consortium R Submissions Working Group 14 | Minutes of 12-07-2020 Meeting prepared by Joseph Rickert 15 | 16 | ## Attendees 17 | 18 | * Joseph Rickert - R Consortium 19 | * Adrian Waddell - Roche 20 | * Doug Kelkhoff - Genentech 21 | * Bob Engle - Biogen 22 | * Juliane Manitz - Emd Serono 23 | * Lyn Taylor - Phastar 24 | * Mike Stackhouse - Atorus 25 | * Paul Schuette - FDA 26 | * Sascha Ahrweiler - Bayer 27 | * Sean Lopp - RStudio 28 | * Tadeusz Lewandowski - Roche 29 | * Yilong Zhang - Merck 30 | * Paulo Bargo - Janssen 31 | * Andy Nicholls - GSK 32 | * Keaven Anderson - Merck 33 | * Kevin Bolger - Procogia 34 | 35 | Joseph Rickert opened the meeting and provided a brief overview of R Consortium Working Group concerned with Pharma related topics: 36 | 37 | * The R Validation Hub is working towards developing standards and tools for assessing the risk associated with working with R packages in regulatory environments 38 | * The RTRS Working Group is working for a standard for developing R packages that will be meet the 21 CFR Part 11 standard for constructing tables for clinical trials and FDA submissions 39 | * This new R Submissions working group will focus on IT and infrastructure issues 40 | 41 | Paul Schuette of the FDA then gave a brief overview of FDA Submissions Infrastructure. He described: 42 | * The Electronic Common Technical Document 43 | * The ECTD Technical Guide 44 | * ECTD M1 to M5 File Structure 45 | * The Study Data Resources Document 46 | * The Study Data Technical Conformance Guide 47 | * CDISC - Clinical Data Interchange Standards Consortium 48 | * STDM - Study Data Tabulation Model 49 | * ADaM - Analysis Data Model 50 | 51 | 52 | Adrian Waddell asked how flexible the FDA was willing to be on this infrastructure and whether it could be negotiated. Paul Schuette indicated that negotiation was possible. Thereafter, followed a robust discussion which was recorded and available at the following link. 53 | 54 | 55 | https://zoom.us/rec/share/d3SRLOn0bYxQhWWeo8ftoZb3uJC97bq3grdgtQ-MOyTToZb4JJoEWtBwQ25QnrPD.hp1xXVfAiYA1Twv4?startTime=1607105048000 56 | 57 | The pass code is yif%qn4p 58 | -------------------------------------------------------------------------------- /_Minutes/p1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Minutes/p1.png -------------------------------------------------------------------------------- /_Minutes/pilot2_shiny.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Minutes/pilot2_shiny.png -------------------------------------------------------------------------------- /_Minutes/sec_4.1.2.10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_Minutes/sec_4.1.2.10.png -------------------------------------------------------------------------------- /_freeze/about/execute-results/html.json: -------------------------------------------------------------------------------- 1 | { 2 | "hash": "a754ce02c0472c0b163d8ffb898e69ac", 3 | "result": { 4 | "markdown": "---\ntitle: \"About\"\n---\n\n\nAbout this site\n\n\n::: {.cell}\n\n```{.r .cell-code}\n1 + 1\n```\n\n::: {.cell-output .cell-output-stdout}\n\n```\n[1] 2\n```\n\n\n:::\n:::\n", 5 | "supporting": [], 6 | "filters": [ 7 | "rmarkdown/pagebreak.lua" 8 | ], 9 | "includes": {}, 10 | "engineDependencies": {}, 11 | "preserve": {}, 12 | "postProcess": true 13 | } 14 | } -------------------------------------------------------------------------------- /_prototyping/README.md: -------------------------------------------------------------------------------- 1 | Use this directory as a sandbox. The contents (other than this README) will not be version controlled nor included in the main website. -------------------------------------------------------------------------------- /_quarto.yml: -------------------------------------------------------------------------------- 1 | project: 2 | type: website 3 | 4 | website: 5 | title: "R Submissions Working Group" 6 | site-url: https://rconsortium.github.io/submissions-wg 7 | repo-url: https://github.com/RConsortium/submissions-wg 8 | repo-actions: [edit, issue] 9 | 10 | navbar: 11 | left: 12 | - href: index.qmd 13 | text: Home 14 | - minutes.qmd 15 | - href: join.qmd 16 | text: How to Join 17 | - text: "Pilots" 18 | menu: 19 | - pilot_background.qmd 20 | - pilot1.qmd 21 | - pilot2.qmd 22 | - pilot3.qmd 23 | - pilot4.qmd 24 | - href: https://rconsortium.github.io/submissions-pilot5-website/ 25 | text: R Submission Pilot 5 26 | - text: "Site Development" 27 | menu: 28 | - href: dev_background.qmd 29 | text: Development Guide 30 | - href: minutes_prep.qmd 31 | text: Minutes Draft Gen 32 | 33 | right: 34 | - icon: github 35 | href: https://github.com/RConsortium/submissions-wg 36 | aria-label: GitHub Site 37 | 38 | format: 39 | html: 40 | theme: 41 | light: flatly 42 | dark: darkly 43 | css: styles.css 44 | toc: true 45 | link-external-newwindow: true 46 | 47 | execute: 48 | freeze: auto 49 | -------------------------------------------------------------------------------- /_submission-package/m1/m1toc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m1/m1toc.pdf -------------------------------------------------------------------------------- /_submission-package/m1/us/12-cov-let/cover.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m1/us/12-cov-let/cover.pdf -------------------------------------------------------------------------------- /_submission-package/m5/53-clin-stud-rep/535-rep-effic-safety-stud/5351-stud-rep-contr/cdiscpilot01/cdiscpilot01.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/53-clin-stud-rep/535-rep-effic-safety-stud/5351-stud-rep-contr/cdiscpilot01/cdiscpilot01.pdf -------------------------------------------------------------------------------- /_submission-package/m5/53-clin-stud-rep/535-rep-effic-safety-stud/5351-stud-rep-contr/cdiscpilot01/narratives.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/53-clin-stud-rep/535-rep-effic-safety-stud/5351-stud-rep-contr/cdiscpilot01/narratives.txt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adae.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adae.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adlbc.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adlbc.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adlbh.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adlbh.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adlbhy.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adlbhy.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adqsadas.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adqsadas.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adqscibc.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adqscibc.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adqsnpix.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adqsnpix.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adsl.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adsl.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adtte.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/adtte.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/advs.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/advs.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/dataguide.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/dataguide.pdf -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/define.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/define.pdf -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/define.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/analysis/adam/datasets/define.xml -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/ae.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/ae.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/blankcrf.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/blankcrf.pdf -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/cm.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/cm.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/define.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/define.pdf -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/dm.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/dm.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/ds.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/ds.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/ex.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/ex.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/lb.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/lb.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/mh.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/mh.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/qs.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/qs.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/relrec.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/relrec.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/sc.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/sc.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/se.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/se.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/suppae.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/suppae.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/suppdm.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/suppdm.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/suppds.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/suppds.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/supplb.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/supplb.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/sv.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/sv.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/ta.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/ta.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/te.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/te.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/ti.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/ti.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/ts.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/ts.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/tv.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/tv.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/vs.xpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/cdiscpilot01/tabulations/sdtm/vs.xpt -------------------------------------------------------------------------------- /_submission-package/m5/datasets/datatoc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/datasets/datatoc.pdf -------------------------------------------------------------------------------- /_submission-package/m5/m5toc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/m5/m5toc.pdf -------------------------------------------------------------------------------- /_submission-package/ndatoc.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/_submission-package/ndatoc.pdf -------------------------------------------------------------------------------- /about.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "About" 3 | --- 4 | 5 | About this site 6 | 7 | ```{r} 8 | 1 + 1 9 | ``` 10 | -------------------------------------------------------------------------------- /data/Errors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/data/Errors.png -------------------------------------------------------------------------------- /data/Errors2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/data/Errors2.png -------------------------------------------------------------------------------- /data/wg_attendees.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/data/wg_attendees.xlsx -------------------------------------------------------------------------------- /dev_background.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Site Development" 3 | --- 4 | 5 | ## Overview 6 | 7 | The Submissions working group website is built using the [Quarto](https://quarto.org/) scientific and technical publishing system based in markdown format and compiled with Pandoc. 8 | 9 | ## Development Setup 10 | 11 | Ensure that you have Quarto version 1.2 or later installed on your development environment. More details on downloading and installing Quarto can be found in the [Get Started](https://quarto.org/docs/get-started/) guide. 12 | 13 | A recent version of R (4.1.2 or later) is sufficient. Ensure that you have the following R packages installed: 14 | 15 | * [`{openxlsx2}`](https://janmarvin.github.io/openxlsx2/) 16 | * [`{dplyr}`](https://dplyr.tidyverse.org/index.html) 17 | 18 | It is recommended to use the RStudio IDE (v2022.07 or later) as your development environment for building and rendering the web site. Refer to the [Creating a Website](https://quarto.org/docs/websites/) guide details on the workflow. 19 | 20 | ## Site Organization 21 | 22 | Most of the site uses standard configurations outlined in the [websites guide](https://quarto.org/docs/websites/). Below are additional details for developing site content based on the type of material: 23 | 24 | ### Pilots 25 | 26 | Each pilot conducted by this working group has its own dedicated page and associated quarto file named `pilotX.qmd` where `X` denotes the index of the pilot. High-level descriptions of all pilots is stored in `pilot_background.qmd`. 27 | 28 | ### Working Group Minutes 29 | 30 | The site uses the blog feature to create a custom listing for working group meeting minutes with all files stored in the `minutes` directory. Within the `minutes` directory, create a directory with the date of the working group meeting (for example `2023-01-23`) and within this directory, create a quarto document named `index.qmd`. Visit the [Minutes Draft Gen](minutes_prep.qmd) page for an interactive way of creating a new Quarto document file used for a future working group meeting minutes, including an easy way to assemble the attendees present at the meeting. -------------------------------------------------------------------------------- /faq.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Q & A" 3 | --- 4 | 5 | ## About R based regulatory submission 6 | 7 | 1. How to submit readable code in R to submit to the FDA? 8 | - Readable code can be submitted as .r files, according to the [R consortium R submission working group pilot 1](https://github.com/RConsortium/submissions-pilot1-to-fda). 9 | - Analytical codes using functions in open-sourced or proprietary packages are considered readable, as long as the proprietary R packages are submitted in the way as mentioned below. 10 | 11 | 12 | 2. How to submit proprietary R packages to FDA? 13 | - Proprietary packages cannot be submitted as a compressed file through the eCTD gateway. 14 | 15 | - However, proprietary R packages can be converted to a plain `.txt` file by [pkglite](https://merck.github.io/pkglite/) then submitted through the eCTD gateway. FDA reviewers can use the same package to convert the `.txt` file back to R package. An example can be found in the [R consortium R submission working group pilot 1](https://github.com/RConsortium/submissions-pilot1-to-fda). 16 | 17 | 3. What if open sourced R packages are used? 18 | - No need to submit source code of open source packages. 19 | - In adrg/programtoc, include package version, its dependency packages and their versions 20 | - In analytical code, include appropriate `library()` calls. 21 | 22 | 23 | 4. If R is used, what is needed for the ADRG? 24 | - If you use R it is recommended to have a section in the ADRG/programtoc that provides the FDA with: 25 | - R version 26 | - Package dependencies and their versions 27 | - Instructions to convert proprietary R package `.txt` files back to the natural format (if proprietary R packages are submitted) 28 | 29 | 30 | 31 | ## About the R consortium R submission working group 32 | 33 | 1. Who can join the working group? How can I get involved? 34 | - anyone can join the working group. The working group meets monthly on Friday. If you are interested in joining the monthly meetings, please contact Joe Rickert at joseph.rickert@rstudio.com. 35 | - You can also monitor our [issue board](https://github.com/RConsortium/submissions-wg/issues). Any contribution/feedback are more than welcome -------------------------------------------------------------------------------- /index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Introduction" 3 | --- 4 | 5 | Welcome to the R Consortium R Submission Working Group! 6 | 7 | The R submission working group is a cross industry pharma working group focusing on improving practices of R-based clinical trial regulatory submissions. 8 | 9 | You can view previous meeting minutes with links to recorded sessions [here](minutes.qmd) 10 | 11 | # Our Mission 12 | 13 | - Easier R-based clinical trial regulatory submissions today 14 | - by showing open examples of using current submission portals 15 | - Easier R-based clinical trial regulatory submissions tomorrow 16 | - by collecting feedback and influencing future industry and agency decisions on system/process setup 17 | 18 | ::: {.callout-tip} 19 | ## Join the Team! 20 | 21 | Please visit the [How to Join](join.qmd) page for complete details on joining our working group. 22 | ::: 23 | -------------------------------------------------------------------------------- /join.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: How to Join 3 | --- 4 | 5 | The Submissions Working Group welcomes all contributors who are passionate about bringing innovation to the clinical submission process! Here are the ways you can join the group's communication platforms and keep up-to-date with additional information: 6 | 7 | * Register for future Submissions Working Group meetings by visiting the [public calendar](https://zoom-lfx.platform.linuxfoundation.org/meetings/rcons?view=week). Our working group meetings typically occur on the first Friday of each month. 8 | * Send a note to our contact email: `operations[at]r-consortium.org`. 9 | * Join the Slock Group using the link [slack.r-consortium.org](https://slack.r-consortium.org) at the channel `#wg-submissions`. 10 | * Review the mailing list at [lists.r-consortium.org/g/Rconsortium-wg-submissions](https://lists.r-consortium.org/g/Rconsortium-wg-submissions) -------------------------------------------------------------------------------- /minutes.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Meeting Minutes" 3 | listing: 4 | contents: minutes # all Quarto docs in the `minutes` directory will be included on your listing page 5 | type: table # or `default` or `table`; each type has its own set of yaml options to include 6 | sort: "date desc" # can also sort on more than one field 7 | feed: true 8 | categories: true # allows you to sort posts by assigned categories 9 | --- 10 | -------------------------------------------------------------------------------- /minutes/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/.DS_Store -------------------------------------------------------------------------------- /minutes/2020-12-04/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2020-12-04" 3 | author: Joseph Rickert 4 | date: 12-04-2020 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - "Joseph Rickert - R Consortium" 11 | - "Adrian Waddell - Roche" 12 | - "Doug Kelkhoff - Genentech" 13 | - "Bob Engle - Biogen" 14 | - "Juliane Manitz - Emd Serono" 15 | - "Lyn Taylor - Phastar" 16 | - "Mike Stackhouse - Atorus" 17 | - "Paul Schuette - FDA" 18 | - "Sascha Ahrweiler - Bayer" 19 | - "Sean Lopp - RStudio" 20 | - "Tadeusz Lewandowski - Roche" 21 | - "Yilong Zhang - Merck" 22 | - "Paulo Bargo - Janssen" 23 | - "Andy Nicholls - GSK" 24 | - "Keaven Anderson - Merck" 25 | - "Kevin Bolger - Procogia" 26 | ::: 27 | 28 | {{< video https://youtu.be/FSctdTm74co >}} 29 | 30 | Joseph Rickert opened the meeting and provided a brief overview of R Consortium Working Group concerned with Pharma related topics: 31 | 32 | * The R Validation Hub is working towards developing standards and tools for assessing the risk associated with working with R packages in regulatory environments 33 | * The RTRS Working Group is working for a standard for developing R packages that will be meet the 21 CFR Part 11 standard for constructing tables for clinical trials and FDA submissions 34 | * This new R Submissions working group will focus on IT and infrastructure issues 35 | 36 | Paul Schuette of the FDA then gave a brief overview of FDA Submissions Infrastructure. He described: 37 | 38 | * The Electronic Common Technical Document 39 | * The ECTD Technical Guide 40 | * ECTD M1 to M5 File Structure 41 | * The Study Data Resources Document 42 | * The Study Data Technical Conformance Guide 43 | * CDISC - Clinical Data Interchange Standards Consortium 44 | * STDM - Study Data Tabulation Model 45 | * ADaM - Analysis Data Model 46 | 47 | Adrian Waddell asked how flexible the FDA was willing to be on this infrastructure and whether it could be negotiated. Paul Schuette indicated that negotiation was possible. Thereafter, followed a robust discussion which was recorded and available on the R Consortium YouTube Channel at [this link](https://youtu.be/FSctdTm74co). 48 | 49 | -------------------------------------------------------------------------------- /minutes/2021-02-05/img/sec_4.1.2.10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/2021-02-05/img/sec_4.1.2.10.png -------------------------------------------------------------------------------- /minutes/2021-02-05/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2021-02-05" 3 | author: Joseph Rickert 4 | date: 02-05-2021 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - "Joseph Rickert - R Consortium" 11 | - "Adrian Waddell - Roche" 12 | - "Paul Schuette - FDA" 13 | - "Bella Feng - EQRx" 14 | - "Yilong Zhang - Merck" 15 | - "Eric Nantz - Lilly" 16 | - "Doug Kelkhoff - Genentech" 17 | - "Mike Stackhouse - Atorus Research" 18 | - "Nan Xiao - Merck" 19 | - "Ning Leng - Genentech" 20 | - "Sascha Ahrweiler - Bayer" 21 | - "Steven Hasendinckx J&J" 22 | - "Phil Bowsher - RStudio" 23 | - "Bob Engle - Biogen" 24 | ::: 25 | 26 | Joseph Rickert brought the meeting to order, reminded everyone of the [GitHub repo](https://github.com/RConsortium/submissions-wg) for the working group where minutes and other documents are kept, and summarized the goal articulated in the previous meeting to develop a pilot environment to study approaches for creating R submissions. 27 | 28 | Mike Stackhouse noted that the group should focus on creating an environment that was "consumable" by the FDA. Paul Schuette agreed with this. He noted that the experiment he did with Eric Nantz two years ago (discussed in a previous meeting) was the closest he has come so far. 29 | 30 | Next followed a discussion about the FDA computing environment, a discussion thread that developed over the course of the meeting. Some key points are: 31 | 32 | * FDA reviewers and researchers work primarily on desktop machines that mostly run Windows 10. 33 | * There are two classes of desktop computers 1) regulatory computers and 2) scientific computers. 34 | * The scientific computers are not as rigorously controlled as the scientific computers. 35 | * Paul has installed Ubuntu on a few of the scientific computers 36 | * Desktop computers run SAS version 9.4, but the build may differ 37 | * A perceived advantage of SAS is that it generally runs older software 38 | * There is a pilot project going on at the FDA to run SAS on a server, which would ensure a common build. 39 | * When asked: If the FDA decided to move to Linux, how long would it take to convert all of the desktops from Windows to Linux, Paul speculated that it would take a minimum of two years. The implication is that Windows is not going away anytime soon. 40 | * When asked about the installation of R packages, Paul indicated that installation was not a problem. 41 | * Obtaining R packages is more of an issue. In theory, the FDA could use MRAN or some other repository which enables downloading specific versions of packages. 42 | 43 | The discussion turned to considering section 4.1.2.10 of the Study Data Technical Performance Guide: 44 | 45 | 46 | 47 | Yilong Zhang stated that his company has focused on the sentence: "The Specific Software utilized should be specified in the ADRG." 48 | Mike Stackhouse explained that the PHUSE working group is is leading is also focused on interpreting this sentence. 49 | 50 | Paul noted that companies are obligated to use the SAS export file format, but are not obligated to use SAS. Bella Feng noted that a recent FDA oncology pilot explicitly specifies SAS. 51 | 52 | Paul noted that many FDA analyses are hybrid using both SAS and R. The group acknowledge that all R submissions were the ultimate goal, we should be planning to work with mixed submissions for some time. 53 | 54 | Mike Stackhouse stated that, although there are many corner cases, we should focus on building a reference platform along the lines of least resistance and go for what is easily achievable. The group generally agreed with this. 55 | 56 | Towards the end of the meeting Paul mentioned that although the that section 4.1.2.10 specifies that programs should be submitted as text files, it does not mean that .txt extensions must be used. This surprised everyone. When asked if the FDA document could be revised to make this more clear, Paul stated that it was possible. He said that the document is generally revised once or twice a year, the most recent revision is dated Nov 20, 2020. 57 | 58 | **Action Item:** Paul stated that if the group wanted to suggest the wording to revise the document, he would submit it to the responsible FDA committee. He noted that to make the change it would need to be approved by the committee and go through the normal approval process. 59 | 60 | Joseph Rickert asked the group to use the repo to submit their suggested changes to the wording about using text, and reminded everyone that the next meeting was set for Friday, March 5, 2021. 61 | 62 | Joseph Rickert, Mike Stackhouse and Sascha Ahrweiler agreed to meet before the next Submissions group meeting to try and map out the various PHUSE, PSI and R Consortium working groups that are doing similar or complementary work. 63 | 64 | The link to the video of the meeting is: 65 | https://zoom.us/rec/share/gFQancacysPFHm0Czrw3sAYOCrt35BgTbcodBdY2LGld4fkykDypkL99PX5o0i-c.JRCWwmZfO9Ui8G6f 66 | 67 | Passcode: `+GKR9&i6` 68 | -------------------------------------------------------------------------------- /minutes/2021-03-05/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2021-03-05" 3 | author: Joseph Rickert 4 | date: 03-05-2021 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - "Joseph Rickert - R Consortium" 11 | - "Adrian Waddell - Roche" 12 | - "Paul Schuette - FDA" 13 | - "Bella Feng - EQRX" 14 | - "Yilong Zhang - Merck" 15 | - "Doug Kelkhoff - Genentech" 16 | - "Mike Stackhouse - Atorus Research" 17 | - "Nan Xiao - Merck" 18 | - "Ning Leng - Genentech" 19 | - "Steven Hasendinckx J&J" 20 | - "Bryant Chen - FDA" 21 | - "Ojesh Upadhyay - GSK" 22 | ::: 23 | 24 | Joseph Rickert opened the meeting at 9AM Pacific Time and pointed to the agenda in the Discussion Section of the GitHub repo. 25 | 26 | Agenda: 27 | 9:00: Doug Kelkhoff - R Validation Hub infrastructure 28 | 9:05: Nan Xiao the R package `pkglite` which packs/unpacks multiple R packages into txt file is going to be used in an upcoming FDA pilot eCTD submission 29 | 9:30: Open discussion 30 | We will continue the discussion initiated by Paul Schuette last time about modifying the software section of the [study data technical conformance guide](https://www.fda.gov/industry/fda-resources-data-standards/study-data-standards-resources) 31 | 32 | Doug provided an overview of the R Validation Hub protect, pointed out areas of overlap with this working group and welcomed those interested to participate in the Validation Hub effort. Detailed information on the R Validation Hub project which is also organized as an R Consortium working group can be found [here](https://www.pharmar.org/). 33 | 34 | Next, Nan Xiao of Merck gave an overview and demo of the R package `pkglite` which is expected to be up on CRAN soon. 35 | 36 | The specific aims of the package are: 37 | 38 | * Aim 1: To provide a tool for packing and restoring R packages as plaintext assets that are easy to store, transfer, and review. 39 | * Aim 2: To provide a grammar for specifying the file packing scope that is functional, precise, and extendable. 40 | * Aim 3: To provide a standard for exchanging the packed asset that is unambiguous, human-friendly, and machine-readable. 41 | 42 | Nan's presentation can be found in the Document's section of the R Consortium/submissions-wg GitHub repository. 43 | 44 | Yilong stated that he hoped that our working group could collaborate on a pilot submission to the FDA. He prepared the following outline to be included in these minutes. 45 | 46 | Goal: prepare a open source dummy submission and submit to eCTD portal 47 | 48 | The information below is to track potential high level action items. Individual issues can be created to track the progress later. 49 | 50 | Data source: [CDISC pilot submission](https://bitbucket.cdisc.org/projects/CED/repos/sdtm-adam-pilot-project/browse) 51 | 52 | R program for table listing and figure (TLFs): [program](program: https://github.com/atorus-research/CDISC_pilot_replication) 53 | 54 | General action items: 55 | 56 | 1. Make sure this use case is following CDISC terms of use 57 | 2. Identify TLFs in-scope for submission 58 | 59 | Action items for dummy submission package preparation: 60 | 61 | 1. Identify point of contact to initiate dummy submission request 62 | 2. Prepare analysis results metadata 63 | 3. Prepare step-by-step instruction to re-run analysis based on code in program folder and put in ADRG: link 64 | 4. Wrap-up submission package 65 | 66 | Those present all seemed to be enthusiastic about cooperating on the generating a dummy submission. 67 | 68 | Next, followed a lengthy discussion on the required process. Mike Stackhouse suggested that we work with PHUSE to suggest people who may be knowledgeable about past CDISC submissions. Paul Schuette recommended Zak Skrivanek. 69 | 70 | Yilong Zhang mentioned that Merck intends to use this package as part of a pilot oncology submission in the near future. Bella mentioned that the FDA Oncology group had specified SAS. 71 | 72 | Ning Leng of mentioned that Roche is preparing a rare disease submission for 2022. 73 | 74 | Joe Rickert asked the group: assuming the `pkglite` dummy submission is successful, how far the this would advance us towards the ultimate goal of an all R submission. 75 | 76 | While everyone was very excited about the `pkglite` all agreed with Doug Kelkhoff's sober assessment that the industry would ultimately want a more "R idiomatic" solution that took advantage of R naming conventions and did not introduce special requirements for R package maintenance. 77 | 78 | Mike Stackhouse noted that a successful `pkglite` submission would break down a significant barrier to R submissions, but that there were other hurdles. Nevertheless, he asserted that it would be a significant advance to be able to say here is "how" you can do an R submission. 79 | 80 | It the few minutes that remained, Joe Rickert brought the groups attention to the copy of the word document containing the text of section 4.1.2.10 of the FDA document pertaining to software and asked everyone mark up the document to suggest ideal wording from the industry's point of view. 81 | 82 | The group agreed to met next on Friday April 1, at 9AM Pacific time. 83 | 84 | The zoom recording of this meeting is available [here](https://zoom.us/rec/share/o-E_jt-wuDDbUCcEhQFwdsE2vZhPlAdcoqo0mgLNmKgqAe3DCDBkxTH_A7eR_MEa.MoCkXvxa5JzE9F2d) (Passcode: `d9N%+JWY`). 85 | -------------------------------------------------------------------------------- /minutes/2021-04-05/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2021-04-05" 3 | author: Joseph Rickert 4 | date: 04-05-2021 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - "Joseph Rickert - R Consortium" 11 | - "Paul Schuette - FDA" 12 | - "Yilong Zhang - Merck" 13 | - "Doug Kelkhoff - Genentech" 14 | - "Nan Xiao - Merck" 15 | - "Ning Leng - Genentech" 16 | - "Bob Engle - Biogen" 17 | - "Eric Nantz - Lilly" 18 | - "Phil Bowsher - RStudio" 19 | - "Sean Lopp - RStudio" 20 | ::: 21 | 22 | Joe Rickert brought the meeting to order and reminded attendees about the action item from the previous meeting to consider the SDTCG_SoftwareProgram document and comment on changes to the language that the group would like the FDA to consider. 23 | 24 | Next, Yilong Zhang shared a proposal that Mike Stackhouse submitted to PHUSE outlining a pilot submission to the FDA prepared for PhUSE for consideration by the group. This document R_eCTD_pilot_submissions.docx. 25 | 26 | For the remainder of the meeting the attendees examined the content of the document and discuss how the project should proceed. The prevailing sentiment was that the pilot submission should be an R Consortium Submissions working group project. 27 | 28 | The video recording for the meeting is available [here](https://zoom.us/rec/share/0uljlGZRgZGx9zZuRgloMOIH92qvtScRq1YtUZhxnC0GF2KxkMXsycDYykqHCIXl.pz5myyWa0kJ5JdnO) (Passcode: `=0yt$%V9`) 29 | 30 | It was agreed that the group would like to go back to meeting on Fridays. The next meeting is scheduled for Friday, May 7, 2021. Thereafter, the group will continue to meet on the first Friday of the month. -------------------------------------------------------------------------------- /minutes/2021-09-03/img/p1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/2021-09-03/img/p1.png -------------------------------------------------------------------------------- /minutes/2021-10-01/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2021-10-01" 3 | author: Joseph Rickert 4 | date: 10-01-2021 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - "Heidi Curinckx" 11 | - "Bob Engle - BioGen" 12 | - "Miriam Fossati - Merck" 13 | - "Christopher Kania - Biogen" 14 | - "Doug Kelkhoff - Genentech" 15 | - "Ning Leng - Genentech" 16 | - "Joseph Rickert - R Consortium" 17 | - "Michael Rimler - GSK" 18 | - "Paul Schuette - FDA" 19 | - "Mike Stackhouse - Atorus" 20 | - "Ben Staub - GSK" 21 | - "Adrian Waddell - Roche" 22 | - "Heng Wang - Genentech" 23 | - "Peikun Wu - Merck" 24 | - "Nan Xiao - Merck" 25 | - "Jiang Xu - FDA" 26 | - "Renping Zhang - FDA" 27 | - "Yilong Zhang - Merck" 28 | - "Jizu Zhi - FDA" 29 | ::: 30 | 31 | Joseph Rickert brought the meeting to order and Ning Leng presented an up update on the status of the Pilot 1 project using the slide deck `Leng_Update_10-1-21.pptx` which is available in the [Documents folder](https://github.com/RConsortium/submissions-wg/tree/main/Documents) of submissions-wg GitHub repository. 32 | 33 | Paul Schuette suggested that Ning revise the document to refer to "FDA staff" rather than "FDA Reviewer" which provides more latitude for getting FDA help. Paul also noted that although it might be possible it is not likely that the FDA could accomplish the task requested before the R/Pharma conference which begins on November 2. Those present agreed that it is not necessary to tie the completion of the FDA activities to R/Pharma. 34 | 35 | After Ning's presentation there was a discussion about suggestions for amending the language to section 4.1.2.10 of the Study Data Technical Performance Guide. Doug Kelkhoff noted that in internal discussions within Genentech the idea was floated that, as written the document may already allow passing R code in tar balls through the FDA gateway. If this were indeed true it would greatly simply R submissions since tar balls are a common method of delivering R code. Jian Xu of the FDA suggest that the way to get an answer to this question is to submit the question to the email address provided on the FDA eCTD webpage: [Submit an eCTD or Standardized Data Sample to the FDA](https://www.fda.gov/drugs/electronic-regulatory-submission-and-review/submit-ectd-or-standardized-data-sample-fda). Joe and Doug took the **action item** to write up the question and submit it. 36 | 37 | The group then considered a proposal by Adrian to organize the several GitHub repositories and activities of the Submissions WG using the Teams features of GitHub. The group agreed that this would be a good idea but suggested waiting until after Pilot1 is completed. Adrian will set up a meeting with Joe and Ning to setup the structure. 38 | 39 | The video of today's 10-1-21 meeting is available [here](https://rstudio.zoom.us/rec/share/q3guNFXtWp5aKRYAMB1f7CkW2miueFj_uSBuKnvsAksQi9vYvy9IsEAgBgy5eNje.p4Qupsx4WuBHZgQM) passcode: &1b4VEkR 40 | 41 | The next meeting of the WG will be at 9AM Pacific Time on Friday November 5, 2021. 42 | -------------------------------------------------------------------------------- /minutes/2021-11-05/img/ectd_slide.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/2021-11-05/img/ectd_slide.png -------------------------------------------------------------------------------- /minutes/2021-11-05/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2021-11-05" 3 | author: Joseph Rickert 4 | date: 11-05-2021 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - "Beverly Chin - Roche" 11 | - "Heidi Curinckx - Johnson & Johnson" 12 | - "Bob Engle - BioGen" 13 | - "Miriam Fossati - Merck" 14 | - "Steven Hassendinckx" 15 | - "Christopher Kania - Biogen" 16 | - "Ning Leng - Genentech" 17 | - "Eric Nantz - Eli Lilly" 18 | - "Tim Powell - Biogen" 19 | - "Joseph Rickert - R Consortium" 20 | - "Paul Schuette - FDA" 21 | - "Ben Staub - GSK" 22 | - "Adrian Waddell - Roche" 23 | - "Heng Wang - Genentech" 24 | - "Peikun Wu - Merck" 25 | - "Nan Xiao - Merck" 26 | - "Hong Yaan - Regeneron" 27 | - "Renping Zhang - FDA" 28 | - "Yilong Zhang - Merck" 29 | - "Jizu Zhi - FDA" 30 | ::: 31 | 32 | Joseph Rickert brought the meeting to order and stated that with Eli’s help he was able to reactivate the FDA gateway and asked those with the passport to login and make sure there are no other impediments. The details are: 33 | 34 | The R Consortium account has been reactivated: 35 | To access the R Consortium account, please, use this URL: https://esgtest.fda 36 | User ID: joseph.rickert@rstudio.com 37 | Password: Please see Ning, Yilong, Eli or Joe 38 | The Portal is : https://esgtest.fda.gov/ 39 | 40 | He then asked Ning to introduce Beverly Chin from Roche and Tim Powell from Biogen who are both experienced in making regulatory submissions. 41 | 42 | Ning provided a brief overview of the project and an update on progress since the last meeting which included recruiting new members for the working group from presenting at the R/Pharma conference and a new project website (https://rconsortium.github.io/submissions-wg/) built by Eric Nantz. 43 | 44 | Asks to members 45 | We need more people to review the submission package. Bob Engle volunteered Nate Mockler (nate.mockler@biogen.com) as a reviewer. 46 | Identify FDA members who can do the review. (Paul said that he has two candidates.) 47 | 48 | Beverly asked some clarifying questions and we confirmed that We are using the test gateway 49 | CDER will be the center 50 | It is a formal package 51 | 52 | Ning then went over some feedback provided by reviewers so far. The most important issue was 53 | Inconsistent table layouts. However, Ning pointed out that using multiple table layouts is intentional. The objective is to have three tables follow different company specific standards in order to cover tables built with different R packages. Ning said that we will add a disclaimer to the report in order to inform the FDA reviewers of our objectives. 54 | 55 | Reviewers also flagged some things that are not consistent with best practices, such as including the number of individuals at risk on Kaplan-Meier curves. 56 | 57 | Joe asked Ning to review the contents of the Pilot 1 repo 58 | 59 | Ning explained the reviewer comments on issue #47 on the Kanban Board. Ning will link the GitHub issue to a google drive containing more material. 60 | 61 | The group then discussed a timeline for the remainder of the project and settled on the key dates: 62 | Package to be complete and ready to go by end of Friday, 2021-11-19 63 | Submission to be made on Monday 2021-11-22 64 | Joe took the action item to setup a zoom meeting for 9AM PT time on 11-22 65 | 66 | There followed an extended discussion about whether to use .txt or .r for the R files. 67 | The FDA documentation says they will accept both, however, some people Including FDA folks might reject a .r file because they misunderstand the documentation. On the other hand, getting a .r through would be a big win on its own. 68 | 69 | It was also suggested to include a separate README document to describe how the package is constructed. 70 | 71 | At **36.33** into the video Paule Schuette of the FDA shares a slide his colleague Heather Crandall presented recently at the R/Pharma conference about eCTD placement of Study Data Files. 72 | 73 | ![CTD triangle](img/ectd_slide.png) 74 | 75 | Note that R Markdown and R are listed among accepted eCTD format types. 76 | 77 | At **44.33**, “Scrum master” Adrian Waddell leads the group through adding new tasks to the Kanban board: https://github.com/RConsortium/submissions-wg/projects/2 78 | 79 | Summary of tasks leading to Submission 80 | 81 | First round of adjudication by 2021-11-11 82 | Decide about submitting .r files 2021-11-11 83 | Adjudication by 2021-11-18 84 | Final Review 2021-11-19 by Eric Nantz, Adrian and Nate Mockler 85 | Ning and Yilong will address the comments. 86 | Submission on 2021-11-22 87 | Ning Leng, Yilong Zhang, Paul Schuette, Beverely Chin, Tim Powell and Joe Rickert will participate in the Submission meeting. The meeting will be recorded. 88 | 89 | Joe took the action item to setup a zoom meeting for 9AM PT time on 2021-11-22 to make the submission. 90 | 91 | The video of today's 11-5-21 meeting is available [here](https://rstudio.zoom.us/rec/share/GjVepePF9_afktodTswA7hzDPBhj1esmAZovgJDzadpK8rjcAyioGvaY56XCRvKA.Dl1NUIc-9DlV30u9) passcode: `L*jwHXn5` 92 | 93 | The next meeting is scheduled for Friday, 2021-12-03. 94 | -------------------------------------------------------------------------------- /minutes/2021-12-03/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2021-12-03" 3 | author: Joseph Rickert 4 | date: 12-03-2021 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - "Phil Bowsher - RStudio" 11 | - "Hye Soo Cho - FDA" 12 | - "Heidi Curinckx - Johnson & Johnson" 13 | - "Miriam Fossati - Merck" 14 | - "Ellis Hughes - GSK" 15 | - "Maria Matilde Kam - FDA" 16 | - "Emily Nguyen - FDA" 17 | - "Jiang Xu - FDA" 18 | - "Steven Hassendinckx" 19 | - "Ning Leng - Genentech" 20 | - "Joseph Rickert - R Consortium" 21 | - "Paul Schuette - FDA" 22 | - "Adrian Waddell - Roche" 23 | - "Nan Xiao - Merck" 24 | - "Hong Yaan - Regeneron" 25 | - "Renping Zhang - FDA" 26 | - "Yilong Zhang - Merck" 27 | - "Jizu Zhi - FDA" 28 | ::: 29 | 30 | The meeting was recorded and the video is available [here](https://rstudio.zoom.us/rec/share/TPhUmx-9fsnaF8ZWkGqSXGgTqIZAuDy8Fvli182eRysycU5WPOse7kOzagdRU7Lh.acdswWFSSBr7m0K0) Passcode: 0tA$D+ev 31 | 32 | **JR** Opened the meeting with the agenda to discuss the recent pilot submission through the FDA gateway and then discuss a request to have interested companies describe what they are currently doing with respect to FDA submissions in R. 33 | 34 | **YZ** reported that the submission went through the gateway, thanked all who participated and was looking forward to receiving feedback from the FDA. 35 | 36 | **JR** reported that there were still issues with employees from different companies using the RC portal. The current solution is to have people accessing the gateway on behalf of the RC use an R Consortium email address. **JR** wants to make sure that this is fine with the FDA. 37 | 38 | At **6:37** into the video **PS** begins providing FDA feedback on the submission. 39 | 40 | The initial findings were: 41 | 42 | * The submission was partially successful 43 | * The were no problems with the .R extensions 44 | * The FDA was unable to find the `Pilot1wrappers` package. However, the * * FDA analyst was able to generate tables 45 | * The FDA thinks there may have been a switch in the high bit vs. bit settings and asked the WG to check this. 46 | * The FDA analyst was able to retrieve the `pilot1wrappers` package from the Submission WG website, download the code and verify the results 47 | * The FDA noted that including a dependency chart showing the 2nd and 3rd order dependencies would be very helpful. 48 | If this were and actual submission, at least one IR request would have been generated. 49 | 50 | On behalf of the WG, **JR** thanked the FDA team for making extra effort to retrieve the wrapper package and run the code. 51 | 52 | In the subsequent discussion it was determined that the FDA analysts overlooked the instructions, buried in the ADRG, to unpack the `pilot1wrappers` from the file r0plg.txt. Once this was clarified **HC**, the FDA analyst, was able to execute the command and unpack the `pilot1wrappers` package. **HY** noted that in her experience, submissions with R code tend to be *ad hoc*, and that it is common to include a README file. The WG took note and agreed that it would be a good idea to include a `README` file that highlights the key steps, maybe by referring to the ADRG. 53 | 54 | Next, followed a discussion about the requested dependency chart tree diagram. **PS** noted that the importance of dependencies depends on the setting. When running from a standard machine, FDA analysts are able to pull packages from the internet. However, when running from one of their scientific enclaves high performance computing environments they must port in the packages. To illustrate the problem with dependencies *HC* showed the error messages generated by calling R packages previously installed on her machine, but which did not have the dependencies installed that are required to run the submission code. 55 | 56 | The problem seems to be that every file in the submission needs the library calls. This problem needs to be addressed. 57 | 58 | **HC** also identified a problem of not having write permissions. 59 | 60 | The temporary solution seems to be to copy the file into a temporary folder. This problem needs more thought. 61 | 62 | It was agreed that the WG should produce a document about establishing the proper environment. **AW** suggests that there are two possible actions: 63 | 1. Copying to a local folder might clear up most of the problems 64 | 2. If not, the WG should look into environment reproducibility 65 | 66 | **PS** noted that the FDA does not need to run on a central server. This implies that we can assume FDA analysts running on their local environments for the near future. 67 | 68 | **PS** suggested having a pre-submission meeting explaining everything that is necessary to get everything installed and working. 69 | 70 | **AW** suggested that we set up the submission assuming that each file will be run from a fresh R session, and that perhaps we could run some tests to see if the environment is as is expected. 71 | 72 | **NL** asked about writing a blog post about the pilot submission. The WG agreed to write up the post and have **PS** and **HC** review. 73 | 74 | **YL** asked about redoing the submission. **PS** said indicated that is common practice. 75 | 76 | 77 | **NL** said that she and **HC** would like to schedule a separate meeting early next year to discuss where the various participating companies are in the process of making R submissions. Please let **NL** know if you are interested in participating. 78 | 79 | 80 | The next meeting is set for 9:00 AM Pacific Time on Friday, January 7, 2022. -------------------------------------------------------------------------------- /minutes/2022-10-07/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2022-10-07" 3 | author: Joseph Rickert 4 | date: 2022-10-07 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Ellis Hughes (GSK) 11 | - Eric Nantz (Eli Lilly) 12 | - Honghong Zhou (Moderna) 13 | - Jizu Zhi (FDA) 14 | - Joseph Rickert (R Consortium) 15 | - Lei Zhao (Roche/Genentech) 16 | - Miriam Fossati (Merck) 17 | - Nan Xiao (Merck) 18 | - Ning Leng (Roche/Genentech) 19 | - Phil Bowsher (Posit) 20 | - Qin Li 21 | - Renping Zhang (Johnson & Johnson) 22 | - Robert Devine (Johnson & Johnson) 23 | - Ryan Johnson (Posit) 24 | - Saghir Bashir (Argenx) 25 | - Sam Parmar (Pfizer) 26 | - Sean Healey (Pfizer) 27 | - Tadeusz Lewandowski (Roche/Genentech) 28 | - Thomas Neitmann (Roche/Genentech) 29 | - Yilong Zhang (Meta) 30 | 31 | ::: 32 | 33 | The meeting was recorded and the [video](https://rstudio.zoom.us/rec/share/OfSxaPPUsLuFJtZeUjzsZ6kZoAtCn7GTAj-N1j4zU9gUXOI8NqU5SUXf7rbeG2eu.q4s5o6ZiCBJHId8h?startTime=1665158653000) is available. 34 | 35 | ### Pilot 2 Submission 36 | Ning began her update on the Pilot 2 Submission by thanking everyone who helped with the testing. 37 | 38 | Status: 39 | The submission package is ready. However, the submissions team must address some last minute comments from the FDA. One comment is a request to mask p-values that are automatically reported. The apparent concern is that by default p-values will automatically be updated and reported when users filter the data by subgroups. Because we only have a small test data set this will likely lead to widely varying p-values which could be misinterpreted by general reviewers who are not statisticians. The FDA may request additional outputs to me masked. These comments appear to address the long term use of interactive applications in submissions and are not related to the technical aspects of making the submission. 40 | 41 | Consequently, the meeting scheduled for next Tuesday which was intended for submitting Pilot 2 will now be devoted to addressing FDA comments. The team will reschedule the submission as soon as possible after addressing the comments. We do not anticipate a long delay. There are no technical difficulties with masking additional default outputs. 42 | 43 | In the meantime more testing would be helpful and appreciated. 44 | Follow the instructions in the Appendix for Pilot 2: https://github.com/RConsortium/submissions-pilot2/issues/46 45 | Put issues here: https://github.com/RConsortium/submissions-pilot2/issues/ 46 | 47 | JBR noted that there is now significant material in the GitHub repository for the working group which is indicative of the scope and scale of cooperation of the project. This is itself a success story of some interest. 48 | 49 | ### Pilot 3 - ADaM 50 | 51 | (**13:51min** into the video) Thomas Neitmann presented a proposal by Joel Laxamana and himself to extend Pilot 1 to include ADaM datasets programmed in R. The purpose of the proposal is to: 52 | 53 | 1. Demonstrate the viability of using R for ADaM dataset creation in the context of an FDA submission 54 | 2. Provide the industry with an end-to-end example of an R-based FDA filing 55 | 56 | Thomas' presentation may be found here: https://github.com/RConsortium/submissions-wg/tree/main/Documents 57 | 58 | Yilong suggested taking ODM XML as the source to make it more representative of an end-to-end process. This started a discussion about the value of beginning end-to-end examples from various different starting file formats. 59 | 60 | The present team team includes: 61 | 62 | * Thomas Neitmann - Roche (Co-Lead) 63 | * Joel Laxamana - Roche (Co-Lead) 64 | * Lei Zhao - Roche 65 | * Nicole Jones - Merck 66 | 67 | The team is looking for additional help. Please have a look at the presentation and contact one of the Co-Leads if you would like to get involved. 68 | 69 | The working group agreed to change the name of the project from Pilot 1-ADaM Extension to Pilot 3-ADaM to emphasize that the project will include a significant amount of new work. 70 | 71 | **Action Item** JBR will make Paul Schuette aware of this new project and ask for his feedback. 72 | 73 | 74 | ## DA Japan Talk 75 | 76 | (**31:30** min into video) Tadeusz presented a Shiny application that he developed for a presentation he is going to make at the [DIA Japan conference](https://www.diaglobal.org/en/conference-listing/meetings/2022/10/19th-dia-japan-annual-meeting-2022) on October 9, 2022. The intention of the presentation is to gauge the interest of the Japanese authorities. The session in which Tadeusz will present is called [LS09 Innovative and Dynamic Statistical Analytics for Regulatory Submissions](https://www.diaglobal.org/en/conference-listing/meetings/2022/10/19th-dia-japan-annual-meeting-2022/agenda/09/ls09?ref=LS09) 77 | 78 | ## Subgroups 79 | The working group discussed writing writing some material for the GitHub repository explaining the purpose of the various subgroups, listing the members, and explaining how newcomers can get involved. 80 | 81 | **Action Item** Joe and Eric will meet after the Pilot 2 submission to discuss documenting the subgroups. 82 | 83 | ## Containers 84 | 85 | The working group agreed that the planned project to submit a containerized version of the Pilot 2 submission will require a significant amount of new work and pose several challenges. It should therefore be new, independent pilot submission, perhaps Pilot 4-Containers. 86 | -------------------------------------------------------------------------------- /minutes/2022-11-04/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2022-11-04" 3 | author: Joseph Rickert 4 | date: 11-04-2022 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Bob Engle (Biogen) 11 | - Ellis Hughes (GSK) 12 | - Eric Nantz (Eli Lilly) 13 | - Gregory Chen (MSD) 14 | - Heng Wang (Roche/Genentech) 15 | - Honghong Zhou (Moderna) 16 | - HyeSoo Cho (FDA) 17 | - Jizu Zhi (FDA) 18 | - Joel Laxamana (Roche/Genentech) 19 | - Joseph Rickert (R Consortium) 20 | - Lei Zhao (Roche/Genentech) 21 | - Nan Xiao (Merck) 22 | - Ning Leng (Roche/Genentech) 23 | - Paul Schuette (FDA) 24 | - Renping Zhang (Johnson & Johnson) 25 | - Robert Devine (Johnson & Johnson) 26 | - Saghir Bashir (Argenx) 27 | - Stephanie Lussier (Moderna) 28 | - Tadeusz Lewandowski (Roche/Genentech) 29 | - Yilong Zhang (Meta) 30 | - Yiwen Luo (Merck) 31 | - Yutong Liu (Moderna) 32 | 33 | ::: 34 | 35 | {{< video https://youtu.be/LDVWXgDhzzo >}} 36 | 37 | The [video](https://www.youtube.com/watch?v=LDVWXgDhzzo) is available on the [R Consortium YouTube Channel](https://www.youtube.com/channel/UC_R5smHVXRYGhZYDJsnXTwg) 38 | 39 | The meeting began with EN summarizing his understanding of the comments made by PS about his preliminary look at the Pilot 2 submission package. 40 | 41 | The following is a summary of Paul's comments which were made in a email exchange with the workgroup members on 10-27-22: 42 | 43 | * Paul has management approval to proceed with Phase 2 of the pilot, with the proviso that the Demographic, Primary and Efficacy Tables are static. 44 | * There is a minor error in the app: the low dose and high dose column settings reversed. 45 | * The efficacy table is slightly misleading as it currently stands. Suggest that the text for the first row read “Study Drug (High Dose)” or similar. 46 | * Suggest an additional table: "Subjects Remaining in the Study" showing how dropouts proceed. (See Paul's email of 10-27-22 for details) This might show how exploratory analysis might add value over the prespecified 47 | * The number of decimal places displayed in the demographic table isn’t always the same as in pilot 1. 48 | * Suggest that the model specification information be included with the efficacy table rather than left to the app information. 49 | * We would like the app to be able to serve as a model for future submissions. 50 | 51 | Eric stated that the updates appear to be relatively minor and would not be difficult to accomplish. 52 | 53 | The submission team agreed to target 11-18-2022 as the target submission date. 54 | 55 | **Action:** JBR to put a 9AM 11-18-2022 on the calendar for the submission. 56 | 57 | JBR asked Paul to describe the boundary between a useful amount and too much interactivity in a shiny submissions app. Paul replied that it a way to think about it would be the difference between exploratory and inferential analysis. The problem with inferential analysis is the usual problem with p-values, subgroups and cherry picking. Inferential analysis is not the way to go. A Kaplan-Meier curve would be considered an exploratory safety analysis. 58 | 59 | When asked if the guidance on interactivity was likely to stand for some time or change with familiarity Paul replied that it was his personal opinion that since it is consistent with the [ASA Statement on p-values](https://www.amstat.org/asa/files/pdfs/p-valuestatement.pdf) it will probably stand. 60 | 61 | Paul also remarked that interactivity facilitating pre-specified analyses would probably be fine. 62 | 63 | The following links may be helpful with understanding potential problems with on subgroup analyses: 64 | 65 | * [Alosh et al. (2015)](https://www.tandfonline.com/doi/full/10.1080/19466315.2015.1077726) 66 | * [Alosh et al. (2016)](https://onlinelibrary.wiley.com/doi/full/10.1002/sim.7167) 67 | 68 | Joel Laxamana gave an update on the Pilot 3 effort: 69 | 70 | * The kickoff meeting for the team was held this past Wednesday 71 | * There are 13 people on the team 72 | * The goal of Pilot 3 is to duplicate Pilot 1 but also generate the ADaM data sets used for the TLGs in Pilot 1 73 | * The idea is to start with the [STDM](https://www.cdisc.org/standards/foundational/sdtm) and show the traceability to [ADaM](https://www.cdisc.org/standards/foundational/adam) 74 | * Pilot 3 will use the `admiral` package along with `tidyverse` packages. 75 | 76 | **Action:** Joel will add this explanatory material to the Pilot 3 GitHub repo. 77 | 78 | **Action:** Joel will send a list of Pilot 3 team members to be added to the invitation list for this meeting. 79 | 80 | **Action:** The Pilot 3 team will submit a written proposal to the FDA describing the process of unpacking the submission and setting up for analysis. The target date is to have this proposal ready for the 12-02-2022 Submissions WG meeting. 81 | 82 | Lei Zhao asked if it would be a problem submitting data in SAS in doing the analysis in R. In general there is no problem, however the FDA suggests using a limited number of tools and using standard tools to do any translations between formats. The FDA prefers using scripted languages as opposed to using a GUI. 83 | 84 | Ning will be presented at the R/Pharma conference on Wednesday, November 9, 2022 at 11AM Eastern Time. 85 | The next meeting of the regular meeting of the group will be at 9AM Pacific Time on Friday, December 2, 2022. -------------------------------------------------------------------------------- /minutes/2022-12-02/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2022-12-02" 3 | author: Joseph Rickert 4 | date: 12-02-2022 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Bob Engle (Biogen) 11 | - Brian Sucher (Unknown) 12 | - Christopher Kania (Biogen) 13 | - Dadong Zhang (Illumina) 14 | - Ellis Hughes (GSK) 15 | - Eric Nantz (Eli Lilly) 16 | - Heidi Curinckx (Johnson & Johnson) 17 | - HyeSoo Cho (FDA) 18 | - Jizu Zhi (FDA) 19 | - Joel Laxamana (Roche/Genentech) 20 | - Joseph Rickert (R Consortium) 21 | - Miriam Fossati (Merck) 22 | - Nan Xiao (Merck) 23 | - Ning Leng (Roche/Genentech) 24 | - Paul Schuette (FDA) 25 | - Phil Bowsher (Posit) 26 | - Renping Zhang (Johnson & Johnson) 27 | - Robert Devine (Johnson & Johnson) 28 | - Ryan Johnson (Posit) 29 | - Sam Parmar (Pfizer) 30 | - Stephanie Lussier (Moderna) 31 | - Thomas Neitmann (Roche/Genentech) 32 | - Xin Qiu (Johnson & Johnson) 33 | - Yilong Zhang (Meta) 34 | 35 | ::: 36 | 37 | ### Pilot 2 38 | * Ning Leng and Eric Nantz reported that Pilot 2 was submitted on 2022-11-18, upholding the working groups tradition of submitting right before Thanksgiving, and that confirmation was received from the the FDA. 39 | * Paul Schuette explained that since Pilot 2 was not a production submission it was necessary to make a manual request to receive it. Consequently, his group did not receive it until earlier this wee. 40 | * Heysoo Cho said that she will begin working on the submission next week. 41 | * Eric Nantz offered to be available to help if the FDA had any questions. 42 | 43 | ### Pilot 3 44 | Joel Laxamana briefly reviewed the preparations his team is making for Pilot 3 and said that the proposal is ready for transmission to the FDA. Paul suggested that the proposal should be sent to everyone in the working group and not just the FDA. Joel agreed to put the proposal in the Submissions WG GitHub repository and send the link to everyone in the working group. 45 | 46 | Joel noted that many of the Pilot 3 concepts were taken from the ADRG for Pilot 2 and mentioned that they are checking against the CDISC versions as they build the package. 47 | 48 | ### Pilot 4 49 | Eric said that he hopes to ramp up Pilot 4 work after the Pilot 2 submission. The simplest plan will be just to containerize the Pilot 2 submission. [Podman](https://podman.io/) is the initial choice for a the container technology. 50 | 51 | The group also expressed about aligning the Pilot 4 project with the Validation Hub to stand up a repo of validated packages. This would open up the possibility of hosting the container on the validation repo. The Validation Hub repo for this project is: https://github.com/pharmaR/regulatory-r-repo-wg 52 | 53 | 54 | 55 | ### Other Business 56 | 57 | JBR asked Ning and Joel for summary paragraphs for Pilot 2 and Pilot 3 for inclusion in end of the year reports on working group status. Ning is working on a blog post about Pilot 2 for inclusion in the R Consortium Blog. 58 | 59 | There were no updates available about efforts to make pilot submissions to other regulatory agencies. 60 | 61 | The following events were identified as possibilities for delivering presentations: 62 | 63 | * The Appsilon Shiny conference: [ShinyConf 2023](https://shinyconf.appsilon.com/) March 15-17 - Ning and Eric are drafting an abstract 64 | * R/Medicine June 8-9 65 | * R/Pharma 66 | * useR! 2023 67 | * Posit::conf 68 | * ASA BioPharma SWE Workshop in September 69 | 70 | #### Other future plans: 71 | 72 | * Ning noted that she and Yilong have been thinking about a journal submission and have begun writing a draft. 73 | * The group also agreed that they would like to do another R - Adoption Webinar. 74 | * Finally, Eric noted that he would like to rework the Submissions repository site and build it on a Quarto framework. 75 | * **The next meeting of the Submissions WG will be at 10AM PST, Friday 13, 2023.** This is a one week delay to move the meeting beyond the holiday season. 76 | -------------------------------------------------------------------------------- /minutes/2023-01-13/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2023-01-13" 3 | date: 2023-01-13 4 | author: Eric Nantz 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Andrew Borgman (Bristol Myers Squibb) 11 | - Ben Straub (GSK) 12 | - Bob Engle (Biogen) 13 | - Doug Kelkhoff (Roche/Genentech) 14 | - Eric Nantz (Eli Lilly) 15 | - HyeSoo Cho (FDA) 16 | - Jizu Zhi (FDA) 17 | - Joel Laxamana (Roche/Genentech) 18 | - Kevin Kunzmann (Boehringer Ingelheim) 19 | - Luke Schantz (R Consortium) 20 | - Paul Schuette (FDA) 21 | - Phil Bowsher (Posit) 22 | - Ryan Johnson (Posit) 23 | - Sean Healey (Pfizer) 24 | - Stephanie Lussier (Moderna) 25 | - Steven Hasendinckx (Johnson & Johnson) 26 | - Tadeusz Lewandowski (Roche/Genentech) 27 | 28 | ::: 29 | 30 | {{< video https://youtu.be/apmL_9HvbAo >}} 31 | 32 | ### Pilot 2 Update 33 | 34 | * Paul Schuette confirms he was able to successfully run the Shiny application 35 | * They encountered a few messages that they ignored, as the app still ran successfully 36 | * HyeSoo Cho followed the ADRG instructions and launched Shiny app, without major issues 37 | * She noticed minor issues around errors and warning messages. As part of her evaluation she tried launching the app in different machines to compare results. She saw some inconsistency between the environments, and they will report specific findings by next month (February) 38 | 39 | ### Pilot 3 Update 40 | 41 | * Proposal was sent in early December for comments and feedback 42 | * Running into minor issues with using the same specifications and define settings as the original Pilot 1 43 | * They are not clear on the most up-to-date specifications to use for the AdAM creation, would like help from others 44 | * Thomas found a Phuse 2017 white paper on transport files for the next generation. This paper proposed new ways to convert the data before transfer 45 | * Additional ideas such as a proposal to use JSON format 46 | * FDA is aware of the proposal, but it is not clear how far the proposal has gone 47 | * Study Data Technical Conformance Guide details the latest specs and what is acceptable in the gateway 48 | * Sponsors can use SAS version 8 of XPT if they choose, most sponsors are using version 5 from their experience 49 | * Other pilots have explored alternative formats, but have not been satisfactory in their opinion 50 | * It is common to convert the XPT format to a more consumable format. For instance as sas7bdat in SAS or R objects inside R. 51 | * Pilot 3 will adhere to the current specifications 52 | * Ben recommended consulting with Sam Hume as a collaborate to push the effort along 53 | * The ectd gateway would need updating to accommodate json formats. That would require discussions with the E-data team in the FDA 54 | 55 | ### R Validation Hub intro to Repositories working group 56 | 57 | * Doug Kelkhoff and Kevin Kunzmann led a an introduction to the mission and goals of the working group 58 | * Mission is to support a transparent cross-industry approach of maintaining a repo of R packages with evidence of their quality and assessment criteria 59 | * Repo structure may resemble pieces of CRAN, R Universe . Still in the early stages of finding what is needed before working on the technical solutions 60 | * Asks for regulatory input: Work with the team to understand how you want to observe quality. Paul mentioned that bigger issues that FDA are thinking about are from the IT perspective (making sure that packages don't compromise infrastructure). Example of haven running into buffer overflow issues. Typically those concerns are raised by security team colleagues. They are not aware of any independent effort to gauge the quality of an R package. Typically they point people to the R Validation Hub. While it is not doing independent replication of algorithms, it is a reasonable attempt in the right direction. Typically all pieces of software give you the "as-is" message and no implied guarantee of quality 61 | * Asks for industry input: Contribute by direct support, or by intermittent feedback as they prepare deliverables 62 | * High level of planned deliverables: List of quality criteria, mockup of the communication portal, and an example of a package server modelling a real-world workflow 63 | * Visit the repository to get involved 64 | * Ben asked about ways to consolidation various validation efforts heard from other sponsors. What existing tools are being implemented in this space? 65 | * Doug shared an article from the ASA Biopharmaceutical Report regarding the risk assessment of R packages: 66 | 67 | -------------------------------------------------------------------------------- /minutes/2023-04-07/repos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/2023-04-07/repos.png -------------------------------------------------------------------------------- /minutes/2023-05-05/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2023-05-05" 3 | author: Joseph Rickert 4 | date: 2023-05-05 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Andrew Borgman (Bristol Myers Squibb) 11 | - Ben Straub (GSK) 12 | - Brandon Suchon 13 | - Eric Nantz (Eli Lilly) 14 | - HyeSoo Cho (FDA) 15 | - Jizu Zhi (FDA) 16 | - Joel Laxamana (Roche/Genentech) 17 | - Joseph Rickert (R Consortium) 18 | - Kui Shen (Bayer) 19 | - Nan Xiao (Merck) 20 | - Nate Mockler (Biogen) 21 | - Paul Schuette (FDA) 22 | - Renping Zhang (Johnson & Johnson) 23 | - Robert Devine (Johnson & Johnson) 24 | - Saghir Bashir (Argenx) 25 | - Sam Parmar (Pfizer) 26 | - Sean Healey (Pfizer) 27 | ::: 28 | 29 | ## Add Minutes Here 30 | 31 | The meeting was recorded and the [video](https://zoom.us/rec/share/OZyp5N4QZCsl-D-8BMBzjuSubI0BYbISZTa6703egO97UUroboDFZVt4hjRudEY.1qe8A653kLrj7tfz?startTime=1683302886000) is available. 32 | 33 | Joseph Rickert brought the meeting to order. 34 | 35 | ### Pilot 2 36 | * Hye Soo Cho of the FDA reported that her review of the Pilot 2 submission is going well and that she expects to be finished in June. 37 | * She is not sure how long it will take the FDA to produce a written response. 38 | 39 | 40 | ### Pilot 3 41 | 42 | Joel Laxamana reported on Pilot 3 work. 43 | 44 | * The execution team isfocusing on documentation particularly in the ADRG. 45 | * The WG confirmed that Pilot 3 should follow the Pilot 2 approach and send the renv.lock file in the R package text. 46 | * There was a question about SDTM Versions. The WG confirmed that the Pilot 3 should use the same 1.4 STDM version as Pilots 1 and 2. See [Issue #104](https://github.com/RConsortium/submissions-pilot3-adam/issues/104). 47 | * QC Findings will go as Appendix to the ADRG. 48 | * The WG agreed that the second week in July would be a good target for Pilot 3 Submission. 49 | 50 | ### Pilot 4 51 | 52 | Eric Nantz provided an update on Pilot 4 work. 53 | 54 | * Eric expects to have the first meeting of the execution team volunteers in the next few weeks. 55 | * Eric is taking a step back from committing to a container based approach in order to consider the possibility of using a self executing app in the browser using [WebAssembly](https://webassembly.org/). 56 | * George Stagg from Posit is providing some assistance. 57 | * The idea is that the R submission code would run in a browser on the local machine of an FDA analyst. 58 | * Paul Schuette indicated that this approach would likely be possible at the FDA. 59 | * Eric said that he would be happy to try and replicate the FDA web browser environment. 60 | * The group suggested that a Webinar for the R Adoption series and other venues would be valuable for promoting the project. 61 | * Paul speculated that this web browser technology might also be useful to FDA field inspectors. 62 | * Eric noted that a selection core of R packages is being identified to have them run run "out of the box". 63 | * If you would like to be part of the Pilot 4 team please contact Eric Nantz. 64 | * Paul noted that recent events where outside entities can re-litigate FDA approvals decades after the approval could make it even more necessary to be able to run analysis software many years after the submission. 65 | * JBR surmised that there may be interest in WG to explore long term archiving. 66 | 67 | 68 | ### Other discussions 69 | 70 | * Nan Xiao brought attention to a [document](https://www.jpma.or.jp/information/evaluation/results/allotment/DS_202204_oss.html) describing the submission WG in Japan. It refers to a survey about open source that mentions the Pilot 1 Submissions. 71 | * We are exploring the possibility of arranging for the authors of this document to participation in an R Adoption Series webinar. 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | The next meeting of the Submissions WG will be on Friday, June 2, 2023 at 9AM Pacific Time. 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /minutes/2023-06-02/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2023-06-02" 3 | author: Joseph Rickert 4 | date: 2023-06-02 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Andrew Borgman (Bristol Myers Squibb) 11 | - Eric Nantz (Eli Lilly) 12 | - Heidi Curinckx (JnJ) 13 | - Jizu Zhi (FDA) 14 | - Joseph Rickert (R Consortium) 15 | - Kui Shen (Bayer) 16 | - Loan Robinson (Bayer) 17 | - Miriam Fossate (Merck) 18 | - Nan Xiao (Merck) 19 | - Nate Mockler (Biogen) 20 | - Paul Schuette (FDA) 21 | - Renping Zhang (Johnson & Johnson) 22 | - Shannon Lewis 23 | - Yilong Zhang 24 | 25 | ::: 26 | 27 | ## Minutes 28 | 29 | The meeting was recorded and the [video](https://zoom.us/rec/share/O_pRQJl9E9Wq5mesygYFA4KTfVAudRq1Y8YIW3VwCrsr6LpV-9Lg4nfD3TuJs1a6.ewHmJNFkhrTRByzx?startTime=1685721888000) is available. 30 | 31 | ### Pilot 2 32 | 33 | JBR brought the meeting to order and Paul Schuette gave an account FDA analyst HyeSoo Cho's findings so far with respect to the Pilot 2 submission. Identified problems include: 34 | 35 | * Warning messages about packages being loaded from GitHub. (Packages should be loaded from CRAN or some other curated repository which archives packages.) 36 | * Footnotes and reference numbers that appeared in the previous Pilot 2 submission were missing from some tables. 37 | * Instead of remaining unchanged, some p-values were replaced with the text "non applicable" when filters were applied to the KM curve. 38 | * It is possible to create an empty data set with conflicting filters. 39 | 40 | Although all agreed that these were relatively minor problems, it was nevertheless agreed that we should ask Hye Soo to stop her review, while Eric Nantz fixes the problems. The end goal is to have Pilot 2 be as clean as possible to serve as a model for future work. 41 | 42 | The new schedule is to arrange for a new Pilot 2 Submission by the end of June. We will arrange for a new submission date when Eric is ready. 43 | 44 | ### Pilot 3 45 | 46 | * There was no Pilot 3 update 47 | * The additional work on Pilot 2 will cause the Pilot 3 submission to be pushed back to the end of July. 48 | 49 | ### Pilot 4 50 | 51 | * Eric Nantz announced that the Pilot 4 kickoff meeting will happen next work 52 | * Engineers from Appsilon will join the Pilot 4 team. They are particularly interested in investigating the [WebAssembly](https://webassembly.org/) technology and have experience with containers. 53 | 54 | The next meeting of the Submissions WG will be at 9AM Pacific Time on Friday, July 7, 2023. -------------------------------------------------------------------------------- /minutes/2023-07-07/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2023-07-07" 3 | author: Joseph Rickert 4 | date: 2023-07-07 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | 9 | ## View Attendee List 10 | 11 | - Eric Nantz (Eli Lilly) 12 | - HyeSoo Cho (FDA) 13 | - Jizu Zhi (FDA) 14 | - Joel Laxamana (Roche/Genentech) 15 | - Joseph Rickert (R Consortium) 16 | - Nan Xiao (Merck) 17 | - Ning Leng (Roche/Genentech) 18 | - Robert Devine (Johnson & Johnson) 19 | - Saghir Bashir (Argenx) 20 | - Yilong Zhang (Meta) 21 | 22 | ::: 23 | 24 | ## Add Minutes Here 25 | 26 | The meeting was recorded and the [video](https://zoom.us/rec/share/UWy0e4rQiG-Qa7LJL8NT6cLCrl0PgqZPr93VAVBZowYbCnqZ01C1hQDf1haXAdM_.n__f-SUm5BcNMzgm?startTime=1688745965000) is available. 27 | 28 | 29 | ### Pilot 2 30 | 31 | Eric Nantz has made progress on addressing the issues identified with the Pilot 2 resubmission. Fixes include: 32 | 33 | * User-friendly warnings when filters result in an empty data set 34 | * Consistent footer and header notes in the tables titles that match the Pilot 1 output 35 | 36 | The only outstanding issue that is still being worked on is the warning message that results from loading the `teal` package from GitHub because it is not yet on CRAN. 37 | 38 | The Pilot 2 resubmission is now scheduled for 9AM PST on Wednesday July 19, 2023. 39 | Note that shortly after the meeting the date was confirmed with Beverly Chin and HyeSoo Cho. 40 | 41 | **Action** 42 | JBR to schedule a zoom meeting and invite: 43 | 44 | * HyeSoo Cho 45 | * Beverly Chin 46 | * Ning Leng 47 | * Eric Nantz 48 | 49 | ### Pilot 3 50 | 51 | Joel Laxamana provide the status update on Pilot 3. 52 | 53 | * The Pilot 3 team is developing the testing process and updating the ADRG to follow the process used in Pilots 1 and 2. 54 | * Developing the testing process is taking longer than expected because they are attempting to develop a hybrid of the different process used in Pilot 1 and Pilot 2. 55 | * The group is now targeting the first or second week of August for the Pilot 3 submission. 56 | 57 | **Action:** 58 | 59 | * Joel will confer with Eric and Nan at a separate meeting. 60 | 61 | ### Pilot 4 62 | 63 | Eric began the status update by thanking the team members from Appsilon who have set up a robust container environment for the Pilot. 64 | 65 | * Saghir has been able to get the container and app running in his environment 66 | * We must work with the FDA to determine the best method of transferring the container 67 | * Team members from Posit are working with the the WebAssembly 68 | * The major technical requirement is that the FDA reviewers will be able to run the [Podman](https://podman.io/) container environment 69 | * Right now, we believe that the WebAssembly, if it works as advertised, would be preferred by the FDA 70 | * It was noted that that FDA primarily works in a Windows environment 71 | 72 | **Action:** 73 | 74 | * JBR to discuss FDA constraints and preferences with Paul Schuette. 75 | 76 | The next meeting of the working group will be at 9AM PST on Friday, August 4, 2023 77 | -------------------------------------------------------------------------------- /minutes/2023-08-04/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2023-08-04" 3 | author: Joseph Rickert 4 | date: 2023-08-04 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Ben Straub (GSK) 11 | - Li Eva 12 | - Christopher Kania (Biogen) 13 | - Eric Nantz (Eli Lilly) 14 | - HyeSoo Cho (FDA) 15 | - Jizu Zhi (FDA) 16 | - Joel Laxamana (Roche/Genentech) 17 | - Joseph Rickert (R Consortium) 18 | - Nan Xiao (Merck) 19 | - Ning Leng (Roche/Genentech) 20 | - Paul Schuette (FDA) 21 | - Sam Parmar (Pfizer) 22 | - Shannon Lewis (BeiGene) 23 | - Stephanie Lussier (Moderna) 24 | ::: 25 | 26 | ## Minutes 27 | 28 | The meeting was recorded and the [video](https://zoom.us/rec/share/iap2ryh7-PHu8_fOMfwtXHpO7D7hB333mCsNODLlxE99EoCGXRTm1MSEFP7ceXzk.DnZSMt89C-TB8tZj?startTime=1691165275000) is available. 29 | 30 | 31 | ### Pilot 2 32 | 33 | * HyeSoo Cho reported that she is working with the Pilot 2 re-submission and that it looks good. 34 | * HyeSoo expects that the final on this pilot submission will be available at the end of August. 35 | * Everyone recognized and thanked Eric Nantz for the sustained and last-minute efforts to get the submission out. 36 | 37 | ### Pilot 3 38 | 39 | * Paul Schuette brought the attention of the group to section 4.i.2.10 of the [Study Data Technical Conformance Guide - Technical Specification Document](https://www.fda.gov/regulatory-information/search-fda-guidance-documents/study-data-technical-conformance-guide-technical-specifications-document) and noted that .r and .zip files are listed as acceptable file formats for submissions. 40 | * After some discussion, the WG decided not to use [pkglite](https://cran.r-project.org/package=pkglite) to wrap the R code as has been done in previous submissions, but to use .r files for the code. (Note that the FDA is unable to use .R internally.) 41 | * Paul noted that the FDA desires submission materials to be applicable for at least six years after submission. This triggered a discussion about possibility of using .zip files to transmit the proprietary packages, instead of pulling them from public GitHub repos as been done so far. 42 | * In the end, the WG decided to stick with using GitHub repos for Pilot 3 as there is currently no apparent advantage testing the .zip file method at this time. 43 | * Joel Laxama reported that preparations for Pilot 3 are going well, and that a submission using .r files could be made in early September. 44 | * The WG thought it best to wait for the report of Pilot 2 before submitting Pilot 3. 45 | 46 | ### Pilot 4 47 | 48 | * Eric reported that thanks to the contributions form the [Appsilon](https://appsilon.com/) team significant progress has been made on using both mechanisms for packaging R submissions: Containers and [Webassembly](https://webassembly.org/). 49 | * Appsilon has a prototype submission using [Podman](https://podman.io/) container technology working and has confirmed that it does not cause files to be written to the hard-drive. 50 | * Eric and the Appsilon engineers also have a very preliminary prototype of using Webassembly to wrap a small Shiny app working. 51 | * Posit engineers have contributed [webr](https://cran.r-project.org/package=webr) expertise. 52 | * Eric is optimistic about having Pilot submissions based on at least one, and possibly both, of these technologies ready by the end of the year. 53 | 54 | **Action** 55 | 56 | * Paul will confer with the proper FDA authorities to determine what preparations need to be undertaken by the FDA to use the Podman containers. 57 | * Eric will invite an Appsilon representative to come to a future meeting of this working group 58 | 59 | ### July 21 San Jose State R Foundation Event 60 | 61 | The R Foundation [SummeR Bay Area R Meetup](https://bbsw-tickets.ticketleap.com/summer-r-day2/) was held at San Jose State University on July 21. Ning Leng reports that: 62 | 63 | * The 50 plus attendees indicated that quite a few Bay Area companies are moving towards using R in preclinical and genomics research. 64 | * Representatives from several small CROs attended the meeting. 65 | * R Consortium recognition was not good among this group. 66 | 67 | **Action** 68 | 69 | * JBR and Ning to discuss what can be done to promote the RC. 70 | 71 | ### Speaking Opportunities 72 | 73 | * Eric will submit an abstract to speak about Pilot 3 and Pilot 4 at the [R/Pharma](https://bbsw-tickets.ticketleap.com/summer-r-day2/) virtual conference to be held in on October 24 - 26. 74 | * Paul will be presenting at both at the [PHUSE/FDA CSS 2023](https://advance.phuse.global/pages/viewpage.action?pageId=70778947) conference on September 18 -20 and the [ASA Biopharmaaceutical Section Regulatory-Industry statistics Workshop](https://advance.phuse.global/pages/viewpage.action?pageId=70778947) to be held in Rockville, MD on September 27 - 29 and intends to use the activities of this working group as a case study. 75 | * Ning suggests that it would be very nice if the RC could hold and R Adoption Webinar in the November time frame to describe the Pilot 2 and Pilot 3 submissions. 76 | 77 | **Action:** 78 | 79 | * Paul will check with FDA authorities to see whether it is appropriate for the R Consortium to publicize his speaking opportunities on social media. 80 | * JBR will work with the RC Community development committee to make preparations for the promoting the speaking activities and setting up the webinar. 81 | 82 | 83 | The next meeting of the Submissions working group will be at 9AM PST on Friday, September 1st. 84 | 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /minutes/2023-11-03/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2023-11-03" 3 | author: Joseph Rickert 4 | date: 2023-11-03 5 | --- 6 | 7 | Minutes prepared by Joseph Rickert and Joel Laxamana. 8 | 9 | ::: {.callout-tip collapse="true"} 10 | ## View Attendee List 11 | 12 | - Ben Straub (GSK) 13 | - Dadong Zhang (Illumina) 14 | - Eric Nantz (Eli Lilly) 15 | - HyeSoo Cho (FDA) 16 | - Jizu Zhi (FDA) 17 | - Joel Laxamana (Roche/Genentech) 18 | - Kaushal Shah 19 | - Nan Xiao (Merck) 20 | - Paul Schuette (FDA) 21 | - Robert Devine (Johnson & Johnson) 22 | - Saghir Bashir (Argenx) 23 | - Sam Parmar (Pfizer) 24 | - Joseph Rickert (ProCogia) 25 | - Ismael Alejandro Rodriguez Rodriguez (Appsilon) 26 | - Yilong Zhang (Meta) 27 | - YounKyeong Chang 28 | ::: 29 | 30 | The meeting was recorded and the [video](https://zoom.us/rec/share/5kFrV4r2DWK1aHHozPu0k8fu_66jV4IRlVsWoEgrnyocLF8ki-KBswB6wBEcBQmm.-txYArp2uGa7KpZp?startTime=1699027425000) is available. 31 | 32 | ### FDASA Presentation 33 | 34 | * The FDASA would like to invite the R Consortium to present at their meeting on December 11, 2023 (3 to 4:30PM ET). 35 | * However the FDASA would like to extend an informal invitation rather than formally sponsor the event which would require administrative approvals 36 | * The RC would still be able to advertise that Paul and HyeSoo are speaking 37 | * The RC would conduct the event which is expected to attract at least 200 of the approximately 400 FDASA participants. 38 | * Registration of the event is not required 39 | * The RC can also invite the general public to attend. (JBR will confer with Ning and Eric about doing this.) 40 | * JBR will s=send an invitation to the event, which should include the abstract, to Paul to be used to invite FDASA members to the event through internal channels. 41 | 42 | ### Pilot 3 43 | 44 | HeySoo and Paul provide some preliminary comments regarding their evaluation of Pilot 3. An extensive discussion starts approximately 12 minutes into the video recording. 45 | 46 | * The ADRG states that Pilot 3 was uses R 4.2.3. However, FDA may test using different R versions. (i.e. one reviewer using Rv4.2.1 and other using Rv4.2.3). There are at least two kinds of workstations used by FDA reviewers. Reviewers with "scientific workstations" may have administrative privileges and be able to install R. Users on other "regulatory" workstations will most likely use the version of R installed by IT. 47 | * The Pilot 3 package didn't install using renv::install() when using Rv4.2.1, but did install when the reviewer was using Rv4.2.3. 48 | * The submissions programs which process ADaM data were not updated to match the file adam-pilot-3.xlsx which was included in the submission. 49 | * Reviewers were able to update the file name. When they did so the adsl.r ran fine. 50 | * Do not include notes to change the path to source data file in the ADRG steps. Instead, keep a note open to advise reviewers that they "should change the source data path to the location where the data was downloaded from the EDR". 51 | * The submission documentation should include the version of the Microsoft operating system used along with the the version of the RStudio IDE used. 52 | * Sponsors using proprietary software also encounter Operating system issues. There may also be issues with respect to a difference in the size and power of the computers used by the sponsors and those available to the FDA reviewers. 53 | * There may be problems if file names using upper case letters are used in submissions. The FDA technical conformance guide specifies the use of lower case letters. 54 | 55 | The next meeting of the Submissions WG will be held at 9AM Pacific Time on Friday, December 8, 2023. -------------------------------------------------------------------------------- /minutes/2023-12-01/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2023-12-01" 3 | author: Joseph Rickert 4 | date: 2023-12-01 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Ben Straub (GSK) 11 | - YounKyeong Chang (FDA) 12 | - Eric Nantz (Eli Lilly) 13 | - HyeSoo Cho (FDA) 14 | - Jizu Zhi (FDA) 15 | - Joel Laxamana (Roche/Genentech) 16 | - Joseph Rickert (R Consortium) 17 | - Nan Xiao (Merck) 18 | - Ning Leng (Roche/Genentech) 19 | - Paul Schuette (FDA) 20 | - Saghir Bashir (Argenx) 21 | - Sam Parmar (Pfizer) 22 | - Steven Hasendinckx (Johnson & Johnson) 23 | ::: 24 | 25 | The meeting was recorded and screen sharing in the [video](https://zoom.us/rec/share/rE68tBPpnjdwl0RhevCYKDzq-FA7Z0kla1dSbHixZXZ9jHzoe4Kud-6Mc3r4-PKd.25fRPYZYgN0MzxxU?startTime=1701450234000) provides information about the FDA review of Pilot 3 which is in progress. 26 | 27 | JBR began the meeting the meeting with a reminder that the meeting is to be conducted in compliance with the Antitrust laws. 28 | 29 | ### Pilot 3 30 | 31 | YounKyeong Chang of the FDA provided and update on he ongoing FDA review. 32 | 33 | * Able to generate identical data sets with the code provided, but there were some minor corrections and issues. 34 | * Incorrect folder name in adlvc.r 35 | * After switching to R.4.2.3, code in ADRG installed pilotutils package instead of pilot3 package. Switching to from R 4.3.1 to R 4.2.3 resolved the problem. 36 | * There was an extensive discussion about missing values. 37 | * The CDISC standard indicates missing values with blanks, and depending on the variable type SAS may also use blanks for missing values. However, R encodes missing values as "NA". 38 | * Decision made to convert missing values to NA when using R, but provide an option to use blanks for analysts using SAS. However, the FDA does not yet have an SOP for this. 39 | * A paragraph will be added to the ADRG to explain that SAS users should convert NAs to blanks. 40 | * The ADaM datasets in Pilot 3 where run through [Pinnacle 21](https://www.lexjansen.com/pharmasug/2019/DS/PharmaSUG-2019-DS-119.pdf) without error. 41 | * the WG should reach out to CDISC to see if they will modify their standard to accommodate R. 42 | * **Action** Ning and Joel will prepare a statement to communicate with CDISC. 43 | * Sharing of the details of any FDA findings, such as as those discussed in the meeting, require clearance 44 | * Some minor differences were observed in how SAS and R interpreted data types. 45 | * The FDA will try to evaluate the submission outputs by next month's meeting 46 | * The WG agreed that after the evaluation of Pilot 3 is completed, it would be interesting to do a Pilot 3B submission using .zip instead of `pkglite` as was used in Pilots 1 and 2. Paul said that in light of the FDA requirement to keep submission materials for 6 years this would provide an alternative for long term preservation of the submission materials. 47 | * The FDA will make a decision whether an additional Pilot 3 submission to correct the problems identified so far is necessary after they have completed their evaluation. 48 | 49 | ### Pilot 4 50 | 51 | Eric Nantz provide a status update on Pilot 4 52 | 53 | * Pilot 4 will take the approach of submitting as a .zip archive. 54 | * An issue is that the default behavior of `WebR` is to install a special version of some commonly used R packages that are optimized for `WebR`. 55 | * We would like to include these special `WebR` binaries in the submission bundle in order to improve performance. 56 | * Because the binaries are not executable without a WebAssembly process running, the WG believes these binaries wold not violate the FDA requirement of not including executables. 57 | 58 | ### Events 59 | 60 | * Eric, Hyesoo, and Paul will be presenting on December 11 in a Webinar that will be broadcast within the FDA as well as available to the public. Ning will lead a panel discussion afterwards. 61 | * The webinar *The use of R in Japan’s Pharma Industry* is scheduled for January 8, 2024. 62 | * Gregory Chen will be presenting the WGs work in person at Department of Clinical Assessment for Market Authorization at Swiss medic in January 63 | 64 | The next meeting will take place at 9AM St on Friday, January 5, 2025. **Action:** JBR will issue a new set of calendar invitations for 2024. Stay tuned for details. 65 | -------------------------------------------------------------------------------- /minutes/2024-01-05/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2024-01-05" 3 | author: Joseph Rickert 4 | date: 2024-01-05 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Ben Straub (GSK) 11 | - Dadong Zhang (Illumina) 12 | - Eva Li 13 | - Heidi Curinckx (Johnson & Johnson) 14 | - HyeSoo Cho (FDA) 15 | - Joseph Rickert (R Consortium) 16 | - Margaret Wishart 17 | - Nan Xiao (Merck) 18 | - Ning Leng (Roche/Genentech) 19 | - Paul Schuette (FDA) 20 | - Robert Devine (Johnson & Johnson) 21 | - Saghir Bashir (Argenx) 22 | - Sam Parmar (Pfizer) 23 | - Steven Hasendinckx (Johnson & Johnson) 24 | - Youn Kyeong Chang 25 | ::: 26 | 27 | The meeting was recorded and the [video](https://zoom.us/rec/share/S3CjidPqOBSwSHo7xW3VUu_aUXKS9RdB3rNb07hWWkeyze5SthawUq5I8gnzQ2th.kpiljMfkqrlkuUJf?startTime=1704474237000) is available. 28 | 29 | 30 | ## Pilot 3 31 | 32 | HyeSoo presented an update on the continuing FDA evaluation of Pilot 3. 33 | 34 | * Using the R generated ADaM data sets, the FDA reviewers were able to replicate the results of the demographic, efficacy, primary and KM plot analyses 35 | 36 | * The reviewers compared the results of the four analyses between Pilot 1 and Pilot 3. 37 | * Primary output in Pilot 3 was different from the Pilot 1 result due to the QC findings 38 | * QC findings in ADRG include all of the discrepancies between the original ADaM datasets 39 | * The discrepancies in ADADAS data sets cause different primary analyses outputs 40 | * This is an issue from the CDISC ADADAS 41 | * It is not clear which CDISC file, the file from Pilot 1 or the file from Pilot 3, is correct. 42 | 43 | The following figure compares the Pilot 1 and Pilot 3 primary output. Discrepancies are highlighted. 44 | 45 | ![Comparison of Primary outputs](output_compare.png) 46 | Additional FDA questions included: 47 | 48 | * Why is there an indication in the output of LOCF, "Last observation carried forward" when the original data set had no missing data? 49 | * There was no statistical analysis plan included in the submission. 50 | * **Action:** Ben Straub will confer with the Pilot 3 team to answer these questions and review the discrepancy problems between Pilots 1 and 3. 51 | * **Action:** Paul and HyeSoo will discuss whether we need to resubmit Pilot 3 52 | The next meeting of the working group will be on Friday, February 2, 2024 at 9AM Pacific Time. 53 | * All agreed that if a resubmission of Pilot 3 is required, we will consider submitting via a .zip file. 54 | 55 | 56 | There was some discussion about what we could do to improve the quality control of our submissions. 57 | 58 | * The group agreed that we should have a check list to compare items against a Pilot 1 source of truth. 59 | * We should have additional reviewers who are not close to the work. 60 | * The idea of using AI tools to check consistency was mentioned. 61 | 62 | ## Upcoming Webinar 63 | 64 | On Monday at 7PM PST Rikimaru Nishimura and Yuki Matsunaga will present a webinar on The Adoption of R in Japan’s Pharma Industry Confirmation. Look [here](https://www.r-consortium.org/r-adoption-series-the-adoption-of-r-in-japans-pharma-industry-confirmation) for details and information on how to register. 65 | 66 | * Ning noted that Yuki expressed an interest in collaborating with the R Consortium. 67 | * **Action:** we will set up a meeting with the Japanese team after the meeting. 68 | 69 | ## PharmaSUG 70 | 71 | * JBR mentioned that it is an R Consortium goal for 2024 to make the opportunity for as many speaking engagements as possible. 72 | * Ben mentioned that [PharmaSUG 2024](https://www.pharmasug.org/us/index.html) is coming up in May, 73 | * **Action:** Ben volunteered to submit an abstract by the January 15 deadline. Ning will provide Ben with some copy used for a previous abstract. 74 | 75 | 76 | The next meeting of the working group will be at 9AM PST on Friday, February 2, 2024. -------------------------------------------------------------------------------- /minutes/2024-01-05/output_compare.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/2024-01-05/output_compare.png -------------------------------------------------------------------------------- /minutes/2024-03-01/KM.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/2024-03-01/KM.png -------------------------------------------------------------------------------- /minutes/2024-03-01/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2024-03-01" 3 | author: Joseph Rickert 4 | date: 2024-03-01 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Eric Nantz (Eli Lilly) 11 | - Heidi Curinckx (Johnson & Johnson) 12 | - HyeSoo Cho (FDA) 13 | - Jizu Zhi (FDA) 14 | - Joel Laxamana (Roche/Genentech) 15 | - Joseph Rickert (JBR) (R Consortium) 16 | - Lovemore Gakava (Novo Nordisk) 17 | - Nan Xiao (Merck) 18 | - Nicholas Masel (JRDUS) 19 | - Ning Leng (Roche/Genentech) 20 | - Paul Schuette (FDA) 21 | - Robert Devine (Johnson & Johnson) 22 | - Saghir Bashir (Argenx) 23 | - Sam Parmar (Pfizer) 24 | - Youn Kyeong Chang (FDA) 25 | 26 | The meeting was recorded and the [video](https://zoom.us/rec/share/_n7MWF0l4R5FsccL4GzRrzXUoGOCXwxQqQuY4Pu2CqF0_DX9xMmpYUoxREA92uX-.m3O3hPqvxfZKyws4?startTime=1709312595000) is available. 27 | 28 | ::: 29 | 30 | ## Plot 3 Update 31 | 32 | ### FDA Recommendation 33 | 34 | HyeSoo stated that the FDA is considering issuing a recommendation that submissions sponsors should use the most recent patch version of the previous minor version of R. 35 | 36 | For example, R versions are organized as V x.y.z where: 37 | 38 | * x is the major version 39 | * y is the minor version or subversion 40 | * z is the patch version 41 | 42 | The WG members affirmed that a policy of using a proven stable version of R and not the latest version is in accordance with the practices of their respective companies. 43 | 44 | After some thought the working group proposed that since patch versions are issued frequently (every couple of months), it might be better to have the FDA guidance suggest the last patch version of the previous subversion. 45 | 46 | 47 | Note that the FDA is considering making a recommendation and not considering setting policy. 48 | 49 | 50 | **Action:** 51 | JBR will confer with Sam, Paul and HyeSoo about the wording of the recommendation and then draft a blog post. The post will go to Paul, HyeSoo, Eric, and Ning for review before being published on the R Consortium Blog. 52 | 53 | ## Pilot 3 Update 54 | 55 | HyeSoo and Joel informed the group that they met last month to resolve the discrepancy between the original CDISC version of the ADADAS ADaM data set and the Pilot 3 version of the ADaM data set. Joel noted that there are 818 available records in the QS domain. The CDISC ADADAS only brought in 799 records and imputed the rest, whereas Pilot 3 brought in all available 818 records into ADADAS and then imputed. When the Pilot 3 team adjusted by subsetting to ANL01FL='Y' records first before doing the LOCF imputation the results matched Pilot 1 and the discrepancy was resolved. Look [here](https://github.com/RConsortium/submissions-pilot3-adam/pull/146) for details. 56 | 57 | 58 | This issue illustrates the level of detail that the FDA must consider in replicating a sponsor's submission environment. This particular issue was due to a statistical decision and not an error. Ning pointed out that most of the work in resolving discrepancies over the three Pilot submissions are due to insufficient documentation and inadequate communication and not to differences between SAS and R. 59 | 60 | **Action:** 61 | Ning and Joe will work on a blog post about this topic. 62 | 63 | 64 | ## R/Medicine 65 | 66 | Joel has been invited to give a talk at the upcoming R/Medicine virtual conference which will be held from June 10 to 14. He is deciding whether to give a talk or a workshop. 67 | 68 | 69 | ## UseR! 2024 70 | The group decided to submit an abstract for useR!2024 which will be held from July 8 to 11 in Salzburg, Austria. the deadline for the CFP is Sunday, 10 March at 11:59 PM CET (UTC+1) 71 | 72 | **Action:** 73 | 74 | Eric and Ning will draft the abstract. 75 | 76 | ## Pilot 4 update 77 | 78 | At **38:38** into the video of the meeting, Eric previews the Pilot 4 WebAssembly (Wasm) submission. To be compatible with Wasm, the submission is being built with the [Rhino](https://appsilon.github.io/rhino/) framework for shiny which is different from what was used in Pilot 2 but is functionally equivalent. The screen capture below references the compiling the `WebR` Wasm submission with `shinylive`. 79 | 80 | ![](pilot4.png) 81 | After the build it is necessary to launch a webserver process on your your local setup. The Wasm version of the Shiny app reproduces the `teal` filters without using teal itself. 82 | 83 | ![](KM.png) 84 | The next step is to test the application on the container approximating the FDA Windows environment. 85 | 86 | ### Immediate Plans 87 | 88 | Joel's team will continue working on Pilot 3 re-submission with the goal of picking a submission date at the April meeting. 89 | 90 | 91 | 92 | ## The Submission Environmnt Setup 93 | 94 | Setting up the submission environment is a manual process. The WG would like to automate at lest some of this process to ease the burden of setup and improve repeatability. It would be nice to have a GitHub action workflow that uses containers and runs on Azure. This will be a topic for the next meeting. 95 | 96 | **Action:** 97 | JBR will promote this task and try to find a company willing to work on it. 98 | 99 | ## Swissmedic WG 100 | JBR asked the WG for help in identifying Europe-based Pharma employees who might be interested in starting a new working group focused on Swissmedic. 101 | 102 | **Action:** 103 | Ning will open an issue on this and tag JBR and Gregory. 104 | 105 | ## Next Meeting 106 | The next meeting of the working group will be on Friday, April 5, 2024 at 9AM PST. 107 | -------------------------------------------------------------------------------- /minutes/2024-03-01/pilot4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/2024-03-01/pilot4.png -------------------------------------------------------------------------------- /minutes/2024-04-05/etcd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/2024-04-05/etcd.png -------------------------------------------------------------------------------- /minutes/2024-05-03/VM.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/2024-05-03/VM.png -------------------------------------------------------------------------------- /minutes/2024-05-03/p3error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/2024-05-03/p3error.png -------------------------------------------------------------------------------- /minutes/2024-06-07/ADRG.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/2024-06-07/ADRG.png -------------------------------------------------------------------------------- /minutes/2024-06-07/dates.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RConsortium/submissions-wg/dbb592bf6ab8f515237f8ddd9bf9d7ee3c9e5983/minutes/2024-06-07/dates.png -------------------------------------------------------------------------------- /minutes/2024-10-04/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2024-10-04" 3 | date: 2024-10-04 4 | author: Eric Nantz & Terry Christiani 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Amanda Martin (R Consortium) 11 | - Andre Couturier (Sanofi) 12 | - Ben Straub (GSK) 13 | - Eric Nantz (Eli Lilly) 14 | - HyeSoo Cho (FDA) 15 | - Joel Laxamana (Roche/Genentech) 16 | - Lovemore Gakava (Novo Nordisk) 17 | - Michael Kane (MD Anderson Cancer Institute) 18 | - Mike Stackhouse (Atorus) 19 | - Nicholas Masel (Johnson & Johnson) 20 | - Ning Leng (Roche/Genentech) 21 | - Paul Schuette (FDA) 22 | - Robert Devine (Johnson & Johnson) 23 | - Saghir Bashir (Argenx) 24 | - Terry Christiani (R Consortium) 25 | ::: 26 | 27 | ## Administrative Updates 28 | 29 | * The working group welcomed Terry Christiani, the new executive director for the R Consortium. We are greatly appreciate the important leadership provided by Joseph Rickert, the previous director. More information can be found on this [R Consortium Blog post](https://r-consortium.org/posts/thank-you-joseph-rickert-a-legacy-of-leadership/). 30 | 31 | ## Submissions Pilot 4 32 | 33 | * The submissins pilot 4 (web-assembly version) was transferred on September 20, 2024 successfully and received by the FDA reviewers through the eCTD gateway transfer. The news was also shared in this [announcement post](https://r-consortium.org/posts/using-r-to-submit-research-to-the-fda-pilot-4-successfully-submitted/) on the R Consortium blog. 34 | * Hye Soo informed that their internal reviewer computers required an update to R version 4.4.1 to match the version used in the pilot submission. She also verified that there were no issues in rendering the Shiny application with the web-assembly method of execution. 35 | * A small issue with column labels being switched on Primary Table was identified (a legacy issue that unfortunately was carried forward from an earlier version of the Pilot 1 output). While this is not a critical issue for the scope of pilot 4, Ben Straub shared how leveraging continuous integration / continuous deployment (CI/CD) for verifying the integrity of the data and outputs could be helpful. 36 | * The full review of the submission is ongoing. 37 | * Eric Nantz shared that the Docker container version of Pilot 4 is undergoing additional testing with the working group, and the goal is to have a transfer to the FDA reviewers in November (exact timing TBD). 38 | * Communication with the Nature journal on a potential manuscript is ongoing. Michael Kane noted that additional journals such as the [Journal of Statistical Software](https://www.jstatsoft.org/index) are another possibility for sharing the concepts of web-assembly in clinical submissions. 39 | 40 | ## Analysis Data Reviewers Guide (ADRG) 41 | 42 | * Joel Laxamana shared the ADRG Phuse working group established quarterly meetings starting on October 16. 43 | * Initial steps involve identifying all of the pieces required for an ADRG, and how can templates be enhanced using the learnings from previous open-source efforts such as the Pilot submissions. 44 | * Additional members from the submmissions working group will be invited to future discussions after the initial brainstorming of high-level ideas. 45 | 46 | ## Repositories Working Group Notes 47 | 48 | * The Repositories Working Group is continuing their efforts to deliver a reproducible environment structure in which each organization can adopt to fit their needs. 49 | * It was recommended to invite Doug Kelkoff to a future submissions WG meeting to stay updated with their efforts 50 | 51 | ## Future Pilot Ideas 52 | 53 | ### Realistic Pilot Submissions 54 | 55 | * The previous pilot submissions have been successful with the higher-level goals of demonstrating open-source software including R is a viable method for producing the typical submission deliverables. 56 | * However, the mock clinical data sets and collection of outputs are a much smaller scale than a typical clinical submission which routinely involve many data sets and clinical TFLs. 57 | * Bringing members outside of this core group to lead a realistic pilot submission grouup could be valuable in many fronts. 58 | * Paul Schutte shared that there isn't a cononical data set applicable to the life sciences space in the way that widely-known data sets such as iris and more recently [Palmer Penguins](https://allisonhorst.github.io/palmerpenguins/articles/intro.html). 59 | * With the lack of public de-identified clinical data sets on the scale of the typical submission, creating these realistic sets via robust simulation could be a path forward. 60 | * Action item: Create a subgroup of those interested in this effort. 61 | * Sources such as [Project Datasphere](https://data.projectdatasphere.org/projectdatasphere/html/home) are a way to obtain ADAM data sets involving treatment and standard of care data sets (with many oncology studies). One must request access before data are shared. 62 | 63 | ## JSON Data Format 64 | 65 | * The group is interested in learning more on the current progress of the [Dataset-JSON](https://www.cdisc.org/dataset-json) effort. 66 | * Mike Stakhouse will share more information on the [`{datasetjson}`](https://atorus-research.github.io/datasetjson/) package in the next submissions working group meeting. 67 | -------------------------------------------------------------------------------- /minutes/2024-11-01/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2024-11-01" 3 | date: 2024-11-01 4 | author: Eric Nantz & Theresa Christiani 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Ben Straub (GSK) 11 | - Eric Nantz (Eli Lilly) 12 | - HyeSoo Cho (FDA) 13 | - Joel Laxamana (Roche/Genentech) 14 | - Katie Harding (Freemore) 15 | - Lovemore Gakava (Novo Nordisk) 16 | - Nicholas Masel (Johnson & Johnson) 17 | - Ning Leng (Roche/Genentech) 18 | - Paul Schuette (FDA) 19 | - Robert Devine (Johnson & Johnson) 20 | - Theresa Christiani (R Consortium) 21 | - Youn Kyeong Chang (FDA) 22 | ::: 23 | 24 | ## Pilot 4 Updates 25 | 26 | * HyeSoo shared concerns regarding the appropriateness of zip file formats for program delivery under FDA specifications. Suggestions to explore new architectures or formats for future submissions were raised, particularly for web-based applications. 27 | * Testing for the Docker container component of Pilot 4 is ongoing, thus far no issues but Eric will share more updates in the next meeting. 28 | 29 | ## Working Group Coordination 30 | 31 | * Theresa Christiani introduced an inter-organization coordination committee to streamline overlapping efforts across working groups (R Consortium, PhUSE, Pharmaverse). This committee will promote efficient use of resources and avoid redundant work. Robert Devine volunteered to present at the next coordination meeting. 32 | * Robert will present at the R Consortium's inter-organization coordination committee to share the group's current projects and align efforts across R Consortium, PhUSE, and Pharmaverse. 33 | * Those interested in attending that coordinating meeting can contact Theresa for getting on the invitation to those meetings 34 | 35 | ## Analysis Data Reviewer’s Guide (ADRG) Updates 36 | 37 | * The PhUSE working group on open-source metadata documentation is working to update the ADRG template to better support open-source submissions, with feedback from FDA and industry representatives. 38 | * Lovemore gave a recap of a presentation about the ADRG workstream at the PHUSE FDA quarterly meeting 39 | * The team discussed the potential use of Quarto (QMD) and HTML formats for more accessible, navigable submissions, as well as potential supplementary documents for open-source submissions. Paul confirmed that R Markdown (`.rmd`) format has already been approved as a file type. 40 | * Joel and Lovemore will set up a future meeting to gather additional industry feedback as a first step to developing a new prototype of the ADRG using enhanced metadata and the Quarto format. 41 | * While Quarto is an approved software package at the FDA, and Quarto file format (`.qmd`) is part of the approved file types within the **Modeling and Simulation** section of the [eCTD Format Specifications](https://www.fda.gov/media/85816/download) document. 42 | * Paul and HyeSoo will touch base with Helena Spiglin (part of the FDA committee currently assessing the `.json` formats as an alternative to SAS Transport `.xpt`) for confirming the Quarto and HTML formats can be added as supported formats for ADRG submissions. 43 | 44 | ## Data Set JSON Format and Alternative Data Formats 45 | 46 | * The team discussed exploring JSON and alternative data formats like [Parquet](https://parquet.apache.org/), given the large data sizes in modern studies. 47 | * Although the industry largely uses JSON, there are concerns about its efficiency for high-volume data. Parquet, used internally by several team members, offers a promising alternative, particularly for big data applications. 48 | * For a comprehensive overview of the benefits and tradeoffs between JSON, Parquet, and other formats, see the recent R/Pharma 2024 presentation **An arrow towards the future: A critical look at data formats for clinical reporting** by Craig Gower-Page ([recording link](https://www.youtube.com/watch?v=ipvcRPr0eig)). 49 | * The FDA has primarily used JSON but is open to considering alternatives like Parquet with further testing and demonstrations. 50 | * Paul mentioned he is not aware of previous efforts to convert the publically available CDISC data sets utilized in the previous pilot submissions to JSON format and then perform statistical analyses. 51 | * __Opportunity for Pilot 5__: Using the structure of Pilot 3 as a template, leverage JSON as the format to assess performance. Ben Straub will draft a proposal for an initial review at the next Submissions Working Group meeting in December. 52 | * Nicholas Masel shared that the [{datasetjson}](https://atorus-research.github.io/datasetjson/) R package will need updates to align with the latest schema (1.1) to ensure compatability with the future Pilot 5. 53 | * In addition, Nick and Ben will gather input from industry colleagues and the FDA to determine if Parquet is a viable alternative to JSON for high-volume data in future pilots, and discuss any potential technical or bureaucratic limitations with FDA. 54 | 55 | ## Future Pilot Projects 56 | 57 | The team plans to prioritize two future pilot projects: 58 | 59 | * Pilot 5: A repeat of Pilot 3 but using JSON format to assess package performance and ADRG requirements. 60 | * Comprehensive Benchmark Pilot: A longer-term pilot for H2 2025 using a large, realistic simulated data set to evaluate JSON vs. Parquet for performance, usability, and compatibility. 61 | -------------------------------------------------------------------------------- /minutes/2025-01-10/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2025-01-10" 3 | date: 2025-01-10 4 | editor: 5 | render-on-save: true 6 | --- 7 | 8 | ::: {.callout-tip collapse="true"} 9 | ## View Attendee List 10 | 11 | - Ben Straub (GSK) 12 | - Eric Nantz (Eli Lilly) 13 | - CG Wang 14 | - Heidi Curinckx (Johnson & Johnson) 15 | - HyeSoo Cho (FDA) 16 | - Joel Laxamana (Roche/Genentech) 17 | - Katie Harding (Freenome) 18 | - Lovemore Gakava (Novo Nordisk) 19 | - Nikolay Nikolaev 20 | - Ning Leng (Roche/Genentech) 21 | - Paul Schuette (FDA) 22 | - Phanikumar Tata (Syneos Health) 23 | - Phil Bowsher (Posit) 24 | - Robert Devine (Johnson & Johnson) 25 | - Sam Parmar (Pfizer) 26 | - Terry Christiani (R Consortium) 27 | ::: 28 | 29 | ## ADRG Working Group Updates 30 | 31 | * Lovemore provided an update on recent ADRG working group discussions, with minutes available at . 32 | * Part of the recent discussions involved sharing the FDA's proposed draft of ADRG content with Christine Rossin at Pfizer (a member of the PHUSE ADRG working group). She had many comments pertaining to keeping the ADRG template language-agnostic. The team will set up a 33 | 34 | ## Pilot 5 (Dataset-JSON) Updates 35 | 36 | * Ben shared a two-stage approach to the pilot that mimics how Pilot 4 was split to two components 37 | * The pilot 5 team will hold weekly 30-minute calls on Fridays (10 AM EST) for discussions and updates. 38 | * At a high level the pilot will have two key stages of development: 39 | * Convert the data sets created in Pilot 3 to JSON. 40 | * Adding additional JSON data sets and possibly new TLGs for additional testing and to pinpoint edge cases. 41 | * Pilot 5 now has the following GitHub repositories created: 42 | * Development repository: 43 | * Submission (eCTD-struture) repository: 44 | * R package repository: 45 | * Pilot web site repository: 46 | 47 | ## Pilot 6 (Realistic Submission Data) 48 | 49 | * Ateendees expressed excitement about the ways this new pilot's data package could help many aspects of future submissions involving open-source software, ranging from the ability to handle larger data sets, to having custom versions of the data tailored to benchmarking statistical methodologies. 50 | * Paul reminded the group that while they have access to a vast set of data from submissions, they are not allowed to share sponsor data externally. 51 | * Ben had a suggestion that part of the pilot 6 deliverables include a mock study protocol and statistical analysis plan. 52 | * Eric will begin to create GitHub repositories for the pilot and set up working group meetings. 53 | 54 | ## Additional items 55 | 56 | * The working group expressed interest in sharing publicly the new Pilot 5 and 6 objectives and goals in a future R Consortium Blog Post. 57 | * Nan Xiao has created a Python version of `{pkglite}` which is now part of the Pharmaverse! Details on the package can be found on GitHub at . 67 | -------------------------------------------------------------------------------- /minutes/2025-02-07/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2025-02-07" 3 | date: 2025-02-07 4 | --- 5 | 6 | ::: {.callout-tip collapse="true"} 7 | ## View Attendee List 8 | 9 | - Ben Straub (GSK) 10 | - Eric Nantz (Eli Lilly) 11 | - Katie Harding (Freenome) 12 | - Lovemore Gakava (Novo Nordisk) 13 | - Phanikumar Tata (Syneos Health) 14 | - Robert Devine (Johnson & Johnson) 15 | - Sam Parmar (Pfizer) 16 | - Terry Christiani (R Consortium) 17 | - Vedha Viyash (Appsilon) 18 | ::: 19 | 20 | ## ADRG metadata group updates 21 | 22 | * Lovemore and Sam shared the group's recent updates progress. They recently invited Christine Rossin (from the Phuse ADRG working group) to their meetings for discussion on the proposed template. 23 | * Christine's recommendations include moving the instructions for executing the R code and setting up the execution environment to a supporting document and not part of the ADRG itself, ensuring the ADRG remains software-agnostic, and separate guidelines be developed for specific software. 24 | * The metadata group will be taking the lead on authoring guidelines for the R exection and environment language 25 | * The group will also establish dialogs with other sub-teams addressing additional sections of ADRG such as such as estimands team to ensure everyone is on the same page for updating the ADRG sections 26 | * Next steps include authoring R scripts to help automate population of certain sections in the ADRG template. 27 | 28 | ## Pilot 4 Container Submission updates 29 | 30 | * Eric shared recent progress on assembling a proposed submission structure for the container portion of Pilot 4. 31 | * Substantial updates have been made to the development repository available at as well as the proposed submission structure at . 32 | * Eric will share the proposed structure with our FDA colleagues to ensure this is aligned with the eCTD guidelines. 33 | * Robert inquired about the specific version of the Windows Subsystem for Linux (WSL) used in the testing environment. Eric believes it is WSL2 but that will be confirmed and the appropriate sections of the ADRG will be updated accordingly. 34 | * Vedha shared that running the command `wsl --version` in the testing environment should output the key metadata on the WSL installation. 35 | 36 | ## Pilot 5 DataSet-JSON updates 37 | 38 | * Sam and Ben shared the recent progress from the Pilot 5 working group 39 | * The team has created GitHub repositories for key portions of the pilot: 40 | + Development repository: 41 | * R package repository: 42 | * Website: https://rconsortium.github.io/submissions-pilot5-website/ 43 | * Submission package structure: 44 | * The data sets and R code used in Pilot 3 will be ported over to the Pilot 5 repositories 45 | * The team is very interested in utilizing GitHub actions to perform quality checks of the data set 46 | * `{datasetjson}` was recently updated to version 1.1 and the new update is available on CRAN. You can view the changelog at . 47 | * As development begins for the quality checks and other testing, they will use the Pilot 3 code as a starting point. 48 | - Robert shared that CDISC will hold a general meeting on February 18, 2025. Members of the working group are encouraged to attend. Additional details and registration are available at the pharma-oversight Slack channel in the February 14, 2025 post. 49 | 50 | -------------------------------------------------------------------------------- /minutes/2025-03-07/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2025-03-07" 3 | author: Eric Nantz 4 | date: 2025-03-07 5 | --- 6 | 7 | ::: {.callout-tip collapse="true"} 8 | ## View Attendee List 9 | 10 | - Ben Straub (GSK) 11 | - Eric Nantz (Eli Lilly) 12 | - HyeSoo Cho (FDA) 13 | - Joel Laxamana (Roche/Genentech) 14 | - Katie Harding (Freenome) 15 | - Lovemore Gakava (Novo Nordisk) 16 | - Nabil Baugher (Baseline AI) 17 | - Nan Xiao (Merck) 18 | - Nicholas Masel (Johnson & Johnson) 19 | - Ning Leng (Roche/Genentech) 20 | - Paul Schuette (FDA) 21 | - Sam Parmar (Pfizer) 22 | - Youn Kyeong Chang (FDA) 23 | ::: 24 | 25 | ## Pilot 4: File Structures 26 | 27 | * Discussions internally are taking place at FDA on whether the Zip file can include the file types that may not be on the approved software list 28 | * In light of the evolving discussions, Paul recommends to explore the use of `{pkglite}` for bunlding the applicaiton files in a similar fashion as Submissions Pilot 2 (traditional shiny application) 29 | * Eric will examine both the web assembly and Docker container versions of Pilot 4 and provide FDA reviewers a list of the essenstial file types necessary to review each submission bundle 30 | * Nan Xiao recommends we examine his recent releaes of [pkglite for Python](https://nanx.me/blog/post/pkglite-python/) in case that could help streamline the process, especially for the web-assembly version 31 | 32 | ## Pilot 5: DatasetJSON 33 | 34 | * The pilot 5 team is aiming to have dataset programming completed by the end of March, and TLF programming completed by end of April. 35 | * Questions remain on whether the JSON format can be included as a submission format, Paul mentioned that they are waiting to confirm with the E-sub group internally 36 | * The R-Consortium Blog recently featured a [fantastic post](https://r-consortium.org/posts/submissions-wg-pilot5-pilot6-and-more/) announcing the launch of Pilot 5: 37 | * Minutes from the Pilot 5 working group meetings are available on the Pilot 5 datasetJSON GitHub repository: 38 | * One potential benefit for education and awareness: Embed an example Shiny application inside the Pilot 5 Quarto website for users to explore contents of dataset JSON files. 39 | + After Robert shared a list of apps created in a recent DatasetJSON Hackathon, the team saw an apportunity with the [Dataset-JSON Viewer](https://github.com/sbarcelo777/Dataset-JSON-hackathon) Shiny application 40 | * Sam Parmer quickly created a novel way to compile the application as a web-assembly Shiny app and dynamically deploy the app on Posit's shinylive-R hosting platform to embed insde the Pilot 5 Quarto website as an iframe! You can view the page at 41 | * In parallel, Eric also experimented with compiling the same Shiny application into web-assembly, and has a prototype automatically deployed on GitHub Pages via GitHub Actions: 42 | * Eric will reach out to the developers of the Dataset-JSON viewer to see if they are receptive to Eric sending a pull request to have the application hosted from their GitHub repository (thus their GitHub Pages account). In addition, Eric will ask the maintainers if they would support having a slightly streamlined version of the application that pre-loads JSON data files created for this pilot. 43 | * Eric and other team members are currently updating GitHub actions for repository maintenance and automation 44 | 45 | ## Phuse Metadata / ADRG Working Group 46 | 47 | * Joel Laxana shared the latest updates: They group is trying to hash out the information to be added in the ADRG for future open source submissions 48 | * During recent discussions, the team is going forward with creating a separate supplement document to go along with the traditional ADRG submission document in the case of an R-based submission 49 | * The group is reviewing feedback provided by FDA reviewers from the past couple of pilots submitted 50 | * The open source metadata working group will continue to leverage metadata from a particular project to populate sections of the document, using R functions that can help fill in manual steps 51 | * The group ponders if generative AI help to populate these templates? 52 | * While AI is not in the scope of the Phuse project, the team is open for exploration 53 | * Indeed there could be an opportunity to spin off a separate pilot on exploring generative AI in a clinical submission documentation preparation such as the ADRG. 54 | * A new channel will be created in the R Consortium Submissions Working Group Slack channel. For details on obtaining access to group, please visit the [How to Join](https://rconsortium.github.io/submissions-wg/join.html) page. 55 | 56 | ## Other Topics 57 | 58 | * Members of the wroking group (Ben, Sam, and Eric) will join Appsilon for the April 29th edition of [Shiny Gatherings](https://events.ringcentral.com/events/shiny-gathering-fda-pilots-key-insights-lessons-learned-and-what-s-next-for-2025-820c22c5-9f84-4bb1-a9ba-9341ecaf791d) discussing the insights, lessons learned, and what's next for 2025. 59 | * The group assured colleagues from FDA that there would not be any direct quotes shared from FDA colleagues inside of this webinar, and any issues from the regulatory side would be discussed in general terms. 60 | 61 | -------------------------------------------------------------------------------- /minutes/2025-05-02/index.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes 2025-05-02" 3 | date: 2025-05-02 4 | --- 5 | 6 | ::: {.callout-tip collapse="true"} 7 | ## View Attendee List 8 | 9 | - Athenkosi Nikonyeni 10 | - Ben Straub (GSK) 11 | - Camilla Calmasini 12 | - HyeSoo Cho (FDA) 13 | - Jared Woolfolk 14 | - Joel Laxamana (Roche/Genentech) 15 | - Keaven Anderson (Merck) 16 | - LG 17 | - Mekhala Acharya (Takeda) 18 | - Paul Schuette (FDA) 19 | - Phanikumar Tata (Syneos Health) 20 | - Phil Bowsher (Posit) 21 | - Robert Devine (Johnson & Johnson) 22 | - Sadchla Mascary 23 | - Saghir Bashir (Argenx) 24 | - Sam Parmar (Pfizer) 25 | - Stephen Knapp 26 | - Yilong Zhang (Meta) 27 | - Youn Kyeong Chang (FDA) 28 | ::: 29 | 30 | ## Pilot 4 31 | 32 | * Eric to set up a informal review meeting with FDA colleagues in Mid May (EDIT: The revised transfer will take place on June 13th, 2025) 33 | * Full speed ahead to use `{pkglite}` for assembling the web assembly and container submission bundles. 34 | 35 | ## Pilot 5 36 | 37 | * Part 1 is moving well (conversion from xpt is complete) 38 | * Team will wait until pilot 4 review is complete before transferring pilot 5 39 | * Paul: Not sure where the evaluation of `datasetjson` currently stands within FDA. The team that was sponsoring it has been affected by recent events. They think the gateway will accept the format, but needs verification. 40 | * Previous transfers in the CDISC effort last year may or may not have been through the ectd gateway 41 | * ESUB team is still having discussion on testing the json format 42 | * Heather Crandle from a previous R Pharma conference is part of that team (2021) 43 | * In current [eCTD instructions](https://www.fda.gov/media/85816/download), JSON is allowed for the PK category 44 | * Ben will follow up with team involved in the CDISC pilot to see how they transferred the JSON files. 45 | * The team has embedded a datasetJSON viewer with shiny and webassebly in the pilot 5 website available at 46 | * Ning had started investing using LLMs to generate portions of the ADRG. There may be an update at the next WG meeting. 47 | 48 | ## ADRG updates 49 | 50 | * The group's approach is to help communicate the version of different packages and how to best display that information in the ADRG template. 51 | * Feedback has been received from FDA reviewers. The group is looking into ways to update templates such that multiple software languages can be supported 52 | * Keeping in close sync with Christine at Pfizer on ensuring the existing standards are being met. 53 | * Also have plans to contact reps from Pinnacle21 on how they populate the ADRG, while ensuring minimal overlap occurs. 54 | * Goal of the group is to reduce manual effort of populating the various tables in the ADRG such as package versions, number of programs, functions used in those programs, which is all a manual effort at the moment 55 | 56 | 57 | ## Other Topics 58 | 59 | * Yilong would like to know if there is interest in a Python-based pilot for submitting tables and listings produced in Python. 60 | * Thus far not as much interest in the office of Biostatistics according to FDA reviewers 61 | * CDRH handles things differently than CDER. CDRH does not require CDISC standards to be followed. A greater range of issues can occur in those cases. 62 | * Meta has utilized standards inspired by CDISC in their initial work 63 | * FDA is open to a more focused discussion, but they will need to pull in internal experts that have more expertise 64 | * Eric asked about the FDA having any preference on the method of rounding as there are rare cases where the default rounding method in R can be different than SAS default method 65 | * FDA does not have a "preferred method" to share at this time, and there may not be a single one answer that's correct in this area 66 | * Pilot 1 had a small issue with rounding dealing with the approximation method for confidence intervals 67 | * We are looking for help on the website! Please get in touch with the working group and browse the [GitHub issue list](https://github.com/RConsortium/submissions-wg/issues) if you would like to contribute. 68 | -------------------------------------------------------------------------------- /minutes/_metadata.yml: -------------------------------------------------------------------------------- 1 | # options specified here will apply to all posts in this folder 2 | 3 | # prevents posts from re-rendering unless explicitly told to do so (for more, see https://quarto.org/docs/projects/code-execution.html#freeze) 4 | freeze: true 5 | 6 | # enable banner style title blocks (see https://quarto.org/docs/authoring/title-blocks.html) 7 | title-block-banner: true 8 | 9 | author: 'Eric Nantz & Theresa Christiani' 10 | 11 | comments: 12 | giscus: 13 | repo: RConsortium/submissions-wg 14 | repo-id: "MDEwOlJlcG9zaXRvcnkzMTk1MjUxNTI=" -------------------------------------------------------------------------------- /minutes_prep.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Minutes Document Writer" 3 | format: 4 | html: 5 | toc: true 6 | echo: false 7 | --- 8 | 9 | ```{r} 10 | #| output: false 11 | 12 | library(openxlsx2) 13 | library(dplyr) 14 | people_df <- read_xlsx("data/wg_attendees.xlsx", sheet = 1) |> 15 | mutate(name2 = paste0(name, " (", affiliation, ")")) |> 16 | arrange(name) |> 17 | select(-affiliation_url) 18 | 19 | ojs_define(people_data = people_df) 20 | ``` 21 | 22 | # Instructions 23 | 24 | This interactive page creates a template Quarto document text for capturing meeting minutes associated with the Submissions working group. 25 | 26 | ## Date Selection 27 | 28 | Use the date input below to choose the date of the meeting. Click the calendar icon in the widget to launch the UI. 29 | 30 | ```{ojs} 31 | viewof meeting_date = Inputs.date({ 32 | label: "Meeting Date", 33 | value: "2024-12-01" 34 | }) 35 | ``` 36 | 37 | ```{ojs} 38 | 39 | meeting_year = (meeting_date.getFullYear()).toString() 40 | meeting_month = (meeting_date.getMonth() + 1).toString().padStart(2, "0") 41 | meeting_day = (meeting_date.getDate() + 1).toString().padStart(2, "0") 42 | 43 | meeting_array = [meeting_year, meeting_month, meeting_day] 44 | meeting_print = meeting_array.join("-") 45 | ``` 46 | 47 | 48 | 49 | ## Attendee Selection 50 | 51 | Use the personnel table below to select the attendees present at the meeting by checking the box that appears once you hover your cursor above a name. The table is ordered alphabetically by first name as default, but you can sort by a different column by clicking the column name. 52 | 53 | ::: {.callout-warning} 54 | If an attendee is not listed in this table, please add their name, affiliation, and affiliation URL to the `wg_attendees.xlsx` spreadsheet located in the `data` subdirectory of this repository. After your edits to the spreadsheet are completed, re-build the Quarto site. 55 | ::: 56 | 57 | ```{ojs} 58 | data2 = transpose(people_data) 59 | ``` 60 | 61 | 62 | ```{ojs} 63 | viewof people_selection = Inputs.table(data2, { 64 | sort: "name", 65 | multiple: true, 66 | rows: 40, 67 | required: false 68 | }) 69 | ``` 70 | 71 | # Output 72 | 73 | ## Setup 74 | 75 | In your local copy of this repository, create a sub-directory within the `minutes` directory to hold files associated with this meeting's minutes. Based on your selection above, create the following directory: 76 | 77 | ```{ojs} 78 | meeting_print 79 | ``` 80 | 81 | Copy the text in the output below into a new Quarto document named `index.qmd`. 82 | 83 | ```{ojs} 84 | silly = html`${Array.from(new Set(people_selection.map(c => c.name2))) 85 | .sort() 86 | .map(c => `- ${c}`) 87 | .join("\n")}` 88 | 89 | silly2 = html`
---
 90 | title: "Minutes ${meeting_print}"
 91 | date: ${meeting_print}
 92 | ---
 93 | 
 94 | ::: {.callout-tip collapse="true"}
 95 | ## View Attendee List
 96 | 
 97 | ${silly}
 98 | :::
 99 | 
100 | ## Add Minutes Here
101 | 
102 |
` 103 | ``` 104 | 105 | ```{ojs} 106 | silly2 107 | ``` -------------------------------------------------------------------------------- /pilot1.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "R Submission Pilot 1" 3 | comments: 4 | giscus: 5 | repo: RConsortium/submissions-wg 6 | repo-id: "MDEwOlJlcG9zaXRvcnkzMTk1MjUxNTI=" 7 | --- 8 | 9 | ## Pilot 1 - Common Analyses 10 | 11 | **Goal**: Example R based submission contains common analyses, submission through the current FDA e-submission system and process. 12 | 13 | **Key evaluation aspects**: 14 | 15 | - For Submitter 16 | - Submission of proprietary R packages 17 | - Preparation of R-based submission materials 18 | - For FDA Staff 19 | - Reproduce analysis results by reconstructing submitted proprietary packages / retrieving open source packages 20 | 21 | **Data and analysis scope**: 22 | 23 | - 4 TLFs 24 | - Simulated data from CDISC Pilot 25 | 26 | 27 | **Success Criteria**: 28 | 29 | - Submitter: 30 | - Successfully submit via eCTD portal 31 | - FDA Reviewer: 32 | - Reproduce analysis results 33 | 34 | 35 | **Timeline**: 36 | 37 | - 2021.11 Initial submission to FDA. 38 | - 2022.12 Received FDA response 39 | - 2022.02 Revised submission to FDA 40 | - 2022.03 Received final FDA response. 41 | 42 | 43 | **Links**: 44 | 45 | [R Submission Pilot 1 Development Repo](https://github.com/RConsortium/submissions-pilot1) 46 | 47 | 48 | [R Submission Pilot 1 eCTD Submission Package to FDA](https://github.com/RConsortium/submissions-pilot1-to-fda) 49 | 50 | **Additional Activities** 51 | - explore pilot submissions to other HAs (Japan, China) 52 | 53 | **Key team members**: 54 | 55 | Developer team: 56 | - Ning Leng, Heng Wang (Roche) 57 | - Mike Stakehouse, Eli Miller (Atorus) 58 | - Yilong Zhang, Peikun Wu (Merck) 59 | 60 | Esub team: 61 | - Beverly Chin (Roche) 62 | - Tim Powell (Biogen) 63 | 64 | FDA Staff: 65 | - Paul Schuette 66 | - Hye Soo Cho -------------------------------------------------------------------------------- /pilot2.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "R Submission Pilot 2" 3 | --- 4 | 5 | ## Pilot 2 - shiny submission 6 | 7 | **Goal**: Example R based submission contains a shiny app, submission through the current FDA e-submission system and process 8 | 9 | **Key evaluation aspects**: 10 | 11 | - For Submitter 12 | - Submission of an shiny app through the eCTD portal 13 | - The shiny codes will be provided as a proprietary R package 14 | - For FDA Reviewer 15 | - Reproduce analysis results by deploying the shiny app on a FDA laptop/server 16 | 17 | **Timeline**: 2022 H1 18 | 19 | **Data and analysis scope**: 20 | 21 | - a shiny app that displays the 4 TLFs submitted in [pilot 1]((https://github.com/RConsortium/submissions-pilot1-to-fda)), with basic data set filtering functionalities. 22 | - Simulated data from CDISC Pilot (the same data set as in pilot 1) 23 | 24 | **Links**: 25 | 26 | [R Submission Pilot 2 Development Repo](https://github.com/RConsortium/submissions-pilot2) 27 | 28 | 29 | [R Submission Pilot 2 eCTD Submission Package to FDA](https://github.com/RConsortium/submissions-pilot2-to-fda) 30 | 31 | 32 | **Key team members**: 33 | 34 | Developer team: 35 | - Ning Leng, Heng Wang (Roche) 36 | - Mike Stakehouse, Eli Miller (Atorus) 37 | - Yilong Zhang, Gregery Chen (Merck) 38 | - Eric Nantz (Eli Lilly) 39 | 40 | 41 | FDA Staff: 42 | - Paul Schuette 43 | - Hye Soo Cho -------------------------------------------------------------------------------- /pilot3.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "R Submission Pilot 3" 3 | --- 4 | 5 | **Goals**: 6 | 7 | - Expand on the work done in [Submission Pilot 1](pilot1.qmd), by now utilizing R to generate ADaM datasets. 8 | - Deliver a publicly accessible R-based Submission to the FDA 9 | 10 | **Key evaluation aspects**: 11 | 12 | - For Submitter 13 | - Generation of a reproducible R environment with a proprietary R package from git and open source R packages using `{renv}` 14 | - Preparation of R-based submission materials 15 | - For FDA Staff 16 | - Reproduce analysis results by reproducing the R environment and retrieving both proprietary & open source packages 17 | 18 | **Timeline**: 19 | 20 | - Submitted to FDA on August 28th, 2023 21 | - Feedback from FDA on February 2nd, 2024 (See feedback [here](https://rconsortium.github.io/submissions-wg/minutes/2024-02-02/)) 22 | - Re-submitted to FDA on April 19th, 2024 23 | 24 | **Data and analysis scope**: 25 | 26 | - ADaMs created using R (SDTM `.xpt` sourced from the CDISC Pilot study) 27 | - TLFs created using R (ADaMs `.xpt` sourced from the ADaMs generated in R by the Pilot 3 team) 28 | 29 | **Links**: 30 | 31 | [R Submission Pilot 3 Development Repo](https://github.com/RConsortium/submissions-pilot3-adam) 32 | 33 | - Development area for Pilot 3 Team 34 | - See Closed Issues discussion for how we worked 35 | - See [Wiki](https://github.com/RConsortium/submissions-pilot3-adam/wiki) for Common Issues we ran into and QC Findings we encountered when comparing the ADaMs generated in the CDISC Pilot study. 36 | - We made use of a [GitHub Project Board](https://docs.github.com/en/issues/planning-and-tracking-with-projects/learning-about-projects/about-projects) (Kanban Style Board) to drive development. 37 | 38 | [R Submission Pilot 3 eCTD Submission Package to FDA](https://github.com/RConsortium/submissions-pilot3-adam-to-fda) 39 | 40 | - Materials submitted to the FDA following eCTD standards 41 | 42 | [R Submission Pilot 3 Website](https://rconsortium.github.io/submissions-pilot3-adam/main/) 43 | 44 | - Breakdown of the Github repositories and files used for analysis development, e-submission and proprietary Pilot 3 R Package 45 | - Renderings of ADaMs, TLFs, ADRG and Cover Letter 46 | - Discussions at Conferences and feedback from FDA 47 | 48 | **Key team members**: 49 | 50 | Developer team: 51 | 52 | - Joel Laxamana-Roche 53 | - Thomas Neitmann-Denali 54 | - Phanikumar Tata-Syneos 55 | - Robert Devine-J&J 56 | - Steven Haesendonckx-J&J 57 | - Lei Zhao-Roche 58 | - Nicole Jones-Denali 59 | - Kangjie Zhang-Bayer 60 | - Benjamin Straub-GSK 61 | - Declan Hodges-GSK 62 | - Bingjun Wang - Merck 63 | 64 | FDA Staff: 65 | 66 | - Paul Schuette\ 67 | - Hye Soo Cho 68 | -------------------------------------------------------------------------------- /pilot4.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "R Submission Pilot 4" 3 | --- 4 | 5 | **Goal**: Utilize alternative methods of distributing a self-contained submission bundle of the Pilot 2 Shiny application with container and web-assembly technologies. 6 | 7 | **Key evaluation aspects**: 8 | 9 | - For Submitter 10 | - Generation of a reproducible execution environment containing both the system and R-specific dependencies using [Podman](https://podman.io/) as the container runtime. 11 | - Generation of a web-assembly version of the Pilot 2 Shiny application (with small modifications) that can be executed in an existing R process running a web server using the `{httpuv}` R package 12 | - Preparation of submission materials with detailed instructions on setting up the execution environments 13 | - For FDA Staff 14 | - Reproduce the ability to use the Shiny application by building the specific Podman container and viewing the web-assembly version of the application using the Edge internet browser. 15 | 16 | **Timeline**: Targeting a first submission in Q2 2024 17 | 18 | **Data and analysis scope**: 19 | 20 | - A shiny app that replicates most of the functionality from the [pilot 2]((https://github.com/RConsortium/submissions-pilot2-to-fda)), with small modifications to account for the web-assembly constraints. 21 | - Simulated data from CDISC Pilot (the same data set as in pilot 1) 22 | 23 | **Links**: 24 | 25 | [R Submission Pilot 4 Container Development Repository](https://github.com/Appsilon/experimental-fda-submission-4-podman) 26 | 27 | [R Submission Pilot 4 Web-Assembly App Repository](https://github.com/RConsortium/submissions-pilot4-webR) 28 | 29 | **Key team members** 30 | 31 | Developer team: 32 | 33 | - Eric Nantz (Eli Lilly) 34 | - Sam Parmar (Pfizer) 35 | - Pedro Silva (Appsilon) 36 | - Ismael Rodgriguez (Appsilon) 37 | - Andre Verissimo (Appsilon) 38 | - Robert Devine (J&J) 39 | - Nate Mockler (Biogen) 40 | - Andrew Borgman (Bristol Myers Squibb) 41 | - Ben Straub (GSK) 42 | 43 | FDA Staff: 44 | 45 | - Paul Schutte 46 | - Hye Soo Cho 47 | 48 | -------------------------------------------------------------------------------- /pilot_background.qmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Background" 3 | --- 4 | 5 | ## Pilot principles: 6 | 7 | Provide open examples of submitting R-based clinical trial data/analysis packages to FDA. 8 | 9 | - All pilots will be submitted from R consortium (non-profit identity) to FDA. Simulated data will be used. 10 | - All data, codes and documents will be fully available to public. 11 | - FDA Staff will be engaged to reproduce analysis results. 12 | - Submitter and FDA Staff will provide feedback on current process/system 13 | - Contributions from multiple companies/organizations are preferred. Avoid bias to any company/organization specific tools. 14 | 15 | You can view more details about each of the current pilots by visiting their dedicated pages below: 16 | 17 | - [Pilot 1](pilot1.qmd): Common analyses 18 | - [Pilot 2](pilot2.qmd): Shiny application submission 19 | - [Pilot 3](pilot3.qmd): ADaM datasets 20 | - [Pilot 4](pilot4.qmd): Alternative formats -------------------------------------------------------------------------------- /styles.css: -------------------------------------------------------------------------------- 1 | /* css styles */ 2 | -------------------------------------------------------------------------------- /submissions-wg.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: Default 4 | SaveWorkspace: Default 5 | AlwaysSaveHistory: Default 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: Sweave 13 | LaTeX: pdfLaTeX 14 | --------------------------------------------------------------------------------