├── .gitignore ├── FP_Linux_Install.sh ├── FP_OSX_Install.sh ├── FP_Windows_Install.ps1 ├── FastProject ├── CLI.py ├── DataTypes.py ├── FileIO.py ├── Filters.py ├── Global.py ├── Housekeeping Genes │ ├── Ensemble Housekeeping.txt │ └── Gene Name Housekeeping.txt ├── HtmlViewer.py ├── Interactive.py ├── NormalizationMethods.py ├── Pipelines.py ├── Projections.py ├── SigScoreMethods.py ├── Signatures.py ├── SubSample.py ├── Tests │ ├── TestFiles │ │ ├── housekeeping.txt │ │ ├── input_projection_RFormat_ok.txt │ │ ├── input_projection_duplicate18and36_bad.txt │ │ ├── input_projection_good.txt │ │ ├── input_projection_header_ok.txt │ │ ├── input_projection_header_ok2.txt │ │ ├── input_projection_missing10and20_bad.txt │ │ ├── input_projection_nan26and27_bad.txt │ │ ├── input_projection_parseError26and28_bad.txt │ │ ├── precomputed_sigs.txt │ │ ├── sigsA.txt │ │ ├── sigsB.txt │ │ ├── sigsSmall.gmt │ │ ├── smallData.txt │ │ └── weights.txt │ ├── __init__.py │ ├── full_battery.py │ ├── test_JSON.py │ └── test_projections.py ├── Transforms.py ├── Utils │ ├── ProgressBar.py │ ├── __init__.py │ ├── em.py │ └── hdt.py ├── Viewer Resources │ ├── Clustered_HeatMap.js │ ├── ColorScatter.js │ ├── HeatMap.js │ ├── QC_Report.html │ ├── Results.html │ ├── Utilities.js │ ├── bootstrap.css │ ├── bootstrap.min.js │ ├── d3.min.js │ ├── d3tip.js │ ├── jquery-2.1.4.min.js │ ├── jszip.min.js │ └── styles.css ├── __init__.py ├── _tsne_fix.py ├── _version.py └── jsonIO.py ├── LICENSE ├── MANIFEST.in ├── README.md ├── SampleOutput.png ├── deploy.sh ├── docs ├── FastProject.DataTypes.rst ├── FastProject.FileIO.rst ├── FastProject.Filters.rst ├── FastProject.Global.rst ├── FastProject.HtmlViewer.rst ├── FastProject.NormalizationMethods.rst ├── FastProject.Pipelines.rst ├── FastProject.Projections.rst ├── FastProject.SigScoreMethods.rst ├── FastProject.Signatures.rst ├── FastProject.SubSample.rst ├── FastProject.Transforms.rst ├── FastProject.Utils.ProgressBar.rst ├── FastProject.Utils.em.rst ├── FastProject.Utils.hdt.rst ├── FastProject.Utils.hdt_python.rst ├── FastProject.Utils.rst ├── FastProject.rst ├── FastProject.tests.rst ├── FastProject.tests.test_projections.rst ├── FastProject.tests.test_split_merge.rst ├── conf.py ├── index.rst └── make.bat ├── ez_setup.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | 56 | # Distrubtion 57 | dist/ 58 | build/ 59 | *.egg-info/ 60 | setuptools*.egg 61 | 62 | #Vim swap files 63 | *.swp 64 | -------------------------------------------------------------------------------- /FP_Linux_Install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Name of application to install 5 | AppName="FastProject" 6 | 7 | # Set your project's install directory name here 8 | InstallDir="FastProject" 9 | 10 | # Dependencies installed by Conda 11 | # Comment out the next line if no Conda dependencies 12 | CondaDeps="numpy=1.11 nomkl scipy=0.18 scikit-learn=0.18.1 pandas=0.19" 13 | 14 | # Install the package from PyPi 15 | # Comment out next line if installing locally 16 | PyPiPackage="FastProject" 17 | 18 | # Local packages to install 19 | # Useful if your application is not in PyPi 20 | # Distribute this with a .tar.gz and use this variable 21 | # Comment out the next line if no local package to install 22 | # LocalPackage="FastProject-0.9.2.tar.gz" 23 | 24 | # Entry points to add to the path 25 | # Comment out the next line of no entry point 26 | # (Though not sure why this script would be useful otherwise) 27 | EntryPoint="fastproject" 28 | 29 | echo 30 | echo "Installing $AppName" 31 | 32 | echo 33 | echo "Installing into: $(pwd)/$InstallDir" 34 | echo 35 | 36 | # Miniconda doesn't work for directory structures with spaces 37 | if [[ $(pwd) == *" "* ]] 38 | then 39 | echo "ERROR: Cannot install into a directory with a space in its path" >&2 40 | echo "Exiting..." 41 | echo 42 | exit 1 43 | fi 44 | 45 | # Test if new directory is empty. Exit if it's not 46 | if [ -d $(pwd)/$InstallDir ]; then 47 | if [ "$(ls -A $(pwd)/$InstallDir)" ]; then 48 | echo "ERROR: Directory is not empty" >&2 49 | echo "If you want to install into $(pwd)/$InstallDir, " 50 | echo "clear the directory first and run this script again." 51 | echo "Exiting..." 52 | echo 53 | exit 1 54 | fi 55 | fi 56 | 57 | set +e 58 | # Download and install Miniconda 59 | curl "https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh" -o Miniconda_Install.sh 60 | if [ $? -ne 0 ]; then 61 | curl "http://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh" -o Miniconda_Install.sh 62 | fi 63 | set -e 64 | 65 | bash Miniconda_Install.sh -b -f -p $InstallDir 66 | 67 | # Activate the new environment 68 | PATH="$(pwd)/$InstallDir/bin":$PATH 69 | 70 | # Make the new python environment completely independent 71 | # Modify the site.py file so that USER_SITE is not imported 72 | python -s << END 73 | import site 74 | site_file = site.__file__.replace(".pyc", ".py"); 75 | with open(site_file) as fin: 76 | lines = fin.readlines(); 77 | for i,line in enumerate(lines): 78 | if(line.find("ENABLE_USER_SITE = None") > -1): 79 | user_site_line = i; 80 | break; 81 | lines[user_site_line] = "ENABLE_USER_SITE = False\n" 82 | with open(site_file,'w') as fout: 83 | fout.writelines(lines) 84 | END 85 | 86 | # Install Conda Dependencies 87 | if [[ $CondaDeps ]]; then 88 | conda install $CondaDeps -y -c defaults --override-channels 89 | fi 90 | 91 | # Install Package from PyPi 92 | if [[ $PyPiPackage ]]; then 93 | pip install $PyPiPackage -q 94 | fi 95 | 96 | # Install Local Package 97 | if [[ $LocalPackage ]]; then 98 | pip install $LocalPackage -q 99 | fi 100 | 101 | # Cleanup 102 | rm Miniconda_Install.sh 103 | conda clean -iltp --yes 104 | 105 | # Add Entry Point to the path 106 | if [[ $EntryPoint ]]; then 107 | 108 | cd $InstallDir 109 | mkdir Scripts 110 | ln -s ../bin/$EntryPoint Scripts/$EntryPoint 111 | 112 | echo "$EntryPoint script installed to $(pwd)/Scripts" 113 | echo 114 | echo "Add folder to path by appending to .bashrc?" 115 | read -p "[y/n] >>> " -r 116 | echo 117 | if [[ ($REPLY == "yes") || ($REPLY == "Yes") || ($REPLY == "YES") || 118 | ($REPLY == "y") || ($REPLY == "Y")]] 119 | then 120 | echo "export PATH=\"$(pwd)/Scripts\":\$PATH" >> ~/.bashrc 121 | echo "Your PATH was updated." 122 | echo "Restart the terminal for the change to take effect" 123 | else 124 | echo "Your PATH was not modified." 125 | fi 126 | 127 | cd .. 128 | fi 129 | 130 | echo 131 | echo "$AppName Install Successfully" 132 | -------------------------------------------------------------------------------- /FP_OSX_Install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Name of application to install 5 | AppName="FastProject" 6 | 7 | # Set your project's install directory name here 8 | InstallDir="FastProject" 9 | 10 | # Dependencies installed by Conda 11 | # Comment out the next line if no Conda dependencies 12 | CondaDeps="numpy=1.11 nomkl scipy=0.18 scikit-learn=0.18.1 pandas=0.19" 13 | 14 | # Install the package from PyPi 15 | # Comment out next line if installing locally 16 | PyPiPackage="FastProject" 17 | 18 | # Local packages to install 19 | # Useful if your application is not in PyPi 20 | # Distribute this with a .tar.gz and use this variable 21 | # Comment out the next line if no local package to install 22 | # LocalPackage="FastProject-0.9.2.tar.gz" 23 | 24 | # Entry points to add to the path 25 | # Comment out the next line of no entry point 26 | # (Though not sure why this script would be useful otherwise) 27 | EntryPoint="fastproject" 28 | 29 | echo 30 | echo "Installing $AppName" 31 | 32 | echo 33 | echo "Installing into: $(pwd)/$InstallDir" 34 | echo 35 | 36 | # Miniconda doesn't work for directory structures with spaces 37 | if [[ $(pwd) == *" "* ]] 38 | then 39 | echo "ERROR: Cannot install into a directory with a space in its path" >&2 40 | echo "Exiting..." 41 | echo 42 | exit 1 43 | fi 44 | 45 | # Test if new directory is empty. Exit if it's not 46 | if [ -d $(pwd)/$InstallDir ]; then 47 | if [ "$(ls -A $(pwd)/$InstallDir)" ]; then 48 | echo "ERROR: Directory is not empty" >&2 49 | echo "If you want to install into $(pwd)/$InstallDir, " 50 | echo "clear the directory first and run this script again." 51 | echo "Exiting..." 52 | echo 53 | exit 1 54 | fi 55 | fi 56 | 57 | set +e 58 | # Download and install Miniconda 59 | curl "https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh" -o Miniconda_Install.sh 60 | if [ $? -ne 0 ]; then 61 | curl "http://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh" -o Miniconda_Install.sh 62 | fi 63 | set -e 64 | 65 | bash Miniconda_Install.sh -b -f -p $InstallDir 66 | 67 | # Activate the new environment 68 | PATH="$(pwd)/$InstallDir/bin":$PATH 69 | 70 | # Make the new python environment completely independent 71 | # Modify the site.py file so that USER_SITE is not imported 72 | python -s << END 73 | import site 74 | site_file = site.__file__.replace(".pyc", ".py"); 75 | with open(site_file) as fin: 76 | lines = fin.readlines(); 77 | for i,line in enumerate(lines): 78 | if(line.find("ENABLE_USER_SITE = None") > -1): 79 | user_site_line = i; 80 | break; 81 | lines[user_site_line] = "ENABLE_USER_SITE = False\n" 82 | with open(site_file,'w') as fout: 83 | fout.writelines(lines) 84 | END 85 | 86 | # Install Conda Dependencies 87 | if [[ $CondaDeps ]]; then 88 | conda install $CondaDeps -y -c defaults --override-channels 89 | fi 90 | 91 | # Install Package from PyPi 92 | if [[ $PyPiPackage ]]; then 93 | pip install $PyPiPackage -q 94 | fi 95 | 96 | # Install Local Package 97 | if [[ $LocalPackage ]]; then 98 | pip install $LocalPackage -q 99 | fi 100 | 101 | # Cleanup 102 | rm Miniconda_Install.sh 103 | conda clean -iltp --yes 104 | 105 | # Add Entry Point to the path 106 | if [[ $EntryPoint ]]; then 107 | 108 | cd $InstallDir 109 | mkdir Scripts 110 | ln -s ../bin/$EntryPoint Scripts/$EntryPoint 111 | 112 | echo "$EntryPoint script installed to $(pwd)/Scripts" 113 | echo 114 | echo "Add folder to path by appending to .bash_profile?" 115 | read -p "[y/n] >>> " -r 116 | echo 117 | if [[ ($REPLY == "yes") || ($REPLY == "Yes") || ($REPLY == "YES") || 118 | ($REPLY == "y") || ($REPLY == "Y")]] 119 | then 120 | echo "export PATH=\"$(pwd)/Scripts\":\$PATH" >> ~/.bash_profile 121 | echo "Your PATH was updated." 122 | echo "Restart the terminal for the change to take effect" 123 | else 124 | echo "Your PATH was not modified." 125 | fi 126 | 127 | cd .. 128 | fi 129 | 130 | echo 131 | echo "$AppName Install Successfully" 132 | -------------------------------------------------------------------------------- /FP_Windows_Install.ps1: -------------------------------------------------------------------------------- 1 | $ErrorActionPreference = "Stop" 2 | 3 | # Name of application to install 4 | $AppName="FastProject" 5 | 6 | # Set your project's install directory name here 7 | $InstallDir="FastProject" 8 | 9 | # Dependencies installed by Conda 10 | # Commend out the next line if no Conda dependencies 11 | $CondaDeps="numpy=1.11","scipy=0.18","scikit-learn=0.18.1","pandas=0.19" 12 | 13 | # Dependencies installed with pip instead 14 | # Comment out the next line if no PyPi dependencies 15 | $PyPiPackage="FastProject" 16 | 17 | # Local packages to install 18 | # Useful if your application is not in PyPi 19 | # Distribute this with a .tar.gz and use this variable 20 | # Comment out the next line if no local package to install 21 | # $LocalPackage="FastProject-0.9.2.tar.gz" 22 | 23 | # Entry points to add to the path 24 | # Comment out the next line of no entry point 25 | # (Though not sure why this script would be useful otherwise) 26 | $EntryPoint="fastproject" 27 | 28 | Write-Host ("`nInstalling $AppName to "+(get-location).path+"\$InstallDir") 29 | 30 | 31 | # Download Latest Miniconda Installer 32 | Write-Host "`nDownloading Miniconda Installer...`n" 33 | 34 | (New-Object System.Net.WebClient).DownloadFile("https://repo.continuum.io/miniconda/Miniconda-latest-Windows-x86_64.exe", "$pwd\Miniconda_Install.exe") 35 | 36 | # Install Python environment through Miniconda 37 | Write-Host "Installing Miniconda...`n" 38 | Start-Process Miniconda_Install.exe "/S /AddToPath=0 /D=$pwd\$InstallDir" -Wait 39 | 40 | # Install Dependences to the new Python environment 41 | $env:Path = "$pwd\$InstallDir\Scripts;" + $env:Path 42 | 43 | # Make the new python environment completely independent 44 | # Modify the site.py file so that USER_SITE is not imported 45 | $site_program = @" 46 | import site 47 | site_file = site.__file__.replace('.pyc', '.py'); 48 | with open(site_file) as fin: 49 | lines = fin.readlines(); 50 | for i,line in enumerate(lines): 51 | if(line.find('ENABLE_USER_SITE = None') > -1): 52 | user_site_line = i; 53 | break; 54 | lines[user_site_line] = 'ENABLE_USER_SITE = False\n' 55 | with open(site_file,'w') as fout: 56 | fout.writelines(lines) 57 | "@ 58 | python -c $site_program 59 | 60 | if(Test-Path variable:CondaDeps) 61 | { 62 | Write-Host "Installing Conda dependencies...`n" 63 | conda install $CondaDeps -y -c defaults --override-channels 64 | } 65 | 66 | if(Test-Path variable:PyPiPackage) 67 | { 68 | Write-Host "Installing PyPi dependencies...`n" 69 | pip install $PyPiPackage 70 | } 71 | 72 | if(Test-Path variable:LocalPackage) 73 | { 74 | Write-Host "Installing Local package...`n" 75 | pip install $LocalPackage 76 | } 77 | 78 | # Cleanup 79 | Remove-Item "Miniconda_Install.exe" 80 | conda clean -iltp --yes 81 | 82 | # Add Entry Point to path 83 | 84 | if(Test-Path variable:EntryPoint) 85 | { 86 | # Move entry-point executable to an isolated folder 87 | $script_folder = "$pwd\$InstallDir\PathScripts" 88 | New-Item $script_folder -type directory | Out-Null 89 | Move-Item $pwd\$InstallDir\Scripts\$EntryPoint.exe $script_folder 90 | 91 | # Ask user if they want to update path 92 | $title = "Update Path" 93 | $message = "`nDo you want to add the $EntryPoint script to your User PATH?" 94 | 95 | $yes = New-Object System.Management.Automation.Host.ChoiceDescription "&Yes", ` 96 | "Prepends the User PATH variable with the location of the $EntryPoint script" 97 | 98 | $no = New-Object System.Management.Automation.Host.ChoiceDescription "&No", ` 99 | "User PATH is not modified" 100 | 101 | $options = [System.Management.Automation.Host.ChoiceDescription[]]($yes, $no) 102 | 103 | $result = $host.ui.PromptForChoice($title, $message, $options, 0) 104 | 105 | if($result -eq 0) 106 | { 107 | # Update the user's path 108 | $old_path = (Get-ItemProperty -Path HKCU:\Environment).Path 109 | $new_path = $script_folder + ";" + $old_path 110 | cmd /c "setx PATH $new_path" 111 | Set-ItemProperty -Path HKCU:\Environment -Name PATH -Value $new_path 112 | Write-Host "User PATH has been updated" 113 | Write-Host "Open a new command prompt to see the change" 114 | } 115 | else 116 | { 117 | Write-Host "User PATH was not modified.`n" 118 | Write-Host "You may want to add the $EntryPoint script to your path." 119 | Write-Host "It is located in: $script_folder`n" 120 | } 121 | } 122 | 123 | Write-Host "`n$AppName Successfully Installed" 124 | 125 | Write-Host "Press any key to continue ..." 126 | 127 | $x = $host.UI.RawUI.ReadKey("NoEcho,IncludeKeyDown") 128 | -------------------------------------------------------------------------------- /FastProject/CLI.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ Entry point for console fastproject script 3 | 4 | - Parses arguments 5 | - Loads data from files 6 | - Runs analysis 7 | - Writes outputs to file 8 | 9 | Then launches the main pipeline 10 | """ 11 | 12 | from __future__ import absolute_import, print_function, division; 13 | import argparse; 14 | import os 15 | import logging; 16 | import numpy as np; 17 | from . import FileIO; 18 | from . import Signatures; 19 | from . import Pipelines; 20 | from . import HtmlViewer; 21 | from .DataTypes import ExpressionData; 22 | from .Global import FP_Output 23 | import FastProject 24 | 25 | 26 | def parseFPArgs(): 27 | """Defines the command-line arguments and parses the FastProject call 28 | 29 | Returns 30 | ------- 31 | argparse.Namespace 32 | 33 | """ 34 | parser = argparse.ArgumentParser(prog="FastProject", description='Analyze a set of expression data with FastProject.'); 35 | 36 | parser.add_argument("data_file", help="Input expression data matrix"); 37 | 38 | parser.add_argument("-k", "--housekeeping", metavar="FILE", default="", 39 | help="Read list of housekeeping genes from FILE. Uses default list if not specified"); 40 | 41 | parser.add_argument("-s", "--signatures", metavar="FILE", nargs='*', 42 | help="Loads signatures from FILE."); 43 | 44 | parser.add_argument("-p", "--precomputed", metavar="FILE", nargs='*', 45 | help="Loads precomputed signature scores from FILE."); 46 | 47 | parser.add_argument("-o", "--output", metavar="DIRECTORY", 48 | help="Name of output directory. Otherwise, output directory is auto-generated"); 49 | 50 | parser.add_argument("--nofilter", action="store_true", 51 | help="Project using all genes."); 52 | 53 | parser.add_argument("--nomodel", action="store_true", 54 | help="No estimation of expression probability or false negative probability"); 55 | 56 | parser.add_argument("--pca_filter", action="store_true", 57 | help="Filters PC principal components that correlate with a calculated QC metric for each sample"); 58 | 59 | parser.add_argument("--qc", action="store_true", 60 | help="Performs a quality check on samples, filtering samples that do not pass"); 61 | 62 | parser.add_argument("--all_sigs", action="store_true", 63 | help="Do not remove insignificant signatures from output."); 64 | 65 | parser.add_argument("--debug", action="store_true", 66 | help="Run FastProject in Debug mode"); 67 | 68 | parser.add_argument("--lean", action="store_true", 69 | help="Run FastProject in Lean mode - subset of analyses to reduce runtime on large data sets"); 70 | 71 | parser.add_argument("--subsample_size", type=int, metavar="N", default=1000, 72 | help="Planned Feature: Number of samples to use when sub_sampling"); 73 | 74 | parser.add_argument("--min_signature_genes", type=int, metavar="N", default=5, 75 | help="Signatures that match less than N genes in the data are discarded"); 76 | 77 | parser.add_argument("--projections", metavar="FILE", nargs='*', 78 | help="Loads projection coordinates from FILE"); 79 | 80 | parser.add_argument("--weights", metavar="FILE", 81 | help="Loads weights from FILE. Use these weights instead of FastProject's FNR calculation"); 82 | 83 | parser.add_argument("--threshold", metavar="N", type=int, 84 | help="Removes transcripts detected in less than N samples. " + 85 | "Default is 20%% of total sample count.") 86 | 87 | parser.add_argument("--sig_norm_method", 88 | choices=["none", "znorm_columns", "znorm_rows", 89 | "znorm_rows_then_columns", "rank_norm_columns"], 90 | default="znorm_rows", 91 | help="Pick a normalization method to be applied to data before evaluating signature scores"); 92 | 93 | parser.add_argument("--sig_score_method", 94 | choices=["naive", "weighted_avg", "imputed", "only_nonzero"], 95 | default="weighted_avg", 96 | help="Pick a method to evaluate signature scores"); 97 | 98 | args = parser.parse_args(); 99 | 100 | args = vars(args); # Convert to a Dictionary 101 | 102 | return args; 103 | 104 | 105 | def loadFilesFromDisk(args): 106 | """Loads files from disk into data structures 107 | 108 | Typically called before calling Pipelines.Analysis() 109 | 110 | Parameters 111 | ---------- 112 | args : dict 113 | Contains file locations on disk 114 | 115 | Returns 116 | ------- 117 | expressionMatrix : ExpressionData 118 | signatures : list of Signatures.Signature 119 | precomputed_signatures : dict 120 | Keys are precomputed signature names (str) 121 | Values are signature levels/scores (SigScoreMethods.SignatureScores) 122 | housekeeping_genes : list of str 123 | input_projections : dict 124 | Keys are of type str, representing projection names 125 | Values are of type 2xN pandas.DataFrame with column names matching 126 | sample names in `expressionMatrix` 127 | input_weights : pandas.DataFrame or None 128 | Same size as expressionMatrix 129 | Values are floats from 0.0 to 1.0 130 | 131 | """ 132 | # Read expression data from file 133 | filename = args["data_file"]; 134 | 135 | if(not os.path.isfile(filename)): 136 | raise ValueError("\n", filename, "not found.\nExiting..."); 137 | 138 | (edata, genes, cells) = FileIO.read_matrix(filename); 139 | expressionMatrix = ExpressionData(edata, genes, cells); 140 | 141 | FP_Output("Imported ", edata.shape[0], " genes across ", edata.shape[1], " samples"); 142 | 143 | # Load Signature files 144 | signatures = []; 145 | if(args["signatures"]): 146 | for sig_file in args["signatures"]: 147 | if(not os.path.isfile(sig_file)): 148 | raise ValueError("Option Error: signature file " + sig_file + " not found.\nExiting..."); 149 | 150 | signatures += Signatures.read_signatures(sig_file); 151 | 152 | # Load Precomputed Sig file 153 | precomputed_signatures = {}; 154 | if(args["precomputed"]): 155 | for precomputed_sig_file in args["precomputed"]: 156 | if(not os.path.isfile(precomputed_sig_file)): 157 | raise ValueError("Option Error: precomputed signature file " + precomputed_sig_file + " not found.\nExiting..."); 158 | precomputed_signatures.update(Signatures.load_precomputed(precomputed_sig_file, cells)); 159 | 160 | if(not args["signatures"] and not args["precomputed"]): # Need one or the other here 161 | raise ValueError( 162 | "Option Error: Must specify either a signature file or a pre-computed signature file.\nExiting..."); 163 | 164 | if(len(signatures) + len(precomputed_signatures) == 0): # Need one or the other here 165 | raise ValueError( 166 | "Option Error: Must specify either a signature file or a pre-computed signature file.\nExiting..."); 167 | 168 | # Load housekeeping genes 169 | housekeeping_genes = FileIO.load_housekeeping_genes(args["housekeeping"]) 170 | 171 | # Load projection coordinates (if provided) 172 | input_projections = {}; 173 | if(args["projections"]): 174 | input_projections = FileIO.load_input_projections(args["projections"], cells); 175 | 176 | # Load input weights (if provided) 177 | input_weights = None; 178 | if(args["weights"]): 179 | input_weights = FileIO.load_input_weights(args["weights"], genes, cells); 180 | 181 | return (expressionMatrix, signatures, precomputed_signatures, 182 | housekeeping_genes, input_projections, input_weights); 183 | 184 | 185 | def createOutputDirectories(args): 186 | """ 187 | Creates the output directory structure 188 | """ 189 | 190 | # Create directory for all outputs 191 | if(args["output"]): 192 | dir_name = args["output"]; 193 | else: 194 | default_dir_name = 'FastProject_Output'; 195 | if(os.path.isdir(default_dir_name)): 196 | i = 1; 197 | while(True): 198 | dir_name = default_dir_name + str(i); 199 | if(not os.path.isdir(dir_name)): 200 | break; 201 | else: 202 | i = i + 1; 203 | else: 204 | dir_name = default_dir_name; 205 | 206 | FileIO.make_dirs(dir_name); 207 | 208 | logger = logging.getLogger("FastProject") 209 | logger.setLevel(logging.INFO); 210 | fh = logging.FileHandler(os.path.join(dir_name, 'fastproject.log')) 211 | fh.setFormatter(logging.Formatter('%(asctime)s %(message)s')) 212 | logger.addHandler(fh) 213 | 214 | logger.info("Running FastProject version " + FastProject.__version__); 215 | logger.info("Using numpy version " + np.__version__); 216 | 217 | for key in args: 218 | logger.info(key + ": " + str(args[key])); 219 | 220 | return dir_name; 221 | 222 | 223 | def entry(): 224 | """Entry point for the fastproject command-line script 225 | """ 226 | 227 | args = parseFPArgs(); 228 | 229 | (expressionMatrix, signatures, precomputed_signatures, 230 | housekeeping_genes, input_projections, 231 | input_weights) = loadFilesFromDisk(args); 232 | 233 | try: 234 | dir_name = createOutputDirectories(args); # Needs to be created first so logging can write here 235 | 236 | models, qc_info = Pipelines.Analysis(expressionMatrix, signatures, precomputed_signatures, 237 | housekeeping_genes, input_projections, input_weights, args); 238 | 239 | FileIO.saveResultstoDisk(models, signatures, qc_info, dir_name); 240 | except: 241 | import traceback; 242 | import sys; 243 | traceback.print_exc(); 244 | 245 | tb_type, value, tb = sys.exc_info(); 246 | 247 | if(args["debug"]): 248 | import pdb; 249 | pdb.post_mortem(tb); 250 | 251 | 252 | -------------------------------------------------------------------------------- /FastProject/DataTypes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Wrapper classes for different types of data 3 | 4 | Here are the wrapper classes for ExpressionData, and PCData 5 | 6 | This was created to organize nuances in how signature 7 | scores, distance matrices, and anything else, is computed 8 | on the different types of data. 9 | 10 | """ 11 | 12 | from __future__ import absolute_import, print_function, division; 13 | import numpy as np; 14 | 15 | 16 | class ExpressionData(np.ndarray): 17 | 18 | def __new__(subtype, data, row_labels=[], col_labels=[]): 19 | 20 | obj = np.asarray(data).view(subtype); 21 | 22 | if(len(row_labels) == 0): 23 | obj.row_labels = [str(i) for i in range(data.shape[0])]; 24 | elif(len(row_labels) != data.shape[0]): 25 | raise ValueError("Number of row labels does not equal number of rows in data"); 26 | else: 27 | obj.row_labels = row_labels; 28 | 29 | if(len(col_labels) == 0): 30 | obj.col_labels = [str(i) for i in range(data.shape[1])]; 31 | elif(len(col_labels) != data.shape[1]): 32 | raise ValueError("Number of column labels does not equal number of columns in data"); 33 | else: 34 | obj.col_labels = col_labels; 35 | 36 | obj.weights = np.ones(data.shape); 37 | obj.filters = dict(); 38 | 39 | return obj; 40 | 41 | def __init__(self, data, row_labels=[], col_labels=[]): 42 | pass; 43 | 44 | def __array_finalize__(self, obj): 45 | if obj is None: 46 | return; 47 | 48 | self.filters = getattr(obj, 'filters' , []); 49 | self.weights = getattr(obj, 'weights' , []); 50 | self.row_labels = getattr(obj, 'row_labels', []); 51 | self.col_labels = getattr(obj, 'col_labels', []); 52 | 53 | def subset_genes(self, indices): 54 | if(issubclass(type(indices), np.ndarray)): 55 | if(indices.dtype == np.bool): 56 | indices = np.nonzero(indices)[0]; 57 | 58 | out = self[indices,:]; 59 | out.weights = out.weights[indices,:]; 60 | out.row_labels = [self.row_labels[i] for i in indices]; 61 | 62 | return(out); 63 | 64 | def subset_samples(self, indices): 65 | if(issubclass(type(indices), np.ndarray)): 66 | if(indices.dtype == np.bool): 67 | indices = np.nonzero(indices)[0]; 68 | 69 | out = self[:, indices]; 70 | out.weights = out.weights[:, indices]; 71 | out.col_labels = [self.col_labels[i] for i in indices]; 72 | return(out); 73 | 74 | def filtered_genes(self, filter_name=None): 75 | """ 76 | Returns the list of genes (in the order used by projection_data or projection_weights) 77 | filtered by the listed filter 78 | 79 | :param filter_name: Name of the filter to use 80 | :return: List of gene (row) names 81 | """ 82 | 83 | if(filter_name is None or filter_name == 'None'): 84 | return self.row_labels; 85 | else: 86 | filter = self.filters[filter_name]; 87 | fg = [gene for gene in self.row_labels if gene in filter] 88 | return fg; 89 | 90 | def projection_data(self, filter_name=None): 91 | """ 92 | Returns the data matrix in self filtered by filters[filter_name] 93 | :return: A sliced copy of the underlying data matrix 94 | """ 95 | if(filter_name is None or filter_name == 'None'): 96 | return self.base; 97 | else: 98 | filter = self.filters[filter_name]; 99 | filter_i = [i for i, gene in enumerate(self.row_labels) if gene in filter]; 100 | return self.base[filter_i, :]; 101 | 102 | def projection_weights(self, filter_name=None): 103 | """ 104 | Returns the weight matrix, filtered by projection_mask 105 | :return: A sliced copy of the weight matrix 106 | """ 107 | 108 | if(filter_name is None or filter_name == 'None'): 109 | return self.weights; 110 | else: 111 | filter = self.filters[filter_name]; 112 | filter_i = [i for i,gene in enumerate(self.row_labels) if gene in filter]; 113 | return self.weights[filter_i,:]; 114 | 115 | def get_normalized_copy(self, norm_method): 116 | """ 117 | Runs norm_method on the data and returns an ExpressionData copy 118 | 119 | norm_method is of the form: 120 | 121 | out = norm_method(input); 122 | 123 | input: numpy.ndarray N x M 124 | out: numpy.ndarray N x M 125 | 126 | """ 127 | 128 | normalized_data = norm_method(self.base); 129 | normalized_edata = ExpressionData(normalized_data, 130 | row_labels=self.row_labels, 131 | col_labels=self.col_labels); 132 | 133 | normalized_edata.weights = self.weights; 134 | normalized_edata.filters = self.filters; 135 | 136 | return normalized_edata; 137 | 138 | def merge_data(self, other): 139 | """ 140 | Merges columns of self and other 141 | :param other: Instance of Expression Data with same rows as self 142 | :return: New Expression Data Instance combining self and other 143 | """ 144 | 145 | #Ensure row labels match 146 | if(len(self.row_labels) != len(other.row_labels)): 147 | raise ValueError("Cant merge ExpressionData objects with different row labels"); 148 | 149 | for x,y in zip(self.row_labels, other.row_labels): 150 | if(x != y): 151 | raise ValueError("Cant merge ExpressionData objects with different row labels"); 152 | 153 | row_labels_merge = self.row_labels; #Could be either, they are ensured to be identical at this point 154 | 155 | data_merge = np.concatenate((self.base, other.base), axis=1); 156 | 157 | #For Filters, combine sets preferring self over other 158 | filters_merge = self.filters.copy(); 159 | for key in other.filters: 160 | if(key not in filters_merge): 161 | filters_merge.update({key: other.filters[key]}); 162 | 163 | weights_merge = np.concatenate((self.weights, other.weights), axis=1); 164 | col_labels_merge = self.col_labels + other.col_labels; 165 | 166 | merge_data = ExpressionData(data_merge, row_labels_merge, col_labels_merge); 167 | merge_data.weights = weights_merge; 168 | merge_data.filters = filters_merge; 169 | return merge_data; 170 | 171 | 172 | class PCData(np.ndarray): 173 | 174 | def __new__(subtype, data, variance, loadings, parent_data): 175 | 176 | obj = np.asarray(data).view(subtype); 177 | 178 | obj.variance = variance; 179 | 180 | obj.row_labels = ["PC"+str(i+1) for i in range(data.shape[0])]; 181 | 182 | obj.loadings = loadings; 183 | 184 | obj.col_labels = parent_data.col_labels; 185 | 186 | obj.parent_data = parent_data; 187 | 188 | return obj; 189 | 190 | #All data values in PCData have the same weight 191 | @property 192 | def weights(self): 193 | return np.ones(self.shape); 194 | 195 | def __array_finalize__(self, obj): 196 | if obj is None: 197 | return; 198 | 199 | self.variance = getattr(obj, 'variance', []); 200 | self.row_labels = getattr(obj, 'row_labels', []); 201 | self.col_labels = getattr(obj, 'col_labels', []); 202 | self.loadings = getattr(obj, 'loadings', []); 203 | self.parent_data = getattr(obj, 'parent_data', []); 204 | 205 | # def eval_signature(self, signature): 206 | # """For a signature, calculate a score against each sample in the data 207 | # 208 | # Parameters 209 | # ---------- 210 | # signature : Signature, 211 | # expression data to evaluate the signature against 212 | # 213 | # Returns 214 | # ------- 215 | # out : 1D ndarray, length = Num_Samples 216 | # result of evaluating this signature on each sample in data 217 | # 218 | # """ 219 | # 220 | # #On Principle Components, just evaluate signature on parent data 221 | # return self.parent_data.eval_signature(signature); 222 | 223 | 224 | def subset_samples(self, indices): 225 | if(issubclass(type(indices), np.ndarray)): 226 | if(indices.dtype == np.bool): 227 | indices = np.nonzero(indices)[0]; 228 | 229 | out = self[:, indices]; 230 | out.col_labels = [self.col_labels[i] for i in indices]; 231 | out.parent_data = out.parent_data.subset_samples(indices); 232 | return(out); 233 | 234 | def subset_components(self, indices): 235 | if(issubclass(type(indices), np.ndarray)): 236 | if(indices.dtype == np.bool): 237 | indices = np.nonzero(indices)[0]; 238 | 239 | out = self[indices,:]; 240 | out.loadings = out.loadings[:,indices]; 241 | out.variance = out.variance[indices]; 242 | out.row_labels = [self.row_labels[i] for i in indices]; 243 | return(out); 244 | 245 | def filtered_genes(self, filter_name=None): 246 | """ 247 | Returns the list of genes (in the order used by projection_data or projection_weights) 248 | filtered by the listed filter 249 | 250 | :param filter_name: Name of the filter to use 251 | :return: List of gene (row) names 252 | """ 253 | 254 | return self.parent_data.filtered_genes(filter_name); 255 | 256 | def projection_data(self, filter_name=None): 257 | """ 258 | Returns the data matrix in self filtered by projection_mask 259 | :return: A sliced copy of self 260 | """ 261 | #PCA Data does not have projection_mask, just return all 262 | return self; 263 | 264 | def projection_weights(self, filter_name=None): 265 | """ 266 | Returns the weight matrix, filtered by projection_mask 267 | :return: A sliced copy of the weight matrix 268 | """ 269 | #PCA Data does not have a projection mask, just return weights 270 | return self.weights; 271 | 272 | 273 | -------------------------------------------------------------------------------- /FastProject/Filters.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Functions that are used to select genes 3 | 4 | Functions here use a variety of criteria to reduce the number 5 | of genes to a more manageable size - ideally extracting the 6 | genes that are more biologically informative. 7 | 8 | """ 9 | from __future__ import absolute_import, print_function, division; 10 | 11 | import numpy as np 12 | from . import Utils 13 | from .Global import FP_Output; 14 | 15 | 16 | def apply_filters(data, threshold, nofilter, lean): 17 | """Applies filters to the ExpressionData object 18 | May remove rows. 19 | Populates the data.filters field 20 | 21 | Parameters 22 | ---------- 23 | data : DataTypes.ExpressionData 24 | Data object to be filtered 25 | threshold : int 26 | Minimum number of samples gene must be detected in to pass 27 | nofilter : boolean 28 | if true, Only filter rows that have all identical values 29 | lean : boolean 30 | if true, skip extra filter methods (HDT, Fano) 31 | 32 | 33 | Returns 34 | ------- 35 | data_out : DataTypes.ExpressionData 36 | Filtered data object 37 | 38 | """ 39 | 40 | if(nofilter): 41 | filter_dict = {}; 42 | data = filter_genes_novar(data); 43 | 44 | filter_dict.update({'No_Filter': set(data.row_labels)}); 45 | data.filters = filter_dict; 46 | 47 | return data; 48 | 49 | else: 50 | filter_dict = {}; 51 | data = filter_genes_threshold(data, threshold); 52 | 53 | filter_dict.update({ 54 | 'Threshold': set(data.row_labels), 55 | }); 56 | 57 | if(not lean): 58 | for name, method in _filter_methods.items(): 59 | FP_Output("Applying filter method:", name); 60 | 61 | mask = method(data); 62 | 63 | if(np.array(mask).sum() > 10): # Only add these filters if they have enough genes 64 | filter_dict.update({ 65 | name: set([data.row_labels[i] for i, x in enumerate(mask) if x]), 66 | }); 67 | 68 | data.filters = filter_dict; 69 | 70 | return data; 71 | 72 | 73 | # This filter is run first, does not need to be registered at the bottom 74 | def filter_genes_threshold(data, threshold): 75 | """Filters out genes that are at least active in 76 | number of samples. 77 | """ 78 | 79 | keep_indices = np.where((data > 0).sum(axis=1) > threshold)[0]; 80 | 81 | return data.subset_genes(keep_indices); 82 | 83 | 84 | # This filter is run when the --nofilter option is selected 85 | # Needed for stability during other methods. 86 | def filter_genes_novar(data): 87 | """ 88 | Filters out genes with 0 variance across samples 89 | :param data: ExpressionData from DataTypes 90 | :return: Subset of data 0-variance rows removed 91 | """ 92 | 93 | row_var = np.var(data, axis=1); 94 | keep_indices = np.where(row_var != 0)[0]; 95 | 96 | return data.subset_genes(keep_indices); 97 | 98 | 99 | # --------------------------------------------------------------------------- # 100 | # # 101 | # Define Filter Methods Here # 102 | # # 103 | # --------------------------------------------------------------------------- # 104 | 105 | # Built-in Methods 106 | 107 | 108 | def filter_genes_fano(data, num_mad=2): 109 | """ 110 | Uses fano filtering on the genes. Splits into quantiles first. 111 | Only retains genes that have fano factor median absolute 112 | deviations in each quantile. 113 | 114 | :param data: numpy.ndarray (Num_Genes x Num_Samples) 115 | :param num_mad: float 116 | :return: numpy.ndarray, bool, (Num_Genes) 117 | True for rows that pass the filter. Otherwise, False; 118 | """ 119 | mu = np.mean(data, axis=1); 120 | sigma = np.std(data, axis=1); 121 | 122 | # slice up mu and sigma into bins, by mu 123 | aa = np.argsort(mu); 124 | mu_sort = mu[aa]; 125 | sigma_sort = sigma[aa]; 126 | 127 | N_QUANTS = 30; 128 | m = mu_sort.size // N_QUANTS; 129 | 130 | gene_passes = np.zeros(data.shape[0]) == 1; 131 | 132 | for i in range(N_QUANTS): 133 | if(i == N_QUANTS - 1): 134 | rr = np.arange(i * m, mu_sort.size) 135 | else: 136 | rr = np.arange(i * m, (i + 1) * m); 137 | 138 | mu_quant = mu_sort[rr]; 139 | mu_quant[mu_quant == 0] = 1; # so we don't divide by zero later 140 | sigma_quant = sigma_sort[rr]; 141 | fano_quant = sigma_quant**2 / mu_quant; 142 | mad_quant = np.median(np.abs(fano_quant - np.median(fano_quant))); 143 | gene_passes_quant = fano_quant > np.median(fano_quant) + num_mad * mad_quant; 144 | gene_passes_quant_i = np.nonzero(gene_passes_quant)[0]; 145 | gene_passes_i = gene_passes_quant_i + i * m; 146 | gene_passes[gene_passes_i] = True; 147 | 148 | # gene_passes are in sorted mu order, need to revert back to original order 149 | original_ii = np.argsort(aa); 150 | gene_passes = gene_passes[original_ii]; 151 | 152 | return gene_passes; 153 | 154 | 155 | def filter_genes_hdt(data, p_val=0.05): 156 | """Filters out genes that pass the Hartigans Dip Test for bimodality 157 | with at least p < p_val""" 158 | # perform Hartigans dip test on the rest of the rows 159 | 160 | (dips, ps, xlows, xups) = Utils.HDT_Sig_batch(data, 1000); 161 | 162 | return ps <= p_val; 163 | 164 | 165 | # Add additional filtering methods here 166 | 167 | # Register filter methods 168 | _filter_methods = dict(); 169 | _filter_methods["HDT"] = filter_genes_hdt; 170 | _filter_methods["Fano"] = filter_genes_fano; 171 | -------------------------------------------------------------------------------- /FastProject/Global.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Contains information that is useful to share 3 | across all modules. 4 | 5 | Right now, mainly command-line arguments, logging 6 | and resource locations. 7 | """ 8 | from __future__ import absolute_import, print_function, division; 9 | import logging; 10 | import sys 11 | import os 12 | import random; 13 | 14 | # Defaults for arguments are defined here 15 | # These are all overwritten if called from the command line 16 | # However, by putting defaults here, it makes it easier to 17 | # call FastProject from within other scripts 18 | # These should be kept in sync with arguments in __main__.py 19 | # args = namedtuple("defalt_args", ["data_file", "housekeeping", 20 | # "signatures", "precomputed", "output", 21 | # "nofilter", "nomodel", "pca_filter", 22 | # "qc", "subsample_size", 23 | # "min_signature_genes", "projections", 24 | # "weights", "threshold"]); 25 | # 26 | # args.data_file = ""; 27 | # args.housekeeping = ""; 28 | # args.signatures = []; 29 | # args.precomputed = []; 30 | # args.output = ""; 31 | # args.nofilter = False; 32 | # args.nomodel = False; 33 | # args.pca_filter = False; 34 | # args.qc = False; 35 | # args.subsample_size = None; 36 | # args.min_signature_genes = 5; 37 | # args.projections = []; 38 | # args.weights = ""; 39 | # args.threshold = None; 40 | 41 | 42 | logger = logging.getLogger("FastProject"); 43 | def FP_Output(*args): 44 | """ 45 | Used to have finer control over outputs. 46 | """ 47 | print(*args); 48 | logmessage = ' '.join([str(a) for a in args]); 49 | if(logmessage.startswith("\n")): 50 | logmessage = logmessage[1:]; 51 | logger.info(logmessage); 52 | 53 | # This section for finding resource files 54 | # Behavior is different depending on whether or not we are running frozen 55 | 56 | if getattr(sys, 'frozen', False): 57 | this_directory = sys._MEIPASS; 58 | else: 59 | this_directory = os.path.dirname(os.path.abspath(__file__)); 60 | 61 | 62 | def get_viewer_resource_dir(): 63 | return os.path.join(this_directory, "Viewer Resources"); 64 | 65 | 66 | def get_housekeeping_dir(): 67 | return os.path.join(this_directory, "Housekeeping Genes"); 68 | 69 | # Chosen by roll of a 2147483648-sided die 70 | # Guaranteed to be random 71 | RANDOM_SEED = 1335607828; 72 | random.seed(RANDOM_SEED); 73 | -------------------------------------------------------------------------------- /FastProject/Housekeeping Genes/Ensemble Housekeeping.txt: -------------------------------------------------------------------------------- 1 | ENSG00000276072 2 | ENSG00000275700 3 | ENSG00000097007 4 | ENSG00000120437 5 | ENSG00000184009 6 | ENSG00000075624 7 | LRG_132 8 | ENSG00000184009 9 | ENSG00000130402 10 | ENSG00000139567 11 | ENSG00000143537 12 | ENSG00000160710 13 | ENSG00000087274 14 | ENSG00000173020 15 | ENSG00000104964 16 | ENSG00000141385 17 | LRG_666 18 | ENSG00000228892 19 | ENSG00000235758 20 | ENSG00000227642 21 | ENSG00000204310 22 | ENSG00000236873 23 | ENSG00000226467 24 | ENSG00000206324 25 | ENSG00000235985 26 | ENSG00000206428 27 | ENSG00000234836 28 | ENSG00000204472 29 | ENSG00000237727 30 | ENSG00000235588 31 | ENSG00000149925 32 | ENSG00000109107 33 | ENSG00000089053 34 | ENSG00000136938 35 | ENSG00000182718 36 | ENSG00000197043 37 | ENSG00000100280 38 | ENSG00000161203 39 | ENSG00000042753 40 | ENSG00000166181 41 | ENSG00000084234 42 | ENSG00000143761 43 | ENSG00000272822 44 | ENSG00000134287 45 | ENSG00000168374 46 | ENSG00000004059 47 | ENSG00000175220 48 | ENSG00000141522 49 | ENSG00000102606 50 | ENSG00000213465 51 | ENSG00000163466 52 | ENSG00000111229 53 | ENSG00000241553 54 | ENSG00000128272 55 | ENSG00000152234 56 | ENSG00000099624 57 | ENSG00000159199 58 | ENSG00000154518 59 | ENSG00000167863 60 | ENSG00000241468 61 | ENSG00000241837 62 | ENSG00000117410 63 | ENSG00000185883 64 | ENSG00000131100 65 | ENSG00000128524 66 | ENSG00000136888 67 | ENSG00000273686 68 | ENSG00000166710 69 | ENSG00000158850 70 | ENSG00000126581 71 | ENSG00000168283 72 | ENSG00000269897 73 | ENSG00000174744 74 | ENSG00000172270 75 | ENSG00000133243 76 | ENSG00000145741 77 | ENSG00000197223 78 | ENSG00000198668 79 | ENSG00000160014 80 | ENSG00000143933 81 | ENSG00000198668 82 | ENSG00000160014 83 | ENSG00000143933 84 | ENSG00000127022 85 | ENSG00000126247 86 | ENSG00000077549 87 | ENSG00000112576 88 | ENSG00000163468 89 | ENSG00000135624 90 | ENSG00000110651 91 | LRG_142 92 | ENSG00000158825 93 | ENSG00000125817 94 | ENSG00000172757 95 | ENSG00000111642 96 | ENSG00000148337 97 | ENSG00000166165 98 | ENSG00000171603 99 | ENSG00000122705 100 | ENSG00000175416 101 | ENSG00000018236 102 | ENSG00000142156 103 | LRG_475 104 | ENSG00000105669 105 | ENSG00000168090 106 | ENSG00000131143 107 | ENSG00000178741 108 | ENSG00000135940 109 | ENSG00000111775 110 | ENSG00000112695 111 | ENSG00000115944 112 | ENSG00000127184 113 | ENSG00000214078 114 | ENSG00000100884 115 | ENSG00000107175 116 | ENSG00000184371 117 | ENSG00000103653 118 | ENSG00000213923 119 | ENSG00000230700 120 | ENSG00000232960 121 | ENSG00000204435 122 | ENSG00000224774 123 | ENSG00000206406 124 | ENSG00000224398 125 | ENSG00000228875 126 | ENSG00000160213 127 | LRG_485 128 | ENSG00000159692 129 | ENSG00000168036 130 | ENSG00000117984 131 | ENSG00000179091 132 | ENSG00000129562 133 | ENSG00000112977 134 | ENSG00000204209 135 | ENSG00000206206 136 | ENSG00000229396 137 | ENSG00000227046 138 | ENSG00000206279 139 | ENSG00000231617 140 | ENSG00000183283 141 | ENSG00000244038 142 | ENSG00000099977 143 | ENSG00000275003 144 | ENSG00000172893 145 | LRG_340 146 | ENSG00000104371 147 | ENSG00000132002 148 | ENSG00000105993 149 | ENSG00000123992 150 | ENSG00000175550 151 | ENSG00000205250 152 | ENSG00000143590 153 | ENSG00000156976 154 | ENSG00000110321 155 | ENSG00000196361 156 | ENSG00000074800 157 | ENSG00000143420 158 | ENSG00000197586 159 | ENSG00000100632 160 | ENSG00000012232 161 | ENSG00000149806 162 | ENSG00000105202 163 | ENSG00000149557 164 | ENSG00000088832 165 | ENSG00000110195 166 | ENSG00000111206 167 | ENSG00000167996 168 | ENSG00000089280 169 | ENSG00000170296 170 | ENSG00000034713 171 | ENSG00000180447 172 | ENSG00000203879 173 | ENSG00000057608 174 | ENSG00000196743 175 | ENSG00000114353 176 | ENSG00000087460 177 | ENSG00000172354 178 | ENSG00000204628 179 | ENSG00000125166 180 | ENSG00000169347 181 | ENSG00000197858 182 | ENSG00000205336 183 | ENSG00000167468 184 | ENSG00000105737 185 | ENSG00000105723 186 | ENSG00000143774 187 | ENSG00000113648 188 | ENSG00000163041 189 | ENSG00000132475 190 | ENSG00000188375 191 | ENSG00000049239 192 | ENSG00000084754 193 | ENSG00000138029 194 | ENSG00000143575 195 | LRG_64 196 | ENSG00000143321 197 | ENSG00000185359 198 | ENSG00000169567 199 | ENSG00000189403 200 | ENSG00000115756 201 | ENSG00000230989 202 | ENSG00000044574 203 | ENSG00000109971 204 | ENSG00000149428 205 | ENSG00000117318 206 | ENSG00000101365 207 | ENSG00000185885 208 | ENSG00000143621 209 | ENSG00000166333 210 | LRG_444 211 | ENSG00000178035 212 | ENSG00000129009 213 | ENSG00000101384 214 | ENSG00000162434 215 | ENSG00000166086 216 | ENSG00000143543 217 | ENSG00000130522 218 | ENSG00000065427 219 | LRG_366 220 | ENSG00000105438 221 | ENSG00000129250 222 | ENSG00000140859 223 | ENSG00000185896 224 | ENSG00000002834 225 | ENSG00000134333 226 | ENSG00000111716 227 | ENSG00000168961 228 | ENSG00000171916 229 | ENSG00000166407 230 | ENSG00000090006 231 | ENSG00000011009 232 | ENSG00000003056 233 | ENSG00000145050 234 | ENSG00000126934 235 | ENSG00000173327 236 | ENSG00000121653 237 | ENSG00000162889 238 | ENSG00000103495 239 | ENSG00000185231 240 | ENSG00000143384 241 | ENSG00000160294 242 | ENSG00000014641 243 | ENSG00000116688 244 | LRG_255 245 | ENSG00000131446 246 | ENSG00000276701 247 | ENSG00000240972 248 | ENSG00000089693 249 | ENSG00000103152 250 | ENSG00000011028 251 | ENSG00000276345 252 | ENSG00000214026 253 | ENSG00000143436 254 | ENSG00000128626 255 | ENSG00000147065 256 | ENSG00000087250 257 | ENSG00000182979 258 | ENSG00000173171 259 | ENSG00000110921 260 | LRG_156 261 | ENSG00000136997 262 | ENSG00000100345 263 | ENSG00000092841 264 | ENSG00000196531 265 | ENSG00000115053 266 | ENSG00000125356 267 | ENSG00000131495 268 | ENSG00000267855 269 | ENSG00000167774 270 | ENSG00000099795 271 | ENSG00000109390 272 | ENSG00000168653 273 | ENSG00000167792 274 | ENSG00000178127 275 | ENSG00000129559 276 | ENSG00000100906 277 | LRG_89 278 | ENSG00000239672 279 | ENSG00000011052 280 | ENSG00000243678 281 | ENSG00000147140 282 | ENSG00000272325 283 | ENSG00000162231 284 | ENSG00000115758 285 | ENSG00000185624 286 | ENSG00000070756 287 | ENSG00000100836 288 | ENSG00000125618 289 | ENSG00000106244 290 | ENSG00000249915 291 | ENSG00000089220 292 | ENSG00000113068 293 | ENSG00000123349 294 | ENSG00000108518 295 | ENSG00000142657 296 | ENSG00000102144 297 | ENSG00000112511 298 | ENSG00000239756 299 | ENSG00000225553 300 | ENSG00000092621 301 | ENSG00000137193 302 | ENSG00000127445 303 | ENSG00000118762 304 | ENSG00000187838 305 | ENSG00000196576 306 | ENSG00000141744 307 | ENSG00000181222 308 | ENSG00000100142 309 | ENSG00000177700 310 | ENSG00000204619 311 | ENSG00000234058 312 | ENSG00000206501 313 | ENSG00000233314 314 | ENSG00000236560 315 | ENSG00000237403 316 | ENSG00000237829 317 | ENSG00000227720 318 | ENSG00000104695 319 | ENSG00000105568 320 | ENSG00000117450 321 | ENSG00000181929 322 | ENSG00000130175 323 | ENSG00000274442 324 | ENSG00000174231 325 | ENSG00000135406 326 | ENSG00000101182 327 | ENSG00000008018 328 | ENSG00000126067 329 | ENSG00000159377 330 | ENSG00000136930 331 | ENSG00000108671 332 | ENSG00000099341 333 | ENSG00000100911 334 | ENSG00000011304 335 | ENSG00000156471 336 | ENSG00000187514 337 | ENSG00000104960 338 | ENSG00000183255 339 | ENSG00000105404 340 | ENSG00000136238 341 | ENSG00000179262 342 | ENSG00000132341 343 | ENSG00000127314 344 | ENSG00000102317 345 | ENSG00000265241 346 | ENSG00000157110 347 | ENSG00000173039 348 | ENSG00000142599 349 | ENSG00000206287 350 | ENSG00000204227 351 | ENSG00000228520 352 | ENSG00000226788 353 | ENSG00000235107 354 | ENSG00000231115 355 | ENSG00000117748 356 | ENSG00000147403 357 | ENSG00000198755 358 | ENSG00000142676 359 | ENSG00000167526 360 | ENSG00000142541 361 | ENSG00000174748 362 | ENSG00000215472 363 | ENSG00000265681 364 | ENSG00000063177 365 | ENSG00000108298 366 | ENSG00000131469 367 | ENSG00000162244 368 | ENSG00000100316 369 | ENSG00000136942 370 | ENSG00000165502 371 | ENSG00000145592 372 | ENSG00000172809 373 | ENSG00000122406 374 | ENSG00000161016 375 | ENSG00000137818 376 | ENSG00000177600 377 | ENSG00000163902 378 | ENSG00000124614 379 | ENSG00000142534 380 | ENSG00000112306 381 | ENSG00000110700 382 | ENSG00000164587 383 | ENSG00000115268 384 | ENSG00000105193 385 | ENSG00000223367 386 | ENSG00000227794 387 | ENSG00000226225 388 | ENSG00000096150 389 | ENSG00000235650 390 | ENSG00000231500 391 | ENSG00000105372 392 | ENSG00000140988 393 | ENSG00000138326 394 | ENSG00000118181 395 | ENSG00000143947 396 | ENSG00000083845 397 | ENSG00000175634 398 | ENSG00000277079 399 | ENSG00000274005 400 | ENSG00000278081 401 | ENSG00000277359 402 | ENSG00000275323 403 | ENSG00000274626 404 | ENSG00000274646 405 | ENSG00000170889 406 | ENSG00000274950 407 | ENSG00000278270 408 | ENSG00000125844 409 | ENSG00000115310 410 | ENSG00000183207 411 | ENSG00000160633 412 | ENSG00000150459 413 | ENSG00000031698 414 | ENSG00000116521 415 | ENSG00000263290 416 | ENSG00000162512 417 | ENSG00000073578 418 | ENSG00000106803 419 | ENSG00000181523 420 | ENSG00000108528 421 | ENSG00000075415 422 | ENSG00000130821 423 | ENSG00000065054 424 | ENSG00000077312 425 | ENSG00000125835 426 | ENSG00000125743 427 | ENSG00000143977 428 | ENSG00000112335 429 | ENSG00000142168 430 | LRG_652 431 | ENSG00000091640 432 | ENSG00000072310 433 | ENSG00000112658 434 | ENSG00000116649 435 | ENSG00000140319 436 | ENSG00000133226 437 | ENSG00000163479 438 | ENSG00000162009 439 | ENSG00000166444 440 | ENSG00000084090 441 | ENSG00000206342 442 | ENSG00000204344 443 | ENSG00000234947 444 | ENSG00000236250 445 | ENSG00000226033 446 | ENSG00000226257 447 | ENSG00000102572 448 | ENSG00000108639 449 | ENSG00000149591 450 | ENSG00000177156 451 | ENSG00000206281 452 | ENSG00000236490 453 | ENSG00000206208 454 | ENSG00000112493 455 | ENSG00000231925 456 | LRG_114 457 | ENSG00000103363 458 | ENSG00000070814 459 | ENSG00000166848 460 | ENSG00000104980 461 | ENSG00000163931 462 | ENSG00000137076 463 | ENSG00000034510 464 | ENSG00000137364 465 | ENSG00000126602 466 | ENSG00000130726 467 | ENSG00000278243 468 | ENSG00000104522 469 | ENSG00000113312 470 | ENSG00000130640 471 | ENSG00000178952 472 | ENSG00000170315 473 | ENSG00000150991 474 | ENSG00000131508 475 | ENSG00000103275 476 | ENSG00000130725 477 | ENSG00000010256 478 | ENSG00000169021 479 | ENSG00000173660 480 | ENSG00000102226 481 | ENSG00000170236 482 | ENSG00000139190 483 | ENSG00000049245 484 | ENSG00000173511 485 | ENSG00000071127 486 | ENSG00000100219 487 | ENSG00000134684 488 | LRG_273 489 | ENSG00000166913 490 | ENSG00000128245 491 | ENSG00000134308 492 | ENSG00000164924 493 | ENSG00000185650 494 | ENSG00000162300 495 | -------------------------------------------------------------------------------- /FastProject/Housekeeping Genes/Gene Name Housekeeping.txt: -------------------------------------------------------------------------------- 1 | RPL10 2 | ACTN4 3 | GPAA1 4 | RPS11 5 | GM9800 6 | PTMA 7 | ACTB 8 | ANXA2 9 | GAS1 10 | STK24 11 | GOT2 12 | KIFC3 13 | SGSH 14 | ENO1 15 | GM4735 16 | GM5506 17 | LOC100047043 18 | TAGLN 19 | GM10036 20 | GM10288 21 | GM5093 22 | GM7384 23 | GM7589 24 | RPL11 25 | SYNGR2 26 | RPS2 27 | TKT 28 | ARPC2 29 | COX7A2L 30 | RTN4 31 | PRKCSH 32 | DDT 33 | ATP6V1F 34 | B4GALT3 35 | STK19 36 | PDAP1 37 | ILK 38 | PDCD6 39 | COX5A 40 | SLC25A3 41 | PTTG1IP 42 | MANF 43 | COPS6 44 | PKM2 45 | RPS18 46 | EG627427 47 | GM13337 48 | GM5385 49 | GM7669 50 | GM7901 51 | GM8096 52 | GM8341 53 | GM9210 54 | GM9252 55 | GM9347 56 | LOC630761 57 | LOC630896 58 | PHGDH 59 | SCAMP3 60 | TIMM44 61 | TSTA3 62 | RAC1 63 | FCER2A 64 | AP2M1 65 | FTH1 66 | ADRBK1 67 | YWHAH 68 | ANXA6 69 | ARF5 70 | LOC100046958 71 | CSK 72 | SFRS2 73 | RPL19 74 | TLN1 75 | CLSTN1 76 | GM6742 77 | GM7993 78 | GM9115 79 | LOC677259 80 | ODC1 81 | JUND 82 | ID3 83 | CIZ1 84 | UBE2M 85 | YARS 86 | UQCRFS1 87 | TUBB5 88 | AES 89 | ATP5A1 90 | ATP5G1 91 | GM10039 92 | ARL2 93 | CLTB 94 | LOC100046457 95 | ARF1 96 | GM5823 97 | GM8230 98 | KARS 99 | LOC631033 100 | ATP5J2 101 | RPL13A 102 | AGPAT1 103 | GABARAP 104 | DRAP1 105 | ATP5D 106 | MCM3AP 107 | MSN 108 | DNAJB1 109 | RPS5 110 | MAP2K2 111 | CSNK1E 112 | RPN1 113 | SRP14 114 | GM9646 115 | LOC634703 116 | RPS19 117 | TADA3L 118 | PSMB2 119 | JAK1 120 | GM13020 121 | GM7379 122 | RPL38 123 | PSMD8 124 | ARHGDIA 125 | NONO 126 | DDOST 127 | JAM3 128 | RBM8A 129 | ENTPD6 130 | GNAI2 131 | MAPKAPK2 132 | CPNE1 133 | BTBD2 134 | GM1821 135 | UBB 136 | RAD23A 137 | CENPB 138 | ILF2 139 | PPP1R11 140 | DAD1 141 | ATP5H 142 | CANX 143 | CYC1 144 | LDHA 145 | ACVRL1 146 | CCBP2 147 | DAP 148 | 4932431P20RIK 149 | GM12587 150 | GM13932 151 | GM2509 152 | GM2676 153 | GM3307 154 | GM3462 155 | GM3899 156 | GM4587 157 | GM5473 158 | GM5842 159 | GM5853 160 | GM5937 161 | GM7890 162 | GM8390 163 | HMG1L1 164 | HMGB1 165 | HMGB1L 166 | LOC100045876 167 | LOC637733 168 | LAMP1 169 | NDUFB7 170 | NME2 171 | GM6573 172 | LOC100044992 173 | RPS13 174 | GM5870 175 | SEC61B 176 | ARHGEF7 177 | GUK1 178 | PKD2 179 | CREB3 180 | GM2A 181 | CSTB 182 | EIF4A2 183 | RPL35 184 | NDUFS5 185 | IMPDH2 186 | ISLR 187 | RPS10 188 | TERF2IP 189 | SRM 190 | CLTA 191 | PSMB7 192 | API5 193 | MTAP4 194 | RAP1B 195 | LOC100046344 196 | LOC100047753 197 | NME1 198 | ALDOC 199 | PTOV1 200 | HADHA 201 | ARF3 202 | RPL8 203 | CSNK2B 204 | TALDO1 205 | B2M 206 | TUBB4 207 | UQCR 208 | COPE 209 | BMI1 210 | CAPNS1 211 | GNAS 212 | LTBP4 213 | SNRPB 214 | CTNNB1 215 | MIF 216 | DAZAP2 217 | GM10080 218 | GM11836 219 | GM5526 220 | MYL6 221 | PFN1 222 | MDH1 223 | AKR1A4 224 | ATP6V0B 225 | MRC2 226 | ARHGAP1 227 | HAX1 228 | UBE2I 229 | KDELR1 230 | TUBGCP2 231 | RING1 232 | SOD1 233 | GM10063 234 | GM12334 235 | GM5462 236 | GM7567 237 | GM9153 238 | LOC676277 239 | RPS12 240 | BRMS1 241 | LOC637711 242 | MVK 243 | DKK4 244 | ZFP36L1 245 | USP11 246 | P4HB 247 | GM10774 248 | POLR2L 249 | USP50 250 | ADD1 251 | COL6A1 252 | VAMP1 253 | POLR2F 254 | GM5805 255 | RPS14 256 | COX4I1 257 | E2F4 258 | GRIK5 259 | DNPEP 260 | WDR1 261 | ATP5G3 262 | RPL15 263 | GM8276 264 | PABPC1 265 | ZFPL1 266 | NACA 267 | PIN1 268 | HDGF 269 | GSK3A 270 | RPS25 271 | GM11539 272 | GM5312 273 | LOC100048449 274 | RPL18 275 | GPX4 276 | NDUFV1 277 | MRPL23 278 | HPCAL1 279 | GM10079 280 | RPS16 281 | PSMB1 282 | NFKBIA 283 | PNMT 284 | GM9844 285 | LOC100048142 286 | TMSB10 287 | SAP18 288 | RPA2 289 | HYOU1 290 | HSPA5 291 | AFG3L2 292 | LOC100048880 293 | BECN1 294 | CSF1 295 | JTB 296 | MGAT1 297 | CFL1 298 | LMO1 299 | SREBF1 300 | YWHAQ 301 | SPAG7 302 | RBM3 303 | GM15451 304 | RPL10A 305 | EIF4G2 306 | RPL17 307 | CNTN1 308 | CKB 309 | ENSA 310 | H6PD 311 | RRBP1 312 | RPLP1 313 | RABAC1 314 | PFDN1 315 | GM10257 316 | GM8524 317 | H3F3A 318 | H3F3C 319 | LOC100045490 320 | ATP6V0C 321 | AATF 322 | PRPF8 323 | ARF4 324 | ACAT2 325 | MYH9 326 | PPP2R1A 327 | LOC100048413 328 | MLF2 329 | LYPLA2 330 | AP1B1 331 | DHCR7 332 | ERH 333 | NXF1 334 | SFRS9 335 | CTSD 336 | FUS 337 | FEZ1 338 | PSMD11 339 | SLC25A11 340 | LYZ1 341 | LYZ2 342 | PSME2 343 | AP2S1 344 | IFITM1 345 | UQCRC1 346 | TUFM 347 | DNAJB6 348 | GNB2L1 349 | MFN2 350 | MYC 351 | RPL3 352 | MTX1 353 | ATP6V1E1 354 | EFNA3 355 | LOC100046031 356 | LDHB 357 | GPI1 358 | PFDN5 359 | MRPS12 360 | LGALS9 361 | ABL1 362 | GDI1 363 | ADAM15 364 | NDUFA2 365 | SNRPA 366 | PSMA7 367 | MT3 368 | CDA 369 | GNB2 370 | SLC9A3R2 371 | CPNE6 372 | APLP2 373 | SLC6A8 374 | RUVBL2 375 | GM11307 376 | GM15500 377 | GM15616 378 | GM7424 379 | GM8934 380 | LOC632230 381 | RPL5 382 | UBE2D2 383 | BTF3 384 | ACTG1 385 | MC2R 386 | COX7A2 387 | CALM1 388 | SARS 389 | GABARAPL2 390 | GP2 391 | RPLP2 392 | TXN1 393 | HINT1 394 | ANP32B 395 | DAXX 396 | RPS24 397 | RPS6KB2 398 | LOC100047647 399 | NUDT3 400 | CTBP1 401 | LOC100045360 402 | PSMB4 403 | PGD 404 | MRPL9 405 | RPL27 406 | MPG 407 | FAU 408 | GM9843 409 | MAP3K11 410 | FOLR1 411 | CCT3 412 | AIF1 413 | 4933413G19RIK 414 | FOXM1 415 | PEBP1 416 | NDUFA1 417 | ALDOA 418 | RPS9 419 | CALM2 420 | NCL 421 | COMT1 422 | LASP1 423 | C1D 424 | SSR2 425 | ATF4 426 | EXTL3 427 | SSTR5 428 | RPL37 429 | JAG1 430 | VAMP3 431 | CD81 432 | IDH3B 433 | HGS 434 | NDUFC1 435 | NDUFV2 436 | FKBP1A 437 | KIF1C 438 | MCL1 439 | GM8186 440 | SNRPG 441 | CHD4 442 | PAX8 443 | HSBP1 444 | RELA 445 | HSPA8 446 | BAT1A 447 | TCOF1 448 | RAB1 449 | TTC1 450 | ATP6V1G1 451 | COX6A1 452 | GM7795 453 | COBRA1 454 | ST5 455 | CCND3 456 | M6PR 457 | SRF 458 | MAZ 459 | ARPC4 460 | GM7204 461 | PRDX1 462 | RPL36AL 463 | GM9354 464 | LOC100042019 465 | RPS27A 466 | ARPC3 467 | TAPBP 468 | FBL 469 | TRAP1 470 | CAPZB 471 | TRIM28 472 | 2400003C14RIK 473 | STARD7 474 | PTBP1 475 | SNX3 476 | GM10709 477 | GM13841 478 | GM5561 479 | GM6344 480 | RPL29 481 | UQCRH 482 | COX7C 483 | GM10012 484 | GM3386 485 | LOC100048613 486 | PHF1 487 | SDHA 488 | RAN 489 | ADAR 490 | HADHB 491 | DIAP1 492 | PLXNB2 493 | PRKAG1 494 | NDUFA7 495 | ANAPC5 496 | GPR56 497 | 4732418C07RIK 498 | ATP5O 499 | GM5436 500 | LOC100047429 501 | VEGFB 502 | GDI2 503 | ELAVL3 504 | XBP1 505 | MTA1 506 | BSG 507 | RBPMS 508 | UBC 509 | GM11401 510 | LOC638798 511 | LOC677628 512 | TCEB2 513 | SAFB 514 | PLSCR3 515 | MAPK8IP1 516 | PRPH 517 | RERE 518 | POLR2A 519 | LOC100044385 520 | PPP2CB 521 | COX5B 522 | TPMT 523 | SRRM1 524 | SDC3 525 | NEDD8 526 | PIM1 527 | PABPN1 528 | PTDSS1 529 | YWHAB 530 | PGK1 531 | YWHAZ 532 | GM10071 533 | GM12918 534 | GM15710 535 | GM5075 536 | GM9026 537 | LOC674921 538 | RPL13 539 | CCT7 540 | SNRPD2 541 | ATP5K 542 | RPS15 543 | H2AFY -------------------------------------------------------------------------------- /FastProject/HtmlViewer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Contains functions for generating result html and js files 3 | 4 | """ 5 | import shutil; 6 | import os; 7 | import sys 8 | import numpy as np; 9 | from .Global import get_viewer_resource_dir; 10 | 11 | if sys.version_info.major == 2: 12 | STR_TYPES = (str,unicode) 13 | else: 14 | STR_TYPES = (str,) 15 | 16 | 17 | RESOURCE_DIR = get_viewer_resource_dir(); 18 | OUTPUT_RESOURCE_DIRECTORY = "_resources"; 19 | 20 | def get_output_js_handle(destination_dir): 21 | fout_js = open(os.path.join(destination_dir,OUTPUT_RESOURCE_DIRECTORY,"FP_data.jsdata"), 'w'); 22 | return fout_js; 23 | 24 | def copy_html_files(destination_dir): 25 | shutil.copy(RESOURCE_DIR + os.sep + "Results.html", destination_dir + os.sep + "Results.html"); 26 | 27 | html_sub_directory = os.path.join(destination_dir,OUTPUT_RESOURCE_DIRECTORY) 28 | 29 | shutil.copy(RESOURCE_DIR + os.sep + "jquery-2.1.4.min.js", html_sub_directory + os.sep + "jquery-2.1.4.min.js"); 30 | shutil.copy(RESOURCE_DIR + os.sep + "bootstrap.css", html_sub_directory + os.sep + "bootstrap.css"); 31 | shutil.copy(RESOURCE_DIR + os.sep + "bootstrap.min.js", html_sub_directory + os.sep + "bootstrap.min.js"); 32 | shutil.copy(RESOURCE_DIR + os.sep + "d3.min.js", html_sub_directory + os.sep + "d3.min.js"); 33 | shutil.copy(RESOURCE_DIR + os.sep + "d3tip.js", html_sub_directory + os.sep + "d3tip.js"); #Extension For tooltips 34 | shutil.copy(RESOURCE_DIR + os.sep + "ColorScatter.js", html_sub_directory + os.sep + "ColorScatter.js"); 35 | shutil.copy(RESOURCE_DIR + os.sep + "HeatMap.js", html_sub_directory + os.sep + "HeatMap.js"); 36 | shutil.copy(RESOURCE_DIR + os.sep + "Clustered_HeatMap.js", html_sub_directory + os.sep + "Clustered_HeatMap.js"); 37 | shutil.copy(RESOURCE_DIR + os.sep + "jszip.min.js", html_sub_directory + os.sep + "jszip.min.js"); 38 | shutil.copy(RESOURCE_DIR + os.sep + "styles.css", html_sub_directory + os.sep + "styles.css"); 39 | shutil.copy(RESOURCE_DIR + os.sep + "Utilities.js", html_sub_directory + os.sep + "Utilities.js"); 40 | 41 | def toJS_variable(variable_name, obj): 42 | return "var " + variable_name + " = " + toJS(obj) + ";\n"; 43 | 44 | def toJS(obj, level=0): 45 | """ 46 | Convert a python object into a javascript format. 47 | Only certain object types are supported: 48 | List >> Javascript Array 49 | Numpy.ndarray >> Javascript Array 50 | Dictionary >> Javascript Object if keys are strings 51 | :param obj: object to be converted 52 | :param level: determines indentation level 53 | :return: string representation of the object in javascript 54 | """ 55 | 56 | try: 57 | obj.to_JSON; 58 | except AttributeError: 59 | pass; 60 | else: 61 | return obj.to_JSON(); 62 | 63 | if(isinstance(obj, STR_TYPES)): 64 | return '"' + obj + '"'; 65 | if(type(obj) is int or type(obj) is float or isinstance(obj, np.number)): 66 | return str(obj); 67 | if(issubclass(type(obj),np.ndarray) and obj.ndim < 3): 68 | return ndarray_to_JS(obj); 69 | if(type(obj) is list or type(obj) is set): 70 | return '[' + ','.join([toJS(x) for x in obj]) + ']'; 71 | if(type(obj) is bool): 72 | if(obj): return 'true'; 73 | else: return 'false'; 74 | if(type(obj) is dict): 75 | pairs = list(); 76 | indentation = "".join(["\t"]*(level+1)); 77 | for key in obj: 78 | if(isinstance(key, STR_TYPES) or type(key) is int or type(key) is float): 79 | pairs.append("\n"+indentation+toJS(key, level+1)+':'+toJS(obj[key], level+1)); 80 | else: 81 | raise ValueError("Non-compatible Value Encountered for Object Key:", type(key)); 82 | return '{' + ','.join(pairs) + '}'; 83 | 84 | raise ValueError("Non-convertible Value Encountered:", type(obj)); 85 | 86 | def ndarray_to_JS(np_array, format_str = "{:.3f}"): 87 | """ 88 | Returns a string representation of np_array in JS list of lists syntax 89 | :param np_array: Array to be converted (1 or 2 dimensional) 90 | :return: String representation. Not a complete JS statement 91 | 92 | For example, output might look like '[[1,2],[3,5]]' 93 | 94 | """ 95 | if(np_array.ndim == 1): 96 | value = "[" + ", ".join([format_str.format(x) for x in np_array]) + "]"; 97 | else: 98 | rows = list(); 99 | for row in np_array: 100 | rows.append("[" + ", ".join([format_str.format(x) for x in row]) + "]"); 101 | 102 | value = "[" + ','.join(rows) + "]"; 103 | 104 | return value; 105 | 106 | def dict_to_JS(py_dict, object_name, transpose_arrays = False): 107 | """ 108 | Formats the dictionary of numpy ndarrays into text that can be written to a .js file for html access 109 | 110 | The result is a javascript object with keys equal to the projection names 111 | 112 | :param py_dict: dictionary with key=String, value = numpy.ndarray 113 | :param object_name: string representing the name of the JS object to create 114 | :param transpose_arrays: bool to indicate whether or not the arrays in dict should be 115 | transposed before being written. 116 | :return: string representation to be written to JS file. 117 | """ 118 | 119 | #Efficient way to do this is to build a list of strings and join it at the end 120 | lines = list(); 121 | lines.append("var " + object_name + " = {};"); #Initialize empty JS object 122 | 123 | for key in py_dict: 124 | line_start = object_name + "['" + key + "'] = "; 125 | np_array = py_dict[key]; 126 | if(transpose_arrays): 127 | np_array = np_array.T; 128 | 129 | value = ndarray_to_JS(np_array); 130 | 131 | whole_line = line_start + value + ';'; 132 | lines.append(whole_line); 133 | 134 | output = '\n'.join(lines) + '\n'; 135 | 136 | return output; 137 | 138 | def string_list_to_JS(stringlist, object_name): 139 | """ 140 | Converts a list of strings into a javascript list of strings; 141 | :param stringlist: list of strings 142 | :param object_name: name of the JS object to be created 143 | :return: text that can be inserted into a js file 144 | """ 145 | 146 | line_start = object_name + " = "; 147 | js_array = "[" + ", ".join(["'"+ss+"'" for ss in stringlist]) + "]"; 148 | output = line_start + js_array + ";\n"; 149 | 150 | return output; 151 | 152 | def generate_js_data_file(filename, projections, sig_scores, sig_proj_matrix, sig_proj_matrix_p, projection_keys, signature_keys): 153 | """ 154 | 155 | :param filename: (String) name of file to write. Should end in .js 156 | :param projections: dictionary with key = (String) name of projection, value = (2xN_samples) numpy.ndarray 157 | :param sig_scores: dictionary with key= (String) name of signature, value = (N_samples) numpy.ndarray 158 | :param sig_proj_matrix: (N_Signatures x N_Projections) numpy.ndarray 159 | :param sig_proj_matrix_p: (N_Signatures x N_Projections) numpy.ndarray 160 | :param projection_keys: List of Strings. Projection names in order of sig_proj_matrix columns 161 | :param signature_keys: List of Strings. Signatures names in order of sig_prog_matrix rows 162 | :return: None 163 | """ 164 | 165 | with open(filename, 'w') as fout: 166 | fout.write(dict_to_JS(projections, "FP_Projections", transpose_arrays=True)); 167 | fout.write(dict_to_JS(sig_scores, "FP_SigScores")) 168 | fout.write("FP_SigProjMatrix = " + ndarray_to_JS(sig_proj_matrix) + ";\n"); 169 | fout.write("FP_SigProjMatrix_p = " + ndarray_to_JS(sig_proj_matrix_p) + ";\n"); 170 | fout.write(string_list_to_JS(projection_keys,"FP_ProjectionKeys")); 171 | fout.write(string_list_to_JS(signature_keys,"FP_SignatureKeys")); 172 | 173 | 174 | 175 | 176 | -------------------------------------------------------------------------------- /FastProject/Interactive.py: -------------------------------------------------------------------------------- 1 | """ 2 | Functions to facilitate interactive use of the library 3 | """ 4 | import numpy as np 5 | import pandas as pd 6 | from . import FileIO 7 | from . import Transforms 8 | from .DataTypes import ExpressionData 9 | from .Projections import perform_weighted_PCA 10 | from .Projections import permutation_wPCA as _permutation_wPCA 11 | from .Signatures import (generate_random_sigs, 12 | calculate_sig_scores as _calculate_sig_scores, 13 | sigs_vs_projections as _sigs_vs_projections, 14 | read_signatures_gmt) 15 | 16 | from .SigScoreMethods import SignatureScores 17 | 18 | 19 | __all__ = ["estimate_weights", "weighted_PCA", 20 | "permutation_wPCA", "generate_random_sigs", 21 | "calculate_sig_scores"] 22 | 23 | 24 | def estimate_weights(data, housekeeping_genes=None): 25 | """ 26 | Data is a pandas dataframe 27 | 28 | Returns: DataFrame of equivalent size with weights 29 | """ 30 | 31 | expressionMatrix = ExpressionData( 32 | data.values, 33 | list(data.index), 34 | list(data.columns) 35 | ) 36 | 37 | if housekeeping_genes is None: 38 | housekeeping_genes = FileIO.load_housekeeping_genes() 39 | 40 | p_nd = Transforms.estimate_non_detection(data) 41 | 42 | (fit_func, params) = Transforms.create_false_neg_map( 43 | data, p_nd, housekeeping_genes) 44 | 45 | weights = Transforms.compute_weights( 46 | fit_func, params, expressionMatrix, p_nd) 47 | 48 | weights = pd.DataFrame(weights, index=expressionMatrix.row_labels, 49 | columns=expressionMatrix.col_labels) 50 | 51 | return weights 52 | 53 | 54 | def weighted_PCA(data, weights, max_components=200): 55 | """ 56 | Wrapper around Projections.perform_weighted_PCA 57 | 58 | Paramters 59 | --------- 60 | 61 | data: pandas dataframe, genes x samples 62 | 63 | weights: pandas dataframe, genes x samples 64 | 65 | max_components: Number of components to compute 66 | - if max_components is smaller than either dimension, 67 | the results are truncated 68 | 69 | Returns 70 | ------- 71 | 72 | pc_data : (Num_Samples x Num_Components) pandas dataframe 73 | Data transformed using PCA 74 | Same sample labels as input 75 | 76 | e_val: 77 | eigenvalues 78 | 79 | e_vec: 80 | eigenvectors 81 | """ 82 | 83 | if (data.shape[0] != weights.shape[0] or 84 | data.shape[1] != weights.shape[1]): 85 | raise ValueError("Input data and weights must be same dimension") 86 | 87 | pc_data, e_val, e_vec = perform_weighted_PCA(data.values, 88 | weights.values, max_components) 89 | 90 | pc_data = pd.DataFrame(pc_data.T, index=data.columns) 91 | e_val = pd.Series(e_val, index=pc_data.columns) 92 | e_vec = pd.DataFrame(e_vec, index=data.index, columns=pc_data.columns) 93 | 94 | return pc_data, e_val, e_vec 95 | 96 | 97 | def permutation_wPCA(data, weights=None, max_components=50, 98 | p_threshold=0.05, verbose=False, debug=False): 99 | """ 100 | Computes weighted PCA on data. Returns only significant components. 101 | 102 | After performing PCA on the data matrix, this method then uses 103 | a permutation procedure based on Buja A and Eyuboglu N (1992) 104 | to asses components for significance. 105 | 106 | :param data: pandas.Dataframe of shape Num_Features x Num_Samples 107 | :param weights: pandas.DataFrame of shape Num_Features x Num_Samples 108 | :param max_components: int, Max components to calculate 109 | :param p_threshold: float, P-value to cutoff components at 110 | :param verbose: bool 111 | 112 | Returns 113 | ------- 114 | 115 | pc_data : (Num_Samples x Num_Components) pandas dataframe 116 | Data transformed using PCA 117 | Same sample labels as input 118 | 119 | e_val: 120 | eigenvalues 121 | 122 | e_vec: 123 | eigenvectors 124 | 125 | """ 126 | if weights is None: 127 | weights = pd.DataFrame( 128 | np.ones_like(data.values), 129 | index=data.index, 130 | columns=data.columns 131 | ) 132 | 133 | pc_data, e_val, e_vec = _permutation_wPCA(data.values, weights.values, max_components, 134 | p_threshold, verbose, debug=debug) 135 | 136 | pc_data = pd.DataFrame(pc_data.T, index=data.columns) 137 | e_val = pd.Series(e_val, index=pc_data.columns) 138 | e_vec = pd.DataFrame(e_vec, index=data.index, columns=pc_data.columns) 139 | 140 | return pc_data, e_val, e_vec 141 | 142 | 143 | def calculate_sig_scores(data, signatures, weights=None, method='weighted_avg', 144 | zero_locations=None, min_signature_genes=0): 145 | """ 146 | Interactive version of Signatures.calculate_sig_scores that 147 | uses DataFrames instead 148 | 149 | Parameters 150 | ---------- 151 | data: pandas.DataFrame 152 | Data matrix to use to calculate signature scores 153 | signatures: list of Signature 154 | Signatures of which scores are computed 155 | weights: pandas.DataFrame 156 | Weight matrix to use for weighted sig-score evaluation 157 | If no weights are provided, all ones are used 158 | method: str 159 | Which scoring method to use 160 | zero_locations: boolean numpy.ndarray 161 | Same size as data 162 | True where 'data' was originally zero 163 | Used for normalization methods that transform zeros to some other value 164 | min_signature_genes: numeric 165 | Signatures that match less that `min_signature_genes` are discarded 166 | 167 | Returns 168 | ------- 169 | num_genes: pandas.Series 170 | The number of genes that matched each signature in 171 | the expression matrix. Size is NUM_SIGNATURES 172 | scores: pandas.DataFrame 173 | Scores for each signature on each cell/sample 174 | Size is NUM_SAMPLES x NUM_SIGNATURES 175 | """ 176 | 177 | expressionMatrix = ExpressionData( 178 | data.values, 179 | list(data.index), 180 | list(data.columns) 181 | ) 182 | 183 | if weights is not None: 184 | expressionMatrix.weights = weights.values 185 | else: 186 | expressionMatrix.weights = np.ones_like(data.values) 187 | 188 | result_dict = _calculate_sig_scores( 189 | expressionMatrix, signatures, 190 | method=method, 191 | zero_locations=zero_locations, 192 | min_signature_genes=min_signature_genes) 193 | 194 | # Transform result_dict to dataframe 195 | 196 | all_series = [pd.Series(x.scores, index=x.sample_labels, name=name) 197 | for name, x in result_dict.items()] 198 | 199 | all_scores = pd.concat(all_series, axis=1) 200 | 201 | num_genes = [] 202 | names = [] 203 | for name, sigscores in result_dict.items(): 204 | names.append(name) 205 | num_genes.append(sigscores.numGenes) 206 | 207 | num_genes = pd.Series(num_genes, index=names) 208 | 209 | return all_scores, num_genes 210 | 211 | 212 | def sigs_vs_projection(projection, sig_scores, num_genes, 213 | random_sig_scores, random_num_genes, 214 | numerical_sig_scores=None, factor_sig_scores=None, 215 | NEIGHBORHOOD_SIZE=0.33): 216 | """ 217 | Interactive wrapper for Signatures.sigs_vs_projections 218 | 219 | Uses dataframes for inputs/outputs 220 | 221 | projection: pandas.DataFrame 222 | Projection coordinates per sample. Shape NUM_SAMPLES x 2 223 | 224 | sig_scores: pandas.DataFrame 225 | Signatures scores per sample. Shape NUM_SAMPLES x NUM_SIGNATURES 226 | 227 | num_genes: pandas.Series 228 | Number of genes that matched for each signature. 229 | Size NUM_SIGNATURES 230 | 231 | random_sig_scores: pandas.DataFrame 232 | Background Signatures scores per sample. 233 | Shape NUM_SAMPLES x NUM_RANDOM_SIGNATURES 234 | 235 | random_num_genes: pandas.Series 236 | Number of genes that matched for each random signature. 237 | Size NUM_RANDOM_SIGNATURES 238 | 239 | numerical_sig_scores: pandas.DataFrame 240 | Precomputed numerical scores per sample. 241 | These use a permutation test instead of background signatures 242 | Shape NUM_SAMPLES x NUM_NUMERICAL_SIGNATURES 243 | 244 | factor_sig_scores: pandas.DataFrame 245 | Precomputed factor levels per sample. 246 | These use a permutation test instead of background signatures 247 | Shape NUM_SAMPLES x NUM_FACTOR_SIGNATURES 248 | 249 | NEIGHBORHOOD_SIZE: float 250 | Size in the projection to use for the kernel 251 | 252 | Returns 253 | ------- 254 | consistences: pandas.DataFrame 255 | Consistency scores for each sig/projection pair 256 | Shape TOTAL_NUM_SIGNATURES x 1 257 | 258 | pvals: pandas.DataFrame 259 | P-vales for each sig/projection pair 260 | Shape TOTAL_NUM_SIGNATURES x 1 261 | 262 | """ 263 | 264 | # Ensure all dataframes are aligned and same size 265 | assert len(projection.index & sig_scores.index) == len(projection.index) 266 | sig_scores = sig_scores.loc[projection.index] 267 | 268 | assert (len(projection.index & random_sig_scores.index) == 269 | len(projection.index)) 270 | random_sig_scores = random_sig_scores.loc[projection.index] 271 | 272 | if numerical_sig_scores is not None: 273 | assert (len(projection.index & numerical_sig_scores.index) == 274 | len(projection.index)) 275 | numerical_sig_scores = numerical_sig_scores.loc[projection.index] 276 | 277 | if factor_sig_scores is not None: 278 | assert (len(projection.index & factor_sig_scores.index) == 279 | len(projection.index)) 280 | factor_sig_scores = factor_sig_scores.loc[projection.index] 281 | 282 | # Convert values 283 | projections = {'proj_name': projection.values.T} 284 | sig_scores_dict = {} 285 | 286 | for name, column in sig_scores.iteritems(): 287 | x = SignatureScores(column.values, name, column.index.tolist(), 288 | isFactor=False, isPrecomputed=False, 289 | numGenes=num_genes[name]) 290 | sig_scores_dict[name] = x 291 | 292 | if numerical_sig_scores is not None: 293 | for name, column in numerical_sig_scores.iteritems(): 294 | x = SignatureScores(column.values, name, column.index.tolist(), 295 | isFactor=False, isPrecomputed=True, 296 | numGenes=-1) 297 | sig_scores_dict[name] = x 298 | 299 | if factor_sig_scores is not None: 300 | for name, column in factor_sig_scores.iteritems(): 301 | x = SignatureScores(column.values.tolist(), name, 302 | column.index.tolist(), 303 | isFactor=True, isPrecomputed=True, 304 | numGenes=-1) 305 | sig_scores_dict[name] = x 306 | 307 | random_sig_scores_dict = {} 308 | for name, column in random_sig_scores.iteritems(): 309 | x = SignatureScores(column.values, name, column.index.tolist(), 310 | isFactor=False, isPrecomputed=False, 311 | numGenes=random_num_genes[name]) 312 | random_sig_scores_dict[name] = x 313 | 314 | result = _sigs_vs_projections(projections, sig_scores_dict, 315 | random_sig_scores_dict, 316 | NEIGHBORHOOD_SIZE=NEIGHBORHOOD_SIZE) 317 | 318 | row_labels, col_labels, sig_proj_matrix, sig_proj_matrix_p, random_sig_proj_matrix, random_sig_score_keys = result 319 | 320 | consistencies = pd.DataFrame(sig_proj_matrix, index=row_labels, 321 | columns=col_labels) 322 | 323 | null_consistencies = pd.DataFrame(random_sig_proj_matrix, 324 | index=random_sig_score_keys, 325 | columns=col_labels) 326 | 327 | pvals = pd.DataFrame(sig_proj_matrix_p, index=row_labels, 328 | columns=col_labels) 329 | 330 | return consistencies, pvals, null_consistencies 331 | -------------------------------------------------------------------------------- /FastProject/NormalizationMethods.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Methods to normalize data 3 | 4 | Different options to normalize the data before the signature 5 | values are calculated. 6 | 7 | Determined using the command-line argument: --sig_norm_method 8 | 9 | """ 10 | from __future__ import absolute_import, print_function, division; 11 | import numpy as np; 12 | from scipy.stats import rankdata; 13 | 14 | # Normalization methods 15 | 16 | 17 | def no_normalization(data): 18 | """ 19 | Does nothing. Returns original data 20 | """ 21 | return data; 22 | 23 | 24 | def col_normalization(data): 25 | """ 26 | Perform z-normalization on all columns 27 | """ 28 | data_mu = data.mean(axis=0, keepdims=True); 29 | data_sigma = data.std(axis=0, keepdims=True); 30 | data_sigma[data_sigma == 0] = 1.0; 31 | return (data - data_mu) / data_sigma; 32 | 33 | 34 | def row_normalization(data): 35 | """ 36 | Perform z-normalization on all rows 37 | """ 38 | data_mu = data.mean(axis=1, keepdims=True); 39 | data_sigma = data.std(axis=1, keepdims=True); 40 | data_sigma[data_sigma == 0] = 1.0; 41 | return (data - data_mu) / data_sigma; 42 | 43 | 44 | def row_and_col_normalization(data): 45 | """ 46 | Normalize rows, then columns 47 | """ 48 | data = row_normalization(data); 49 | data = col_normalization(data); 50 | return data; 51 | 52 | 53 | def col_rank_normalization(data): 54 | """ 55 | Create new version of data that has ranks (column-wise) instead of values 56 | """ 57 | rdata = np.zeros(data.shape); 58 | for i in range(data.shape[1]): 59 | rdata[:, i] = rankdata(data[:, i], method="min"); 60 | 61 | return rdata; 62 | -------------------------------------------------------------------------------- /FastProject/SigScoreMethods.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ Different ways to evlauate the signature score 3 | 4 | Each method should have the same signature so they can be swapped 5 | 6 | Right now, each takes in a wrapped data object and a signature object 7 | 8 | Specified by the command-line argument: --sig_score_method 9 | 10 | """ 11 | 12 | from __future__ import division, absolute_import 13 | import numpy as np 14 | from scipy.stats import rankdata 15 | from . import HtmlViewer 16 | 17 | 18 | def naive_eval_signature(self, signature, zeros, min_signature_genes): 19 | """ 20 | Naive eval signature just sums the columns * sign 21 | Equivalent to all weights = 1 22 | """ 23 | sig_vector = signature.sig_indices(self.row_labels); 24 | ii = np.nonzero(sig_vector)[0]; 25 | 26 | num_matched_genes = ii.size; 27 | 28 | if(num_matched_genes == 0): 29 | raise ValueError("No genes match signature"); 30 | 31 | if(num_matched_genes < min_signature_genes): 32 | raise ValueError("Too few genes match signature"); 33 | 34 | data = self.base[ii, :]; 35 | weights = np.ones(data.shape); 36 | sig_vector = sig_vector[ii, :]; 37 | 38 | pdata = data * sig_vector * weights; 39 | 40 | sig_scores = pdata.sum(axis=0); 41 | sig_scores /= np.sum(np.abs(sig_vector) * weights, axis=0); # Only normalize by weights in the signature 42 | 43 | sig_obj = SignatureScores(sig_scores, signature.name, 44 | self.col_labels, isFactor=False, 45 | isPrecomputed=False, 46 | numGenes=num_matched_genes) 47 | 48 | return sig_obj; 49 | 50 | 51 | def weighted_eval_signature(self, signature, zeros, min_signature_genes): 52 | """ 53 | Weighted average using weights in self.weights 54 | """ 55 | sig_vector = signature.sig_indices(self.row_labels); 56 | ii = np.nonzero(sig_vector)[0]; 57 | 58 | num_matched_genes = ii.size; 59 | 60 | if(num_matched_genes == 0): 61 | raise ValueError("No genes match signature"); 62 | 63 | if(num_matched_genes < min_signature_genes): 64 | raise ValueError("Too few genes match signature"); 65 | 66 | weights = self.weights[ii, :]; 67 | data = self.base[ii, :]; 68 | sig_vector = sig_vector[ii, :]; 69 | 70 | pdata = data * sig_vector * weights; 71 | 72 | sig_scores = pdata.sum(axis=0); 73 | norm_factor = np.sum(np.abs(sig_vector) * weights, axis=0); # Only normalize by weights in the signature 74 | norm_factor[norm_factor == 0] = 1.0; 75 | sig_scores /= norm_factor; 76 | 77 | sig_obj = SignatureScores(sig_scores, signature.name, 78 | self.col_labels, isFactor=False, 79 | isPrecomputed=False, 80 | numGenes=num_matched_genes) 81 | 82 | return sig_obj; 83 | 84 | 85 | def imputed_eval_signature(self, signature, zeros, min_signature_genes): 86 | """ 87 | Imputes likely values for zeros using the weight 88 | Weights represent p(not expressed | not detected) 89 | """ 90 | sig_vector = signature.sig_indices(self.row_labels) 91 | ii = np.nonzero(sig_vector)[0] 92 | 93 | num_matched_genes = ii.size 94 | 95 | if(num_matched_genes == 0): 96 | raise ValueError("No genes match signature") 97 | 98 | if(num_matched_genes < min_signature_genes): 99 | raise ValueError("Too few genes match signature") 100 | 101 | weights = self.weights[ii, :] 102 | data = self.base[ii, :] 103 | sig_vector = sig_vector[ii, :] 104 | zeros = zeros[ii, :] 105 | not_zeros = 1-zeros 106 | 107 | # impute values 108 | # weight represents p(not e | not d) 109 | mu = ((data * not_zeros).sum(axis=1, keepdims=True) / 110 | not_zeros.sum(axis=1, keepdims=True)) 111 | mu[np.isnan(mu)] = 0.0 # All zeros, mu is zero 112 | 113 | pe_nd = 1 - weights # Probability expressed | not detected 114 | pne_nd = weights 115 | 116 | sig_scores_real = (data * not_zeros * sig_vector).sum(axis=0) 117 | sig_scores_imputed = np.sum(pe_nd * mu * sig_vector * zeros + 118 | pne_nd * data * sig_vector * zeros, 119 | axis=0) 120 | sig_scores = sig_scores_real + sig_scores_imputed 121 | sig_scores /= data.shape[0] # Just divide number of genes 122 | 123 | sig_obj = SignatureScores(sig_scores, signature.name, 124 | self.col_labels, isFactor=False, 125 | isPrecomputed=False, 126 | numGenes=num_matched_genes) 127 | 128 | return sig_obj 129 | 130 | 131 | def nonzero_eval_signature(self, signature, zeros, min_signature_genes): 132 | """ 133 | Evalute the signature scores using only nonzero entries 134 | Sum columns and normalize by # of nonzero items 135 | """ 136 | sig_vector = signature.sig_indices(self.row_labels) 137 | ii = np.nonzero(sig_vector)[0] 138 | 139 | num_matched_genes = ii.size 140 | 141 | if(num_matched_genes == 0): 142 | raise ValueError("No genes match signature") 143 | 144 | if(num_matched_genes < min_signature_genes): 145 | raise ValueError("Too few genes match signature") 146 | 147 | data = self.base[ii, :] 148 | sig_vector = sig_vector[ii, :] 149 | zeros = zeros[ii, :] 150 | not_zeros = 1-zeros 151 | 152 | sig_scores_real = ((data * not_zeros) * sig_vector).sum(axis=0) 153 | norm_factor = not_zeros.sum(axis=0) 154 | norm_factor[norm_factor == 0] = 1.0 155 | sig_scores = sig_scores_real / norm_factor 156 | 157 | sig_obj = SignatureScores(sig_scores, signature.name, 158 | self.col_labels, isFactor=False, 159 | isPrecomputed=False, 160 | numGenes=num_matched_genes) 161 | 162 | return sig_obj 163 | 164 | 165 | class SignatureScores: 166 | """ 167 | Represents a Signature evaluated on a set of samples 168 | """ 169 | 170 | @property 171 | def ranks(self): 172 | if(self.isFactor): 173 | raise Exception("Factor signature scores have no rank") 174 | 175 | if(self._ranks is None): 176 | self._ranks = rankdata(self.scores, method="average") 177 | 178 | return self._ranks 179 | 180 | def __init__(self, scores, name, sample_labels, isFactor, isPrecomputed, numGenes): 181 | self.scores = scores 182 | self.name = name 183 | self.sample_labels = sample_labels 184 | self.isFactor = isFactor 185 | self.isPrecomputed = isPrecomputed 186 | self._ranks = None 187 | self.numGenes = numGenes 188 | 189 | def to_JSON(self): 190 | """ 191 | Construct a dictionary of certain parameters. 192 | Parse that to JSON using HTMLViewer and return 193 | :return: String with JSON representation of the SignatureScores instance 194 | """ 195 | 196 | out = dict({ 197 | "name": self.name, 198 | "scores": self.scores, 199 | "isFactor": self.isFactor, 200 | "isPrecomputed": self.isPrecomputed, 201 | }) 202 | 203 | if(not self.isFactor): 204 | out.update({"ranks": self.ranks}) 205 | 206 | return HtmlViewer.toJS(out) 207 | -------------------------------------------------------------------------------- /FastProject/SubSample.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ This module handles the splitting and joining of the data if sub-sampling is enabled 3 | 4 | This process is likely out of date - needs attention 5 | before re-enabling this feature. 6 | """ 7 | from __future__ import absolute_import, print_function, division; 8 | from .Utils import ProgressBar; 9 | from . import Signatures; 10 | from . import SigScoreMethods; 11 | from . import Projections; 12 | from . import Transforms; 13 | from .DataTypes import ExpressionData; 14 | from .Global import FP_Output, RANDOM_SEED; 15 | from scipy.spatial.distance import cdist; 16 | import numpy as np; 17 | 18 | def split_samples(data, subsample_size): 19 | np.random.seed(RANDOM_SEED); #Set seed so outputs are repeatable 20 | sub_ii = np.random.choice(data.shape[1], subsample_size, replace=False); 21 | holdouts_ii = np.setxor1d(np.arange(data.shape[1]), sub_ii); 22 | 23 | subset = data.subset_samples(sub_ii); 24 | holdouts = data.subset_samples(holdouts_ii); 25 | 26 | return holdouts, subset; 27 | 28 | def merge_samples(all_data, Models, sigs, prob_params, args): 29 | #Remove genes that aren't in the Expression data set 30 | 31 | edata = Models["Expression"]["Data"]; 32 | keep_genes = set(edata.row_labels); 33 | keep_genes_i = [i for i,gene in enumerate(all_data.row_labels) if gene in keep_genes]; 34 | edata_all = all_data.subset_genes(keep_genes_i); 35 | edata_all.filters = edata.filters; 36 | 37 | #First need to recompute prob info 38 | #TODO: Ensure order of genes is the same as the ordering in mu, st, etc 39 | #M Step of EM Algorithm 40 | if(not args["nomodel"]): 41 | FP_Output("Extending probability model to holdouts"); 42 | (mu_h, mu_l, st_h, Pi) = prob_params; 43 | zmat = edata_all.base; 44 | p_low = np.exp(-1*zmat/mu_l)/mu_l; 45 | 46 | p_high = np.exp(-1 * (zmat - mu_h)**2 / (2*st_h**2)) / st_h / np.sqrt(2*np.pi); 47 | 48 | p_low[~np.isfinite(p_low)] = 1e-5; 49 | p_low[p_low < 1e-5] = 1e-5; 50 | 51 | p_high[~np.isfinite(p_high)] = 1e-5; 52 | p_high[p_high<1e-5] = 1e-5; 53 | 54 | gamma = (Pi * p_high) / ( ((1-Pi)*p_low) + (Pi*p_high)); 55 | pdata_all = Transforms.make_monotonic(gamma, zmat); 56 | (fit_func, params) = Transforms.create_false_neg_map(edata_all, args["housekeeping"]); 57 | (pdata_all, fn_prob) = Transforms.correct_for_fn(pdata_all, mu_h, fit_func, params); 58 | fn_prob[edata_all > 0] = 0; 59 | 60 | edata_all.weights = 1-fn_prob; 61 | sample_passes_qc, sample_qc_scores = Transforms.quality_check(params); 62 | 63 | #Need to make sure we retain all the original labels 64 | sample_passes_labels = {label for label,val in zip(edata_all.col_labels, sample_passes_qc) if val}; 65 | sample_passes_labels = sample_passes_labels.union(edata.col_labels); 66 | sample_passes_qc = np.array([i for i,x in enumerate(edata_all.col_labels) if x in sample_passes_labels]); 67 | 68 | #If specified, remove items that did not pass qc check 69 | if(args["qc"]): 70 | edata_all = edata_all.subset_samples(sample_passes_qc); 71 | 72 | Transforms.z_normalize(edata_all); 73 | 74 | model = Models["Expression"]; 75 | sub_data = model["Data"]; 76 | all_data = edata_all; 77 | 78 | sub_data_cols = set(sub_data.col_labels); 79 | all_cols = set(all_data.col_labels); 80 | holdout_cols = all_cols.difference(sub_data_cols); 81 | holdout_indices = [i for i,x in enumerate(all_data.col_labels) if x in holdout_cols]; 82 | 83 | holdout_data = all_data.subset_samples(holdout_indices); 84 | model["Holdout_Data"] = holdout_data; 85 | 86 | #Merge in projections 87 | #Re calculate clusters 88 | for name, model in Models.items(): 89 | sub_data = model["Data"]; 90 | holdout_data = model["Holdout_Data"]; 91 | 92 | FP_Output("Calculating projection coordinates and cluster assignments for Holdouts -", name); 93 | 94 | for projData in model["projectionData"]: 95 | #Merge projection back in 96 | projData["projections"] = merge_projections(projData["projections"], sub_data, holdout_data); 97 | 98 | #Re-cluster 99 | clusters = Projections.define_clusters(projData["projections"]); 100 | projData["clusters"] = clusters; 101 | 102 | 103 | #Then merge back in data 104 | FP_Output("Merging holdout data matrices back into sub-sample"); 105 | if(not args["nomodel"]): 106 | model = Models["Probability"]; 107 | merge_pdata = model["Data"].merge_data(model["Holdout_Data"]); 108 | merge_edata = merge_pdata.expression_data; 109 | Models["Expression"]["Data"] = merge_edata; 110 | Models["Probability"]["Data"] = merge_pdata; 111 | else: 112 | model = Models["Expression"]; 113 | merge_edata = model["Data"].merge_data(model["Holdout_Data"]); 114 | Models["Expression"]["Data"] = merge_edata; 115 | 116 | 117 | #Some cleanup and fix the labels 118 | for name, model in Models.items(): 119 | model.pop("Holdout_Data"); 120 | model["sampleLabels"] = model["Data"].col_labels; 121 | 122 | 123 | #Merge sig scores back in by re-calculating them 124 | for name, model in Models.items(): 125 | data = model["Data"]; 126 | 127 | #Evaluate Signatures 128 | FP_Output("Calculating Signature Scores for Holdouts -", name); 129 | 130 | sig_scores_dict = dict(); 131 | old_sig_scores_dict = model["signatureScores"]; 132 | 133 | pbar = ProgressBar(len(old_sig_scores_dict)); 134 | for sig in sigs: 135 | if(sig.name in old_sig_scores_dict): 136 | try: 137 | sig_scores_dict[sig.name] = data.eval_signature(sig); 138 | except ValueError: #Only thrown when the signature has no genes in the data 139 | pass #Just discard the signature then 140 | pbar.update(); 141 | pbar.complete(); 142 | 143 | if(args["precomputed"]): 144 | for precomputed_file in args["precomputed"]: 145 | precomputed_sig_scores = Signatures.load_precomputed(precomputed_file, data.col_labels); 146 | sig_scores_dict.update(precomputed_sig_scores); 147 | 148 | #Adds in quality score as a pre-computed signature 149 | if(not args["nomodel"]): 150 | sig_scores_dict["FP_Quality"] = SigScoreMethods.SignatureScores(sample_qc_scores,"FP_Quality",data.col_labels,isFactor=False, isPrecomputed=True); 151 | 152 | model["signatureScores"] = sig_scores_dict; 153 | 154 | 155 | 156 | def merge_projections(sub_projections, sub_data, holdout_data): 157 | """ 158 | Finds approximate locations for held-out data in 2d projections 159 | 160 | :param sub_coordinates: 161 | :param sub_data: 162 | :param holdout_data: 163 | :return: 164 | """ 165 | 166 | all_projections = dict(); 167 | 168 | for key,p in sub_projections.items(): 169 | K=10; 170 | subsample_dist = cdist(holdout_data.T, sub_data.T, metric='euclidean'); 171 | #subsample_dist shape is not_sub_data N_Samples x sub_data N_Samples 172 | 173 | subsample_dist_arg = subsample_dist.argsort(axis=1); 174 | subsample_dist_arg = subsample_dist_arg[:, np.arange(K)]; #Take closest K 175 | 176 | x_coords = p[0,:][subsample_dist_arg]; 177 | y_coords = p[1,:][subsample_dist_arg]; 178 | 179 | #New X,Y coordinate is average of nearest neighbors 180 | holdout_x = np.mean(x_coords, axis=1); 181 | holdout_y = np.mean(y_coords, axis=1); 182 | 183 | NUM_SAMPLES = sub_data.shape[1] + holdout_data.shape[1]; 184 | all_data = np.zeros((2,NUM_SAMPLES)); 185 | all_data[0,:sub_data.shape[1]] = p[0,:]; 186 | all_data[1,:sub_data.shape[1]] = p[1,:]; 187 | all_data[0,sub_data.shape[1]:] = holdout_x; 188 | all_data[1,sub_data.shape[1]:] = holdout_y; 189 | all_projections[key] = all_data; 190 | 191 | return all_projections; 192 | 193 | 194 | -------------------------------------------------------------------------------- /FastProject/Tests/TestFiles/housekeeping.txt: -------------------------------------------------------------------------------- 1 | TSC22D2 2 | P2RY14 3 | GMPS 4 | GFM1 5 | PDCD10 6 | FAM198B 7 | ARFIP1 8 | ARHGEF11 9 | ARHGAP18 10 | ROBLD3 11 | 1500004A13RIK 12 | MTX1 13 | CKS1B 14 | 9330159F19RIK 15 | RPS27 16 | ILF2 17 | CENPW 18 | SNX27 19 | PIP5K1A 20 | LASS2 21 | MCL1 22 | ANP32E 23 | RNU1B2 24 | POLR3C 25 | PHGDH 26 | ATP1A1 27 | DENND2C 28 | SLC16A1 29 | WDR77 30 | HBXIP 31 | GNAI3 32 | CLCC1 33 | DPH5 34 | AGL 35 | FNBP1L 36 | ALPK1 37 | CCDC109B 38 | AIMP1 39 | MANBA 40 | AK002733 41 | PDLIM5 42 | AMD2 43 | GM10636 44 | FUBP1 45 | ACADM 46 | TMEM68 47 | FAM110B 48 | PLEKHF2 49 | FAM92A 50 | AK171183 51 | USP45 52 | MAP3K7 53 | RNGTT 54 | MOBKL2B 55 | APTX 56 | AQP7 57 | OSTM1 58 | GM7819 59 | TESK1 60 | ATG5 61 | EXOSC3 62 | XPA 63 | NR4A3 64 | SMC2 65 | ASCC3 66 | AI314180 67 | GOPC 68 | AKNA 69 | 3110001D03RIK 70 | ACER2 71 | D630037F22RIK 72 | PLAA 73 | TM2D1 74 | CCDC138 75 | USP24 76 | ANKRD57 77 | ZCCHC11 78 | AK148461 79 | PDZK1IP1 80 | RAD54L 81 | MMACHC 82 | PSAP 83 | 4632428N05RIK 84 | ERMAP 85 | SMAP2 86 | MACF1 87 | INPP5B 88 | FTL2 89 | TRAPPC3 90 | SFPQ 91 | PPA1 92 | HDAC1 93 | PTP4A2 94 | LAPTM5 95 | SNHG3 96 | SMPDL3B 97 | TMEM222 98 | ARID1A 99 | HK1 100 | D4WSU53E 101 | CNR2 102 | LUZP1 103 | USP48 104 | MUL1 105 | PQLC2 106 | PADI2 107 | FBLIM1 108 | PRDM2 109 | GM13051 110 | RPS19-PS3 111 | MAD2L2 112 | NMNAT1 113 | RERE 114 | ACOT7 115 | 2610002J02RIK 116 | MRPL20 117 | ISG15 118 | CYP51 119 | DMTF1 120 | RSBN1L 121 | RUFY2 122 | ENSMUSG00000073777 123 | FASTK 124 | GALNT11 125 | HERC4 126 | EPT1 127 | EMILIN1 128 | NRBP1 129 | PISD 130 | TACC3 131 | RNF4 132 | HDH 133 | CNO 134 | RAB28 135 | QDPR 136 | 1810013D10RIK 137 | TLR6 138 | PDS5A 139 | SLC30A9 140 | FRYL 141 | EXOC1 142 | POLR2B 143 | YTHDC1 144 | DCK 145 | THAP6 146 | SCARB2 147 | ANXA3 148 | HNRNPD 149 | MRPS18C 150 | NUDT9 151 | LRRC8D 152 | DR1 153 | AK019690 154 | ANKLE2 155 | CISD1 156 | PDCL3 157 | SELPLG 158 | GIT2 159 | OASL1 160 | DYNLL1 161 | RAB35 162 | AK158973 163 | OAS1B 164 | MAPKAPK5 165 | RAD9B 166 | MKIAA0787 167 | MLXIP 168 | ABCB9 169 | TMED2 170 | SCARB1 171 | SRSF8 172 | D1BWG0212E 173 | GATSL2 174 | WBSCR22 175 | POR 176 | PRKRIP1 177 | SRRT 178 | AGFG2 179 | ZKSCAN1 180 | FAM20C 181 | MAFK 182 | FOXK1 183 | FSCN1 184 | RAC1 185 | SMURF1 186 | ZFP655 187 | D5ERTD605E 188 | USPL1 189 | BC062895 190 | HEPACAM2 191 | RPA3 192 | TMEM168 193 | CAV2 194 | FAM3C 195 | SND1 196 | AHCYL2 197 | LSS 198 | AKR1B10 199 | MTPN 200 | LUC7L2 201 | BRAF 202 | ITGB2 203 | PDIA4 204 | TMEM176A 205 | CYCS 206 | 9430076C15RIK 207 | LSM5 208 | HERC6 209 | MKIAA1122 210 | IL12RB2 211 | VPS24 212 | 0610030E20RIK 213 | TGOLN2 214 | AK164315 215 | MRPL53 216 | DGUOK 217 | ZFML 218 | PCYOX1 219 | NFU1 220 | CNBP 221 | RUVBL1 222 | SLC41A3 223 | NR2C2 224 | ARL6IP5 225 | AK145340 226 | EDEM1 227 | TADA3 228 | SEC13 229 | TMCC1 230 | BMS1 231 | IL17RA 232 | APOBEC1 233 | C030046I01RIK 234 | LPCAT3 235 | LEPREL2 236 | MRPL51 237 | NDUFA9 238 | STK11 239 | RPS15 240 | CLEC2G 241 | CLEC7A 242 | CSDA 243 | CREBL2 244 | H2AFJ 245 | MGST1 246 | ABCC9 247 | ITPR2 248 | PPFIBP1 249 | AK134400 250 | MBOAT7 251 | LENG8 252 | RPL28 253 | VMN1R65 254 | ZFP329 255 | CRX 256 | ZC3H4 257 | CALM3 258 | SYMPK 259 | CD3EAP 260 | APOE 261 | DOHH 262 | 1500002O20RIK 263 | GSK3A 264 | B9D2 265 | BC024978 266 | C030039L03RIK 267 | ZFP36 268 | SIRT2 269 | MAP4K1 270 | BC027344 271 | TBCB 272 | PRODH2 273 | TXNRD1 274 | UBA2 275 | CEBPA 276 | AK081136 277 | NCK2 278 | AK184603 279 | PNKP 280 | RPL13A-PS1 281 | BAX 282 | CYTH2 283 | SAAL1 284 | TMEM86A 285 | NIPA2 286 | TRPM1 287 | CHSY1 288 | ARRDC4 289 | MRPS11 290 | ZFP710 291 | MAN2A2 292 | WHAMM 293 | MESDC1 294 | CTSC 295 | TMEM126A 296 | INTS4 297 | 2210018M11RIK 298 | POLD3 299 | FAM168A 300 | IL18BP 301 | TRIM6 302 | ILK 303 | AK172683 304 | TMEM41B 305 | EIF4G2 306 | COPB1 307 | ARL6IP1 308 | THUMPD1 309 | USP31 310 | TNRC6A 311 | SPNS1 312 | NUPR1 313 | INO80E 314 | SPN 315 | SRCAP 316 | BCKDK 317 | TIAL1 318 | NSMCE4A 319 | FAM53B 320 | 5830432E09RIK 321 | ZFP511 322 | BC024386 323 | GNPTAB 324 | EFCAB4A 325 | 2700078K21RIK 326 | NAP1L4 327 | PPFIA1 328 | 2900053A13RIK 329 | LRRC8E 330 | ABHD13 331 | TUBGCP3 332 | TMCO3 333 | MYOM2 334 | AK166824 335 | POLB 336 | GINS4 337 | DDHD2 338 | ZFP703 339 | AK052414 340 | LEPROTL1 341 | TUSC3 342 | PCM-1 343 | SLC25A4 344 | CDKN2AIP 345 | AK141659 346 | DDX60 347 | AK038827 348 | D130040H23RIK 349 | SF4 350 | PLXNC1 351 | FKBP8 352 | PDE4C 353 | HAUS8 354 | DDA1 355 | UNC13A 356 | AP1M1 357 | AK166041 358 | TMEM184C 359 | ABCE1 360 | ELMOD2 361 | DDX39 362 | AK037841 363 | ATP2B1 364 | TNPO2 365 | ITFG1 366 | ADCY7 367 | LPCAT2 368 | HERPUD1 369 | CCL17 370 | SLC38A7 371 | PDP2 372 | E2F4 373 | NUTF2 374 | SLC7A6OS 375 | NFAT5 376 | ATXN1L 377 | AARS 378 | ADAT1 379 | ATMIN 380 | COTL1 381 | E2F7 382 | AK209590 383 | CBFA2T3 384 | CDK10 385 | AK076318 386 | GNPAT 387 | IRF2BP2 388 | AASDHPPT 389 | TMEM123 390 | MTMR2 391 | 4931406C07RIK 392 | TMEM19 393 | ZFP266 394 | MRPL4 395 | CDKN2D 396 | LDLR 397 | PIGYL 398 | 7-Sep 399 | FLI1 400 | VCPIP1 401 | CCT2 402 | ZFP202 403 | SORL1 404 | DPAGT1 405 | CCDC84 406 | ATP5L 407 | PCSK7 408 | GM5617 409 | AU019823 410 | RDX 411 | TNFAIP8L3 412 | UBE2Q2 413 | HMG20A 414 | COX5A 415 | PML 416 | TLE3 417 | AAGAB 418 | 2010321M09RIK 419 | IRAK3 420 | SNX1 421 | RPS27L 422 | TBK1 423 | MYO1E 424 | NotAGene1 425 | NotAGene2 426 | NotAGene3 427 | -------------------------------------------------------------------------------- /FastProject/Tests/TestFiles/input_projection_RFormat_ok.txt: -------------------------------------------------------------------------------- 1 | "X" "Y" 2 | "Cell1" 0.924527185511 0.42532771365 3 | "Cell2" 0.480852788146 0.322262420197 4 | "Cell3" 0.271190884062 0.269978869876 5 | "Cell4" 0.649348900299 0.55101997885 6 | "Cell5" 0.480855339255 0.256109498879 7 | "Cell6" 0.989466044121 0.240790975978 8 | "Cell7" 0.249687399328 0.0633126380694 9 | "Cell8" 0.703935870481 0.828249322374 10 | "Cell9" 0.966181091242 0.960637550987 11 | "Cell10" 0.321391680442 0.837687121147 12 | "Cell11" 0.741162688617 0.766189056797 13 | "Cell12" 0.410862391148 0.724354020287 14 | "Cell13" 0.0830523835192 0.20979331846 15 | "Cell14" 0.789534903337 0.364840537957 16 | "Cell15" 0.267501797719 0.31449647973 17 | "Cell16" 0.145788186141 0.0813016591073 18 | "Cell17" 0.547968991293 0.745485837507 19 | "Cell18" 0.712970613478 0.00714841560045 20 | "Cell19" 0.910164232137 0.472036566736 21 | "Cell20" 0.715482689871 0.318981321379 22 | "Cell21" 0.397690872353 0.597185193682 23 | "Cell22" 0.42490877541 0.344402878297 24 | "Cell23" 0.879132423733 0.979023315986 25 | "Cell24" 0.33647325425 0.838780308927 26 | "Cell25" 0.762221253192 0.12570830314 27 | "Cell26" 0.259455501873 0.743227352208 28 | "Cell27" 0.822063529539 0.137417495029 29 | "Cell28" 0.863208341244 0.806395843415 30 | "Cell29" 0.14403630973 0.984446645546 31 | "Cell30" 0.0179417895508 0.16959923087 32 | "Cell31" 0.981164032469 0.705625967211 33 | "Cell32" 0.60606348008 0.968978372108 34 | "Cell33" 0.512175697965 0.664786218627 35 | "Cell34" 0.204908768988 0.645517924689 36 | "Cell35" 0.0276632815426 0.0862824988066 37 | "Cell36" 0.835968863729 0.571804773781 38 | "Cell37" 0.5678219983 0.429415342953 39 | "Cell38" 0.760488914111 0.0090888818767 40 | "Cell39" 0.15362819029 0.744383097087 41 | "Cell40" 0.0829598297928 0.465148701435 42 | "Cell41" 0.732377012194 0.356997629997 43 | "Cell42" 0.291002819494 0.694912541254 44 | "Cell43" 0.705031314759 0.44014852423 45 | "Cell44" 0.570060122121 0.577470290667 46 | "Cell45" 0.828873009015 0.369201726608 47 | "Cell46" 0.655041029469 0.649742102674 48 | "Cell47" 0.254317712109 0.524081800309 49 | "Cell48" 0.107850214319 0.552021040268 50 | "Cell49" 0.472895854467 0.822788155725 51 | "Cell50" 0.850415760725 0.806137344172 52 | "Cell51" 0.111074683548 0.0778938723521 53 | "Cell52" 0.38283288889 0.0886971845659 54 | "Cell53" 0.827736110722 0.222318248238 55 | "Cell54" 0.116459547019 0.464917517221 56 | "Cell55" 0.0259443546963 0.742584904307 57 | "Cell56" 0.337385966582 0.509549972341 58 | "Cell57" 0.827712255974 0.959431924582 59 | "Cell58" 0.521009233806 0.569480364403 60 | "Cell59" 0.250726373946 0.373274559656 61 | "Cell60" 0.0477638893547 0.0816771814089 62 | "Cell61" 0.056150861885 0.655139005778 63 | "Cell62" 0.946165489209 0.0817185662244 64 | "Cell63" 0.941604364558 0.552906649137 65 | "Cell64" 0.670048760766 0.95681322441 66 | "Cell65" 0.0106351496024 0.752957547817 67 | -------------------------------------------------------------------------------- /FastProject/Tests/TestFiles/input_projection_duplicate18and36_bad.txt: -------------------------------------------------------------------------------- 1 | Cell1 0.924527185511 0.42532771365 2 | Cell2 0.480852788146 0.322262420197 3 | Cell3 0.271190884062 0.269978869876 4 | Cell4 0.649348900299 0.55101997885 5 | Cell5 0.480855339255 0.256109498879 6 | Cell6 0.989466044121 0.240790975978 7 | Cell7 0.249687399328 0.0633126380694 8 | Cell8 0.703935870481 0.828249322374 9 | Cell9 0.966181091242 0.960637550987 10 | Cell10 0.321391680442 0.837687121147 11 | Cell11 0.741162688617 0.766189056797 12 | Cell12 0.410862391148 0.724354020287 13 | Cell13 0.0830523835192 0.20979331846 14 | Cell14 0.789534903337 0.364840537957 15 | Cell15 0.267501797719 0.31449647973 16 | Cell16 0.145788186141 0.0813016591073 17 | Cell17 0.547968991293 0.745485837507 18 | Cell18 0.712970613478 0.00714841560045 19 | Cell19 0.910164232137 0.472036566736 20 | Cell20 0.715482689871 0.318981321379 21 | Cell21 0.397690872353 0.597185193682 22 | Cell22 0.42490877541 0.344402878297 23 | Cell23 0.879132423733 0.979023315986 24 | Cell24 0.33647325425 0.838780308927 25 | Cell25 0.762221253192 0.12570830314 26 | Cell26 0.259455501873 0.743227352208 27 | Cell27 0.822063529539 0.137417495029 28 | Cell28 0.863208341244 0.806395843415 29 | Cell29 0.14403630973 0.984446645546 30 | Cell30 0.0179417895508 0.16959923087 31 | Cell31 0.981164032469 0.705625967211 32 | Cell32 0.60606348008 0.968978372108 33 | Cell33 0.512175697965 0.664786218627 34 | Cell34 0.204908768988 0.645517924689 35 | Cell35 0.0276632815426 0.0862824988066 36 | Cell36 0.835968863729 0.571804773781 37 | Cell37 0.5678219983 0.429415342953 38 | Cell38 0.760488914111 0.0090888818767 39 | Cell39 0.15362819029 0.744383097087 40 | Cell40 0.0829598297928 0.465148701435 41 | Cell41 0.732377012194 0.356997629997 42 | Cell42 0.291002819494 0.694912541254 43 | Cell43 0.705031314759 0.44014852423 44 | Cell44 0.570060122121 0.577470290667 45 | Cell45 0.828873009015 0.369201726608 46 | Cell46 0.655041029469 0.649742102674 47 | Cell18 0.712970613478 0.00714841560045 48 | Cell47 0.254317712109 0.524081800309 49 | Cell48 0.107850214319 0.552021040268 50 | Cell49 0.472895854467 0.822788155725 51 | Cell50 0.850415760725 0.806137344172 52 | Cell51 0.111074683548 0.0778938723521 53 | Cell52 0.38283288889 0.0886971845659 54 | Cell53 0.827736110722 0.222318248238 55 | Cell54 0.116459547019 0.464917517221 56 | Cell36 0.835968863729 0.571804773781 57 | Cell55 0.0259443546963 0.742584904307 58 | Cell56 0.337385966582 0.509549972341 59 | Cell57 0.827712255974 0.959431924582 60 | Cell58 0.521009233806 0.569480364403 61 | Cell59 0.250726373946 0.373274559656 62 | Cell60 0.0477638893547 0.0816771814089 63 | Cell61 0.056150861885 0.655139005778 64 | Cell62 0.946165489209 0.0817185662244 65 | Cell63 0.941604364558 0.552906649137 66 | Cell64 0.670048760766 0.95681322441 67 | Cell65 0.0106351496024 0.752957547817 68 | -------------------------------------------------------------------------------- /FastProject/Tests/TestFiles/input_projection_good.txt: -------------------------------------------------------------------------------- 1 | Cell1 0.924527185511 0.42532771365 2 | Cell2 0.480852788146 0.322262420197 3 | Cell3 0.271190884062 0.269978869876 4 | Cell4 0.649348900299 0.55101997885 5 | Cell5 0.480855339255 0.256109498879 6 | Cell6 0.989466044121 0.240790975978 7 | Cell7 0.249687399328 0.0633126380694 8 | Cell8 0.703935870481 0.828249322374 9 | Cell9 0.966181091242 0.960637550987 10 | Cell10 0.321391680442 0.837687121147 11 | Cell11 0.741162688617 0.766189056797 12 | Cell12 0.410862391148 0.724354020287 13 | Cell13 0.0830523835192 0.20979331846 14 | Cell14 0.789534903337 0.364840537957 15 | Cell15 0.267501797719 0.31449647973 16 | Cell16 0.145788186141 0.0813016591073 17 | Cell17 0.547968991293 0.745485837507 18 | Cell18 0.712970613478 0.00714841560045 19 | Cell19 0.910164232137 0.472036566736 20 | Cell20 0.715482689871 0.318981321379 21 | Cell21 0.397690872353 0.597185193682 22 | Cell22 0.42490877541 0.344402878297 23 | Cell23 0.879132423733 0.979023315986 24 | Cell24 0.33647325425 0.838780308927 25 | Cell25 0.762221253192 0.12570830314 26 | Cell26 0.259455501873 0.743227352208 27 | Cell27 0.822063529539 0.137417495029 28 | Cell28 0.863208341244 0.806395843415 29 | Cell29 0.14403630973 0.984446645546 30 | Cell30 0.0179417895508 0.16959923087 31 | Cell31 0.981164032469 0.705625967211 32 | Cell32 0.60606348008 0.968978372108 33 | Cell33 0.512175697965 0.664786218627 34 | Cell34 0.204908768988 0.645517924689 35 | Cell35 0.0276632815426 0.0862824988066 36 | Cell36 0.835968863729 0.571804773781 37 | Cell37 0.5678219983 0.429415342953 38 | Cell38 0.760488914111 0.0090888818767 39 | Cell39 0.15362819029 0.744383097087 40 | Cell40 0.0829598297928 0.465148701435 41 | Cell41 0.732377012194 0.356997629997 42 | Cell42 0.291002819494 0.694912541254 43 | Cell43 0.705031314759 0.44014852423 44 | Cell44 0.570060122121 0.577470290667 45 | Cell45 0.828873009015 0.369201726608 46 | Cell46 0.655041029469 0.649742102674 47 | Cell47 0.254317712109 0.524081800309 48 | Cell48 0.107850214319 0.552021040268 49 | Cell49 0.472895854467 0.822788155725 50 | Cell50 0.850415760725 0.806137344172 51 | Cell51 0.111074683548 0.0778938723521 52 | Cell52 0.38283288889 0.0886971845659 53 | Cell53 0.827736110722 0.222318248238 54 | Cell54 0.116459547019 0.464917517221 55 | Cell55 0.0259443546963 0.742584904307 56 | Cell56 0.337385966582 0.509549972341 57 | Cell57 0.827712255974 0.959431924582 58 | Cell58 0.521009233806 0.569480364403 59 | Cell59 0.250726373946 0.373274559656 60 | Cell60 0.0477638893547 0.0816771814089 61 | Cell61 0.056150861885 0.655139005778 62 | Cell62 0.946165489209 0.0817185662244 63 | Cell63 0.941604364558 0.552906649137 64 | Cell64 0.670048760766 0.95681322441 65 | Cell65 0.0106351496024 0.752957547817 66 | -------------------------------------------------------------------------------- /FastProject/Tests/TestFiles/input_projection_header_ok.txt: -------------------------------------------------------------------------------- 1 | 0 1 2 | Cell1 0.924527185511 0.42532771365 3 | Cell2 0.480852788146 0.322262420197 4 | Cell3 0.271190884062 0.269978869876 5 | Cell4 0.649348900299 0.55101997885 6 | Cell5 0.480855339255 0.256109498879 7 | Cell6 0.989466044121 0.240790975978 8 | Cell7 0.249687399328 0.0633126380694 9 | Cell8 0.703935870481 0.828249322374 10 | Cell9 0.966181091242 0.960637550987 11 | Cell10 0.321391680442 0.837687121147 12 | Cell11 0.741162688617 0.766189056797 13 | Cell12 0.410862391148 0.724354020287 14 | Cell13 0.0830523835192 0.20979331846 15 | Cell14 0.789534903337 0.364840537957 16 | Cell15 0.267501797719 0.31449647973 17 | Cell16 0.145788186141 0.0813016591073 18 | Cell17 0.547968991293 0.745485837507 19 | Cell18 0.712970613478 0.00714841560045 20 | Cell19 0.910164232137 0.472036566736 21 | Cell20 0.715482689871 0.318981321379 22 | Cell21 0.397690872353 0.597185193682 23 | Cell22 0.42490877541 0.344402878297 24 | Cell23 0.879132423733 0.979023315986 25 | Cell24 0.33647325425 0.838780308927 26 | Cell25 0.762221253192 0.12570830314 27 | Cell26 0.259455501873 0.743227352208 28 | Cell27 0.822063529539 0.137417495029 29 | Cell28 0.863208341244 0.806395843415 30 | Cell29 0.14403630973 0.984446645546 31 | Cell30 0.0179417895508 0.16959923087 32 | Cell31 0.981164032469 0.705625967211 33 | Cell32 0.60606348008 0.968978372108 34 | Cell33 0.512175697965 0.664786218627 35 | Cell34 0.204908768988 0.645517924689 36 | Cell35 0.0276632815426 0.0862824988066 37 | Cell36 0.835968863729 0.571804773781 38 | Cell37 0.5678219983 0.429415342953 39 | Cell38 0.760488914111 0.0090888818767 40 | Cell39 0.15362819029 0.744383097087 41 | Cell40 0.0829598297928 0.465148701435 42 | Cell41 0.732377012194 0.356997629997 43 | Cell42 0.291002819494 0.694912541254 44 | Cell43 0.705031314759 0.44014852423 45 | Cell44 0.570060122121 0.577470290667 46 | Cell45 0.828873009015 0.369201726608 47 | Cell46 0.655041029469 0.649742102674 48 | Cell47 0.254317712109 0.524081800309 49 | Cell48 0.107850214319 0.552021040268 50 | Cell49 0.472895854467 0.822788155725 51 | Cell50 0.850415760725 0.806137344172 52 | Cell51 0.111074683548 0.0778938723521 53 | Cell52 0.38283288889 0.0886971845659 54 | Cell53 0.827736110722 0.222318248238 55 | Cell54 0.116459547019 0.464917517221 56 | Cell55 0.0259443546963 0.742584904307 57 | Cell56 0.337385966582 0.509549972341 58 | Cell57 0.827712255974 0.959431924582 59 | Cell58 0.521009233806 0.569480364403 60 | Cell59 0.250726373946 0.373274559656 61 | Cell60 0.0477638893547 0.0816771814089 62 | Cell61 0.056150861885 0.655139005778 63 | Cell62 0.946165489209 0.0817185662244 64 | Cell63 0.941604364558 0.552906649137 65 | Cell64 0.670048760766 0.95681322441 66 | Cell65 0.0106351496024 0.752957547817 67 | -------------------------------------------------------------------------------- /FastProject/Tests/TestFiles/input_projection_header_ok2.txt: -------------------------------------------------------------------------------- 1 | 0 1 2 | Cell1 0.924527185511 0.42532771365 3 | Cell2 0.480852788146 0.322262420197 4 | Cell3 0.271190884062 0.269978869876 5 | Cell4 0.649348900299 0.55101997885 6 | Cell5 0.480855339255 0.256109498879 7 | Cell6 0.989466044121 0.240790975978 8 | Cell7 0.249687399328 0.0633126380694 9 | Cell8 0.703935870481 0.828249322374 10 | Cell9 0.966181091242 0.960637550987 11 | Cell10 0.321391680442 0.837687121147 12 | Cell11 0.741162688617 0.766189056797 13 | Cell12 0.410862391148 0.724354020287 14 | Cell13 0.0830523835192 0.20979331846 15 | Cell14 0.789534903337 0.364840537957 16 | Cell15 0.267501797719 0.31449647973 17 | Cell16 0.145788186141 0.0813016591073 18 | Cell17 0.547968991293 0.745485837507 19 | Cell18 0.712970613478 0.00714841560045 20 | Cell19 0.910164232137 0.472036566736 21 | Cell20 0.715482689871 0.318981321379 22 | Cell21 0.397690872353 0.597185193682 23 | Cell22 0.42490877541 0.344402878297 24 | Cell23 0.879132423733 0.979023315986 25 | Cell24 0.33647325425 0.838780308927 26 | Cell25 0.762221253192 0.12570830314 27 | Cell26 0.259455501873 0.743227352208 28 | Cell27 0.822063529539 0.137417495029 29 | Cell28 0.863208341244 0.806395843415 30 | Cell29 0.14403630973 0.984446645546 31 | Cell30 0.0179417895508 0.16959923087 32 | Cell31 0.981164032469 0.705625967211 33 | Cell32 0.60606348008 0.968978372108 34 | Cell33 0.512175697965 0.664786218627 35 | Cell34 0.204908768988 0.645517924689 36 | Cell35 0.0276632815426 0.0862824988066 37 | Cell36 0.835968863729 0.571804773781 38 | Cell37 0.5678219983 0.429415342953 39 | Cell38 0.760488914111 0.0090888818767 40 | Cell39 0.15362819029 0.744383097087 41 | Cell40 0.0829598297928 0.465148701435 42 | Cell41 0.732377012194 0.356997629997 43 | Cell42 0.291002819494 0.694912541254 44 | Cell43 0.705031314759 0.44014852423 45 | Cell44 0.570060122121 0.577470290667 46 | Cell45 0.828873009015 0.369201726608 47 | Cell46 0.655041029469 0.649742102674 48 | Cell47 0.254317712109 0.524081800309 49 | Cell48 0.107850214319 0.552021040268 50 | Cell49 0.472895854467 0.822788155725 51 | Cell50 0.850415760725 0.806137344172 52 | Cell51 0.111074683548 0.0778938723521 53 | Cell52 0.38283288889 0.0886971845659 54 | Cell53 0.827736110722 0.222318248238 55 | Cell54 0.116459547019 0.464917517221 56 | Cell55 0.0259443546963 0.742584904307 57 | Cell56 0.337385966582 0.509549972341 58 | Cell57 0.827712255974 0.959431924582 59 | Cell58 0.521009233806 0.569480364403 60 | Cell59 0.250726373946 0.373274559656 61 | Cell60 0.0477638893547 0.0816771814089 62 | Cell61 0.056150861885 0.655139005778 63 | Cell62 0.946165489209 0.0817185662244 64 | Cell63 0.941604364558 0.552906649137 65 | Cell64 0.670048760766 0.95681322441 66 | Cell65 0.0106351496024 0.752957547817 67 | -------------------------------------------------------------------------------- /FastProject/Tests/TestFiles/input_projection_missing10and20_bad.txt: -------------------------------------------------------------------------------- 1 | Cell1 0.924527185511 0.42532771365 2 | Cell2 0.480852788146 0.322262420197 3 | Cell3 0.271190884062 0.269978869876 4 | Cell4 0.649348900299 0.55101997885 5 | Cell5 0.480855339255 0.256109498879 6 | Cell6 0.989466044121 0.240790975978 7 | Cell7 0.249687399328 0.0633126380694 8 | Cell8 0.703935870481 0.828249322374 9 | Cell9 0.966181091242 0.960637550987 10 | Cell11 0.741162688617 0.766189056797 11 | Cell12 0.410862391148 0.724354020287 12 | Cell13 0.0830523835192 0.20979331846 13 | Cell14 0.789534903337 0.364840537957 14 | Cell15 0.267501797719 0.31449647973 15 | Cell16 0.145788186141 0.0813016591073 16 | Cell17 0.547968991293 0.745485837507 17 | Cell18 0.712970613478 0.00714841560045 18 | Cell19 0.910164232137 0.472036566736 19 | Cell21 0.397690872353 0.597185193682 20 | Cell22 0.42490877541 0.344402878297 21 | Cell23 0.879132423733 0.979023315986 22 | Cell24 0.33647325425 0.838780308927 23 | Cell25 0.762221253192 0.12570830314 24 | Cell26 0.259455501873 0.743227352208 25 | Cell27 0.822063529539 0.137417495029 26 | Cell28 0.863208341244 0.806395843415 27 | Cell29 0.14403630973 0.984446645546 28 | Cell30 0.0179417895508 0.16959923087 29 | Cell31 0.981164032469 0.705625967211 30 | Cell32 0.60606348008 0.968978372108 31 | Cell33 0.512175697965 0.664786218627 32 | Cell34 0.204908768988 0.645517924689 33 | Cell35 0.0276632815426 0.0862824988066 34 | Cell36 0.835968863729 0.571804773781 35 | Cell37 0.5678219983 0.429415342953 36 | Cell38 0.760488914111 0.0090888818767 37 | Cell39 0.15362819029 0.744383097087 38 | Cell40 0.0829598297928 0.465148701435 39 | Cell41 0.732377012194 0.356997629997 40 | Cell42 0.291002819494 0.694912541254 41 | Cell43 0.705031314759 0.44014852423 42 | Cell44 0.570060122121 0.577470290667 43 | Cell45 0.828873009015 0.369201726608 44 | Cell46 0.655041029469 0.649742102674 45 | Cell47 0.254317712109 0.524081800309 46 | Cell48 0.107850214319 0.552021040268 47 | Cell49 0.472895854467 0.822788155725 48 | Cell50 0.850415760725 0.806137344172 49 | Cell51 0.111074683548 0.0778938723521 50 | Cell52 0.38283288889 0.0886971845659 51 | Cell53 0.827736110722 0.222318248238 52 | Cell54 0.116459547019 0.464917517221 53 | Cell55 0.0259443546963 0.742584904307 54 | Cell56 0.337385966582 0.509549972341 55 | Cell57 0.827712255974 0.959431924582 56 | Cell58 0.521009233806 0.569480364403 57 | Cell59 0.250726373946 0.373274559656 58 | Cell60 0.0477638893547 0.0816771814089 59 | Cell61 0.056150861885 0.655139005778 60 | Cell62 0.946165489209 0.0817185662244 61 | Cell63 0.941604364558 0.552906649137 62 | Cell64 0.670048760766 0.95681322441 63 | Cell65 0.0106351496024 0.752957547817 64 | -------------------------------------------------------------------------------- /FastProject/Tests/TestFiles/input_projection_nan26and27_bad.txt: -------------------------------------------------------------------------------- 1 | Cell1 0.924527185511 0.42532771365 2 | Cell2 0.480852788146 0.322262420197 3 | Cell3 0.271190884062 0.269978869876 4 | Cell4 0.649348900299 0.55101997885 5 | Cell5 0.480855339255 0.256109498879 6 | Cell6 0.989466044121 0.240790975978 7 | Cell7 0.249687399328 0.0633126380694 8 | Cell8 0.703935870481 0.828249322374 9 | Cell9 0.966181091242 0.960637550987 10 | Cell10 0.321391680442 0.837687121147 11 | Cell11 0.741162688617 0.766189056797 12 | Cell12 0.410862391148 0.724354020287 13 | Cell13 0.0830523835192 0.20979331846 14 | Cell14 0.789534903337 0.364840537957 15 | Cell15 0.267501797719 0.31449647973 16 | Cell16 0.145788186141 0.0813016591073 17 | Cell17 0.547968991293 0.745485837507 18 | Cell18 0.712970613478 0.00714841560045 19 | Cell19 0.910164232137 0.472036566736 20 | Cell20 0.715482689871 0.318981321379 21 | Cell21 0.397690872353 0.597185193682 22 | Cell22 0.42490877541 0.344402878297 23 | Cell23 0.879132423733 0.979023315986 24 | Cell24 0.33647325425 0.838780308927 25 | Cell25 0.762221253192 0.12570830314 26 | Cell26 nan 0.743227352208 27 | Cell27 nan 0.137417495029 28 | Cell28 0.863208341244 0.806395843415 29 | Cell29 0.14403630973 0.984446645546 30 | Cell30 0.0179417895508 0.16959923087 31 | Cell31 0.981164032469 0.705625967211 32 | Cell32 0.60606348008 0.968978372108 33 | Cell33 0.512175697965 0.664786218627 34 | Cell34 0.204908768988 0.645517924689 35 | Cell35 0.0276632815426 0.0862824988066 36 | Cell36 0.835968863729 0.571804773781 37 | Cell37 0.5678219983 0.429415342953 38 | Cell38 0.760488914111 0.0090888818767 39 | Cell39 0.15362819029 0.744383097087 40 | Cell40 0.0829598297928 0.465148701435 41 | Cell41 0.732377012194 0.356997629997 42 | Cell42 0.291002819494 0.694912541254 43 | Cell43 0.705031314759 0.44014852423 44 | Cell44 0.570060122121 0.577470290667 45 | Cell45 0.828873009015 0.369201726608 46 | Cell46 0.655041029469 0.649742102674 47 | Cell47 0.254317712109 0.524081800309 48 | Cell48 0.107850214319 0.552021040268 49 | Cell49 0.472895854467 0.822788155725 50 | Cell50 0.850415760725 0.806137344172 51 | Cell51 0.111074683548 0.0778938723521 52 | Cell52 0.38283288889 0.0886971845659 53 | Cell53 0.827736110722 0.222318248238 54 | Cell54 0.116459547019 0.464917517221 55 | Cell55 0.0259443546963 0.742584904307 56 | Cell56 0.337385966582 0.509549972341 57 | Cell57 0.827712255974 0.959431924582 58 | Cell58 0.521009233806 0.569480364403 59 | Cell59 0.250726373946 0.373274559656 60 | Cell60 0.0477638893547 0.0816771814089 61 | Cell61 0.056150861885 0.655139005778 62 | Cell62 0.946165489209 0.0817185662244 63 | Cell63 0.941604364558 0.552906649137 64 | Cell64 0.670048760766 0.95681322441 65 | Cell65 0.0106351496024 0.752957547817 66 | -------------------------------------------------------------------------------- /FastProject/Tests/TestFiles/input_projection_parseError26and28_bad.txt: -------------------------------------------------------------------------------- 1 | Cell1 0.924527185511 0.42532771365 2 | Cell2 0.480852788146 0.322262420197 3 | Cell3 0.271190884062 0.269978869876 4 | Cell4 0.649348900299 0.55101997885 5 | Cell5 0.480855339255 0.256109498879 6 | Cell6 0.989466044121 0.240790975978 7 | Cell7 0.249687399328 0.0633126380694 8 | Cell8 0.703935870481 0.828249322374 9 | Cell9 0.966181091242 0.960637550987 10 | Cell10 0.321391680442 0.837687121147 11 | Cell11 0.741162688617 0.766189056797 12 | Cell12 0.410862391148 0.724354020287 13 | Cell13 0.0830523835192 0.20979331846 14 | Cell14 0.789534903337 0.364840537957 15 | Cell15 0.267501797719 0.31449647973 16 | Cell16 0.145788186141 0.0813016591073 17 | Cell17 0.547968991293 0.745485837507 18 | Cell18 0.712970613478 0.00714841560045 19 | Cell19 0.910164232137 0.472036566736 20 | Cell20 0.715482689871 0.318981321379 21 | Cell21 0.397690872353 0.597185193682 22 | Cell22 0.42490877541 0.344402878297 23 | Cell23 0.879132423733 0.979023315986 24 | Cell24 0.33647325425 0.838780308927 25 | Cell25 0.762221253192 0.12570830314 26 | Cell26 popsicle 0.743227352208 27 | Cell27 0.822063529539 0.137417495029 28 | Cell28 0.86320q341244 0.806395843415 29 | Cell29 0.14403630973 0.984446645546 30 | Cell30 0.0179417895508 0.16959923087 31 | Cell31 0.981164032469 0.705625967211 32 | Cell32 0.60606348008 0.968978372108 33 | Cell33 0.512175697965 0.664786218627 34 | Cell34 0.204908768988 0.645517924689 35 | Cell35 0.0276632815426 0.0862824988066 36 | Cell36 0.835968863729 0.571804773781 37 | Cell37 0.5678219983 0.429415342953 38 | Cell38 0.760488914111 0.0090888818767 39 | Cell39 0.15362819029 0.744383097087 40 | Cell40 0.0829598297928 0.465148701435 41 | Cell41 0.732377012194 0.356997629997 42 | Cell42 0.291002819494 0.694912541254 43 | Cell43 0.705031314759 0.44014852423 44 | Cell44 0.570060122121 0.577470290667 45 | Cell45 0.828873009015 0.369201726608 46 | Cell46 0.655041029469 0.649742102674 47 | Cell47 0.254317712109 0.524081800309 48 | Cell48 0.107850214319 0.552021040268 49 | Cell49 0.472895854467 0.822788155725 50 | Cell50 0.850415760725 0.806137344172 51 | Cell51 0.111074683548 0.0778938723521 52 | Cell52 0.38283288889 0.0886971845659 53 | Cell53 0.827736110722 0.222318248238 54 | Cell54 0.116459547019 0.464917517221 55 | Cell55 0.0259443546963 0.742584904307 56 | Cell56 0.337385966582 0.509549972341 57 | Cell57 0.827712255974 0.959431924582 58 | Cell58 0.521009233806 0.569480364403 59 | Cell59 0.250726373946 0.373274559656 60 | Cell60 0.0477638893547 0.0816771814089 61 | Cell61 0.056150861885 0.655139005778 62 | Cell62 0.946165489209 0.0817185662244 63 | Cell63 0.941604364558 0.552906649137 64 | Cell64 0.670048760766 0.95681322441 65 | Cell65 0.0106351496024 0.752957547817 66 | -------------------------------------------------------------------------------- /FastProject/Tests/TestFiles/precomputed_sigs.txt: -------------------------------------------------------------------------------- 1 | Type Cell1 Cell2 Cell3 Cell4 Cell5 Cell6 Cell7 Cell8 Cell9 Cell10 Cell11 Cell12 Cell13 Cell14 Cell15 Cell16 Cell17 Cell18 Cell19 Cell20 Cell21 Cell22 Cell23 Cell24 Cell25 Cell26 Cell27 Cell28 Cell29 Cell30 Cell31 Cell32 Cell33 Cell34 Cell35 Cell36 Cell37 Cell38 Cell39 Cell40 Cell41 Cell42 Cell43 Cell44 Cell45 Cell46 Cell47 Cell48 Cell49 Cell50 Cell51 Cell52 Cell53 Cell54 Cell55 Cell56 Cell57 Cell58 Cell59 Cell60 Cell61 Cell62 Cell63 Cell64 Cell65 2 | TIME_PRECOMPUTED Numerical 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 4 4 4 4 4 4 4 4 4 4 6 6 6 6 6 6 6 6 6 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 3 | Stimulus Factor LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC PAM LPS PIC 4 | -------------------------------------------------------------------------------- /FastProject/Tests/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'David' 2 | -------------------------------------------------------------------------------- /FastProject/Tests/full_battery.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, division, absolute_import 2 | import os; 3 | import logging; 4 | from ..Pipelines import Analysis; 5 | from ..CLI import loadFilesFromDisk, createOutputDirectories; 6 | from ..FileIO import saveResultstoDisk; 7 | 8 | this_directory = os.path.dirname(os.path.abspath(__file__)); 9 | TEST_FILE_DIR = os.path.join(this_directory, "TestFiles"); 10 | 11 | 12 | def get_default_args(): 13 | args = { 14 | "data_file": os.path.join(TEST_FILE_DIR, "smallData.txt"), 15 | "signatures": [os.path.join(TEST_FILE_DIR, "sigsSmall.gmt")], 16 | "precomputed": [os.path.join(TEST_FILE_DIR, "precomputed_sigs.txt")], 17 | "housekeeping": "", 18 | "output": "", 19 | "nofilter": False, 20 | "nomodel": False, 21 | "pca_filter": False, 22 | "qc": False, 23 | "subsample_size": None, 24 | "min_signature_genes": 5, 25 | "projections": [], 26 | "weights": None, 27 | "threshold": None, 28 | "all_sigs": False, 29 | "debug": False, 30 | "lean": False, 31 | "sig_norm_method": "znorm_rows", 32 | "sig_score_method": "weighted_avg"} 33 | 34 | return args; 35 | 36 | 37 | def test(func): 38 | 39 | def test_func(): 40 | args = func() 41 | try: 42 | run_test(args); 43 | except: 44 | with open('errors.log', 'a') as fout: 45 | fout.write("Errors Found During Test: " + func.__name__ + "\n"); 46 | print("Errors Found During Test: " + func.__name__ + "\n"); 47 | fout.write(" ARG DUMP:\n"); 48 | for key in args: 49 | fout.write(" " + key + ": " + str(args[key]) + "\n"); 50 | 51 | import traceback 52 | fout.write(" EXCEPTION:") 53 | for line in traceback.format_exc().splitlines(): 54 | fout.write(" " + line + "\n"); 55 | else: 56 | with open('errors.log', 'a') as fout: 57 | fout.write("Test Passed: " + func.__name__ + "\n"); 58 | print("Test Passed: " + func.__name__ + "\n"); 59 | 60 | return test_func; 61 | 62 | 63 | def run_test(args): 64 | 65 | dir_name = createOutputDirectories(args); # Needs to be created first so logging can write here 66 | 67 | (expressionMatrix, signatures, precomputed_signatures, 68 | housekeeping_genes, input_projections, 69 | input_weights) = loadFilesFromDisk(args); 70 | 71 | models, qc_info = Analysis(expressionMatrix, signatures, precomputed_signatures, 72 | housekeeping_genes, input_projections, input_weights, args); 73 | 74 | saveResultstoDisk(models, signatures, qc_info, dir_name); 75 | 76 | # Cleanup 77 | import shutil 78 | import time 79 | 80 | # Close logger 81 | logger = logging.getLogger("FastProject") 82 | for handler in logger.handlers: 83 | handler.close() 84 | logger.removeHandler(handler) 85 | 86 | for x in range(10): # Solution to Dropbox locking files. 87 | try: 88 | shutil.rmtree(dir_name); 89 | except: 90 | pass; 91 | else: 92 | break; 93 | time.sleep(0.1); 94 | 95 | 96 | @test 97 | def test_simple(): 98 | args = get_default_args(); 99 | 100 | return args; 101 | 102 | 103 | @test 104 | def test_hk(): 105 | args = get_default_args(); 106 | 107 | args["housekeeping"] = os.path.join(TEST_FILE_DIR, "housekeeping.txt"); 108 | 109 | return args; 110 | 111 | 112 | @test 113 | def test_text_sigs(): 114 | args = get_default_args(); 115 | 116 | args["signatures"] = [os.path.join(TEST_FILE_DIR, "sigsA.txt")]; 117 | 118 | return args; 119 | 120 | 121 | @test 122 | def test_text_sigsB(): 123 | args = get_default_args(); 124 | 125 | args["signatures"] = [os.path.join(TEST_FILE_DIR, "sigsB.txt")]; 126 | 127 | return args; 128 | 129 | 130 | @test 131 | def test_multi_sigs(): 132 | args = get_default_args(); 133 | 134 | args["signatures"] = [os.path.join(TEST_FILE_DIR, "sigsA.txt"), 135 | os.path.join(TEST_FILE_DIR, "sigsSmall.gmt")]; 136 | 137 | return args; 138 | 139 | 140 | @test 141 | def test_no_precomputed(): 142 | args = get_default_args(); 143 | 144 | args["precomputed"] = []; 145 | 146 | return args; 147 | 148 | 149 | @test 150 | def test_no_sigs(): 151 | args = get_default_args(); 152 | 153 | args["signatures"] = []; 154 | 155 | return args; 156 | 157 | 158 | @test 159 | def test_qc(): 160 | args = get_default_args(); 161 | 162 | args["qc"] = True 163 | 164 | return args; 165 | 166 | 167 | @test 168 | def test_nofilter(): 169 | args = get_default_args(); 170 | 171 | args["nofilter"] = True; 172 | 173 | return args; 174 | 175 | 176 | @test 177 | def test_nomodel(): 178 | args = get_default_args(); 179 | 180 | args["nomodel"] = True; 181 | 182 | return args; 183 | 184 | 185 | @test 186 | def test_pca_filter(): 187 | args = get_default_args(); 188 | 189 | args["pca_filter"] = True 190 | 191 | return args; 192 | 193 | 194 | @test 195 | def test_weights(): 196 | args = get_default_args(); 197 | 198 | args["weights"] = os.path.join(TEST_FILE_DIR, "weights.txt"); 199 | 200 | return args; 201 | 202 | 203 | @test 204 | def test_projections(): 205 | args = get_default_args(); 206 | 207 | args["projections"] = [os.path.join(TEST_FILE_DIR, "input_projection_good.txt")]; 208 | 209 | return args; 210 | 211 | 212 | @test 213 | def test_many_projections(): 214 | args = get_default_args(); 215 | 216 | projection_files = ["input_projection_good.txt", 217 | "input_projection_RFormat_ok.txt", 218 | "input_projection_header_ok.txt", 219 | "input_projection_header_ok2.txt"]; 220 | 221 | args["projections"] = [os.path.join(TEST_FILE_DIR, x) for x in projection_files]; 222 | 223 | return args; 224 | 225 | 226 | @test 227 | def test_threshold(): 228 | args = get_default_args(); 229 | 230 | args["threshold"] = 10; 231 | 232 | return args; 233 | 234 | 235 | @test 236 | def test_all_sigs(): 237 | args = get_default_args(); 238 | 239 | args["all_sigs"] = True; 240 | 241 | return args; 242 | 243 | 244 | @test 245 | def test_debug(): 246 | args = get_default_args(); 247 | 248 | args["debug"] = True; 249 | 250 | return args; 251 | 252 | 253 | @test 254 | def test_lean(): 255 | args = get_default_args(); 256 | 257 | args["lean"] = True; 258 | 259 | return args; 260 | 261 | 262 | @test 263 | def test_sig_norm_method_none(): 264 | args = get_default_args(); 265 | 266 | args["sig_norm_method"] = "none"; 267 | 268 | return args; 269 | 270 | 271 | @test 272 | def test_sig_norm_method_znorm_columns(): 273 | args = get_default_args(); 274 | 275 | args["sig_norm_method"] = "znorm_columns"; 276 | 277 | return args; 278 | 279 | 280 | @test 281 | def test_sig_norm_method_znorm_rows(): 282 | args = get_default_args(); 283 | 284 | args["sig_norm_method"] = "znorm_rows"; 285 | 286 | return args; 287 | 288 | 289 | @test 290 | def test_sig_norm_method_znorm_rows_then_columns(): 291 | args = get_default_args(); 292 | 293 | args["sig_norm_method"] = "znorm_rows_then_columns"; 294 | 295 | return args; 296 | 297 | 298 | @test 299 | def test_sig_norm_method_rank_norm_columns(): 300 | args = get_default_args(); 301 | 302 | args["sig_norm_method"] = "rank_norm_columns"; 303 | 304 | return args; 305 | 306 | 307 | @test 308 | def test_sig_score_method_naive(): 309 | args = get_default_args(); 310 | 311 | args["sig_score_method"] = "naive"; 312 | 313 | return args; 314 | 315 | 316 | @test 317 | def test_sig_score_method_weighted_avg(): 318 | args = get_default_args(); 319 | 320 | args["sig_score_method"] = "weighted_avg"; 321 | 322 | return args; 323 | 324 | 325 | @test 326 | def test_sig_score_method_imputed(): 327 | args = get_default_args(); 328 | 329 | args["sig_score_method"] = "imputed"; 330 | 331 | return args; 332 | 333 | 334 | @test 335 | def test_sig_score_method_only_nonzero(): 336 | args = get_default_args(); 337 | 338 | args["sig_score_method"] = "only_nonzero"; 339 | 340 | return args; 341 | 342 | 343 | @test 344 | def test_output(): 345 | args = get_default_args(); 346 | 347 | args["output"] = "outfolder"; 348 | 349 | return args; 350 | 351 | 352 | def run_all(): 353 | 354 | test_simple(); 355 | test_hk(); 356 | test_text_sigs(); 357 | test_text_sigsB(); 358 | test_multi_sigs(); 359 | test_no_precomputed(); 360 | test_no_sigs(); 361 | test_qc(); 362 | test_nofilter(); 363 | test_nomodel(); 364 | test_pca_filter(); 365 | test_weights(); 366 | test_projections(); 367 | test_many_projections(); 368 | test_threshold(); 369 | test_all_sigs(); 370 | test_debug(); 371 | test_lean(); 372 | test_sig_norm_method_none(); 373 | test_sig_norm_method_znorm_columns(); 374 | test_sig_norm_method_znorm_rows(); 375 | test_sig_norm_method_znorm_rows_then_columns(); 376 | test_sig_norm_method_rank_norm_columns(); 377 | test_sig_score_method_naive(); 378 | test_sig_score_method_weighted_avg(); 379 | test_sig_score_method_imputed(); 380 | test_sig_score_method_only_nonzero(); 381 | test_output(); 382 | 383 | 384 | -------------------------------------------------------------------------------- /FastProject/Tests/test_JSON.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test conversions of FastProject datatypes to/from JSON 3 | """ 4 | import numpy as np 5 | import unittest 6 | from ..DataTypes import ExpressionData 7 | from ..Signatures import Signature 8 | from ..SigScoreMethods import SignatureScores 9 | from .. import jsonIO 10 | 11 | SAMPLE_EXPRESSION_DATA_JSON = """ 12 | {"col_labels": ["sample1", "sample2", "sample3"], 13 | "row_labels": ["feature1", "feature2", "feature3"], 14 | "data": [[1.2302, 3.43, 2.34], 15 | [3.23, 5.93, 1.03], 16 | [4.12, 9.23, 1.00]] 17 | } 18 | """ 19 | 20 | SAMPLE_SIGNATURE_JSON = """ 21 | [{ 22 | "sig_dict": {"gene1": 1, "gene2": -1, "gene3": 1}, 23 | "signed": true, 24 | "source": "mysignaturefile.gmt", 25 | "name": "sample signature" 26 | }]""" 27 | 28 | SAMPLE_PRECOMPUTED_SIGNATURE_JSON = """ 29 | {"sample precomputed": { 30 | "scores": [3.42, 5.04, 5.00], 31 | "name": "sample precomputed", 32 | "sample_labels": ["sample1", "sample2", "sample3"], 33 | "isFactor": false, 34 | "isPrecomputed": true, 35 | "numGenes": 0 36 | } 37 | }""" 38 | 39 | SAMPLE_PRECOMPUTED_SIGNATURE2_JSON = """ 40 | {"sample precomputed": { 41 | "scores": ["level1", "level2", "level1"], 42 | "name": "sample precomputed", 43 | "sample_labels": ["sample1", "sample2", "sample3"], 44 | "isFactor": true, 45 | "isPrecomputed": true, 46 | "numGenes": 0 47 | } 48 | }""" 49 | 50 | 51 | class TestJsonIO(unittest.TestCase): 52 | 53 | def testExpressionData_roundTrip(self): 54 | """ 55 | Test roundtrip conversion 56 | """ 57 | 58 | data = np.random.randint(0, 100, size=(3, 3)) / 100.0; 59 | row_labels = ['a', 'b', 'c']; 60 | col_labels = ['d', 'e', 'f']; 61 | 62 | testData = ExpressionData(data, 63 | row_labels=row_labels, 64 | col_labels=col_labels); 65 | 66 | json_str = jsonIO.expressionData_to_JSON(testData); 67 | 68 | testData2 = jsonIO.JSON_to_ExpressionData(json_str); 69 | 70 | data2 = testData2.base; 71 | row_labels2 = testData2.row_labels; 72 | col_labels2 = testData2.col_labels; 73 | 74 | for rl1, rl2 in zip(row_labels, row_labels2): 75 | self.assertEqual(rl1, rl2) 76 | 77 | for cl1, cl2 in zip(col_labels, col_labels2): 78 | self.assertEqual(cl1, cl2) 79 | 80 | self.assertEqual(data.shape[0], data2.shape[0]); 81 | self.assertEqual(data.shape[1], data2.shape[1]); 82 | 83 | for i in range(data.shape[0]): 84 | for j in range(data.shape[1]): 85 | self.assertEqual(data[i, j], data2[i, j]); 86 | 87 | def testJSON_to_Signature_roundTrip(self): 88 | """ 89 | Test roundtrip conversion 90 | """ 91 | 92 | sig_dict = {'gene1': 1, 'gene2': -1, 'gene3': 1}; 93 | signed = False; 94 | source = 'myfile.gmt'; 95 | name = 'test signature'; 96 | 97 | testSig = Signature(sig_dict, signed, source, name); 98 | 99 | testSigs = [testSig]; 100 | 101 | testJson = jsonIO.signatures_to_JSON(testSigs); 102 | 103 | resultSigs = jsonIO.JSON_to_Signatures(testJson); 104 | 105 | for a, b in zip(testSigs, resultSigs): 106 | self.assertEqual(a.signed, b.signed) 107 | self.assertEqual(a.source, b.source) 108 | self.assertEqual(a.name, b.name) 109 | 110 | self.assertEqual(a.sig_dict, b.sig_dict); 111 | 112 | def testJSON_to_SignatureScore_roundTrip(self): 113 | """ 114 | Test roundtrip conversion 115 | """ 116 | 117 | scores = [3.42, 10, 2.3] 118 | name = 'sigName' 119 | sample_labels = ['sample1', 'sample2', 'sample3'] 120 | isFactor = False 121 | isPrecomputed = True 122 | numGenes = 4 123 | 124 | testSigScores = SignatureScores(scores, name, sample_labels, 125 | isFactor, isPrecomputed, numGenes); 126 | 127 | testSigScoresDict = {name: testSigScores}; 128 | 129 | testJson = jsonIO.precomputed_signatures_to_JSON(testSigScoresDict); 130 | 131 | resultSigScoresDict = jsonIO.JSON_to_SignatureScores(testJson); 132 | 133 | for key in testSigScoresDict: 134 | orig = testSigScoresDict[key]; 135 | result = resultSigScoresDict[key] 136 | self.assertEqual(orig.scores, result.scores); 137 | self.assertEqual(orig.name, result.name); 138 | self.assertEqual(orig.sample_labels, result.sample_labels); 139 | self.assertEqual(orig.isFactor, result.isFactor); 140 | self.assertEqual(orig.isPrecomputed, result.isPrecomputed); 141 | self.assertEqual(orig.numGenes, result.numGenes); 142 | 143 | def testJSON_to_ExpressionData(self): 144 | 145 | json_str = SAMPLE_EXPRESSION_DATA_JSON; 146 | 147 | result = jsonIO.JSON_to_ExpressionData(json_str); 148 | 149 | self.assertIsInstance(result, ExpressionData) 150 | 151 | def testJSON_to_Signature(self): 152 | 153 | json_str = SAMPLE_SIGNATURE_JSON; 154 | 155 | result = jsonIO.JSON_to_Signatures(json_str); 156 | 157 | self.assertIsInstance(result, list); 158 | self.assertIsInstance(result[0], Signature); 159 | 160 | def testJSON_to_SignatureScore(self): 161 | 162 | # Test first example str 163 | json_str = SAMPLE_PRECOMPUTED_SIGNATURE_JSON; 164 | 165 | result = jsonIO.JSON_to_SignatureScores(json_str); 166 | 167 | self.assertIsInstance(result, dict); 168 | self.assertIsInstance(result.values()[0], SignatureScores); 169 | 170 | # Test second example string (for a factor precomputed sig) 171 | json_str = SAMPLE_PRECOMPUTED_SIGNATURE2_JSON; 172 | 173 | result = jsonIO.JSON_to_SignatureScores(json_str); 174 | 175 | self.assertIsInstance(result, dict); 176 | self.assertIsInstance(result.values()[0], SignatureScores); 177 | -------------------------------------------------------------------------------- /FastProject/Tests/test_projections.py: -------------------------------------------------------------------------------- 1 | import unittest; 2 | 3 | 4 | class TestProjectionMethods(unittest.TestCase): 5 | pass; 6 | -------------------------------------------------------------------------------- /FastProject/Utils/ProgressBar.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Simple progress bar used throughout FastProject 3 | 4 | """ 5 | from __future__ import absolute_import, print_function, division; 6 | 7 | import sys 8 | 9 | class ProgressBar: 10 | 11 | width = 40; #Number of spaces to use 12 | 13 | def __init__(self, max_i): 14 | self.max_i = max_i; 15 | self.i = 0; 16 | 17 | if(self.max_i == 0): 18 | self.max_i = 1; 19 | 20 | sys.stdout.write(self.build_str()); 21 | sys.stdout.flush(); 22 | 23 | def build_str(self): 24 | #Determine number of hashes 25 | n_hash = int(self.i / self.max_i * ProgressBar.width); 26 | 27 | #Determine percentage to display 28 | pcent = int(self.i / self.max_i * 100); 29 | pcent_str = '{:3d}'.format(pcent) 30 | 31 | out_string = '|'; 32 | out_string = out_string + ''.join(['#']*n_hash); 33 | out_string = out_string + ''.join(['-']*(ProgressBar.width-n_hash)); 34 | out_string = out_string + '| ' + pcent_str + '%'; 35 | 36 | return out_string; 37 | 38 | def update(self): 39 | if(self.i < self.max_i): 40 | self.i = self.i + 1; 41 | 42 | #Only update stdout if number of hashes has changed 43 | if(int(self.i / self.max_i * ProgressBar.width) > int((self.i - 1) / self.max_i * ProgressBar.width)): 44 | 45 | new_str = self.build_str(); 46 | 47 | sys.stdout.write(''.join(['\b']*len(new_str))); #Erase old 48 | sys.stdout.write(new_str); 49 | 50 | if(self.i == self.max_i): 51 | sys.stdout.write('\n'); 52 | 53 | sys.stdout.flush(); 54 | 55 | def complete(self): 56 | #Only does anything if we haven't completed yet 57 | if(self.i != self.max_i): 58 | self.i = self.max_i-1; 59 | self.update(); 60 | -------------------------------------------------------------------------------- /FastProject/Utils/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | 3 | from .ProgressBar import ProgressBar 4 | 5 | from .em import em_exp_norm_mixture; 6 | 7 | from .hdt import HDT_Sig_batch; 8 | -------------------------------------------------------------------------------- /FastProject/Utils/em.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """EM Algorithm 3 | 4 | Used to fit data to an exponential/normal mixture of 5 | distributions 6 | 7 | Derived from Nir Yosef's MATLAB code 8 | 9 | """ 10 | from __future__ import absolute_import, print_function, division; 11 | import numpy as np; 12 | from . import ProgressBar 13 | 14 | #zmat is Genes x Samples 15 | #cutoff is Genes x 1 - represents the initial cutoff between normal and exponential distributions 16 | 17 | def em_exp_norm_mixture(zmat, cutoff, progressbar = True): 18 | with np.errstate(divide='ignore', invalid='ignore'): #nans and infs are handled explicitly, don't need warning 19 | max_iter = 150; 20 | 21 | if(progressbar): pbar = ProgressBar(max_iter); 22 | 23 | #promote to 2d if single gene given 24 | if(zmat.ndim == 1): 25 | zmat = zmat.reshape((1, zmat.shape[0])); 26 | cutoff = np.array([cutoff]); 27 | 28 | #Make sure cutoff is 2d 29 | cutoff = cutoff.reshape((cutoff.shape[0],1)); 30 | 31 | cutoffs = np.tile(cutoff, (1,zmat.shape[1])); 32 | gamma = zmat > cutoffs; 33 | Pi = np.mean(gamma,axis=1).reshape((zmat.shape[0],1)); 34 | Pi[Pi == 0] = 1 / zmat.shape[1]; 35 | 36 | mu_l=weighted_mean(zmat,1-gamma); 37 | mu_h=weighted_mean(zmat,gamma); 38 | st_h=weighted_std(zmat,gamma, mu_h); 39 | 40 | for niter in range(max_iter): 41 | #E 42 | prev_gamma = gamma; 43 | p_low = np.exp(-1*zmat/mu_l)/mu_l; 44 | 45 | p_high = np.exp(-1 * (zmat - mu_h)**2 / (2*st_h**2)) / st_h / np.sqrt(2*np.pi); 46 | 47 | p_low[~np.isfinite(p_low)] = 1e-5; 48 | p_low[p_low < 1e-5] = 1e-5; 49 | 50 | p_high[~np.isfinite(p_high)] = 1e-5; 51 | p_high[p_high<1e-5] = 1e-5; 52 | 53 | gamma = (Pi * p_high) / ( ((1-Pi)*p_low) + (Pi*p_high)); 54 | 55 | #M 56 | Pi = np.mean(gamma,axis=1).reshape((zmat.shape[0],1)); 57 | mu_l=weighted_mean(zmat,1-gamma); 58 | mu_h=weighted_mean(zmat,gamma); 59 | st_h=weighted_std(zmat,gamma, mu_h); 60 | 61 | 62 | if(progressbar): pbar.update(); 63 | 64 | if(niter % 10 == 0): 65 | biggest_change = np.max(np.abs(gamma - prev_gamma)); 66 | if(biggest_change < 0.01): 67 | break; 68 | 69 | 70 | #if niter == 1: print mu_l, mu_h, st_l, st_h 71 | #print 'Iteration: ', niter, ' L: ', sum(L); 72 | #if d>0.95: 73 | #break; 74 | if(progressbar): pbar.complete(); 75 | 76 | L = np.sum(gamma * np.log(p_high) + (1-gamma)*np.log(p_low),axis=1); 77 | st_l=weighted_std(zmat,1-gamma, mu_l); 78 | 79 | #Final E 80 | prev_gamma = gamma; 81 | p_low = np.exp(-1*zmat/mu_l)/mu_l; 82 | 83 | p_high = np.exp(-1 * (zmat - mu_h)**2 / (2*st_h**2)) / st_h / np.sqrt(2*np.pi); 84 | 85 | p_low[~np.isfinite(p_low)] = 1e-5; 86 | p_low[p_low < 1e-5] = 1e-5; 87 | 88 | p_high[~np.isfinite(p_high)] = 1e-5; 89 | p_high[p_high<1e-5] = 1e-5; 90 | 91 | gamma = (Pi * p_high) / ( ((1-Pi)*p_low) + (Pi*p_high)); 92 | 93 | return (gamma, mu_l, mu_h, st_l, st_h, Pi, L); 94 | 95 | #y and gamma same size 96 | #returns average of each row, weighted by gammas 97 | def weighted_mean(y, gamma): 98 | mu = np.sum(gamma * y, axis=1) / np.sum(gamma, axis=1); 99 | mu = mu.reshape((mu.shape[0], 1)); #make it 2 dimensional 100 | return mu 101 | 102 | #y and gamma same size 103 | #returns std dev of each row, weighted by gammas 104 | def weighted_std(y, gamma, mu=None): 105 | if(mu is None): 106 | mu = weighted_mean(y,gamma); 107 | 108 | wstd = np.sqrt(np.sum((gamma * (y-mu)**2), axis=1) / np.sum(gamma, axis=1)); 109 | wstd = wstd.reshape((wstd.shape[0],1)); 110 | return wstd 111 | -------------------------------------------------------------------------------- /FastProject/Viewer Resources/ColorScatter.js: -------------------------------------------------------------------------------- 1 | /* 2 | Initializes a zoomable scatter plot in the element "parent" 3 | parent = , for example, "#chart_div" 4 | */ 5 | function ColorScatter(parent, colorbar) 6 | { 7 | if(colorbar === undefined) { this.colorbar_enabled = false;} 8 | else {this.colorbar_enabled = colorbar;} 9 | 10 | var colorbar_height = 20; 11 | var colorbar_width = 200; 12 | 13 | var self = this; 14 | var xdomain = [-2, 2]; 15 | var ydomain = [-2, 2]; 16 | 17 | this.margin = {top: 20, right: 20, bottom: 15, left: 40}; 18 | this.width = $(parent).width() - this.margin.right - this.margin.left; 19 | this.height = $(parent).height() - this.margin.top - this.margin.bottom; 20 | this.circle_radius = 4.0 21 | 22 | if(this.colorbar_enabled){ 23 | this.height -= colorbar_height; 24 | this.height -= 15; 25 | } 26 | 27 | this.x = d3.scale.linear() 28 | .domain(xdomain) 29 | .range([0, self.width]); 30 | 31 | this.y = d3.scale.linear() 32 | .domain(ydomain) 33 | .range([self.height, 0]); 34 | 35 | this.xAxis = d3.svg.axis() 36 | .scale(self.x) 37 | .orient("bottom") 38 | .tickSize(-self.height); 39 | 40 | this.yAxis = d3.svg.axis() 41 | .scale(self.y) 42 | .orient("left") 43 | .ticks(5) 44 | .tickSize(-self.width); 45 | 46 | this.zoom = d3.behavior.zoom() 47 | .x(self.x) 48 | .y(self.y) 49 | .scaleExtent([0.2, 32]) 50 | .on("zoom", self.redraw()); 51 | 52 | //This gets overwritten when you set data 53 | this.colorScale = d3.scale.linear() 54 | .domain([0,0.5,1]) 55 | .range(["blue", "green", "red"]); 56 | 57 | this.tip = d3.tip() 58 | .attr('class', 'd3-tip') 59 | .offset([-10, 0]) 60 | .html(function(d){ return d[3]+": "+d[2]; }); 61 | 62 | this.svg = d3.select(parent).append("svg") 63 | .attr("width", self.width + self.margin.left + self.margin.right) 64 | .attr("height", self.height + self.margin.top + self.margin.bottom) 65 | .append("g") 66 | .attr("transform", "translate(" + self.margin.left + "," + self.margin.top + ")") 67 | .call(self.zoom) 68 | .call(self.tip); 69 | 70 | this.svg.append("rect") 71 | .attr("width", self.width) 72 | .attr("height", self.height); 73 | 74 | this.svg.append("g") 75 | .attr("class", "x axis") 76 | .attr("transform", "translate(0," + self.height + ")") 77 | .call(self.xAxis); 78 | 79 | this.svg.append("g") 80 | .attr("class", "y axis") 81 | .call(self.yAxis); 82 | 83 | if(this.colorbar_enabled){ 84 | this.colorbar_svg = d3.select(parent).append("svg") 85 | .attr("width", this.width + this.margin.left + this.margin.right) 86 | .attr("height", colorbar_height+20); 87 | 88 | this.colorbar = this.colorbar_svg.append("rect") 89 | .attr("x", this.width + this.margin.left + this.margin.right - colorbar_width - 50) 90 | .attr("y", 0) 91 | .attr("width", colorbar_width) 92 | .attr("height", colorbar_height); 93 | } 94 | 95 | this.points = []; 96 | 97 | this.selected = -1; 98 | this.selected_links = []; 99 | 100 | this.hovered = -1; 101 | this.hovered_links = []; 102 | 103 | this.last_event = -1; 104 | } 105 | 106 | ColorScatter.prototype.setData = function(points, isFactor) 107 | { 108 | this.points = points; 109 | var cvals = points.map(function(e){return e[2];}); //extract 3rd column 110 | 111 | //Adjust circle size based on number of points 112 | //Max is 5, Min is 2 113 | //5 at 100 points, 2 at 2000 points 114 | var m = (2-5) / (Math.log10(2000) - Math.log10(100)); 115 | var b = 5 - m*Math.log10(100); 116 | var new_radius = m*Math.log10(points.length) + b 117 | new_radius = Math.max(2, new_radius) 118 | new_radius = Math.min(5, new_radius) 119 | this.circle_radius = new_radius 120 | 121 | 122 | 123 | if(isFactor) 124 | { 125 | //Find unique values 126 | var unique = d3.set(cvals).values(); 127 | if(unique.length <= 10) { this.colorScale = d3.scale.category10();} 128 | else { this.colorScale = d3.scale.category20();} 129 | 130 | this.colorScale.domain(unique); 131 | this.setColorBar(); 132 | } 133 | else 134 | { 135 | cvals.sort(d3.ascending); // Needed for quantile 136 | var low = d3.quantile(cvals, 0.1); 137 | var high = d3.quantile(cvals, 0.9); 138 | var mid = d3.mean(cvals); 139 | 140 | this.colorScale = d3.scale.linear() 141 | .domain([low, mid, high]) 142 | .range(["blue", "green", "red"]); 143 | 144 | // Format the bound labels 145 | var label_low = parseFloat(low.toFixed(2)).toString(); 146 | var label_high = parseFloat(high.toFixed(2)).toString(); 147 | 148 | this.setColorBar(this.colorScale.range(), 149 | label_low, 150 | label_high); 151 | } 152 | 153 | this.redraw(true)(); 154 | }; 155 | 156 | ColorScatter.prototype.setColorBar = function(colors, label_low, label_high) 157 | { 158 | if(!this.colorbar_enabled){ return; } 159 | 160 | this.colorbar_svg.selectAll("text").remove(); 161 | this.colorbar_svg.select("defs").remove(); 162 | 163 | if(colors === undefined){ 164 | // Clear color bar - useful for factors 165 | this.colorbar 166 | .style("fill","rgba(0,0,0,0)") 167 | .style("stroke","none"); 168 | } 169 | else 170 | { 171 | var gradient = this.colorbar_svg.append("svg:defs") 172 | .append("svg:linearGradient") 173 | .attr("id", "gradient") 174 | .attr("x1", "0%") 175 | .attr("y1", "0%") 176 | .attr("x2", "100%") 177 | .attr("y2", "0%") 178 | .attr("spreadMethod", "pad"); 179 | 180 | for(var i = 0; i < colors.length; i++){ 181 | var stop_percentage = Math.floor((100 / (colors.length-1))*i); 182 | 183 | gradient.append("svg:stop") 184 | .attr("offset", stop_percentage.toString() + "%") 185 | .attr("stop-color", colors[i]) 186 | .attr("stop-opacity", 0.8); 187 | } 188 | 189 | this.colorbar 190 | .style("fill", "url(#gradient)") 191 | .style("stroke", "black") 192 | .style("stroke-width", "1px"); 193 | 194 | var label_low_x = this.colorbar.attr("x"); 195 | var label_high_x = (parseInt(this.colorbar.attr("x")) + 196 | parseInt(this.colorbar.attr("width"))).toString(); 197 | 198 | this.colorbar_svg.append("text") 199 | .attr("text-anchor", "middle") 200 | .attr("x", label_low_x) 201 | .attr("y", 30) 202 | .attr("font-size", "10px") 203 | .text(label_low); 204 | 205 | this.colorbar_svg.append("text") 206 | .attr("text-anchor", "middle") 207 | .attr("x", label_high_x) 208 | .attr("y", 30) 209 | .attr("font-size", "10px") 210 | .text(label_high); 211 | } 212 | }; 213 | 214 | ColorScatter.prototype.setSelected = function(selected_index, event_id) 215 | { 216 | if(event_id === undefined){ 217 | event_id = Math.random(); 218 | } 219 | 220 | //Needed to prevent infinite loops with linked hover and select events 221 | if(this.last_event !== event_id) { 222 | this.last_event = event_id; 223 | this.selected = selected_index; 224 | this.redraw()(); 225 | this.selected_links.forEach(function (e) { 226 | e.setSelected(selected_index, event_id); 227 | }); 228 | } 229 | }; 230 | 231 | ColorScatter.prototype.setHovered = function(hovered_indices, event_id) 232 | { 233 | if(event_id === undefined){ 234 | event_id = Math.random(); 235 | } 236 | 237 | //test for single index, and wrap in list 238 | if(typeof(hovered_indices) === "number"){hovered_indices = [hovered_indices];} 239 | if(hovered_indices.length === 0){hovered_indices = [-1];} 240 | 241 | //Needed to prevent infinite loops with linked hover and select events 242 | if(this.last_event !== event_id) { 243 | this.last_event = event_id; 244 | this.hover_col = hovered_indices; 245 | if(hovered_indices.length === 1 && hovered_indices[0] === -1){ 246 | //Clear the hover 247 | this.svg.selectAll("circle") 248 | .classed("point-faded", false) 249 | .classed("point-hover", false); 250 | } 251 | else{ 252 | this.svg.selectAll("circle") 253 | .classed("point-faded", true) 254 | .classed("point-hover", function (d, i) { 255 | return hovered_indices.indexOf(i) > -1; 256 | }); 257 | } 258 | 259 | this.hovered_links.forEach(function (e) { 260 | e.setHovered(hovered_indices, event_id); 261 | }); 262 | } 263 | }; 264 | 265 | ColorScatter.prototype.redraw = function(performTransition) { 266 | var self = this; 267 | return function(){ 268 | self.svg.select(".x.axis").call(self.xAxis); 269 | self.svg.select(".y.axis").call(self.yAxis); 270 | var circles = self.svg.selectAll("circle") 271 | .data(self.points); 272 | 273 | circles.enter().append("circle").attr("r",self.circle_radius); 274 | circles.style("fill", function(d){return self.colorScale(d[2]);}) 275 | .on("click", function(d,i){self.setSelected(i);}) 276 | .on("mouseover", function(d,i){self.tip.show(d,i); self.setHovered(i);}) 277 | .on("mouseout", function(d,i){self.tip.hide(d,i); self.setHovered(-1);}) 278 | .classed("point-selected", function(d, i){return i === self.selected;}); 279 | 280 | if(performTransition !== undefined && performTransition === true) 281 | { 282 | circles 283 | .transition() 284 | .duration(1000) 285 | .attr("cx", function(d){return self.x(d[0]);}) 286 | .attr("cy", function(d){return self.y(d[1]);}); 287 | } 288 | else 289 | { 290 | circles 291 | .attr("cx", function(d){return self.x(d[0]);}) 292 | .attr("cy", function(d){return self.y(d[1]);}); 293 | } 294 | 295 | circles.exit().remove(); 296 | }; 297 | }; 298 | -------------------------------------------------------------------------------- /FastProject/Viewer Resources/HeatMap.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by David DeTomaso on 6/24/2015. 3 | */ 4 | 5 | function HeatMap(parent) 6 | { 7 | var self = this; 8 | this.h = 1; //height of row 9 | this.w = 4; //width of column 10 | 11 | this.width = 600; 12 | this.height = 450; 13 | 14 | this.svg = d3.select(parent).append("svg") 15 | .attr("width", self.width) 16 | .attr("height", self.height); 17 | 18 | var offset = 0; 19 | 20 | this.labels = this.svg.append("g"); 21 | 22 | this.labels.append("text") 23 | .classed("col_label", true) 24 | .attr("x", 0) 25 | .attr("y", offset + 20) 26 | .attr("font-size", "20px"); 27 | 28 | this.labels.append("text") 29 | .classed("row_label", true) 30 | .attr("x", 0) 31 | .attr("y", offset + 40) 32 | .attr("font-size", "20px"); 33 | 34 | this.labels.append("rect") 35 | .attr("x", this.width-40) 36 | .attr("y", offset) 37 | .attr("width", 40) 38 | .attr("height", 40) 39 | .style("fill", "white"); 40 | 41 | this.labels.append("text") 42 | .classed("rect_label", true) 43 | .attr("text-anchor", "middle") 44 | .attr("x", this.width-20) 45 | .attr("y", offset+23) 46 | .attr("font-size", "10px"); 47 | 48 | offset += 40; 49 | offset += 10; //Some margin 50 | 51 | 52 | this.cluster_bar = this.svg.append("g"); 53 | this.cluster_bar.on("mouseleave", function(){ 54 | self.setHovered(-1); 55 | }); 56 | this.cluster_bar 57 | .attr("transform", "translate(0," + 58 | (offset)+")"); 59 | 60 | this.cluster_bar_props = { 61 | height: 10, 62 | colors: d3.scale.category10().domain(d3.range(10)) 63 | }; 64 | 65 | offset += this.cluster_bar_props.height; 66 | offset += 10; //Some margin 67 | 68 | this.grid = this.svg.append("g"); 69 | this.grid 70 | .attr("transform", "translate(0," + 71 | (offset)+")"); 72 | 73 | offset += this.height; 74 | this.svg.attr("height", offset); 75 | 76 | //define a color scale using the min and max expression values 77 | this.colorScale = d3.scale.linear() 78 | .domain([-0.6, 0, 0.6]) 79 | .range(["steelblue", "white", "lightcoral"]); 80 | 81 | this.data = []; 82 | this.selected = -1; 83 | this.selected_links = []; 84 | 85 | this.hover_cols = -1; 86 | this.hovered_links = []; 87 | 88 | this.hover_rows = -1; 89 | 90 | this.last_event = 0; 91 | 92 | this.col_clusters = null; //Cluster assignment for each column in data matrix 93 | this.col_order = null; //Position for each column in data matrix 94 | 95 | this.row_labels = []; 96 | this.col_labels = []; 97 | 98 | } 99 | 100 | HeatMap.prototype.cluster_columns = function(assignments) 101 | { 102 | var self = this; 103 | this.col_clusters = assignments; 104 | 105 | //Argsort the col_clusters 106 | var rr = d3.range(0,this.col_clusters.length); 107 | rr.sort(function(a,b){return self.col_clusters[a] - self.col_clusters[b];}); 108 | //Argsort again to get rank 109 | var rr2 = d3.range(0,rr.length); 110 | rr2.sort(function(a,b){return rr[a] - rr[b];}); 111 | 112 | this.col_order = rr2; 113 | this.redraw()(); 114 | }; 115 | 116 | HeatMap.prototype.setData = function(data, render) 117 | { 118 | var formatted_data = data.map(function(row,row_i){ 119 | return row.map(function(value, col_i){ 120 | return {"row":row_i, "col":col_i, "value":value}; 121 | }); 122 | }); 123 | 124 | this.data = formatted_data; 125 | var N_ROWS = data.length; 126 | var N_COLS = data[0].length; 127 | 128 | this.w = Math.floor(this.width/N_COLS); 129 | if(this.w === 0) {this.w = 1;} 130 | this.h = Math.floor(this.height/N_ROWS); 131 | if(this.h === 0) {this.h = 1;} 132 | 133 | this.col_clusters = Array.apply(null, Array(N_COLS)).map(Number.prototype.valueOf,0); 134 | this.col_order = d3.range(0, N_COLS); //Initial sorting 135 | 136 | if(render){ 137 | this.redraw(true)(); 138 | } 139 | }; 140 | 141 | HeatMap.prototype.setSelected = function(selected_index, event_id) 142 | { 143 | if(event_id === undefined){ 144 | event_id = Math.random(); 145 | } 146 | 147 | //Needed to prevent infinite loops with linked hover and select events 148 | if(this.last_event !== event_id) { 149 | this.last_event = event_id; 150 | this.selected = selected_index; 151 | this.redraw()(); 152 | this.selected_links.forEach(function (e) { 153 | e.setSelected(selected_index, event_id); 154 | }); 155 | } 156 | }; 157 | 158 | HeatMap.prototype.setHovered = function(hovered_indices, event_id) 159 | { 160 | if(event_id === undefined){ 161 | event_id = Math.random(); 162 | } 163 | 164 | //test for single index, and wrap in list 165 | if(typeof(hovered_indices) === "number"){hovered_indices = [hovered_indices];} 166 | 167 | //Needed to prevent infinite loops with linked hover and select events 168 | if(this.last_event !== event_id) { 169 | this.last_event = event_id; 170 | this.hover_cols = hovered_indices; 171 | this.grid.selectAll("g").selectAll("rect") 172 | .classed("heatmap-hover", function (d, i) { 173 | return hovered_indices.indexOf(i) > -1; 174 | }); 175 | this.hovered_links.forEach(function (e) { 176 | e.setHovered(hovered_indices, event_id); 177 | }); 178 | if(this.hover_cols.length === 1) 179 | { 180 | var ii = this.hover_cols[0]; 181 | this.labels.select(".col_label").text(this.col_labels[ii]); 182 | } 183 | } 184 | }; 185 | 186 | HeatMap.prototype.setHoveredRow = function(hovered_row_indices) 187 | { 188 | if(typeof(hovered_row_indices) === "number"){ 189 | hovered_row_indices = [hovered_row_indices]; 190 | } 191 | 192 | this.hover_rows = hovered_row_indices; 193 | 194 | if(this.hover_rows.length === 1) 195 | { 196 | var ii = this.hover_rows[0]; 197 | this.labels.select(".row_label").text(this.row_labels[ii]); 198 | } 199 | 200 | }; 201 | 202 | //Sets the text and color for the square upper-right indicator 203 | HeatMap.prototype.setHoveredIndicator = function(data_val) 204 | { 205 | if(data_val !== undefined) 206 | { 207 | this.labels.select("rect") 208 | .style("fill", this.colorScale(data_val)); 209 | this.labels.select(".rect_label") 210 | .text(data_val); 211 | } 212 | else 213 | { 214 | this.labels.select("rect") 215 | .style("fill", "white"); 216 | this.labels.select(".rect_label") 217 | .text(""); 218 | } 219 | 220 | }; 221 | 222 | HeatMap.prototype.redraw = function(performTransition) { 223 | var self = this; 224 | return function(){ 225 | //self.svg.select(".x.axis").call(self.xAxis); 226 | //self.svg.select(".y.axis").call(self.yAxis); 227 | 228 | //Draw cluster-bar 229 | var clusterRects = self.cluster_bar.selectAll("rect") 230 | .data(self.col_clusters); 231 | 232 | clusterRects.enter() 233 | .append("rect") 234 | .attr('height', self.cluster_bar_props.height) 235 | .attr('y', self.cluster_bar_props.offset); 236 | 237 | clusterRects 238 | .attr('width', self.w) 239 | .style('fill',function(d) { 240 | return self.cluster_bar_props.colors(d);}) 241 | .attr('x', function(d,i){ 242 | return (self.col_order[i] * self.w); }) 243 | .on("mouseover", function(d) { 244 | var ii = d3.range(self.col_clusters.length); 245 | var selected_i = ii.filter(function(e,j){ 246 | return self.col_clusters[j] === d;}); 247 | self.setHovered(selected_i); 248 | }); 249 | 250 | clusterRects.exit().remove(); 251 | 252 | 253 | //generate heatmap rows 254 | var heatmapRow = self.grid.selectAll("g") 255 | .data(self.data); 256 | 257 | heatmapRow.enter() 258 | .append("g"); 259 | 260 | //heatmapRow.attr("transform", function(d, j){ 261 | // return "translate(0," + (j*self.h)+ ")"}); 262 | 263 | heatmapRow.exit().remove(); 264 | 265 | //generate heatmap columns 266 | var heatmapRects = heatmapRow 267 | .selectAll("rect") 268 | .data(function(d) { 269 | return d; 270 | }); 271 | 272 | heatmapRects.enter().append("rect") 273 | .on("mouseover", function(d){self.setHovered(d.col); self.setHoveredRow(d.row); self.setHoveredIndicator(d.value);}); 274 | 275 | self.svg 276 | .on("mouseleave", function(d){self.setHovered(-1); self.setHoveredRow(-1); self.setHoveredIndicator();}); 277 | 278 | heatmapRects.style('fill',function(d) { 279 | return self.colorScale(d.value);}) 280 | .attr('width',self.w) 281 | .attr('height',self.h) 282 | .attr('y',function(d){ return d.row*self.h;}) 283 | .attr('x', function(d) { 284 | return (self.col_order[d.col] * self.w);}); 285 | 286 | heatmapRects.exit().remove(); 287 | 288 | }; 289 | }; 290 | -------------------------------------------------------------------------------- /FastProject/Viewer Resources/QC_Report.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | FastProject Quality Report 4 | 40 | 41 | 99 | 100 | 101 | 102 |

Quality Report

103 |
104 | <% table %> 105 | 106 | 107 | -------------------------------------------------------------------------------- /FastProject/Viewer Resources/Utilities.js: -------------------------------------------------------------------------------- 1 | /* 2 | A library for general purpose utility functions 3 | Requires: 4 | d3.js 5 | 6 | */ 7 | 8 | 9 | 10 | /* 11 | Exports a zip with data in it 12 | To be used with the FastProject results viewer 13 | */ 14 | function exportSigProj() 15 | { 16 | var zip = new JSZip(); 17 | 18 | var data = getDataContext(); 19 | 20 | var choice = $('#cluster_select').val(); 21 | var sig_key = global_status.plotted_signature; 22 | var proj_key = global_status.plotted_projection; 23 | 24 | //Convert the data that's in the scatter plot to a tab-delimited table 25 | 26 | var proj = data.Projections[proj_key]; 27 | var sig = data.SigScores[sig_key]; 28 | var cluster_assignments = data.Clusters[proj_key][choice]; 29 | 30 | var table; 31 | if(sig.isFactor){ 32 | table = [proj[0], proj[1], sig.scores, cluster_assignments];} 33 | else{ 34 | table = [proj[0], proj[1], sig.scores, sig.ranks, cluster_assignments];} 35 | 36 | 37 | table = d3.transpose(table); 38 | if(sig.isFactor){ 39 | table = [["X", "Y", "Signature Score", "Cluster: "+choice]].concat(table);} 40 | else{ 41 | table = [["X", "Y", "Signature Score", "Signature Rank", "Cluster: "+choice]].concat(table);} 42 | 43 | table = table.map(function(x){ return x.join("\t");}); 44 | var scatter_csv_str = table.join("\n"); 45 | zip.file("Scatter.txt", scatter_csv_str); 46 | 47 | //Convert the heatmap into a tab-delimited table 48 | if(!sig.isPrecomputed) 49 | { 50 | var heat_data_plus = global_heatmap.data_plus; 51 | var heat_data_minus = global_heatmap.data_minus; 52 | var heat_data = []; 53 | 54 | //stitch together heat_data_plus and heat_data_minus to make heat_data 55 | for(var i=0; i= 2: 103 | m = "[t-SNE] Iteration %d: error = %.7f, gradient norm = %.7f" 104 | print(m % (i + 1, error, grad_norm)) 105 | 106 | if error < best_error: 107 | best_error = error 108 | best_iter = i 109 | elif i - best_iter > n_iter_without_progress: 110 | if verbose >= 2: 111 | print("[t-SNE] Iteration %d: did not make any progress " 112 | "during the last %d episodes. Finished." 113 | % (i + 1, n_iter_without_progress)) 114 | break 115 | if grad_norm <= min_grad_norm: 116 | if verbose >= 2: 117 | print("[t-SNE] Iteration %d: gradient norm %f. Finished." 118 | % (i + 1, grad_norm)) 119 | break 120 | if error_diff <= min_error_diff: 121 | if verbose >= 2: 122 | m = "[t-SNE] Iteration %d: error difference %f. Finished." 123 | print(m % (i + 1, error_diff)) 124 | break 125 | 126 | if new_error is not None: 127 | error = new_error 128 | 129 | return p, error, i 130 | 131 | 132 | sk_ver = [] 133 | for c in sklearn.__version__.split("."): 134 | try: 135 | ic = int(c) 136 | sk_ver.append(ic) 137 | except ValueError: 138 | pass 139 | sk_ver = tuple(sk_ver) 140 | 141 | if sk_ver < (0, 19, 0): 142 | from sklearn.manifold import t_sne 143 | t_sne._gradient_descent = _gradient_descent 144 | -------------------------------------------------------------------------------- /FastProject/_version.py: -------------------------------------------------------------------------------- 1 | # Store the version here to be imported by both the 2 | # setup.py file and the package's init module 3 | 4 | __version__ = "1.1.4"; 5 | -------------------------------------------------------------------------------- /FastProject/jsonIO.py: -------------------------------------------------------------------------------- 1 | """ 2 | A module for functions which convert 3 | FastProject datatypes to/from JSON 4 | """ 5 | 6 | from .DataTypes import ExpressionData 7 | from .Signatures import Signature 8 | from .SigScoreMethods import SignatureScores 9 | import json 10 | import numpy as np 11 | 12 | 13 | def expressionData_to_JSON(expressionData): 14 | """Converts a Datatypes.ExpressionData into its 15 | JSON representation 16 | 17 | Only uses the data values, row labels, and column labels. 18 | 19 | An expressionData in JSON form is specified as: 20 | 21 | {'col_labels': ['sample1', 'sample2', 'sample3', ...], 22 | 'row_labels': ['feature1', 'feature2', 'feature3', ...], 23 | 'data': [[1.2302, 3.43, 2.34, 5.432, ...], 24 | [3.23, 5.93, 1.03, 4.34, ...], 25 | ... 26 | ] 27 | } 28 | 29 | 30 | Parameters 31 | ---------- 32 | expressionData : DataTypes.ExpressionData 33 | The object to convert 34 | 35 | Returns 36 | ------- 37 | str 38 | The JSON representation of the input object 39 | 40 | """ 41 | if(not isinstance(expressionData, ExpressionData)): 42 | raise ValueError("Input is not of type FastProject.DataTypes.ExpressionData"); 43 | 44 | obj = { 45 | "row_labels": expressionData.row_labels, 46 | "col_labels": expressionData.col_labels, 47 | "data": expressionData.base.tolist() 48 | } 49 | 50 | return json.dumps(obj); 51 | 52 | 53 | def signatures_to_JSON(signatures): 54 | """Returns the JSON representation of the 55 | input signatures 56 | 57 | Parameters 58 | ---------- 59 | signatures : list of Signatures.Signature 60 | The object to convert 61 | 62 | Returns 63 | ------- 64 | str 65 | The JSON representation of the input object 66 | 67 | """ 68 | result = [] 69 | for signature in signatures: 70 | if(not isinstance(signature, Signature)): 71 | raise ValueError("Input is not of type FastProject.Signatures.Signature"); 72 | 73 | obj = {'sig_dict': signature.sig_dict, 74 | 'signed': signature.signed, 75 | 'source': signature.source, 76 | 'name': signature.name}; 77 | 78 | result.append(obj); 79 | 80 | return json.dumps(result); 81 | 82 | 83 | def precomputed_signatures_to_JSON(precomputed_signatures): 84 | """Returns the JSON representation of the input precomputed 85 | signature dictionary 86 | 87 | Parameters 88 | ---------- 89 | precomputed_signature : dict 90 | Key: Name of Signature 91 | Value: SigScoreMethods.SignatureScores 92 | The object to convert 93 | 94 | Returns 95 | ------- 96 | str 97 | The JSON representation of the input object 98 | 99 | """ 100 | result = {}; 101 | 102 | for key in precomputed_signatures: 103 | precomputed_signature = precomputed_signatures[key] 104 | 105 | if(not isinstance(precomputed_signature, SignatureScores)): 106 | raise ValueError("Input is not of type FastProject.SigScoreMethods.SignatureScores"); 107 | 108 | obj = { 109 | "scores": precomputed_signature.scores, 110 | "name": precomputed_signature.name, 111 | "sample_labels": precomputed_signature.sample_labels, 112 | "isFactor": precomputed_signature.isFactor, 113 | "isPrecomputed": precomputed_signature.isPrecomputed, 114 | "numGenes": precomputed_signature.numGenes, 115 | } 116 | 117 | # Convert to a python list if its a numpy array 118 | if(isinstance(obj['scores'], np.ndarray)): 119 | obj['scores'] = obj['scores'].tolist(); 120 | 121 | result[key] = obj; 122 | 123 | return json.dumps(result); 124 | 125 | 126 | def JSON_to_ExpressionData(json_str): 127 | """Converts `json_str` into a FastProject ExpressionData object 128 | 129 | Parameters 130 | ---------- 131 | json_str : str 132 | String representation of an ExpressionData Object to be converted 133 | 134 | Returns 135 | ------- 136 | DataTypes.ExpressionData 137 | Resulting ExpressionData Object 138 | 139 | """ 140 | json_obj = json.loads(json_str); 141 | 142 | data = json_obj['data']; 143 | data = np.array(data); 144 | row_labels = json_obj['row_labels']; 145 | col_labels = json_obj['col_labels']; 146 | 147 | return ExpressionData(data, row_labels, col_labels); 148 | 149 | 150 | def JSON_to_Signatures(json_str): 151 | """Converts `json_str` into a list of FastProject Signature objects 152 | 153 | Parameters 154 | ---------- 155 | json_str : str 156 | String representation of Signature objects to be converted 157 | 158 | Returns 159 | ------- 160 | list of Signatures.Signature 161 | Resulting Signature Objects 162 | 163 | """ 164 | json_obj = json.loads(json_str); 165 | 166 | result = []; 167 | 168 | for sig in json_obj: 169 | sig_dict = sig['sig_dict']; 170 | signed = sig['signed']; 171 | source = sig['source']; 172 | name = sig['name']; 173 | sig_obj = Signature(sig_dict, signed, source, name); 174 | 175 | result.append(sig_obj); 176 | 177 | return result; 178 | 179 | 180 | def JSON_to_SignatureScores(json_str): 181 | """Converts `json_str` into a FastProject SignatureScore object 182 | 183 | Parameters 184 | ---------- 185 | json_str : str 186 | String representation of a dictionary of 187 | SignatureScore objects to be converted 188 | 189 | Returns 190 | ------- 191 | dict 192 | key: Name of Signature 193 | value: Signatures.SignatureScore 194 | 195 | """ 196 | json_obj = json.loads(json_str); 197 | result = {}; 198 | 199 | for key in json_obj: 200 | 201 | sig_score_dict = json_obj[key] 202 | 203 | scores = sig_score_dict['scores']; 204 | name = sig_score_dict['name']; 205 | sample_labels = sig_score_dict['sample_labels']; 206 | isFactor = sig_score_dict['isFactor']; 207 | isPrecomputed = sig_score_dict['isPrecomputed']; 208 | numGenes = sig_score_dict['numGenes']; 209 | 210 | sig_score_obj = SignatureScores(scores, name, sample_labels, isFactor, isPrecomputed, numGenes); 211 | 212 | result[key] = sig_score_obj; 213 | 214 | return result; 215 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright ©2016. The Regents of the University of California (Regents). All Rights Reserved. Permission to use, copy, modify, and distribute this software and its documentation for educational, research, and not-for-profit purposes, without fee and without a signed licensing agreement, is hereby granted, provided that the above copyright notice, this paragraph and the following two paragraphs appear in all copies, modifications, and distributions. Contact The Office of Technology Licensing, UC Berkeley, 2150 Shattuck Avenue, Suite 510, Berkeley, CA 94720-1620, (510) 643-7201, otl@berkeley.edu, http://ipira.berkeley.edu/industry-info for commercial licensing opportunities. 2 | 3 | Created by David DeTomaso and Nir Yosef, University of California, Berkeley 4 | 5 | IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 6 | 7 | REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. 8 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include */Housekeeping?Genes *.txt 2 | recursive-include */Viewer?Resources *.* 3 | recursive-include */Tests/TestFiles *.* 4 | include ez_setup.py 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | FastProject 2 | =========== 3 | 4 | **Note: This repository is no longer undergoing active maintainence.** 5 | 6 | **Instead, see [VISION](https://github.com/yoseflab/vision) for the successor to this project released in 2019.** 7 | - [https://github.com/yoseflab/VISION](https://github.com/yoseflab/vision) 8 | 9 | Here we present FastProject, a software package for two-dimensional visualization of single cell data, which utilizes a plethora of projection methods and provides a way to systematically investigate the biological relevance of these low dimensional representations by incorporating domain knowledge. 10 | 11 | FastProject analyzes a gene expression matrix and produces a dynamic output report in which two-dimensional projections of the data can be explored. Annotated gene sets (referred to as 'signatures') are incorporated so that features in the projections can be understood in relation to the biological processes they might represent. FastProject provides a novel method of scoring each cell against a gene signature so as to minimize the effect of missed transcripts as well as a method to rank signature-projection pairings so that meaningful associations can be quickly identified. Additionally, FastProject is written with a modular architecture and designed to serve as a platform for incorporating and comparing new projection methods and gene selection algorithms. 12 | 13 | 14 | Installing FastProject 15 | ----------------- 16 | FastProject is written in Python (2.7 or 3.3+) and has a few dependencies. 17 | 18 | To simplify the install process, we have created an install file for each platform that will take care of everything. This method is recommended for users that do not regularly use Python. 19 | 20 | *Download the file, and move into the folder where you'd like to install the application before running* 21 | 22 | - [Linux](https://rawgit.com/YosefLab/FastProject/master/FP_Linux_Install.sh) (Install using `bash FP_Linux_Install.sh`) 23 | - [OSX](https://rawgit.com/YosefLab/FastProject/master/FP_OSX_Install.sh) (Install using `bash FP_OSX_Install.sh`) 24 | - [Windows](https://rawgit.com/YosefLab/FastProject/master/FP_Windows_Install.ps1) (Right click and select "Run with PowerShell") 25 | 26 | This installs FastProject and all dependencies as an isolated Python environment. See [this page](https://github.com/YosefLab/FastProject/wiki/Install-Instructions) in the wiki for instructions on how to install into an existing Python environment for development. 27 | 28 | ### Installing into an existing Python environment 29 | A stable version of FastProject is maintained on the Python Package Index [https://pypi.python.org/pypi/FastProject/](https://pypi.python.org/pypi/FastProject/) and can be installed with: 30 | 31 | pip install FastProject 32 | 33 | 34 | How to Run FastProject 35 | ---------------------- 36 | 37 | Sample FastProject command: 38 | 39 | fastproject -s -p 40 | 41 | FastProject must be run with either the -s input or the -p input, though both can be used together. 42 | For signature files and precomputed signature files, multiple files can be specified, separated by a space. 43 | 44 | To display help and additional options: 45 | fastproject -h 46 | 47 | Sample Output 48 | ------------- 49 | ![FastProject Output Sample Image](/SampleOutput.png?raw=true) 50 | 51 | 52 | Additional Info 53 | --------------- 54 | All additional documentation is in the [FastProject Wiki](https://github.com/YosefLab/FastProject/wiki) 55 | -------------------------------------------------------------------------------- /SampleOutput.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/YosefLab/FastProject/f87027b5355403f7f0b6390ccf6f3eb477ded63d/SampleOutput.png -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | python setup.py sdist --formats=gztar upload 2 | -------------------------------------------------------------------------------- /docs/FastProject.DataTypes.rst: -------------------------------------------------------------------------------- 1 | FastProject.DataTypes module 2 | ============================ 3 | 4 | .. automodule:: FastProject.DataTypes 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.FileIO.rst: -------------------------------------------------------------------------------- 1 | FastProject.FileIO module 2 | ========================= 3 | 4 | .. automodule:: FastProject.FileIO 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.Filters.rst: -------------------------------------------------------------------------------- 1 | FastProject.Filters module 2 | ========================== 3 | 4 | .. automodule:: FastProject.Filters 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.Global.rst: -------------------------------------------------------------------------------- 1 | FastProject.Global module 2 | ========================= 3 | 4 | .. automodule:: FastProject.Global 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.HtmlViewer.rst: -------------------------------------------------------------------------------- 1 | FastProject.HtmlViewer module 2 | ============================= 3 | 4 | .. automodule:: FastProject.HtmlViewer 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.NormalizationMethods.rst: -------------------------------------------------------------------------------- 1 | FastProject.NormalizationMethods module 2 | ======================================= 3 | 4 | .. automodule:: FastProject.NormalizationMethods 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.Pipelines.rst: -------------------------------------------------------------------------------- 1 | FastProject.Pipelines module 2 | ============================ 3 | 4 | .. automodule:: FastProject.Pipelines 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.Projections.rst: -------------------------------------------------------------------------------- 1 | FastProject.Projections module 2 | ============================== 3 | 4 | .. automodule:: FastProject.Projections 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.SigScoreMethods.rst: -------------------------------------------------------------------------------- 1 | FastProject.SigScoreMethods module 2 | ================================== 3 | 4 | .. automodule:: FastProject.SigScoreMethods 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.Signatures.rst: -------------------------------------------------------------------------------- 1 | FastProject.Signatures module 2 | ============================= 3 | 4 | .. automodule:: FastProject.Signatures 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.SubSample.rst: -------------------------------------------------------------------------------- 1 | FastProject.SubSample module 2 | ============================ 3 | 4 | .. automodule:: FastProject.SubSample 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.Transforms.rst: -------------------------------------------------------------------------------- 1 | FastProject.Transforms module 2 | ============================= 3 | 4 | .. automodule:: FastProject.Transforms 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.Utils.ProgressBar.rst: -------------------------------------------------------------------------------- 1 | FastProject.Utils.ProgressBar module 2 | ==================================== 3 | 4 | .. automodule:: FastProject.Utils.ProgressBar 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.Utils.em.rst: -------------------------------------------------------------------------------- 1 | FastProject.Utils.em module 2 | =========================== 3 | 4 | .. automodule:: FastProject.Utils.em 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.Utils.hdt.rst: -------------------------------------------------------------------------------- 1 | FastProject.Utils.hdt module 2 | ============================ 3 | 4 | .. automodule:: FastProject.Utils.hdt 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.Utils.hdt_python.rst: -------------------------------------------------------------------------------- 1 | FastProject.Utils.hdt_python module 2 | =================================== 3 | 4 | .. automodule:: FastProject.Utils.hdt_python 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.Utils.rst: -------------------------------------------------------------------------------- 1 | FastProject.Utils package 2 | ========================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | .. toctree:: 8 | 9 | FastProject.Utils.ProgressBar 10 | FastProject.Utils.em 11 | FastProject.Utils.hdt 12 | FastProject.Utils.hdt_python 13 | 14 | Module contents 15 | --------------- 16 | 17 | .. automodule:: FastProject.Utils 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | -------------------------------------------------------------------------------- /docs/FastProject.rst: -------------------------------------------------------------------------------- 1 | FastProject package 2 | =================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | FastProject.Utils 10 | FastProject.tests 11 | 12 | Submodules 13 | ---------- 14 | 15 | .. toctree:: 16 | 17 | FastProject.DataTypes 18 | FastProject.FileIO 19 | FastProject.Filters 20 | FastProject.Global 21 | FastProject.HtmlViewer 22 | FastProject.NormalizationMethods 23 | FastProject.Pipelines 24 | FastProject.Projections 25 | FastProject.SigScoreMethods 26 | FastProject.Signatures 27 | FastProject.SubSample 28 | FastProject.Transforms 29 | 30 | Module contents 31 | --------------- 32 | 33 | .. automodule:: FastProject 34 | :members: 35 | :undoc-members: 36 | :show-inheritance: 37 | -------------------------------------------------------------------------------- /docs/FastProject.tests.rst: -------------------------------------------------------------------------------- 1 | FastProject.tests package 2 | ========================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | .. toctree:: 8 | 9 | FastProject.tests.test_projections 10 | FastProject.tests.test_split_merge 11 | 12 | Module contents 13 | --------------- 14 | 15 | .. automodule:: FastProject.tests 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | -------------------------------------------------------------------------------- /docs/FastProject.tests.test_projections.rst: -------------------------------------------------------------------------------- 1 | FastProject.tests.test_projections module 2 | ========================================= 3 | 4 | .. automodule:: FastProject.tests.test_projections 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/FastProject.tests.test_split_merge.rst: -------------------------------------------------------------------------------- 1 | FastProject.tests.test_split_merge module 2 | ========================================= 3 | 4 | .. automodule:: FastProject.tests.test_split_merge 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. FastProject documentation master file, created by 2 | sphinx-quickstart on Thu Jan 21 23:55:35 2016. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to FastProject's documentation! 7 | ======================================= 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 4 13 | 14 | FastProject 15 | 16 | 17 | Indices and tables 18 | ================== 19 | 20 | * :ref:`genindex` 21 | * :ref:`modindex` 22 | * :ref:`search` 23 | 24 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | echo. coverage to run coverage check of the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | 51 | REM Check if sphinx-build is available and fallback to Python version if any 52 | %SPHINXBUILD% 1>NUL 2>NUL 53 | if errorlevel 9009 goto sphinx_python 54 | goto sphinx_ok 55 | 56 | :sphinx_python 57 | 58 | set SPHINXBUILD=python -m sphinx.__init__ 59 | %SPHINXBUILD% 2> nul 60 | if errorlevel 9009 ( 61 | echo. 62 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 63 | echo.installed, then set the SPHINXBUILD environment variable to point 64 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 65 | echo.may add the Sphinx directory to PATH. 66 | echo. 67 | echo.If you don't have Sphinx installed, grab it from 68 | echo.http://sphinx-doc.org/ 69 | exit /b 1 70 | ) 71 | 72 | :sphinx_ok 73 | 74 | 75 | if "%1" == "html" ( 76 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 80 | goto end 81 | ) 82 | 83 | if "%1" == "dirhtml" ( 84 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 88 | goto end 89 | ) 90 | 91 | if "%1" == "singlehtml" ( 92 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 93 | if errorlevel 1 exit /b 1 94 | echo. 95 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 96 | goto end 97 | ) 98 | 99 | if "%1" == "pickle" ( 100 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can process the pickle files. 104 | goto end 105 | ) 106 | 107 | if "%1" == "json" ( 108 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 109 | if errorlevel 1 exit /b 1 110 | echo. 111 | echo.Build finished; now you can process the JSON files. 112 | goto end 113 | ) 114 | 115 | if "%1" == "htmlhelp" ( 116 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 117 | if errorlevel 1 exit /b 1 118 | echo. 119 | echo.Build finished; now you can run HTML Help Workshop with the ^ 120 | .hhp project file in %BUILDDIR%/htmlhelp. 121 | goto end 122 | ) 123 | 124 | if "%1" == "qthelp" ( 125 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 129 | .qhcp project file in %BUILDDIR%/qthelp, like this: 130 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\FastProject.qhcp 131 | echo.To view the help file: 132 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\FastProject.ghc 133 | goto end 134 | ) 135 | 136 | if "%1" == "devhelp" ( 137 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. 141 | goto end 142 | ) 143 | 144 | if "%1" == "epub" ( 145 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 149 | goto end 150 | ) 151 | 152 | if "%1" == "latex" ( 153 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 157 | goto end 158 | ) 159 | 160 | if "%1" == "latexpdf" ( 161 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 162 | cd %BUILDDIR%/latex 163 | make all-pdf 164 | cd %~dp0 165 | echo. 166 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdfja" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf-ja 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "text" ( 181 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 182 | if errorlevel 1 exit /b 1 183 | echo. 184 | echo.Build finished. The text files are in %BUILDDIR%/text. 185 | goto end 186 | ) 187 | 188 | if "%1" == "man" ( 189 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 190 | if errorlevel 1 exit /b 1 191 | echo. 192 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 193 | goto end 194 | ) 195 | 196 | if "%1" == "texinfo" ( 197 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 198 | if errorlevel 1 exit /b 1 199 | echo. 200 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 201 | goto end 202 | ) 203 | 204 | if "%1" == "gettext" ( 205 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 206 | if errorlevel 1 exit /b 1 207 | echo. 208 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 209 | goto end 210 | ) 211 | 212 | if "%1" == "changes" ( 213 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 214 | if errorlevel 1 exit /b 1 215 | echo. 216 | echo.The overview file is in %BUILDDIR%/changes. 217 | goto end 218 | ) 219 | 220 | if "%1" == "linkcheck" ( 221 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 222 | if errorlevel 1 exit /b 1 223 | echo. 224 | echo.Link check complete; look for any errors in the above output ^ 225 | or in %BUILDDIR%/linkcheck/output.txt. 226 | goto end 227 | ) 228 | 229 | if "%1" == "doctest" ( 230 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Testing of doctests in the sources finished, look at the ^ 234 | results in %BUILDDIR%/doctest/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "coverage" ( 239 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of coverage in the sources finished, look at the ^ 243 | results in %BUILDDIR%/coverage/python.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "xml" ( 248 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 252 | goto end 253 | ) 254 | 255 | if "%1" == "pseudoxml" ( 256 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 257 | if errorlevel 1 exit /b 1 258 | echo. 259 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 260 | goto end 261 | ) 262 | 263 | :end 264 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import ez_setup 2 | ez_setup.use_setuptools() 3 | 4 | # Parse version string 5 | import os 6 | this_directory = os.path.dirname(os.path.abspath(__file__)) 7 | version_file = os.path.join(this_directory, "FastProject", "_version.py") 8 | exec(open(version_file).read()) 9 | 10 | from setuptools import setup, find_packages 11 | 12 | setup( 13 | name="FastProject", 14 | version=__version__, 15 | packages=find_packages(), 16 | 17 | entry_points={'console_scripts': ['fastproject = FastProject.CLI:entry']}, 18 | 19 | install_requires=['numpy>=1.11', 'scipy>=0.18.0', 20 | 'scikit-learn>=0.18.1', 'pandas>=0.19.0'], 21 | 22 | include_package_data=True, 23 | 24 | author="David DeTomaso", 25 | author_email="David.DeTomaso@berkeley.edu", 26 | description="A python module for evaulating dimensionality reduction techniques on gene expression data.", 27 | keywords="Bioinformatics dimensionality reduction projection rna-seq", 28 | url="https://github.com/YosefLab/FastProject", 29 | license="Free for Academic and Not-For-Profit Use, See Repository for Details" 30 | ) 31 | --------------------------------------------------------------------------------