├── servalcat ├── refine │ ├── __init__.py │ ├── cgsolve.py │ └── spa.py ├── refmac │ ├── __init__.py │ └── exte.py ├── spa │ ├── __init__.py │ ├── shiftback.py │ ├── translate.py │ ├── realspcc_from_var.py │ └── localcc.py ├── xtal │ ├── __init__.py │ ├── run_refmac_small.py │ └── twin.py ├── __init__.py ├── utils │ ├── __init__.py │ ├── logger.py │ └── generate_operators.py └── __main__.py ├── tests ├── .gitignore ├── 1l2h │ ├── 1l2h.cif.gz │ └── 1l2h.mtz.gz ├── 5e5z │ ├── 5e5z.mtz.gz │ └── 5e5z.pdb.gz ├── 1v9g │ ├── 1v9g-sf.cif.gz │ └── 1v9g-spk.cif.gz ├── 6mw0 │ └── 6mw0-sf.cif.gz ├── biotin │ ├── biotin_talos.mtz │ ├── biotin_talos.ins │ └── biotin_talos.pdb ├── test_for_ci.py ├── test_misc.py ├── test_xtal.py └── test_refine.py ├── docs ├── requirements.txt ├── spa_examples │ ├── chrmine_figs │ │ ├── coot_113-fs8.png │ │ ├── refined_fsc_1.png │ │ ├── refined_fsc_2.png │ │ └── ccpem_input-fs8.png │ ├── p4_figs │ │ ├── coot_fofc_omit_4sigma.png │ │ └── pymol_fofc_omit_4sigma.png │ ├── index.rst │ ├── omitmap.rst │ └── chrmine.rst ├── commands.rst ├── Makefile ├── index.rst ├── help │ ├── trim.txt │ ├── fofc.txt │ └── refine_spa.txt ├── conf.py ├── refmac_keywords.rst ├── spa.rst └── overview.rst ├── .gitmodules ├── commit.sh ├── .readthedocs.yaml ├── src ├── array.h ├── lambertw.hpp ├── refine │ ├── ncsr.hpp │ └── cgsolve.hpp ├── ext.cpp ├── amplitude.cpp └── math.hpp ├── scripts ├── metal_to_json.py ├── test_fc_with_refmac.py ├── test_fw.py └── refine_xtal_true_phase.py ├── pyproject.toml ├── README.md ├── .github └── workflows │ └── ci.yml ├── .gitignore └── CMakeLists.txt /servalcat/refine/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /servalcat/refmac/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /servalcat/spa/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /servalcat/xtal/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | 7dy0/ 2 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx_rtd_theme >= 1.0.0 2 | sphinx >= 2.4.4 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "eigen"] 2 | path = eigen 3 | url = https://gitlab.com/libeigen/eigen.git 4 | -------------------------------------------------------------------------------- /tests/1l2h/1l2h.cif.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/tests/1l2h/1l2h.cif.gz -------------------------------------------------------------------------------- /tests/1l2h/1l2h.mtz.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/tests/1l2h/1l2h.mtz.gz -------------------------------------------------------------------------------- /tests/5e5z/5e5z.mtz.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/tests/5e5z/5e5z.mtz.gz -------------------------------------------------------------------------------- /tests/5e5z/5e5z.pdb.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/tests/5e5z/5e5z.pdb.gz -------------------------------------------------------------------------------- /tests/1v9g/1v9g-sf.cif.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/tests/1v9g/1v9g-sf.cif.gz -------------------------------------------------------------------------------- /tests/1v9g/1v9g-spk.cif.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/tests/1v9g/1v9g-spk.cif.gz -------------------------------------------------------------------------------- /tests/6mw0/6mw0-sf.cif.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/tests/6mw0/6mw0-sf.cif.gz -------------------------------------------------------------------------------- /tests/biotin/biotin_talos.mtz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/tests/biotin/biotin_talos.mtz -------------------------------------------------------------------------------- /docs/spa_examples/chrmine_figs/coot_113-fs8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/docs/spa_examples/chrmine_figs/coot_113-fs8.png -------------------------------------------------------------------------------- /docs/spa_examples/chrmine_figs/refined_fsc_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/docs/spa_examples/chrmine_figs/refined_fsc_1.png -------------------------------------------------------------------------------- /docs/spa_examples/chrmine_figs/refined_fsc_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/docs/spa_examples/chrmine_figs/refined_fsc_2.png -------------------------------------------------------------------------------- /docs/spa_examples/chrmine_figs/ccpem_input-fs8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/docs/spa_examples/chrmine_figs/ccpem_input-fs8.png -------------------------------------------------------------------------------- /docs/spa_examples/p4_figs/coot_fofc_omit_4sigma.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/docs/spa_examples/p4_figs/coot_fofc_omit_4sigma.png -------------------------------------------------------------------------------- /docs/spa_examples/p4_figs/pymol_fofc_omit_4sigma.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/keitaroyam/servalcat/HEAD/docs/spa_examples/p4_figs/pymol_fofc_omit_4sigma.png -------------------------------------------------------------------------------- /docs/spa_examples/index.rst: -------------------------------------------------------------------------------- 1 | SPA examples 2 | ============ 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | chrmine 8 | ab42 9 | omitmap 10 | 11 | -------------------------------------------------------------------------------- /servalcat/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | 9 | __version__ = '0.4.127' 10 | __date__ = '2025-11-24' 11 | -------------------------------------------------------------------------------- /commit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | newv=`python -c 'v={}; exec(open("servalcat/__init__.py").read(), v); v=v["__version__"].split("."); print(".".join(v[:-1])+".{}".format(int(v[-1])+1))'` 3 | date=`date "+%Y-%m-%d"` 4 | sed -i "s/__version__.*/__version__ = '$newv'/; s/__date__.*/__date__ = '$date'/" servalcat/__init__.py 5 | git add servalcat/__init__.py 6 | git commit 7 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-20.04 11 | tools: 12 | python: "3.9" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/conf.py 17 | 18 | # Optionally declare the Python requirements required to build your docs 19 | python: 20 | install: 21 | - requirements: docs/requirements.txt 22 | -------------------------------------------------------------------------------- /docs/commands.rst: -------------------------------------------------------------------------------- 1 | Servalcat commands 2 | ================== 3 | 4 | SPA 5 | --- 6 | 7 | refine_spa_norefmac 8 | ~~~~~~~~~~~~~~~~~~~ 9 | 10 | Refinement pipeline for SPA. The weighted and sharpened Fo-Fc map is calculated after the refinement. For details please see the reference. 11 | 12 | .. literalinclude:: help/refine_spa.txt 13 | :language: console 14 | 15 | fofc 16 | ~~~~ 17 | 18 | .. literalinclude:: help/fofc.txt 19 | :language: console 20 | 21 | trim 22 | ~~~~ 23 | 24 | .. literalinclude:: help/trim.txt 25 | :language: console 26 | 27 | 28 | Utilities 29 | --------- 30 | 31 | TBD 32 | -------------------------------------------------------------------------------- /src/array.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | namespace nb = nanobind; 4 | 5 | template 6 | using np_array = nb::ndarray, nb::device::cpu>; 7 | 8 | // gemmi python/array.h 9 | template 10 | auto make_numpy_array(std::initializer_list size, 11 | std::initializer_list strides={}) { 12 | size_t total_size = 1; 13 | for (size_t i : size) 14 | total_size *= i; 15 | T* c_array = new T[total_size]; 16 | nb::capsule owner(c_array, [](void* p) noexcept { delete [] static_cast(p); }); 17 | return nb::ndarray(c_array, size, owner, strides); 18 | } 19 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /tests/test_for_ci.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import unittest 10 | import json 11 | import os 12 | import sys 13 | from servalcat import utils 14 | from servalcat.refine.refine import RefineParams 15 | from servalcat import ext 16 | root = os.path.abspath(os.path.dirname(__file__)) 17 | 18 | class TestCI(unittest.TestCase): 19 | def test_ext(self): 20 | pdbin = os.path.join(root, "biotin", "biotin_talos.pdb") 21 | st = utils.fileio.read_structure(pdbin) 22 | params = RefineParams(st, refine_xyz=True) 23 | g = ext.Geometry(st, params) 24 | 25 | 26 | if __name__ == '__main__': 27 | unittest.main() 28 | 29 | -------------------------------------------------------------------------------- /servalcat/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | from . import logger 10 | from . import symmetry 11 | from . import fileio 12 | from . import hkl 13 | from . import model 14 | from . import maps 15 | from . import refmac 16 | from . import restraints 17 | from . import commands 18 | 19 | def make_loggraph_str(df, main_title, title_labs, s2=None, float_format=None): 20 | if s2 is not None: 21 | df = df.copy() 22 | df.insert(0, "1/resol^2", s2) 23 | ret = "$TABLE: {} :\n".format(main_title) 24 | ret += "$GRAPHS\n" 25 | all_labs = list(df.columns) 26 | for t, labs in title_labs: 27 | if s2 is not None: labs = ["1/resol^2"] + labs 28 | ret += ": {} :A:{}:\n".format(t, ",".join(str(all_labs.index(l)+1) for l in labs)) 29 | ret += "$$\n" 30 | lines = df.to_string(index=False, index_names=False, header=True, float_format=float_format).splitlines() 31 | ret += lines[0] + "\n$$\n$$\n" 32 | ret += "\n".join(lines[1:]) + "\n$$\n" 33 | return ret 34 | # make_loggraph_str() 35 | 36 | -------------------------------------------------------------------------------- /tests/biotin/biotin_talos.ins: -------------------------------------------------------------------------------- 1 | TITL xscale_a.res in P2(1)2(1)2(1) 2 | CELL 0.0251 5.18 10.28 20.85 90 90 90 3 | ZERR 1 0 0 0 0 0 0 4 | LATT -1 5 | SYMM 0.5-X,-Y,0.5+Z 6 | SYMM -X,0.5+Y,0.5-Z 7 | SYMM 0.5+X,0.5-Y,-Z 8 | SFAC C H N O S 9 | UNIT 40 56 8 12 4 10 | 11 | L.S. 10 12 | PLAN 15 2 13 | CONF 14 | list 6 15 | MORE -1 16 | fmap 2 17 | acta 18 | WGHT 0.2 0 19 | 20 | S001 5 0.56014 0.63151 0.45348 11.00000 0.02654 21 | C002 1 0.52025 0.39483 0.67882 11.00000 0.01706 22 | C003 1 0.55196 0.49761 0.28968 11.00000 0.01916 23 | O004 4 0.42000 0.42808 0.25457 11.00000 0.03010 24 | O005 4 0.46838 0.51248 0.69065 11.00000 0.02996 25 | C006 1 0.72724 0.68200 0.33435 11.00000 0.01880 26 | C007 1 0.87577 0.56469 0.35940 11.00000 0.01434 27 | O008 4 0.38056 0.31104 0.70029 11.00000 0.03472 28 | N009 3 0.53733 0.62905 0.29314 11.00000 0.02731 29 | N00A 3 0.74185 0.45550 0.32910 11.00000 0.01293 30 | C00B 1 0.92304 0.42400 0.45856 11.00000 0.02531 31 | C00C 1 0.61553 0.75380 0.39404 11.00000 0.02046 32 | C00D 1 0.86724 0.56087 0.43238 11.00000 0.01495 33 | C00E 1 0.95269 0.40377 0.52660 11.00000 0.02156 34 | C00F 1 0.70514 0.41716 0.56941 11.00000 0.02811 35 | C00G 1 0.75866 0.37504 0.63883 11.00000 0.02045 36 | HKLF 4 37 | 38 | END 39 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. servalcat documentation master file, created by 2 | sphinx-quickstart on Fri Apr 22 07:49:31 2022. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Servalcat - refinement and map calculation 7 | ========================================== 8 | 9 | Servalcat (for **S**\ tructur\ **e** **r**\ efinement and **val**\ idation for **c**\ rystallography and single p\ **a**\ r\ **t**\ icle analysis) is a software package for atomic model refinement and map calculation. 10 | 11 | Source code repository: https://github.com/keitaroyam/servalcat 12 | 13 | References 14 | ---------- 15 | 16 | | Cryo-EM SPA refinement: 17 | | `Yamashita, K., Palmer, C. M., Burnley, T., Murshudov, G. N. (2021) "Cryo-EM single particle structure refinement and map calculation using Servalcat" Acta Cryst. D77, 1282-1291 `_ 18 | 19 | | Refmacat and restraint generation using GEMMI: 20 | | `Yamashita, K., Wojdyr, M., Long, F., Nicholls, R. A., Murshudov, G. N. (2023) "GEMMI and Servalcat restrain REFMAC5" Acta Cryst. D79, 368-373 `_ 21 | 22 | Table of Contents 23 | ----------------- 24 | .. toctree:: 25 | :maxdepth: 2 26 | 27 | overview 28 | spa 29 | spa_examples/index 30 | xtal 31 | refmac_keywords 32 | commands 33 | -------------------------------------------------------------------------------- /scripts/metal_to_json.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import pandas 4 | import gemmi 5 | 6 | csv_in, json_out = sys.argv[1:] 7 | df = pandas.read_csv(csv_in) 8 | ret = {} 9 | for x in df.to_dict(orient="records"): 10 | metal = gemmi.Element(x["Metal"]).name 11 | ligand = gemmi.Element(x["Ligand"]).name 12 | l = ret.setdefault(metal, {}).setdefault(ligand, []) 13 | modes = [] 14 | for i in range(1, 4): 15 | m = x["mode{}".format(i)] 16 | s = x["std{}".format(i)] 17 | if m == m and s == s: # not nan 18 | modes.append({"mode": m, "std": s}) 19 | l.append({"coord": x["Coordination"], 20 | "median": x["median"], 21 | "mad": x["mad"], 22 | "mean": x["mean"], 23 | "std": x["std"] if x["std"] == x["std"] else 0, 24 | "count": x["count"], 25 | "modes": modes}) 26 | 27 | json.dump({"version": 1, "date": "2023-05-05", "metal_coordination":ret}, open(json_out, "w"), indent=1, allow_nan=False) 28 | 29 | """ 30 | {'Metal': 'Ba', 31 | 'Ligand': 'Cl', 32 | 'Coordination': 8, 33 | 'median': 3.12194718739944, 34 | 'mad': 0.0398797514650128, 35 | 'mean': 3.1440235683357343, 36 | 'std': 0.0460297586743337, 37 | 'count': 20, 38 | 'nmodes': 0, 39 | 'mode1': nan, 40 | 'std1': nan, 41 | 'mode2': nan, 42 | 'std2': nan, 43 | 'mode3': nan, 44 | 'std3': nan} 45 | """ 46 | -------------------------------------------------------------------------------- /tests/biotin/biotin_talos.pdb: -------------------------------------------------------------------------------- 1 | CRYST1 5.140 10.300 20.840 90.00 90.00 90.00 P 21 21 21 2 | HETATM 1 C10 BTN A 1 1.354 11.664 -2.848 1.00 2.45 C 3 | HETATM 2 C11 BTN A 1 0.152 11.332 -3.698 1.00 2.41 C 4 | HETATM 3 C2 BTN A 1 1.893 9.694 1.421 1.00 2.42 C 5 | HETATM 4 C3 BTN A 1 0.339 10.331 4.374 1.00 1.13 C 6 | HETATM 5 C4 BTN A 1 1.973 9.606 2.938 1.00 2.02 C 7 | HETATM 6 C5 BTN A 1 1.178 8.404 3.443 1.00 1.43 C 8 | HETATM 7 C6 BTN A 1 0.482 7.721 2.275 1.00 2.58 C 9 | HETATM 8 C7 BTN A 1 1.986 11.140 0.931 1.00 3.38 C 10 | HETATM 9 C8 BTN A 1 2.313 11.254 -0.557 1.00 2.85 C 11 | HETATM 10 C9 BTN A 1 1.050 11.268 -1.410 1.00 3.09 C 12 | HETATM 11 N1 BTN A 1 0.211 9.007 4.338 1.00 1.87 N 13 | HETATM 12 N2 BTN A 1 1.304 10.731 3.555 1.00 1.94 N 14 | HETATM 13 O11 BTN A 1 -0.052 10.130 -3.963 1.00 2.57 O 15 | HETATM 14 O12 BTN A 1 -0.592 12.256 -4.097 1.00 3.32 O 16 | HETATM 15 O3 BTN A 1 -0.324 11.079 5.073 1.00 2.98 O 17 | HETATM 16 S1 BTN A 1 0.360 8.930 0.995 1.00 2.38 S 18 | END 19 | -------------------------------------------------------------------------------- /tests/test_misc.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import unittest 10 | import numpy 11 | import pandas 12 | import json 13 | import gemmi 14 | import os 15 | import shutil 16 | import sys 17 | import tempfile 18 | import hashlib 19 | from servalcat import utils 20 | 21 | root = os.path.abspath(os.path.dirname(__file__)) 22 | 23 | class RestrTests(unittest.TestCase): 24 | def test_merge_dict(self): 25 | cifs = [os.path.join(root, "dict", f) for f in ("acedrg_link_4D4-MS6.cif", "acedrg_link_MS6-GLY.cif")] 26 | tmpfd, tmpf = tempfile.mkstemp(prefix="servalcat_merged_", suffix=".cif") 27 | os.close(tmpfd) 28 | utils.fileio.merge_ligand_cif(cifs, tmpf) 29 | 30 | doc = gemmi.cif.read(tmpf) 31 | names = set(["comp_list", "link_list", "mod_list", "comp_4D4", "comp_MS6", 32 | "mod_4D4m1", "mod_MS6m1", "mod_MS6m1-0", "mod_GLYm1", 33 | "link_MS6-GLY", "link_4D4-MS6"]) 34 | self.assertTrue(names.issubset([x.name for x in doc])) 35 | # TODO need to test hydrogen generation etc? 36 | os.remove(tmpf) 37 | 38 | # test_merge_dict() 39 | 40 | # class RestrTests 41 | 42 | if __name__ == '__main__': 43 | unittest.main() 44 | 45 | -------------------------------------------------------------------------------- /docs/help/trim.txt: -------------------------------------------------------------------------------- 1 | $ servalcat trim -h 2 | usage: servalcat trim [-h] [--maps MAPS [MAPS ...]] [--mask MASK] 3 | [--pixel_size PIXEL_SIZE] [--model MODEL [MODEL ...]] 4 | [--no_expand_ncs] [--padding PADDING] 5 | [--mask_cutoff MASK_CUTOFF] [--noncubic] [--noncentered] 6 | [--no_shift] [--no_shift_keep_cell] 7 | [--force_cell FORCE_CELL FORCE_CELL FORCE_CELL FORCE_CELL FORCE_CELL FORCE_CELL] 8 | [--disable_cell_check] [--shifts SHIFTS] 9 | 10 | Trim maps and shift models into a small new box. 11 | 12 | optional arguments: 13 | -h, --help show this help message and exit 14 | --maps MAPS [MAPS ...] 15 | Input map file(s) 16 | --mask MASK Mask file 17 | --pixel_size PIXEL_SIZE 18 | Override pixel size (A) 19 | --model MODEL [MODEL ...] 20 | Input atomic model file(s) 21 | --no_expand_ncs Do not expand strict NCS in MTRIX or _struct_ncs_oper 22 | --padding PADDING padding in angstrom unit (default: 10.0) 23 | --mask_cutoff MASK_CUTOFF 24 | Mask value cutoff to define boundary (default: 0.5) 25 | --noncubic 26 | --noncentered If specified non-centered trimming is performed. Not 27 | recommended if having some symmetry 28 | --no_shift If specified resultant maps will have shifted origin 29 | and overlap with the input maps. 30 | --no_shift_keep_cell Keep original unit cell when --no_shift is given 31 | --force_cell FORCE_CELL FORCE_CELL FORCE_CELL FORCE_CELL FORCE_CELL FORCE_CELL 32 | Use specified unit cell parameter 33 | --disable_cell_check Turn off unit cell consistency test 34 | --shifts SHIFTS Specify shifts.json to use precalculated parameters 35 | 36 | If --mask is provided, a boundary is decided using the mask and --padding. 37 | Otherwise the model is used. 38 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | import os 7 | 8 | # -- Path setup -------------------------------------------------------------- 9 | 10 | # If extensions (or modules to document with autodoc) are in another directory, 11 | # add these directories to sys.path here. If the directory is relative to the 12 | # documentation root, use os.path.abspath to make it absolute, like shown here. 13 | # 14 | # import os 15 | # import sys 16 | # sys.path.insert(0, os.path.abspath('.')) 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | 21 | project = 'Servalcat' 22 | copyright = '2021-2024, Keitaro Yamashita' 23 | author = 'Keitaro Yamashita, Garib Murshudov' 24 | 25 | 26 | # -- General configuration --------------------------------------------------- 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | ] 33 | 34 | # Add any paths that contain templates here, relative to this directory. 35 | templates_path = ['_templates'] 36 | 37 | # List of patterns, relative to source directory, that match files and 38 | # directories to ignore when looking for source files. 39 | # This pattern also affects html_static_path and html_extra_path. 40 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 41 | 42 | 43 | # -- Options for HTML output ------------------------------------------------- 44 | 45 | # The theme to use for HTML and HTML Help pages. See the documentation for 46 | # a list of builtin themes. 47 | # 48 | if not os.environ.get('READTHEDOCS'): 49 | html_theme = 'sphinx_rtd_theme' 50 | 51 | # Add any paths that contain custom static files (such as style sheets) here, 52 | # relative to this directory. They are copied after the builtin static files, 53 | # so a file named "default.css" will overwrite the builtin "default.css". 54 | html_static_path = ['_static'] 55 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["scikit-build-core==0.11.6", "nanobind==2.9.2"] 3 | build-backend = "scikit_build_core.build" 4 | 5 | # https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ 6 | [project] 7 | name = "servalcat" 8 | dynamic = ["version"] 9 | requires-python = ">=3.8" 10 | dependencies = ['packaging', 'numpy>=1.15', 'scipy', 'pandas>=1.1.0', 'omegaconf==2.3.0', 'gemmi==0.7.4'] 11 | description="Structure refinement and validation for crystallography and single particle analysis" 12 | readme = "README.md" 13 | authors = [ 14 | { name = "Keitaro Yamashita", email = "" }, 15 | { name = "Garib N. Murshudov", email = "" }, 16 | ] 17 | urls.repository = "https://github.com/keitaroyam/servalcat" 18 | license = {text = "MPL-2.0"} 19 | 20 | [project.scripts] 21 | servalcat = "servalcat.__main__:main" 22 | refmacat = "servalcat.refmac.refmac_wrapper:command_line" 23 | 24 | [tool.scikit-build] 25 | #build-dir = "/tmp/gemmi_build2/{wheel_tag}" 26 | wheel.expand-macos-universal-tags = true # not sure if this is useful 27 | cmake.build-type = "Release" 28 | build.verbose = true 29 | sdist.include = ["eigen/Eigen", "eigen/README.md", "eigen/COPYING*"] 30 | sdist.exclude = [".*", "eigen/", "scripts/"] 31 | 32 | [tool.scikit-build.metadata.version] 33 | provider = "scikit_build_core.metadata.regex" 34 | input = "servalcat/__init__.py" 35 | 36 | [tool.cibuildwheel] 37 | # increase pip debugging output 38 | build-verbosity = 2 39 | test-command = "python {project}/tests/test_for_ci.py" # comment out to test with gemmi build 40 | [tool.cibuildwheel.environment] 41 | SKBUILD_CMAKE_ARGS = '-DGENERATE_STUBS=OFF' # -DINSTALL_GEMMI_IF_BUILT=1 to test with gemmi build 42 | 43 | # Needed for full C++17 support on macOS 44 | [tool.cibuildwheel.macos.environment] 45 | MACOSX_DEPLOYMENT_TARGET = "10.14" 46 | SKBUILD_CMAKE_ARGS = '-DGENERATE_STUBS=OFF' # -DINSTALL_GEMMI_IF_BUILT=1 to test with gemmi build 47 | 48 | [tool.codespell] 49 | skip = './eigen,./gemmi,*.pdb,*.crd,*.cif,*.mmcif,*.ent,*.log*,*.dic,tags,*.bak,*build*,*~' 50 | # codespell apparently requires that the words here are lowercase 51 | ignore-words-list = 'fo,fom,dum,varn,readd,alph,valu,chec,buil' 52 | 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Servalcat 2 | [![Build](https://github.com/keitaroyam/servalcat/workflows/CI/badge.svg)](https://github.com/keitaroyam/servalcat/actions/workflows/ci.yml) 3 | [![PyPI](https://img.shields.io/pypi/v/servalcat?color=blue)](https://pypi.org/project/servalcat/) 4 | 5 | **S**tructur**e** **r**efinement and **val**idation for **c**rystallography and single p**a**r**t**icle analysis 6 | 7 | Servalcat implements pipelines that use Refmac5: 8 | * `servalcat refine_spa`: cryo-EM SPA refinement pipeline 9 | * `servalcat refine_cx`: small molecule crystallography 10 | 11 | and a Refmac5 controller 12 | * `refmacat`: behaves as Refmac, but uses GEMMI for restraint generation instead of MAKECIF 13 | 14 | Now “No Refmac5” refinement programs have been actively developed: 15 | * `servalcat refine_geom`: geometry optimization 16 | * `servalcat refine_spa_norefmac`: "No Refmac" version of refine\_spa 17 | * `servalcat refine_xtal_norefmac`: crystallographic refinement 18 | 19 | Also, it has several utility commands: `servalcat util`. 20 | 21 | ## Installation 22 | 23 | ``` 24 | pip install servalcat 25 | ``` 26 | will install the stable version. 27 | 28 | The required GEMMI version is now [v0.7.4](https://github.com/project-gemmi/gemmi/releases/tag/v0.7.4). It may not work with the latest gemmi code from the github. The policy is in the main branch I only push the code that works with the latest package of GEMMI. 29 | 30 | To use the Refmac5 related commands, you also need to install [CCP4](https://www.ccp4.ac.uk/). For "No Refmac5" commands, you may just need [the monomer library](https://github.com/MonomerLibrary/monomers) if CCP4 is not installed. 31 | 32 | **Notice:** 33 | From ver. 0.4.6, Servalcat is no longer python-only package and has some C++ code. If you build Servalcat by yourself, probably you also need to build GEMMI using the same compiler. 34 | 35 | ## Usage 36 | Please read the documentation: https://servalcat.readthedocs.io/en/latest/ 37 | 38 | ## References 39 | * [Yamashita, K., Wojdyr, M., Long, F., Nicholls, R. A., Murshudov, G. N. (2023) "GEMMI and Servalcat restrain REFMAC5" *Acta Cryst.* D**79**, 368-373](https://doi.org/10.1107/S2059798323002413) 40 | * [Yamashita, K., Palmer, C. M., Burnley, T., Murshudov, G. N. (2021) "Cryo-EM single particle structure refinement and map calculation using Servalcat" *Acta Cryst. D***77**, 1282-1291](https://doi.org/10.1107/S2059798321009475) 41 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build_wheels: 7 | name: Wheels on ${{ matrix.os }} 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [ubuntu-22.04, macos-15-intel, macos-14, windows-2022] 12 | 13 | steps: 14 | - uses: actions/checkout@v5 15 | with: 16 | submodules: true 17 | - name: Build wheels 18 | uses: pypa/cibuildwheel@v3.3.0 19 | env: 20 | CIBW_BUILD: 'cp*' 21 | CIBW_SKIP: '*musllinux* *_i686 *-win32' 22 | CIBW_TEST_SKIP: 'cp38*_arm64' 23 | 24 | - run: ls -lh wheelhouse 25 | shell: bash 26 | 27 | - uses: actions/upload-artifact@v4 28 | with: 29 | name: wheels2-${{ matrix.os }} 30 | path: wheelhouse/*.whl 31 | 32 | merge: 33 | runs-on: ubuntu-latest 34 | needs: build_wheels 35 | steps: 36 | - name: Merge Artifacts 37 | uses: actions/upload-artifact/merge@v4 38 | with: 39 | name: wheels2 40 | pattern: wheels2-* 41 | delete-merged: true 42 | 43 | standard: 44 | strategy: 45 | fail-fast: false 46 | matrix: 47 | os: [ubuntu-22.04, ubuntu-24.04, macos-15-intel, macos-14, macos-15, windows-latest] 48 | py: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.14t'] 49 | name: ${{ matrix.os }} py${{ matrix.py }} 50 | runs-on: ${{ matrix.os }} 51 | needs: merge 52 | if: "!contains(github.event.head_commit.message, '[skip ci]')" 53 | steps: 54 | - uses: actions/checkout@v5 55 | with: 56 | submodules: true 57 | 58 | - name: checkout monomers 59 | uses: actions/checkout@v5 60 | with: 61 | repository: MonomerLibrary/monomers 62 | path: monomers 63 | 64 | - uses: actions/download-artifact@v4 65 | with: 66 | name: wheels2 67 | path: wheels2 68 | - uses: actions/setup-python@v6 69 | with: 70 | python-version: ${{ matrix.py }} 71 | - name: Python Setup 72 | run: python -m pip install --find-links=wheels2 servalcat 73 | # add gemmi here to test with gemmi build 74 | - name: Python Tests 75 | continue-on-error: true 76 | env: 77 | CLIBD_MON: ${{runner.workspace}}/servalcat/monomers 78 | run: | 79 | cd tests/ 80 | python3 -m unittest discover -v -s . 81 | cd .. 82 | -------------------------------------------------------------------------------- /docs/help/fofc.txt: -------------------------------------------------------------------------------- 1 | $ servalcat fofc -h 2 | usage: servalcat fofc [-h] [--halfmaps HALFMAPS HALFMAPS | --map MAP] 3 | [--pixel_size PIXEL_SIZE] --model MODEL -d RESOLUTION 4 | [-m MASK] [-r MASK_RADIUS] [--no_check_mask_with_model] 5 | [-B B] [--half1_only] [--normalized_map] 6 | [--no_fsc_weights] [--sharpening_b SHARPENING_B] 7 | [--trim] [--trim_mtz] [--monlib MONLIB] [--omit_proton] 8 | [--omit_h_electron] [--source {electron,xray,neutron}] 9 | [-o OUTPUT_PREFIX] [--keep_charges] 10 | 11 | Fo-Fc map calculation based on model and data errors 12 | 13 | options: 14 | -h, --help show this help message and exit 15 | --halfmaps HALFMAPS HALFMAPS 16 | --map MAP Use only if you really do not have half maps. 17 | --pixel_size PIXEL_SIZE 18 | Override pixel size (A) 19 | --model MODEL Input atomic model file 20 | -d RESOLUTION, --resolution RESOLUTION 21 | -m MASK, --mask MASK mask file 22 | -r MASK_RADIUS, --mask_radius MASK_RADIUS 23 | mask radius (not used if --mask is given) 24 | --no_check_mask_with_model 25 | Disable mask test using model 26 | -B B Estimated blurring 27 | --half1_only Only use half 1 for map calculation (use half 2 only 28 | for noise estimation) 29 | --normalized_map Write normalized map in the masked region. Now this is 30 | on by default. 31 | --no_fsc_weights Just for debugging purpose: turn off FSC-based 32 | weighting 33 | --sharpening_b SHARPENING_B 34 | Use B value (negative value for sharpening) instead of 35 | standard deviation of the signal 36 | --trim Write trimmed maps 37 | --trim_mtz Write trimmed mtz 38 | --monlib MONLIB Monomer library path. Default: $CLIBD_MON 39 | --omit_proton Omit proton from model in map calculation 40 | --omit_h_electron Omit hydrogen electrons from model in map calculation 41 | --source {electron,xray,neutron} 42 | -o OUTPUT_PREFIX, --output_prefix OUTPUT_PREFIX 43 | output file name prefix (default: diffmap) 44 | --keep_charges Use scattering factor for charged atoms. Use it with 45 | care. 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | \#* 2 | .#* 3 | .*.swp 4 | *~ 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # files generated by cmake 15 | /CMakeFiles/ 16 | /Testing/ 17 | /CMakeCache.txt 18 | /cmake_install.cmake 19 | /CTestTestfile.cmake 20 | /Makefile 21 | /compile_commands.json 22 | /install_manifest.txt 23 | 24 | # Distribution / packaging 25 | .Python 26 | build*/ 27 | develop-eggs/ 28 | dist/ 29 | downloads/ 30 | eggs/ 31 | .eggs/ 32 | lib/ 33 | lib64/ 34 | parts/ 35 | sdist/ 36 | var/ 37 | wheels/ 38 | pip-wheel-metadata/ 39 | share/python-wheels/ 40 | *.egg-info/ 41 | .installed.cfg 42 | *.egg 43 | MANIFEST 44 | 45 | # PyInstaller 46 | # Usually these files are written by a python script from a template 47 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 48 | *.manifest 49 | *.spec 50 | 51 | # Installer logs 52 | pip-log.txt 53 | pip-delete-this-directory.txt 54 | 55 | # Unit test / coverage reports 56 | htmlcov/ 57 | .tox/ 58 | .nox/ 59 | .coverage 60 | .coverage.* 61 | .cache 62 | nosetests.xml 63 | coverage.xml 64 | *.cover 65 | *.py,cover 66 | .hypothesis/ 67 | .pytest_cache/ 68 | 69 | # Translations 70 | *.mo 71 | *.pot 72 | 73 | # Django stuff: 74 | *.log 75 | local_settings.py 76 | db.sqlite3 77 | db.sqlite3-journal 78 | 79 | # Flask stuff: 80 | instance/ 81 | .webassets-cache 82 | 83 | # Scrapy stuff: 84 | .scrapy 85 | 86 | # Sphinx documentation 87 | docs/_build/ 88 | 89 | # PyBuilder 90 | target/ 91 | 92 | # Jupyter Notebook 93 | .ipynb_checkpoints 94 | 95 | # IPython 96 | profile_default/ 97 | ipython_config.py 98 | 99 | # pyenv 100 | .python-version 101 | 102 | # pipenv 103 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 104 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 105 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 106 | # install all needed dependencies. 107 | #Pipfile.lock 108 | 109 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 110 | __pypackages__/ 111 | 112 | # Celery stuff 113 | celerybeat-schedule 114 | celerybeat.pid 115 | 116 | # SageMath parsed files 117 | *.sage.py 118 | 119 | # Environments 120 | .env 121 | .venv 122 | env/ 123 | venv/ 124 | ENV/ 125 | env.bak/ 126 | venv.bak/ 127 | 128 | # Spyder project settings 129 | .spyderproject 130 | .spyproject 131 | 132 | # Rope project settings 133 | .ropeproject 134 | 135 | # mkdocs documentation 136 | /site 137 | 138 | # mypy 139 | .mypy_cache/ 140 | .dmypy.json 141 | dmypy.json 142 | 143 | # Pyre type checker 144 | .pyre/ 145 | -------------------------------------------------------------------------------- /src/lambertw.hpp: -------------------------------------------------------------------------------- 1 | #ifndef ROBERTDJ_LAMBERTW_HPP_ 2 | #define ROBERTDJ_LAMBERTW_HPP_ 3 | #include 4 | // This code was taken from https://github.com/robertdj/LambertW.jl 5 | 6 | // The LambertW.jl package is licensed under the MIT "Expat" License: 7 | // Copyright (c) 2014: Robert Dahl Jacobsen. 8 | // Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 9 | // The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 10 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 11 | 12 | namespace lambertw { 13 | inline double lambertw_approx(double x) { 14 | const double e = std::exp(1.); 15 | if (x <= 1) { 16 | const double sqrt2 = std::sqrt(2.); 17 | const double sqeta = std::sqrt(2 + 2 * e * x); 18 | const double N2 = 3 * sqrt2 + 6 - (((2237. + 1457 * sqrt2) * e - 4108 * sqrt2 - 5764) * sqeta) / ((215. + 199 * sqrt2) * e - 430 * sqrt2 - 796); 19 | const double N1 = (1. - 1. / sqrt2) * (N2 + sqrt2); 20 | return -1 + sqeta / (1. + N1 * sqeta / (N2 + sqeta)); 21 | } else 22 | return std::log( 6 * x / (5 * std::log( 12. / 5. * (x / std::log(1. + 12 * x / 5.))))); 23 | } 24 | 25 | // may not work if x < exp(-36) 26 | inline double lambertw(double x, double prec) { 27 | if (x < -std::exp(-1.)) 28 | return NAN; 29 | 30 | // First approximation 31 | double W = lambertw_approx(x); 32 | 33 | // Compute residual using logarithms to avoid numerical overflow 34 | // When x == 0, r = NaN, but here the approximation is exact and 35 | // the while loop below is unnecessary 36 | double r = std::abs(W - std::log(std::abs(x)) + std::log(std::abs(W))); 37 | 38 | // Apply Fritsch’s method to increase precision 39 | for (int i = 0; i < 5; ++i) { 40 | if (r <= prec) break; 41 | const double z = std::log(x / W) - W; 42 | const double q = 2 * (1. + W) * (1 + W + 2. / 3. * z); 43 | const double epsilon = z * (q - z) / ((1 + W) * (q - 2 * z)); 44 | W *= 1 + epsilon; 45 | r = std::abs(W - std::log(std::abs(x)) + std::log(std::abs(W))); 46 | } 47 | return W; 48 | } 49 | 50 | } 51 | #endif 52 | -------------------------------------------------------------------------------- /src/refine/ncsr.hpp: -------------------------------------------------------------------------------- 1 | // Author: "Keitaro Yamashita, Garib N. Murshudov" 2 | // MRC Laboratory of Molecular Biology 3 | 4 | #ifndef SERVALCAT_REFINE_NCSR_HPP_ 5 | #define SERVALCAT_REFINE_NCSR_HPP_ 6 | 7 | #include 8 | #include // for Structure, Atom 9 | #include // for AlignmentResult 10 | #include "../math.hpp" 11 | #include 12 | 13 | namespace servalcat { 14 | 15 | struct NcsList { 16 | struct Ncs { 17 | Ncs(const gemmi::AlignmentResult &al, 18 | gemmi::ConstResidueSpan fixed, gemmi::ConstResidueSpan movable, 19 | const std::string &chain_fixed, const std::string &chain_movable) 20 | : chains(std::make_pair(chain_fixed, chain_movable)) { 21 | auto it1 = fixed.first_conformer().begin(); 22 | auto it2 = movable.first_conformer().begin(); 23 | n_atoms.push_back(0); 24 | for (const auto &item : al.cigar) { 25 | char op = item.op(); 26 | for (uint32_t i = 0; i < item.len(); ++i) { 27 | if (op == 'M' && it1->name == it2->name) { 28 | for (const gemmi::Atom& a1 : it1->atoms) 29 | if (const gemmi::Atom* a2 = it2->find_atom(a1.name, a1.altloc, a1.element)) 30 | atoms.emplace_back(&a1, a2); 31 | seqids.emplace_back(it1->seqid, it2->seqid); 32 | n_atoms.push_back(atoms.size()); 33 | } 34 | if (op == 'M' || op == 'I') 35 | ++it1; 36 | if (op == 'M' || op == 'D') 37 | ++it2; 38 | } 39 | } 40 | } 41 | 42 | void calculate_local_rms(int nlen) { 43 | local_rms.clear(); 44 | if (seqids.size() < nlen) 45 | return; 46 | for (int i = 0; i < seqids.size() - nlen; ++i) { 47 | const int n = n_atoms.at(i + nlen) - n_atoms.at(i); 48 | Eigen::MatrixXd x(n, 3), y(n, 3); 49 | for (int j = 0, m = 0; j < nlen; ++j) 50 | for (int k = n_atoms.at(i+j); k < n_atoms.at(i+j+1); ++k, ++m) 51 | for (int l = 0; l < 3; ++l) { 52 | x(m, l) = atoms[k].first->pos.at(l); 53 | y(m, l) = atoms[k].second->pos.at(l); 54 | } 55 | local_rms.push_back(procrust_dist(x, y)); 56 | } 57 | } 58 | 59 | std::vector> atoms; 60 | std::vector> seqids; 61 | std::pair chains; 62 | std::vector n_atoms; 63 | std::vector local_rms; 64 | }; 65 | 66 | void set_pairs() { 67 | all_pairs.clear(); 68 | for (const auto &ncs : ncss) { 69 | all_pairs.emplace_back(); 70 | for (const auto &p : ncs.atoms) 71 | all_pairs.back().emplace(p.first, p.second); 72 | } 73 | } 74 | 75 | std::vector ncss; 76 | std::vector> all_pairs; 77 | }; 78 | } // namespace servalcat 79 | #endif 80 | -------------------------------------------------------------------------------- /docs/refmac_keywords.rst: -------------------------------------------------------------------------------- 1 | Supported Refmac keywords 2 | ========================= 3 | 4 | In Servalcat "No Refmac" refinement programs, some Refmac keywords are supported. 5 | 6 | External restraints 7 | ------------------- 8 | 9 | (Almost?) all the external restraint keywords are supported. 10 | 11 | EXTErnal UNDEfined [IGNOre | STOP] 12 | Default: stop. When ``igno`` is given, the program continues even when it encounters external restraints defined for non-existent atoms. 13 | 14 | EXTErnal WEIGht SCALe [] [DISTance ] [ANGLe ] [TORSion ] [CHIRal ] [PLANe ] [INTErval ] 15 | Default: 1.0. The sigma values in subsequent keywords will be divided by the specified value. 16 | 17 | EXTErnal WEIGht [SGMN ] [SGMX ] 18 | Default: no capping. Cap the sigma values in subsequent keywords (only for dist, angl, chir, tors). 19 | 20 | EXTErnal ALPHall 21 | Default: 1.0. Change the default alpha value for subsequent keywords. 22 | 23 | EXTErnal TYPEAll 24 | Default: 2. Change the default external restraint types (0, 1, 2) for subsequent keywords. 25 | 26 | EXTErnal [SYMAll | SYMAll [Yes | No] [EXCLude [SELF]]] 27 | Default: no symmetry. Change whether symmetry should be considered or not for subsequent keywords. 28 | 29 | EXTErnal [DMIN ] [DMAX ] 30 | Default: no capping. Cap the ideal distance values in subsequent keywords. 31 | 32 | EXTErnal [DISTance | ANGLe | PLANe | CHIRal | TORSion] [TYPE ] [SYMM Y|N] [VALUe ] [SIGMa ] [ALPHa ] 33 | Define distance/angle/plane/chiral/torsion restraints. SYMM is only supported for distance and angle. ALPHA is only for distances. See `Barron (2019) `_ for alpha. 34 | 35 | EXTErnal INTErval [DMIN ] [DMAX ] [SMIN ] [SMAX ] 36 | Define an interval distance restraint. 37 | 38 | EXTErnal STACking PLANe 1 PLANe 2 [DISTance ] [SDDI ] [ANGLe ] [SDAN ] 39 | Define a stacking restraint. 40 | 41 | EXTErnal HARMonic [SIGMa ] 42 | Define a positional harmonic restraint. 43 | 44 | 45 | Restraint weights 46 | ----------------- 47 | 48 | DISTance 49 | Default: 1.0. The bond length sigma is effectively divided by this number. 50 | 51 | ANGLe 52 | Default: 1.0. The bond angle sigma is effectively divided by this number. 53 | 54 | TORSion 55 | Default: 1.0. The torsion angle sigma is effectively divided by this number. 56 | 57 | PLANe 58 | Default: 1.0. The planarity sigma is effectively divided by this number. 59 | 60 | CHIRal 61 | Default: 1.0. The chirality sigma is effectively divided by this number. 62 | 63 | [VDWr | VANDerwaal | NONBonding] 64 | Default: 1.0. The nonbonding interaction sigma is effectively divided by this number. 65 | 66 | 67 | Controlling restraints 68 | ---------------------- 69 | 70 | RESTraint TORSion [INCLude | EXCLude] [NONE | RESIdue | GROUp | LINK ] [NAME ] [VALUe ] [SIGMa ] [PERIod ] 71 | Update torsion angle restraints. For example, ``restr tors include resi ARG name chi5 sigma 2.0`` will make ARG's chi5 torsion angle sigma 2.0 (ideal and period unchanged). -------------------------------------------------------------------------------- /docs/spa.rst: -------------------------------------------------------------------------------- 1 | Single particle analysis (SPA) 2 | ============================== 3 | 4 | Half maps 5 | --------- 6 | TBD 7 | 8 | Point group symmetry 9 | -------------------- 10 | In SPA maps may have point group symmetry: :math:`C_n, D_n, T, O,` or :math:`I`. In such case atomic model must follow the symmetry. Our recommendation is to refine a model in the asymmetric unit (ASU) with symmetry constraints. In crystallography we always refine ASU model and refinement programs internally take symmetry copies into account. We want to do the same thing in SPA. 11 | 12 | The ASU model can be deposited to PDB together with symmetry annotations (e.g. `7a4m `_, `7a5v `_, `7w9w `_). Both ASU model and biological assembly are available at the PDB website. Strictly speaking, biological assembly is not always the same as symmetry in reconstruction. I will explain this later. 13 | 14 | Symmetry information (list of rotation matrix and translation vector) is described in `MTRIX `_ header of PDB file or in `_struct_ncs_oper `_ of mmCIF file. 15 | 16 | For example, in 7w9w.pdb: 17 | :: 18 | 19 | MTRIX1 1 1.000000 0.000000 0.000000 0.00000 1 20 | MTRIX2 1 0.000000 1.000000 0.000000 0.00000 1 21 | MTRIX3 1 0.000000 0.000000 1.000000 0.00000 1 22 | MTRIX1 2 -0.500000 -0.866025 0.000000 180.75898 23 | MTRIX2 2 0.866025 -0.500000 0.000000 48.43422 24 | MTRIX3 2 0.000000 0.000000 1.000000 0.00000 25 | MTRIX1 3 -0.500000 0.866025 0.000000 48.43422 26 | MTRIX2 3 -0.866025 -0.500000 0.000000 180.75898 27 | MTRIX3 3 0.000000 0.000000 1.000000 0.00000 28 | 29 | in 7w9w.cif: 30 | :: 31 | 32 | loop_ 33 | _struct_ncs_oper.id 34 | _struct_ncs_oper.code 35 | _struct_ncs_oper.matrix[1][1] 36 | _struct_ncs_oper.matrix[1][2] 37 | _struct_ncs_oper.matrix[1][3] 38 | _struct_ncs_oper.vector[1] 39 | _struct_ncs_oper.matrix[2][1] 40 | _struct_ncs_oper.matrix[2][2] 41 | _struct_ncs_oper.matrix[2][3] 42 | _struct_ncs_oper.vector[2] 43 | _struct_ncs_oper.matrix[3][1] 44 | _struct_ncs_oper.matrix[3][2] 45 | _struct_ncs_oper.matrix[3][3] 46 | _struct_ncs_oper.vector[3] 47 | _struct_ncs_oper.details 48 | 1 given 1 0 0 0 0 1 0 0 0 0 1 0 ? 49 | 2 generate -0.5 -0.866025404 0 180.758982 0.866025404 -0.5 0 48.4342232 0 0 1 0 ? 50 | 3 generate -0.5 0.866025404 0 48.4342232 -0.866025404 -0.5 0 180.758982 0 0 1 0 ? 51 | 52 | Using this information, symmetry-expanded model can be generated. Various programs can do this: 53 | 54 | .. code-block:: console 55 | 56 | $ gemmi convert --expand-ncs=new 7w9w.pdb 7w9w_expanded.pdb 57 | $ servalcat util expand --model 7w9w.pdb 58 | $ phenix.pdb.mtrix_reconstruction 7w9w.pdb 59 | 60 | For the refinement with point group symmetry, please see :doc:`ChRmine example `. 61 | 62 | Helical symmetry 63 | ---------------- 64 | In helical reconstruction, axial symmetry (:math:`C_n` or :math:`D_n`), twist (in degree), and rise (in Å) parameters define the symmetry. 65 | 66 | For the refinement with helical symmetry, please see :doc:`amyloid-β 42 example `. 67 | -------------------------------------------------------------------------------- /servalcat/refine/cgsolve.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | from servalcat.utils import logger 10 | import numpy 11 | 12 | def cgsolve_rm(A, v, M, gamma=0., ncycl=2000, toler=1.e-4): 13 | gamma_save = None 14 | step_flag = False 15 | gamma_flag = False 16 | conver_flag = False 17 | #f = numpy.zeros(len(v)) 18 | dv = numpy.zeros(len(v)) 19 | dv_save = numpy.zeros(len(v)) 20 | 21 | # preconditioning 22 | A = M.T.dot(A).dot(M) 23 | #print("precond_A=") 24 | #print(A.toarray()) 25 | v = M.T.dot(v) 26 | 27 | vnorm = numpy.sqrt(numpy.dot(v, v)) 28 | test_lim = toler * vnorm 29 | max_gamma_cyc = 500 30 | step = 0.05 31 | 32 | for gamma_cyc in range(max_gamma_cyc): 33 | if gamma_cyc != 0: gamma += step 34 | 35 | logger.writeln("Trying gamma equal {:.4e}".format(gamma)) 36 | r = v - (A.dot(dv) + gamma * dv) 37 | rho = [numpy.dot(r, r)] 38 | if rho[0] < toler: 39 | break 40 | 41 | exit_flag = False 42 | for itr in range(ncycl): 43 | if itr == 0: 44 | p = r 45 | beta = 0. 46 | else: 47 | beta = rho[-1] / rho[-2] 48 | p = r + beta * p 49 | 50 | f = A.dot(p) + gamma * p 51 | alpha = rho[-1] / numpy.dot(p, f) 52 | dv += alpha * p 53 | if itr%20 == 0: 54 | r = v - (A.dot(dv) + gamma * dv) 55 | else: 56 | r = r - alpha * f 57 | 58 | rho.append(numpy.dot(r, r)) 59 | #print("rho=", rho) 60 | if numpy.sqrt(rho[-1]) > 2 * numpy.sqrt(rho[-2]): 61 | logger.writeln("Not converging with gamma equal {:.4e}".format(gamma)) 62 | step *= 1.05 63 | break 64 | 65 | if numpy.sqrt(rho[-1]) < test_lim: 66 | if not gamma_flag: 67 | logger.writeln("Convergence reached with no gamma cycles") 68 | exit_flag = True 69 | break # goto 120 70 | elif conver_flag: 71 | logger.writeln("Convergence reached with gamma equal {:.4e}".format(gamma)) 72 | step *= 1.01 73 | exit_flag = True 74 | break # goto 120 75 | else: 76 | conver_flag = True 77 | step_flag = True 78 | gamma_save = gamma 79 | dv_save = numpy.copy(dv) 80 | gamma = max(0, gamma - step/5.) 81 | step = max(step/1.1, 0.0001) 82 | logger.writeln("Gamma decreased to {:.4e}".format(gamma)) 83 | exit_flag = True 84 | break 85 | # end of inner loop 86 | if exit_flag: break 87 | 88 | gamma_flag = True 89 | if not conver_flag: 90 | dv[:] = 0. 91 | else: 92 | dv = numpy.copy(dv_save) 93 | gamma = gamma_save 94 | logger.writeln("Back to gamma equal {:.4e}".format(gamma)) 95 | 96 | 97 | # postconditioning 98 | dv = M.dot(dv) 99 | return dv, gamma 100 | # cgsolve_rm() 101 | -------------------------------------------------------------------------------- /src/ext.cpp: -------------------------------------------------------------------------------- 1 | // Author: "Keitaro Yamashita, Garib N. Murshudov" 2 | // MRC Laboratory of Molecular Biology 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include // CustomCoef 14 | 15 | namespace nb = nanobind; 16 | void add_refine(nb::module_& m); // refine.cpp 17 | void add_intensity(nb::module_& m); // intensity.cpp 18 | void add_amplitude(nb::module_& m); // amplitude.cpp 19 | void add_twin(nb::module_& m); // twin.cpp 20 | 21 | template // radius in A^-1 unit 22 | gemmi::FPhiGrid hard_sphere_kernel_recgrid(std::tuple size, 23 | const gemmi::UnitCell &unit_cell, 24 | T radius) { 25 | gemmi::Grid grid; 26 | grid.set_size(std::get<0>(size), std::get<1>(size), std::get<2>(size)); 27 | grid.set_unit_cell(unit_cell); 28 | grid.spacegroup = gemmi::find_spacegroup_by_number(1); // would not work in other space group 29 | // box should be sufficiently large. 30 | for (int w = -grid.nw/2; w < grid.nw/2; ++w) 31 | for (int v = -grid.nv/2; v < grid.nv/2; ++v) 32 | for (int u = -grid.nu/2; u < grid.nu/2; ++u) { 33 | const size_t idx = grid.index_near_zero(u, v, w); 34 | const gemmi::Position delta = grid.unit_cell.orthogonalize_difference(grid.get_fractional(u, v, w)); 35 | const double t = 2 * gemmi::pi() * delta.length() * radius; 36 | if (t == 0) 37 | grid.data[idx] = 1; 38 | else 39 | grid.data[idx] = 3. * (-t * std::cos(t) + std::sin(t)) / (t * t * t); 40 | } 41 | auto rg = gemmi::transform_map_to_f_phi(grid, false); 42 | const T rg_sum = rg.sum().real(); 43 | for (auto &x : rg.data) 44 | x /= rg_sum; 45 | return rg; 46 | } 47 | 48 | template 49 | void soft_mask_from_model(gemmi::Grid &grid, gemmi::Model &model, 50 | float radius, float soft_width) { 51 | gemmi::mask_points_in_constant_radius(grid, model, radius, 1., false, false); 52 | const float max_r = radius + soft_width; 53 | gemmi::NeighborSearch ns(model, grid.unit_cell, max_r); 54 | ns.populate(); 55 | for (int w = 0; w < grid.nw; ++w) 56 | for (int v = 0; v < grid.nv; ++v) 57 | for (int u = 0; u < grid.nu; ++u) { 58 | size_t idx = grid.index_q(u, v, w); 59 | if (grid.data[idx] > 0) continue; 60 | const gemmi::Position p = grid.get_position(u, v, w); 61 | const auto result = ns.find_nearest_atom_within_k(p, 1, max_r); 62 | const float d = std::sqrt(result.second) - radius; 63 | if (d < 0) continue; // should not happen 64 | grid.data[idx] = T(0.5 + 0.5 * std::cos(gemmi::pi() * d / soft_width)); 65 | } 66 | } 67 | 68 | NB_MODULE(ext, m) { 69 | m.doc() = "Servalcat extension"; 70 | // importing gemmi allows to output gemmi types from nanobind.stubgen. 71 | nb::module_::import_("gemmi"); 72 | 73 | add_refine(m); 74 | add_intensity(m); 75 | add_amplitude(m); 76 | add_twin(m); 77 | m.def("hard_sphere_kernel_recgrid", hard_sphere_kernel_recgrid); 78 | m.def("soft_mask_from_model", soft_mask_from_model); 79 | // gemmi/python/elem.cpp 80 | using CustomCoef = gemmi::CustomCoef; 81 | m.def("set_custom_form_factors", [](const std::vector& pp) { 82 | CustomCoef::data.clear(); 83 | CustomCoef::Coef item; 84 | for (const auto& p : pp) { 85 | item.set_coefs(p); 86 | CustomCoef::data.push_back(item); 87 | } 88 | }); 89 | m.def("IT92_normalize_etc", [](const gemmi::Element &el_x){ 90 | gemmi::IT92::normalize(); 91 | gemmi::IT92::ignore_charge = false; 92 | const auto coefs = gemmi::IT92::get(el_x, 0).coefs; 93 | gemmi::IT92::get(gemmi::El::X, 0).set_coefs(coefs); 94 | }); 95 | } 96 | -------------------------------------------------------------------------------- /scripts/test_fc_with_refmac.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import gemmi 10 | import numpy 11 | import time 12 | import os 13 | import argparse 14 | import subprocess 15 | from servalcat.utils import logger 16 | from servalcat import utils 17 | 18 | def parse_args(arg_list): 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument('--model', required=True) 21 | parser.add_argument('--refmacmtz') 22 | parser.add_argument('--auto_box_with_padding', type=float, help="Determine box size from model with specified padding") 23 | parser.add_argument('-d', '--resolution', type=float, required=True) 24 | parser.add_argument("--source", choices=["electron", "xray"], default="electron") 25 | return parser.parse_args(arg_list) 26 | # parse_args() 27 | 28 | def run_refmac(pdbin, d_min, source): 29 | cmd = ["refmac5", "xyzin", pdbin] 30 | ofs = open("refmac_sfcalc.log", "w") 31 | p = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE, stdout=ofs, stderr=ofs, 32 | universal_newlines=True) 33 | p.stdin.write("MODE SFCALC\n") 34 | p.stdin.write("make hydr yes\n") 35 | if source == "electron": p.stdin.write("source em mb\n") 36 | p.stdin.write("Sfcalc cr2f\n") 37 | p.stdin.write("Reso {}\n".format(d_min)) 38 | p.stdin.close() 39 | print("exit with", p.wait()) 40 | return "sfcalc_from_crd.mtz" 41 | 42 | def main(args): 43 | logger.set_file("servalcat_test_fc.log") 44 | st = utils.fileio.read_structure(args.model) 45 | st.expand_ncs(gemmi.HowToNameCopiedChain.Dup) 46 | if not st.cell.is_crystal() and args.auto_box_with_padding is not None: 47 | st.cell = utils.model.box_from_model(st[0], args.auto_box_with_padding) 48 | if not st.cell.is_crystal(): 49 | logger.error("ERROR: No unit cell information. Give --cell.") 50 | return 51 | 52 | if not args.refmacmtz: 53 | xyzin = "for_refmac.pdb" 54 | st.write_pdb(xyzin) 55 | args.refmacmtz = run_refmac(xyzin, args.resolution, args.source) 56 | fc_refmac = utils.fileio.read_asu_data_from_mtz(args.refmacmtz, ["Fout0", "Pout0"]) 57 | else: 58 | fc_refmac = utils.fileio.read_asu_data_from_mtz(args.refmacmtz, ["FC", "PHIC"]) 59 | hkldata = utils.hkl.hkldata_from_asu_data(fc_refmac, "FC_refmac") 60 | 61 | monlib = utils.restraints.load_monomer_library(st) 62 | 63 | ofs = open("fc_refmac_gemmi.dat", "w") 64 | ofs.write("blur cutoff rate d_max d_min fsc\n") 65 | for blur in (None,):#(0, 20, 40, 60): 66 | for cutoff in (1e-5, 1e-6, 1e-7, 1e-8, 1e-9): 67 | #for cutoff in (1e-3,1e-4,): 68 | for rate in (1.5,): 69 | t0 = time.time() 70 | fc_asu = utils.model.calc_fc_fft(st, args.resolution, cutoff=cutoff, rate=rate, 71 | mott_bethe=args.source == "electron", 72 | monlib=monlib, source=args.source, blur=blur) 73 | tt = time.time() - t0 74 | if "FC" in hkldata.df: del hkldata.df["FC"] 75 | hkldata.merge_asu_data(fc_asu, "FC") 76 | print("SIZE==", len(hkldata.df.index)) 77 | hkldata.setup_relion_binning() 78 | for i_bin, idxes in hkldata.binned(): 79 | bin_d_min = hkldata.binned_df.d_min[i_bin] 80 | bin_d_max = hkldata.binned_df.d_max[i_bin] 81 | Fc = hkldata.df.FC.to_numpy()[idxes] 82 | Fcr = hkldata.df.FC_refmac.to_numpy()[idxes] 83 | fsc = numpy.real(numpy.corrcoef(Fc, Fcr)[1,0]) 84 | ofs.write("{} {:.1e} {:.1f} {:7.3f} {:7.3f} {:.4f}\n".format(blur, cutoff, rate, 85 | bin_d_max, bin_d_min, 86 | fsc)) 87 | 88 | cc = numpy.real(numpy.corrcoef(Fc, Fcr)[1,0]) 89 | logger.writeln("blur= {} cutoff= {:.1e} rate={:.1f} CC={:.4f} time={:.2f} s".format(blur, cutoff, rate, cc, tt)) 90 | 91 | if __name__ == "__main__": 92 | import sys 93 | args = parse_args(sys.argv[1:]) 94 | main(args) 95 | -------------------------------------------------------------------------------- /scripts/test_fw.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | from servalcat.xtal.french_wilson import process_input, determine_Sigma_and_aniso, french_wilson 10 | from servalcat import utils 11 | import argparse 12 | import subprocess 13 | import gemmi 14 | import time 15 | import plotly.express as px 16 | 17 | def add_arguments(parser): 18 | parser.add_argument('--hklin', required=True, 19 | help='Input MTZ file') 20 | parser.add_argument('--labin', required=True, 21 | help='MTZ column for I,SIGI') 22 | parser.add_argument('--d_min', type=float) 23 | parser.add_argument('--d_max', type=float) 24 | parser.add_argument('--nbins', type=int, 25 | help="Number of bins (default: auto)") 26 | parser.add_argument('-o','--output_prefix', default="servalcat_fw", 27 | help='output file name prefix (default: %(default)s)') 28 | # add_arguments() 29 | 30 | def run_ctruncate(hklin, labin): 31 | hklout = "ctruncate.mtz" 32 | cmd = ["ctruncate", "-hklin", hklin, "-colin", "/*/*/[{}]".format(labin), "-hklout", hklout] 33 | #ofs = open("ctruncate.log", "w") 34 | t0 = time.time() 35 | p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, 36 | universal_newlines=True) 37 | read_mat = False 38 | mat = [] 39 | for l in iter(p.stdout.readline, ""): 40 | if "Anisotropic B scaling (orthogonal coords):" in l: 41 | read_mat = True 42 | elif read_mat and l.strip(): 43 | mat.append([float(x) for x in l.split()[1:4]]) 44 | if len(mat) == 3: 45 | read_mat = False 46 | print("exit with", p.wait()) 47 | t = time.time() - t0 48 | B = gemmi.SMat33d(mat[0][0], mat[1][1], mat[2][2], mat[0][1], mat[0][2], mat[1][2]) 49 | b_iso = B.trace() / 3 50 | b_aniso = B.added_kI(-b_iso) 51 | mtz = gemmi.read_mtz_file(hklout) 52 | hkldata = utils.hkl.hkldata_from_mtz(mtz, ["F", "SIGF"], newlabels=["F_ct", "SIGF_ct"], require_types=["F", "Q"]) 53 | return b_aniso, hkldata, t 54 | 55 | def main(args): 56 | B_ct, hkldata_ct, t_ct = run_ctruncate(args.hklin, args.labin) 57 | t0 = time.time() 58 | hkldata, _, _, _ = process_input(hklin=args.hklin, 59 | labin=args.labin.split(","), 60 | n_bins=args.nbins, 61 | free=None, 62 | xyzins=[], 63 | source=None, 64 | d_min=args.d_min, 65 | n_per_bin=500, 66 | max_bins=30) 67 | 68 | B_aniso = determine_Sigma_and_aniso(hkldata) 69 | french_wilson(hkldata, B_aniso) 70 | t_se = time.time() - t0 71 | 72 | hkldata.merge(hkldata_ct.df) 73 | hkldata.df["snr"] = hkldata.df.I / hkldata.df.SIGI 74 | print(hkldata.df) 75 | 76 | fig = px.scatter(hkldata.df, x="F_ct", y="F", symbol="centric", 77 | color="snr", 78 | facet_col='bin', facet_col_wrap=4, 79 | render_mode='webgl', 80 | #log_y=True, log_x=True, 81 | trendline="ols", trendline_scope="overall") 82 | #fig.update_yaxes(matches=None) 83 | #fig.update_xaxes(matches=None) 84 | fig.show() 85 | #results = px.get_trendline_results(fig) 86 | #print(results) 87 | fig = px.scatter(hkldata.df, x="SIGF_ct", y="SIGF", symbol="centric", render_mode='webgl', 88 | facet_col='bin', facet_col_wrap=4, 89 | color="snr", 90 | #log_y=True, log_x=True, 91 | trendline="ols", trendline_scope="overall") 92 | fig.show() 93 | 94 | print("B_aniso") 95 | print("serval: ", B_aniso) 96 | print("ctruncate:", B_ct) 97 | print() 98 | print("Time") 99 | print("serval: ", t_se) 100 | print("ctruncate:", t_ct) 101 | 102 | if __name__ == "__main__": 103 | parser = argparse.ArgumentParser() 104 | add_arguments(parser) 105 | args = parser.parse_args() 106 | main(args) 107 | -------------------------------------------------------------------------------- /servalcat/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import argparse 10 | import sys 11 | import traceback 12 | import platform 13 | import gemmi 14 | import numpy 15 | import scipy 16 | import pandas 17 | import servalcat.spa.shiftback 18 | import servalcat.spa.run_refmac 19 | import servalcat.spa.fsc 20 | import servalcat.spa.fofc 21 | import servalcat.spa.shift_maps 22 | import servalcat.spa.translate 23 | import servalcat.spa.localcc 24 | import servalcat.xtal.run_refmac_small 25 | import servalcat.xtal.sigmaa 26 | import servalcat.refmac.refmac_wrapper 27 | import servalcat.xtal.french_wilson 28 | import servalcat.utils.commands 29 | import servalcat.refine.refine_geom 30 | import servalcat.refine.refine_spa 31 | import servalcat.refine.refine_xtal 32 | 33 | from servalcat.utils import logger 34 | 35 | def test_installation(): 36 | import packaging.version 37 | vers = logger.dependency_versions() 38 | pandas_ver = packaging.version.parse(vers["pandas"]) 39 | numpy_ver = packaging.version.parse(vers["numpy"]) 40 | msg_unknown = "Unexpected error occurred (related to numpy+pandas). Please report to authors with the result of servalcat -v." 41 | msg_skip = "If you want to ignore this error, please specify --skip_test." 42 | ret = True 43 | 44 | try: 45 | x = pandas.DataFrame(dict(x=[2j])) 46 | x.merge(x) 47 | except TypeError: 48 | ret = False 49 | if pandas_ver >= packaging.version.parse("1.3.0") and numpy_ver < packaging.version.parse("1.19.1"): 50 | print("There is a problem in pandas+numpy. Please update numpy to 1.19.1 or newer (or use pandas < 1.3.0).") 51 | else: 52 | print(traceback.format_exc()) 53 | print(msg_unknown) 54 | except: 55 | print(traceback.format_exc()) 56 | print(msg_unknown) 57 | ret = False 58 | 59 | if not ret: 60 | print(msg_skip) 61 | 62 | return ret 63 | # test_installation() 64 | 65 | def main(): 66 | parser = argparse.ArgumentParser(prog="servalcat", 67 | description="A tool for model refinement and map calculation for crystallography and cryo-EM SPA.") 68 | parser.add_argument("--skip_test", action="store_true", help="Skip installation test") 69 | parser.add_argument("-v", "--version", action="version", 70 | version=logger.versions_str()) 71 | parser.add_argument("--logfile", default="servalcat.log") 72 | subparsers = parser.add_subparsers(dest="command") 73 | 74 | modules = dict(shiftback=servalcat.spa.shiftback, 75 | refine_spa=servalcat.spa.run_refmac, 76 | refine_cx=servalcat.xtal.run_refmac_small, 77 | fsc=servalcat.spa.fsc, 78 | fofc=servalcat.spa.fofc, 79 | trim=servalcat.spa.shift_maps, 80 | translate=servalcat.spa.translate, 81 | localcc=servalcat.spa.localcc, 82 | sigmaa=servalcat.xtal.sigmaa, 83 | fw=servalcat.xtal.french_wilson, 84 | #show=servalcat.utils.show, 85 | util=servalcat.utils.commands, 86 | refmac5=servalcat.refmac.refmac_wrapper, 87 | refine_geom=servalcat.refine.refine_geom, 88 | refine_spa_norefmac=servalcat.refine.refine_spa, 89 | refine_xtal_norefmac=servalcat.refine.refine_xtal, 90 | ) 91 | 92 | for n in modules: 93 | p = subparsers.add_parser(n) 94 | modules[n].add_arguments(p) 95 | 96 | args = parser.parse_args() 97 | 98 | if not args.skip_test and not test_installation(): 99 | return 100 | 101 | if args.command == "util" and not args.subcommand: 102 | print("specify subcommand.") 103 | elif args.command in modules: 104 | logger.set_file(args.logfile) 105 | logger.write_header() 106 | try: 107 | modules[args.command].main(args) 108 | except SystemExit as e: 109 | logger.error(str(e)) 110 | sys.exit(1) 111 | logger.exit_success() 112 | logger.close() 113 | else: 114 | parser.print_help() 115 | 116 | # main() 117 | 118 | if __name__ == "__main__": 119 | main() 120 | 121 | -------------------------------------------------------------------------------- /src/amplitude.cpp: -------------------------------------------------------------------------------- 1 | // Author: "Keitaro Yamashita, Garib N. Murshudov" 2 | // MRC Laboratory of Molecular Biology 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "math.hpp" 10 | #include "array.h" 11 | namespace nb = nanobind; 12 | using namespace servalcat; 13 | 14 | // ML amplitude target; -log(|Fo|; Fc) without constants 15 | // https://doi.org/10.1107/S0907444911001314 eqn (4) 16 | // S: must include epsilon. 17 | // Fc = |sum D*Fc| 18 | double ll_amp(double Fo, double sigFo, double k_ani, double S, double Fc, int c) { 19 | assert(c == 1 || c == 2); // c=1: acentric, 2: centric 20 | if (std::isnan(Fo) || S <= 0) return NAN; 21 | const double Fo_iso = Fo / k_ani; 22 | const double Sigma = (3 - c) * sq(sigFo / k_ani) + S; 23 | const double log_ic0 = log_i0_or_cosh(Fo_iso * Fc / Sigma, c); 24 | const double tmp = (c == 1) ? std::log(2.) + std::log(Fo_iso) : std::log(2 / gemmi::pi()) / c; // don't need this 25 | return std::log(Sigma) / c + (sq(Fo_iso) + sq(Fc)) / (Sigma * c) - log_ic0 - tmp; 26 | } 27 | 28 | auto 29 | ll_amp_der1_params_py(np_array Fo, np_array sigFo, np_array k_ani, 30 | double S, np_array, 2> Fcs, std::vector Ds, 31 | np_array c, np_array eps, np_array w) { 32 | auto Fo_ = Fo.view(); 33 | auto sigFo_ = sigFo.view(); 34 | auto k_ani_ = k_ani.view(); 35 | auto Fcs_ = Fcs.view(); 36 | auto c_ = c.view(); 37 | auto eps_ = eps.view(); 38 | auto w_ = w.view(); 39 | if (Ds.size() != Fcs_.shape(1)) throw std::runtime_error("Fc and D shape mismatch"); 40 | const size_t n_models = Fcs_.shape(1); 41 | const size_t n_ref = Fcs_.shape(0); 42 | const size_t n_cols = n_models + 1; //for_DS ? n_models + 1 : 1; 43 | 44 | // der1 wrt D1, D2, .., S //, or k_ani 45 | auto ret = make_numpy_array({n_ref, n_cols}); 46 | double* ptr = ret.data(); 47 | auto sum_Fc = [&](int i) { 48 | std::complex s = Fcs_(i, 0) * Ds[0]; 49 | for (size_t j = 1; j < n_models; ++j) 50 | s += Fcs_(i, j) * Ds[j]; 51 | return s; 52 | }; 53 | for (size_t i = 0; i < n_ref; ++i) { 54 | if (S <= 0 || std::isnan(Fo_(i))) { 55 | for (size_t j = 0; j < n_cols; ++j) 56 | ptr[i * n_cols + j] = NAN; 57 | continue; 58 | } 59 | const double Fo_iso = Fo_(i) / k_ani_(i); 60 | const double Sigma = (3 - c_(i)) * sq(sigFo_(i) / k_ani_(i)) + S * eps_(i); 61 | const std::complex Fc_total_conj = std::conj(sum_Fc(i)); 62 | const double Fc_abs = std::abs(Fc_total_conj); 63 | if (1){ //for_DS) { 64 | const double m = fom(Fo_iso * Fc_abs / Sigma, c_(i)); 65 | for (size_t j = 0; j < n_models; ++j) { 66 | const double r_fcj_fc = (Fcs_(i, j) * Fc_total_conj).real(); 67 | // wrt Dj 68 | ptr[i*n_cols + j] = w_(i) * 2 * r_fcj_fc / (Sigma * c_(i)) * (1. - m * Fo_iso / Fc_abs); 69 | } 70 | // wrt S 71 | const double tmp = (sq(Fo_iso) + sq(Fc_abs)) / c_(i) - m * (3 - c_(i)) * Fo_iso * Fc_abs; 72 | ptr[i*n_cols + n_models] = w_(i) * eps_(i) * (1. / (c_(i) * Sigma) - tmp / sq(Sigma)); 73 | } 74 | else { 75 | // k_aniso * d/dk_aniso -log p(Io; Fc) 76 | // note k_aniso is multiplied to the derivative 77 | // not implemented 78 | } 79 | } 80 | return ret; 81 | } 82 | 83 | void add_amplitude(nb::module_& m) { 84 | m.def("ll_amp", [](np_array Fo, np_array sigFo, np_array k_ani, 85 | np_array S, np_array Fc, np_array c, np_array w) { 86 | auto Fo_ = Fo.view(); 87 | auto sigFo_ = sigFo.view(); 88 | auto k_ani_ = k_ani.view(); 89 | auto S_ = S.view(); 90 | auto Fc_ = Fc.view(); 91 | auto c_ = c.view(); 92 | auto w_ = w.view(); 93 | size_t len = Fo_.shape(0); 94 | if (len != sigFo_.shape(0) || len != k_ani_.shape(0) || len != S_.shape(0) || 95 | len != Fc_.shape(0) || len != c_.shape(0) || len != w_.shape(0)) 96 | throw std::runtime_error("ll_amp: shape mismatch"); 97 | auto ret = make_numpy_array({len}); 98 | double* retp = ret.data(); 99 | for (size_t i = 0; i < len; ++i) 100 | retp[i] = w_(i) * ll_amp(Fo_(i), sigFo_(i), k_ani_(i), S_(i), Fc_(i), c_(i)); 101 | return ret; 102 | }, 103 | nb::arg("Fo"), nb::arg("sigFo"), nb::arg("k_ani"), nb::arg("S"), nb::arg("Fc"), nb::arg("c"), nb::arg("w")); 104 | m.def("ll_amp_der1_DS", &ll_amp_der1_params_py); 105 | //m.def("ll_int_der1_ani", &ll_int_der1_params_py); 106 | } 107 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15...3.22) 2 | 3 | # get version string from servalcat/__init__.py 4 | file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/servalcat/__init__.py" 5 | serval_version_line REGEX "__version__ = '") 6 | string(REGEX REPLACE "__version__ = '(.+)'" "\\1" serval_version_str ${serval_version_line}) 7 | 8 | project(servalcat LANGUAGES C CXX VERSION ${serval_version_str}) 9 | message(STATUS "Servalcat version ${PROJECT_VERSION}") 10 | 11 | option(SEARCH_INSTALLED_GEMMI "Search for gemmi-config.cmake and use it if found" ON) 12 | option(INSTALL_GEMMI_IF_BUILT "Install also Python module gemmi, if it was built" OFF) 13 | 14 | include(GNUInstallDirs) 15 | 16 | if (DEFINED SKBUILD) # building with scikit-build-core (pip install) 17 | set(PYTHON_INSTALL_DIR "${SKBUILD_PLATLIB_DIR}") 18 | #set(CMAKE_INSTALL_BINDIR "${SKBUILD_SCRIPTS_DIR}") 19 | endif() 20 | 21 | set(CMAKE_CXX_EXTENSIONS OFF) 22 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 23 | 24 | # Set default build mode to Release, unless we got CXXFLAGS=... 25 | if (DEFINED ENV{CXXFLAGS}) 26 | set(USING_ENV_CXXFLAGS ON CACHE BOOL "" FORCE) 27 | endif() 28 | if (NOT CMAKE_BUILD_TYPE AND NOT USING_ENV_CXXFLAGS) 29 | set(CMAKE_BUILD_TYPE Release CACHE STRING 30 | "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." 31 | FORCE) 32 | endif() 33 | 34 | 35 | # CMake >=3.18 has subcomponent Development.Module, scikit-build-core also has it 36 | if (${CMAKE_VERSION} VERSION_LESS 3.18 AND NOT SKBUILD) 37 | find_package(Python ${PYTHON_VERSION} REQUIRED COMPONENTS Interpreter Development) 38 | else() 39 | find_package(Python ${PYTHON_VERSION} REQUIRED COMPONENTS Interpreter Development.Module) 40 | endif() 41 | 42 | execute_process( 43 | COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir 44 | OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE NB_DIR) 45 | list(APPEND CMAKE_PREFIX_PATH "${NB_DIR}") 46 | find_package(nanobind 2.7.0 CONFIG REQUIRED) 47 | message(STATUS "Found nanobind ${nanobind_VERSION}: ${NB_DIR}") 48 | 49 | nanobind_add_module(ext src/ext.cpp src/intensity.cpp src/amplitude.cpp src/refine.cpp src/twin.cpp) 50 | 51 | if (EXISTS "${CMAKE_HOME_DIRECTORY}/eigen/Eigen") 52 | include_directories("${CMAKE_CURRENT_SOURCE_DIR}/eigen") 53 | message(STATUS "Using ${CMAKE_HOME_DIRECTORY}/eigen (internal copy).") 54 | else() 55 | find_package (Eigen3 3.4 CONFIG REQUIRED) 56 | message(STATUS "Found Eigen3 version ${EIGEN3_VERSION_STRING}") 57 | target_link_libraries(ext PRIVATE Eigen3::Eigen) 58 | endif() 59 | 60 | # We need either gemmi C++ development files (headers, library, cmake config) 61 | # or gemmi sources. If we have the former with a shared library, it may require 62 | # extra effort to make sure that the shared library is found at runtime. 63 | if (SEARCH_INSTALLED_GEMMI) 64 | find_package(gemmi 0.7.3 CONFIG) 65 | endif() 66 | if (gemmi_FOUND) 67 | message(STATUS " based on config from ${gemmi_DIR}") 68 | get_target_property(gemmi_TYPE gemmi::gemmi_cpp TYPE) 69 | if (${gemmi_TYPE} STREQUAL "SHARED_LIBRARY") 70 | message(STATUS "** Servalcat Python module will be linked with gemmi shared library. **") 71 | message(STATUS "** The module may require 'repairing' to find the library at runtime. **") 72 | endif() 73 | else() 74 | set(USE_PYTHON ON CACHE BOOL "" FORCE) 75 | set(BUILD_GEMMI_PROGRAM OFF CACHE BOOL "" FORCE) 76 | set(INSTALL_DEV_FILES OFF CACHE BOOL "" FORCE) 77 | set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) 78 | message(STATUS "") 79 | message(STATUS "Configuring GEMMI...") 80 | if (INSTALL_GEMMI_IF_BUILT AND NOT gemmi_FOUND) 81 | set(exclude_or_not) 82 | else() 83 | set(exclude_or_not EXCLUDE_FROM_ALL) 84 | endif() 85 | if (EXISTS "${CMAKE_HOME_DIRECTORY}/gemmi/include/gemmi") 86 | message(STATUS "Using ${CMAKE_HOME_DIRECTORY}/gemmi (internal copy).") 87 | add_subdirectory(gemmi ${exclude_or_not}) 88 | else() 89 | message(STATUS "Using FetchContent...") 90 | include(FetchContent) 91 | FetchContent_Declare( 92 | gemmi 93 | GIT_REPOSITORY https://github.com/project-gemmi/gemmi.git 94 | GIT_TAG v0.7.4 95 | ) 96 | FetchContent_GetProperties(gemmi) 97 | if (NOT gemmi_POPULATED) 98 | FetchContent_Populate(gemmi) 99 | add_subdirectory(${gemmi_SOURCE_DIR} ${gemmi_BINARY_DIR} ${exclude_or_not}) 100 | endif() 101 | endif() 102 | add_dependencies(ext gemmi_py) 103 | endif() 104 | target_link_libraries(ext PRIVATE gemmi::gemmi_cpp) 105 | 106 | if (DEFINED PYTHON_INSTALL_DIR) 107 | message(STATUS "Install directory for Python module: ${PYTHON_INSTALL_DIR}") 108 | set(Python_SITEARCH "${PYTHON_INSTALL_DIR}") 109 | endif() 110 | file(TO_CMAKE_PATH "${Python_SITEARCH}" Python_SITEARCH) 111 | install(DIRECTORY servalcat/ DESTINATION "${Python_SITEARCH}/servalcat" 112 | FILES_MATCHING PATTERN "*.py") 113 | install(TARGETS ext DESTINATION "${Python_SITEARCH}/servalcat") 114 | -------------------------------------------------------------------------------- /servalcat/utils/logger.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import sys 10 | import datetime 11 | import platform 12 | import getpass 13 | import traceback 14 | import shlex 15 | import servalcat 16 | 17 | class Logger(object): 18 | def __init__(self, file_out=None, append=True): 19 | self.ofs = None 20 | self.stopped = False 21 | self.prefix = "" 22 | if file_out: 23 | self.set_file(file_out, append) 24 | # __init__() 25 | def stop_logging(self): self.stopped = True 26 | def start_logging(self): self.stopped = False 27 | def set_prefix(self, p): self.prefix = p 28 | def clear_prefix(self): self.prefix = "" 29 | 30 | def set_file(self, file_out, append=True): 31 | try: 32 | self.ofs = open(file_out, "a" if append else "w") 33 | except: 34 | print("Error: Cannot open log file to write") 35 | # set_file() 36 | 37 | def write(self, l, end="", flush=True, fs=None, print_fs=sys.stdout): 38 | if self.stopped: return 39 | if self.prefix: 40 | l = "".join(self.prefix + x for x in l.splitlines(keepends=True)) 41 | print(l, end=end, file=print_fs, flush=flush) 42 | for f in (self.ofs, fs): 43 | if f is not None: 44 | f.write(l) 45 | f.write(end) 46 | if flush: f.flush() 47 | # write() 48 | 49 | def writeln(self, l, flush=True, fs=None, print_fs=sys.stdout): 50 | self.write(l, end="\n", flush=flush, fs=fs, print_fs=print_fs) 51 | # writeln() 52 | 53 | def error(self, l, end="\n", flush=True, fs=None): 54 | self.write(l, end, flush, fs, print_fs=sys.stderr) 55 | # error() 56 | 57 | def close(self): 58 | if self.ofs is not None: 59 | self.ofs.close() 60 | self.ofs = None 61 | # close() 62 | 63 | def flush(self): # to act as a file object 64 | if self.ofs: 65 | self.ofs.flush() 66 | # class Logger 67 | 68 | _logger = Logger() # singleton 69 | set_file = _logger.set_file 70 | write = _logger.write 71 | writeln = _logger.writeln 72 | error = _logger.error 73 | close = _logger.close 74 | flush = _logger.flush 75 | stop = _logger.stop_logging 76 | start = _logger.start_logging 77 | set_prefix = _logger.set_prefix 78 | clear_prefix = _logger.clear_prefix 79 | 80 | def with_prefix(prefix): 81 | class WithPrefix(object): # should keep original prefix and restore? 82 | def __enter__(self): 83 | _logger.set_prefix(prefix) 84 | return _logger 85 | def __exit__(self, exc_type, exc_val, exc_tb): 86 | _logger.clear_prefix() 87 | return WithPrefix() 88 | 89 | def silent(): 90 | class Silent(object): 91 | def write(self, *args, **kwargs): 92 | pass 93 | def flush(self): 94 | pass 95 | return Silent() 96 | 97 | def dependency_versions(): 98 | import gemmi 99 | import scipy 100 | import numpy 101 | import pandas 102 | return dict(gemmi=gemmi.__version__, 103 | scipy=scipy.version.full_version, 104 | numpy=numpy.version.full_version, 105 | pandas=pandas.__version__) 106 | # dependency_versions() 107 | 108 | def versions_str(): 109 | tmpl = "Servalcat {servalcat} with Python {python} ({deps})" 110 | return tmpl.format(servalcat=servalcat.__version__, 111 | python=platform.python_version(), 112 | deps=", ".join([x[0]+" "+x[1] for x in dependency_versions().items()])) 113 | # versions_str() 114 | 115 | def write_header(command="servalcat"): 116 | writeln("# Servalcat ver. {} (Python {})".format(servalcat.__version__, platform.python_version())) 117 | writeln("# Library vers. {}".format(", ".join([x[0]+" "+x[1] for x in dependency_versions().items()]))) 118 | writeln("# Started on {}".format(datetime.datetime.now())) 119 | writeln("# Host: {} User: {}".format(platform.node(), getpass.getuser())) 120 | writeln("# Command-line:") 121 | writeln("# {} {}".format(command, " ".join(map(lambda x: shlex.quote(x), sys.argv[1:])))) 122 | # write_header() 123 | 124 | def exit_success(): 125 | _logger.writeln("\n# Finished on {}\n".format(datetime.datetime.now())) 126 | 127 | def handle_exception(exc_type, exc_value, exc_traceback): 128 | if issubclass(exc_type, KeyboardInterrupt): 129 | sys.__excepthook__(exc_type, exc_value, exc_traceback) 130 | return 131 | 132 | name = type(exc_value).__name__ if hasattr(type(exc_value), "__name__") else "(unknown)" 133 | #_logger.writeln("Uncaught exception: {}: {}".format(name, exc_value)) 134 | _logger.error("".join(traceback.format_exception(exc_type, exc_value, exc_traceback))) 135 | _logger.writeln("# Abnormally finished on {}\n".format(datetime.datetime.now())) 136 | _logger.close() 137 | 138 | # handle_exception() 139 | 140 | sys.excepthook = handle_exception 141 | -------------------------------------------------------------------------------- /servalcat/spa/shiftback.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import os 10 | import gemmi 11 | import numpy 12 | import argparse 13 | import json 14 | from servalcat.utils import logger 15 | from servalcat import utils 16 | 17 | def add_arguments(parser): 18 | parser.description = 'Shift back for Refmac local refinement results' 19 | parser.add_argument('--model', 20 | help='Atomic model file that needs shift back') 21 | parser.add_argument('--refine_mtz', 22 | help='Local-refined Refmac mtz file that needs shift back') 23 | parser.add_argument('--shifts', 24 | required=True, 25 | default="shifts.json", 26 | help='Shift information file') 27 | parser.add_argument('--output_prefix', 28 | help='output file prefix') 29 | # add_arguments() 30 | 31 | def parse_args(arg_list): 32 | parser = argparse.ArgumentParser() 33 | add_arguments(parser) 34 | return parser.parse_args(arg_list) 35 | # parse_args() 36 | 37 | def refmac_mtz_in_original_cell(org_cell, org_grid_size, new_grid_size, shifts, mtz_in, mtz_out): 38 | targets = (("FWT", "PHWT"), ("DELFWT", "PHDELWT")) 39 | 40 | shifts_frac = org_cell.fractionalize(gemmi.Position(*shifts)).tolist() 41 | 42 | # Output mtz 43 | mtz = gemmi.Mtz() 44 | mtz.spacegroup = gemmi.SpaceGroup("P1") 45 | mtz.cell = org_cell 46 | mtz.add_dataset('HKL_base') 47 | for l in ["H", "K", "L"]: mtz.add_column(l, "H") 48 | 49 | data = None 50 | for i in range(len(targets)): 51 | d_min, m = utils.fileio.read_map_from_mtz(mtz_in, targets[i], new_grid_size) 52 | F = numpy.fft.fftn(m, org_grid_size).conj() 53 | grid = gemmi.ReciprocalComplexGrid(F.astype(numpy.complex64), mtz.cell, mtz.spacegroup) 54 | asu = grid.prepare_asu_data(dmin=d_min) 55 | if data is None: 56 | data = numpy.empty((len(asu), 3+2*len(targets))) 57 | data[:,:3] = asu.miller_array 58 | 59 | shift_factor = numpy.exp(-2j*numpy.pi*numpy.dot(asu.miller_array, shifts_frac)) 60 | F_shift = asu.value_array * shift_factor 61 | data[:,3+2*i] = numpy.absolute(F_shift) 62 | data[:,3+2*i+1] = numpy.angle(F_shift, deg=True) 63 | 64 | mtz.add_column(targets[i][0], "F") 65 | mtz.add_column(targets[i][1], "P") 66 | 67 | mtz.set_data(data) 68 | mtz.write_to_file(mtz_out) 69 | # refmac_mtz_in_original_cell() 70 | 71 | def shift_back_model(st, shifts): 72 | # shifts must be Vec3 object 73 | 74 | for model in st: 75 | for cra in model.all(): 76 | cra.atom.pos -= shifts 77 | 78 | if len(st.ncs) > 0: 79 | for n in st.ncs: 80 | newv = n.tr.vec - shifts + n.tr.mat.multiply(shifts) 81 | n.tr.vec.fromlist(newv.tolist()) 82 | # shift_back_model() 83 | 84 | def shift_back_tls(tlsgroups, shifts): 85 | for g in tlsgroups: 86 | if g["origin"] is not None: 87 | g["origin"] -= shifts 88 | # shift_back_tls() 89 | 90 | def shift_back(xyz_in, shifts_json, refine_mtz=None, out_prefix=None): 91 | logger.writeln("Reading shifts info from {}".format(shifts_json)) 92 | info = json.load(open(shifts_json)) 93 | for k in info: 94 | logger.writeln(" {}= {}".format(k, info[k])) 95 | 96 | org_cell = gemmi.UnitCell(*info["cell"]) 97 | shifts = gemmi.Position(*info["shifts"]) 98 | 99 | if refine_mtz: 100 | logger.writeln("Transforming MTZ: {}".format(refine_mtz)) 101 | if out_prefix: 102 | mtz_out = out_prefix+".mtz" 103 | else: 104 | mtz_out = utils.fileio.splitext(os.path.basename(refine_mtz))[0] + "_shiftback.mtz" 105 | 106 | refmac_mtz_in_original_cell(org_cell, 107 | info["grid"], 108 | info["new_grid"], 109 | info["shifts"], 110 | refine_mtz, 111 | mtz_out) 112 | 113 | if xyz_in: 114 | logger.writeln("Shifting back model: {}".format(xyz_in)) 115 | st, cif_ref = utils.fileio.read_structure_from_pdb_and_mmcif(xyz_in) 116 | 117 | st.cell = org_cell 118 | shift_back_model(st, shifts) 119 | prefix = out_prefix if out_prefix else utils.fileio.splitext(os.path.basename(xyz_in))[0] + "_shiftback" 120 | utils.fileio.write_model(st, prefix, 121 | pdb=True, cif=True, cif_ref=cif_ref) 122 | 123 | 124 | # shift_back() 125 | 126 | 127 | def main(args): 128 | if not args.model and not args.refine_mtz: 129 | raise SystemExit("ERROR: give --model and/or --refine_mtz") 130 | 131 | shift_back(args.model, args.shifts, args.refine_mtz, args.output_prefix) 132 | 133 | if __name__ == "__main__": 134 | import sys 135 | args = parse_args(sys.argv[1:]) 136 | main(args) 137 | 138 | -------------------------------------------------------------------------------- /docs/overview.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | ======== 3 | 4 | Servalcat offers various functionalities for refinement and map calculations in crystallography and single particle analysis (SPA). 5 | 6 | * SPA 7 | * refinement (refine_spa_norefmac) 8 | * sharpened and weighted map calculation (Fo and Fo-Fc) 9 | * map trimming tool 10 | * crystallography 11 | * amplitude or intensity based refinement (refine_xtal_norefmac) 12 | * map calculation from ML parameter estimation (sigmaa) 13 | * others/general 14 | * REFMAC5 wrapper ("refmacat") 15 | * geometry optimisation 16 | 17 | The basic usage is: 18 | 19 | .. code-block:: console 20 | 21 | $ servalcat 22 | 23 | The most common commands are listed below. To see all arguments for a specific command, run: 24 | 25 | .. code-block:: console 26 | 27 | $ servalcat -h 28 | 29 | 30 | Command examples for cryo-EM SPA 31 | -------------------------------- 32 | 33 | Refinement 34 | ~~~~~~~~~~ 35 | Servalcat performs reciprocal space refinement for single particle analysis. The weighted and sharpened Fo-Fc map is calculated after the refinement. For details please see the reference. 36 | Make a new directory and run: 37 | 38 | .. code-block:: console 39 | 40 | $ servalcat refine_spa_norefmac \ 41 | --model input.pdb --resolution 2.5 \ 42 | --halfmaps ../half_map_1.mrc ../half_map_2.mrc \ 43 | --ncycle 10 [--pg C2] \ 44 | [--mask_for_fofc mask.mrc] [-o output_prefix] 45 | 46 | Provide unsharpened and unweighted half maps (e.g., from RELION's Refine3D) after ``--halfmaps``. 47 | 48 | If map has been symmetrised with a point group, use ``--pg`` to specify the point group symbol along with the asymmetric unit model. 49 | The centre of the box is assumed to be the origin of symmetry. The axis convention follows `RELION `_. 50 | 51 | Other useful options: 52 | * ``--ligand lig.cif`` : specify restraint dictionary (.cif) file(s) 53 | * ``--mask_for_fofc mask.mrc`` : specify mask file for Fo-Fc map calculation 54 | * ``--jellybody`` : turn on jelly body refinement 55 | * ``--weight value`` : specify the weight. By default Servalcat determines it from resolution and mask/box ratio 56 | * ``--keyword_file file`` : specify any refmac keyword file(s) (e.g. prosmart restraint file) Note that not all refmac keywords are supported 57 | * ``--pixel_size value`` : override pixel size of map 58 | 59 | Output files: 60 | * ``prefix.pdb``: refined model (legacy PDB format) 61 | * ``prefix.mmcif``: refined model (mmCIF format) 62 | * ``prefix_expanded.pdb``: symmetry-expanded version 63 | * ``prefix_expanded.mmcif``: symmetry-expanded version 64 | * ``prefix_diffmap.mtz``: can be auto-opened with coot. sharpened and weighted Fo map and Fo-Fc map 65 | * ``prefix_diffmap_normalized_fofc.mrc``: Fo-Fc map normalised within a mask. Look at raw values 66 | 67 | Fo-Fc map calculation 68 | ~~~~~~~~~~~~~~~~~~~~~ 69 | It is crucial to refine individual atomic B values with electron scattering factors for a meaningful Fo-Fc map. 70 | While the ``refine_spa_norefmac`` command calculates the Fo-Fc map (explained above), you can use the fofc command to calculate specific maps, like omit maps. 71 | 72 | .. code-block:: console 73 | 74 | $ servalcat fofc \ 75 | --model input.pdb --resolution 2.5 \ 76 | --halfmaps ../half_map_1.mrc ../half_map_2.mrc \ 77 | [--mask mask.mrc] [-o output_prefix] [-B B value] 78 | 79 | 80 | ``-B`` is to calculate weighted maps based on local B estimate. It may be useful for model building in noisy region. 81 | 82 | Map trimming 83 | ~~~~~~~~~~~~ 84 | Maps from single particle analysis are often large due to unnecessary regions outside the molecule. Use trim to save disk space by removing these regions. 85 | 86 | .. code-block:: console 87 | 88 | $ servalcat trim \ 89 | --maps postprocess.mrc halfmap1.mrc halfmap2.mrc \ 90 | [--mask mask.mrc] [--model model.pdb] [--padding 10] 91 | 92 | Maps specified with ``--maps`` are trimmed. The boundary is decided by ``--mask`` or ``--model`` if mask is not available. 93 | Model(s) are shifted into a new box. 94 | By default new boundary is centred on the original map and cubic, but they can be turned off with ``--noncentered`` and ``--noncubic``. 95 | If you do not want to shift maps and models, specify ``--no_shift`` to keep origin. 96 | 97 | Command examples for crystallography 98 | ------------------------------------ 99 | 100 | Refinement 101 | ~~~~~~~~~~ 102 | To perform crystallographic refinement with Servalcat, it is necessary to specify an input model (PDB, mmCIF or smCIF), diffraction data (MTZ or CIF format) and type radiation source (xray, neutron or electron). 103 | 104 | .. code-block:: console 105 | 106 | $ servalcat refine_xtal_norefmac \ 107 | --model input.pdb --hklin ../data.mtz \ 108 | -s xray --ncycle 10 \ 109 | [-o output_prefix] 110 | 111 | Output files: 112 | * ``prefix.pdb``: refined model (legacy PDB format) 113 | * ``prefix.mmcif``: refined model (mmCIF format) 114 | * ``prefix.mtz``: 2Fo-Fc and Fo-Fc maps which can be auto-opened with coot. 115 | 116 | More details about crystallographic refinement with Servalcat can be found `on a separate page `_. -------------------------------------------------------------------------------- /servalcat/spa/translate.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import gemmi 10 | import numpy 11 | import pandas 12 | from servalcat.utils import logger 13 | from servalcat import utils 14 | from servalcat import spa 15 | 16 | def add_arguments(parser): 17 | parser.description = 'Find translation of the model in the map' 18 | parser.add_argument('--model', 19 | required=True, 20 | help="") 21 | group = parser.add_mutually_exclusive_group() 22 | group.add_argument("--halfmaps", nargs=2, help="Input half map files") 23 | group.add_argument("--map", help="Use this only if you really do not have half maps.") 24 | parser.add_argument('--mask', 25 | help='Mask file') 26 | parser.add_argument('--pixel_size', type=float, 27 | help='Override pixel size (A)') 28 | parser.add_argument('-d', '--resolution', 29 | type=float, 30 | required=True, 31 | help='') 32 | parser.add_argument('--no_interpolation', action="store_true", 33 | help="No interpolation in peak finding of translation function") 34 | parser.add_argument('-o', '--output_prefix', default="translated") 35 | 36 | # add_arguments() 37 | 38 | def parse_args(arg_list): 39 | parser = argparse.ArgumentParser() 40 | add_arguments(parser) 41 | return parser.parse_args(arg_list) 42 | # parse_args() 43 | 44 | def calc_fsc(hkldata, lab1, lab2): 45 | stats = hkldata.binned_df["ml"][["d_min", "d_max"]].copy() 46 | stats["ncoeffs"] = 0 47 | stats["fsc"] = 0. 48 | for i_bin, idxes in hkldata.binned("ml"): 49 | stats.loc[i_bin, "ncoeffs"] = len(idxes) 50 | stats.loc[i_bin, "fsc"] = numpy.real(numpy.corrcoef(hkldata.df[lab1].to_numpy()[idxes], 51 | hkldata.df[lab2].to_numpy()[idxes])[1,0]) 52 | 53 | sum_n = sum(stats.ncoeffs) 54 | fscavg = sum(stats.ncoeffs*stats.fsc)/sum_n 55 | return stats, fscavg 56 | # calc_fsc() 57 | 58 | def find_peak(tf_map, ini_pos): 59 | logger.writeln("Finding peak using interpolation..") 60 | 61 | x = tf_map.unit_cell.fractionalize(ini_pos) 62 | logger.writeln(" x0: [{}, {}, {}]".format(*x.tolist())) 63 | logger.writeln(" f0: {}".format(-tf_map.interpolate_value(x, order=3))) 64 | 65 | res = scipy.optimize.minimize(fun=lambda x:-tf_map.interpolate_value(gemmi.Fractional(*x), order=3), 66 | x0=x.tolist(), 67 | jac=lambda x:-numpy.array(tf_map.tricubic_interpolation_der(gemmi.Fractional(*x))[1:])) 68 | logger.writeln(str(res)) 69 | final_pos = tf_map.unit_cell.orthogonalize(gemmi.Fractional(*res.x)) 70 | logger.writeln(" Move from initial: [{:.3f}, {:.3f}, {:.3f}] A".format(*(final_pos-ini_pos).tolist())) 71 | return final_pos 72 | # find_peak() 73 | 74 | def main(args): 75 | if args.halfmaps: 76 | maps = utils.fileio.read_halfmaps(args.halfmaps, pixel_size=args.pixel_size) 77 | assert maps[0][0].shape == maps[1][0].shape 78 | assert maps[0][0].unit_cell == maps[1][0].unit_cell 79 | assert maps[0][1] == maps[1][1] 80 | else: 81 | maps = [utils.fileio.read_ccp4_map(args.map, pixel_size=args.pixel_size)] 82 | 83 | model_format = utils.fileio.check_model_format(args.model) 84 | st = utils.fileio.read_structure(args.model) 85 | st.cell = maps[0][0].unit_cell 86 | st.spacegroup_hm = "P1" 87 | 88 | if args.mask: 89 | mask = utils.fileio.read_ccp4_map(args.mask)[0] 90 | else: 91 | mask = None 92 | 93 | hkldata = utils.maps.mask_and_fft_maps(maps, args.resolution, mask=None) 94 | hkldata.df["FC"] = utils.model.calc_fc_fft(st, args.resolution - 1e-6, source="electron", 95 | miller_array=hkldata.miller_array()) 96 | hkldata.setup_relion_binning("ml") 97 | 98 | stats, fscavg = calc_fsc(hkldata, "FP", "FC") 99 | logger.writeln(stats.to_string()) 100 | logger.writeln("FSCaverage before translation = {:.4f}".format(fscavg)) 101 | 102 | hkldata.df["TF"] = hkldata.df.FP.to_numpy() * numpy.conj(hkldata.df.FC.to_numpy()) 103 | 104 | tf_map = hkldata.fft_map("TF") 105 | max_idx = numpy.unravel_index(numpy.argmax(tf_map), tf_map.shape) 106 | shift = tf_map.get_position(*max_idx) 107 | 108 | if not args.no_interpolation: 109 | shift = utils.maps.optimize_peak(tf_map, shift) 110 | 111 | logger.writeln("shift= {:.4f}, {:.4f}, {:.4f} ".format(*shift)) 112 | 113 | # phase shift for translation 114 | hkldata.df.FC *= numpy.exp(2.j*numpy.pi*numpy.dot(hkldata.miller_array(), 115 | hkldata.cell.fractionalize(shift).tolist())) 116 | stats, fscavg = calc_fsc(hkldata, "FP", "FC") 117 | logger.writeln(stats.to_string()) 118 | logger.writeln("FSCaverage after translation = {:.4f}".format(fscavg)) 119 | 120 | tr = gemmi.Transform(gemmi.Mat33(), shift) 121 | st[0].transform_pos_and_adp(tr) 122 | utils.model.translate_into_box(st) 123 | utils.fileio.write_model(st, file_name=args.output_prefix+model_format) 124 | # main() 125 | 126 | if __name__ == "__main__": 127 | import sys 128 | args = parse_args(sys.argv[1:]) 129 | main(args) 130 | -------------------------------------------------------------------------------- /servalcat/spa/realspcc_from_var.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import gemmi 10 | import numpy 11 | import scipy.integrate 12 | import os 13 | import argparse 14 | from servalcat.utils import logger 15 | from servalcat import utils 16 | from servalcat import spa 17 | 18 | def add_arguments(parser): 19 | parser.description = 'Calculate real space correlation radius from variances in reciprocal space' 20 | parser.add_argument("--halfmaps", nargs=2) 21 | parser.add_argument('--pixel_size', type=float, 22 | help='Override pixel size (A)') 23 | parser.add_argument("-d", '--resolution', type=float, required=True) 24 | parser.add_argument('-m', '--mask', help="mask file") 25 | parser.add_argument('-w', '--weight', help="weight") 26 | parser.add_argument('-f', choices=["noise", "signal", "total"], required=True) 27 | parser.add_argument('--sharpen_signal', action="store_true", help="") 28 | parser.add_argument('--x_max', type=float, default=20) 29 | parser.add_argument("-B", type=float, help="Sharpening (negative)/blurring (positive) B value") 30 | parser.add_argument('--find_B_at', type=float, help="") 31 | parser.add_argument('-o','--output_prefix', default="cc", 32 | help='output file name prefix') 33 | # add_arguments() 34 | 35 | def parse_args(arg_list): 36 | parser = argparse.ArgumentParser() 37 | add_arguments(parser) 38 | return parser.parse_args(arg_list) 39 | # parse_args() 40 | 41 | def integ_var(x, s_list, w_list, B): 42 | tinv = numpy.exp(-B*s_list**2/2) if B is not None else 1. 43 | return scipy.integrate.simps(s_list**2 * tinv * w_list * numpy.sinc(2*numpy.abs(s_list)*numpy.abs(x)), 44 | s_list) 45 | 46 | def calc_var(hkldata, kind="noise", sharpen_signal=False, weight=None): 47 | wsq = None 48 | if kind == "noise": 49 | wsq = hkldata.binned_df["ml"].var_noise.to_numpy(copy=True) 50 | elif kind == "signal": 51 | wsq = hkldata.binned_df["ml"].var_signal.to_numpy(copy=True) 52 | elif kind == "total": 53 | wsq = hkldata.binned_df["ml"].var_signal.to_numpy() + hkldata.binned_df["ml"].var_noise.to_numpy() 54 | else: 55 | raise RuntimeError("unknown kind") 56 | 57 | if sharpen_signal: 58 | wsq /= hkldata.binned_df["ml"].var_signal.to_numpy() 59 | 60 | if weight is None: 61 | wsq *= 1. 62 | elif weight == "fscfull": 63 | wsq *= hkldata.binned_df["ml"].FSCfull.to_numpy()**2 64 | else: 65 | raise RuntimeError("unknown weight") 66 | 67 | return wsq 68 | # calc_var() 69 | 70 | def find_b(hkldata, smax, x, kind="noise", sharpen_signal=False, weight=None): # XXX unfinished 71 | bin_s = 0.5*(1./hkldata.binned_df["ml"][["d_min", "d_max"]]).sum(axis=1).to_numpy() # 0.5 * (1/d_max + 1/d_min) 72 | logger.writeln("kind= {} sharpen_signal={}, weight={}".format(kind, sharpen_signal, weight)) 73 | wsq = calc_var(hkldata, kind, sharpen_signal, weight) 74 | for B in -numpy.arange(0,100,5): 75 | cov_xx = integ_var(0., bin_s, wsq, B) 76 | cov_xy = integ_var(x, bin_s, wsq, B) 77 | print(B, cov_xy/cov_xx) 78 | # find_b() 79 | 80 | def calc_cc_from_var(hkldata, x_list, kind="noise", sharpen_signal=False, weight=None, B=None): 81 | bin_s = 0.5*(1./hkldata.binned_df["ml"][["d_min", "d_max"]]).sum(axis=1).to_numpy() # 0.5 * (1/d_max + 1/d_min) 82 | logger.writeln("kind= {} sharpen_signal={}, weight={}".format(kind, sharpen_signal, weight)) 83 | wsq = calc_var(hkldata, kind, sharpen_signal, weight) 84 | cov_xx = integ_var(0., bin_s, wsq, B) 85 | cov_xy = integ_var(x_list[:,None], bin_s, wsq, B) 86 | cc_all = cov_xy / cov_xx 87 | return cc_all 88 | # calc_cc_from_var() 89 | 90 | def main(args): 91 | maps = utils.fileio.read_halfmaps(args.halfmaps, pixel_size=args.pixel_size) 92 | if args.mask: 93 | mask = utils.fileio.read_ccp4_map(args.mask)[0] 94 | else: 95 | mask = None 96 | 97 | hkldata = utils.maps.mask_and_fft_maps(maps, args.resolution, mask) 98 | hkldata.setup_relion_binning("ml") 99 | utils.maps.calc_noise_var_from_halfmaps(hkldata) 100 | 101 | smax = 1. / args.resolution 102 | if args.find_B_at is not None: 103 | find_b(hkldata, smax, args.find_B_at, 104 | kind=args.f, sharpen_signal=args.sharpen_signal, weight=args.weight) 105 | else: 106 | x_all = numpy.arange(0, args.x_max, 0.1) 107 | cc_all = calc_cc_from_var(hkldata, x_list=x_all, kind=args.f, 108 | sharpen_signal=args.sharpen_signal, weight=args.weight, 109 | B=args.B) 110 | 111 | ofs = open("{}.dat".format(args.output_prefix), "w") 112 | ofs.write("# smax= {}\n".format(smax)) 113 | ofs.write("# halfmaps= {}\n".format(*args.halfmaps)) 114 | ofs.write("# mask= {}\n".format(args.mask)) 115 | ofs.write("# weight= {}\n".format(args.weight)) 116 | ofs.write("# f= {}\n".format(args.f)) 117 | ofs.write("x cc dmin weight f sharpen b\n") 118 | for x, cc in zip(x_all, cc_all): 119 | ofs.write("{:.2f} {:.4f} {:.2f} {} {} {} {}\n".format(x, cc, args.resolution, args.weight, args.f, 120 | "TRUE" if args.sharpen_signal else "FALSE", 121 | args.B)) 122 | # main() 123 | 124 | if __name__ == "__main__": 125 | import sys 126 | args = parse_args(sys.argv[1:]) 127 | main(args) 128 | 129 | -------------------------------------------------------------------------------- /docs/spa_examples/omitmap.rst: -------------------------------------------------------------------------------- 1 | Calculating Fo-Fc omit map 2 | =============================== 3 | 4 | This guide demonstrates how to calculate a :math:`F_{\rm o}-F_{\rm c}` omit map using Servalcat. A :math:`F_{\rm o}-F_{\rm c}` omit map is a difference map calculated after removing (omitting) certain atoms from the model. This is useful for visualising the presence of ligands or other features not explicitly included in the model. 5 | 6 | Before proceeding, you will need 1) unweighted and unsharpened half maps, 2) mask, 3) model files. 7 | 8 | Refine the model first! 9 | ----------------------- 10 | It is important to refine the model before calculating the :math:`F_{\rm o}-F_{\rm c}` map. At least ADP (B-values) should be refined for a meaningful result. 11 | If you plan to use a PDB model deposited in a public archive, be aware that it might have been refined in a way not ideal for map calculation, such as with grouped ADPs. 12 | Please refer to the :doc:`ChRmine example ` for the refinement tutorial. 13 | 14 | .. _normalisation-within-mask: 15 | 16 | Normalisation within mask 17 | ------------------------- 18 | In crystallography, maps are often "sigma-scaled" and displayed with contours at a certain number of sigma (σ). For example, you might see references to ":math:`mF_{\rm o}-DF_{\rm c}` omit map contoured at 3σ" in the literature. This sigma value represents the standard deviation of map values within the unit cell. 19 | 20 | However, in SPA, sigma calculated in this way cannot be directly applied. Box size is arbitrary, and everything outside the mask is set to zero. A larger box with more zero-valued pixels results in a smaller sigma, leading to inflated sigma-scaled peak heights. 21 | For details, refer to section 3.3 of `Yamashita et al. (2021) `_. 22 | 23 | When you provide a mask file to Servalcat, it calculates maps normalised using a sigma derived from within the mask. Be cautious of sigma-scaled values displayed by other visualisation programs. For example, in Coot, ignore map values with the "rmsd" unit. Focus on the raw map values themselves. 24 | In figure captions, consider using a phrase like ":math:`F_{\rm o}-F_{\rm c}` map contoured at XXσ (where σ is the standard deviation within the mask)." 25 | 26 | Tutorial 27 | --------- 28 | 29 | This tutorial uses the AlF\ :sub:`4`\ \ :sup:`-`\ -ADP–bound state of P4-ATPase flippase (`PDB 6k7k `_, `EMD-9937 `_) from `Hiraizumi et al. 2019 `_ as an example. 30 | We will need the pdb (or mmcif) file, half maps, and mask: 31 | :: 32 | 33 | wget https://files.rcsb.org/download/6k7k.pdb 34 | wget https://files.wwpdb.org/pub/emdb/structures/EMD-9937/other/emd_9937_half_map_1.map.gz 35 | wget https://files.wwpdb.org/pub/emdb/structures/EMD-9937/other/emd_9937_half_map_2.map.gz 36 | wget https://files.wwpdb.org/pub/emdb/structures/EMD-9937/masks/emd_9937_msk_1.map 37 | 38 | First, refine the model. 39 | 40 | .. code-block:: console 41 | 42 | $ servalcat refine_spa_norefmac \ 43 | --model ../6k7k.pdb \ 44 | --halfmaps ../emd_9937_half_map_1.map.gz ../emd_9937_half_map_2.map.gz \ 45 | --mask_for_fofc ../emd_9937_msk_1.map \ 46 | --resolution 2.9 47 | 48 | Check the refinement result. To calculate the omit map properly, the overall model quality should be high enough. 49 | 50 | Then remove the atoms that you want to see in the omit map. Here, I removed ADP, ALF, and Mg ions (A/1201-1204) using Coot and saved the model as ``refined_omit.pdb``. 51 | 52 | Here is the command to calculate the :math:`F_{\rm o}-F_{\rm c}` map: 53 | 54 | .. code-block:: console 55 | 56 | $ servalcat fofc \ 57 | --model refined_omit.pdb \ 58 | --halfmaps ../emd_9937_half_map_1.map.gz ../emd_9937_half_map_2.map.gz \ 59 | --mask ../emd_9937_msk_1.map \ 60 | --resolution 2.9 \ 61 | -o diffmap_omit 62 | 63 | * ``--halfmaps`` should be unsharpened and unweighted half maps (you used the same half maps for refinement). 64 | * ``--mask`` (not ``--mask_for_fofc``) is used for FSC weighting and normalisation of map values. 65 | * ``--resolution`` likewise, it is always good to specify a bit higher resolution than the global one (as determined by the FSC=0.143 criterion). 66 | * ``-o`` is the prefix of the output file name. Default is ``diffmap``, which would overwrite the file written by ``refine_spa_norefmac`` job if you run in the same directory. 67 | 68 | 69 | Check omit map with Coot 70 | ~~~~~~~~~~~~~~~~~~~~~~~~ 71 | Open ``refined.pdb`` and Auto-open ``diffmap_omit.mtz`` in Coot. 72 | In Display Manager, turn off the ``FWT PHWT`` map and adjust the contour level of the ``DELFWT PHDELWT`` map (this is :math:`F_{\rm o}-F_{\rm c}` map). 73 | Then you see: 74 | 75 | .. image:: p4_figs/coot_fofc_omit_4sigma.png 76 | :align: center 77 | :scale: 30% 78 | 79 | Here at the top, "4.000 e/A^3 (11.87rmsd)" is shown. You may think this is contoured at 11.87σ, but no, this is actually at 4σ. In Coot, a value outside the brackets is a raw map value (ignore e/A^3 or V unit!). See `above <#normalisation-within-mask>`_ also. 80 | Note that this normalisation within the mask only happens when ``--mask`` is given. 81 | 82 | Check omit map with PyMOL 83 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 84 | PyMOL by default scales maps with sigma (calculated using all pixels) upon reading of map files. It should be turned off before reading map files. So first start PyMOL with the model file only, 85 | 86 | .. code-block:: console 87 | 88 | $ pymol refined.pdb 89 | 90 | and then turn off normalisation in PyMOL: 91 | :: 92 | 93 | set normalize_ccp4_maps, off 94 | load diffmap_omit_normalized_fofc.mrc 95 | isomesh msh_fofc, diffmap_omit_normalized_fofc, 4 96 | 97 | You see: 98 | 99 | .. image:: p4_figs/pymol_fofc_omit_4sigma.png 100 | :align: center 101 | :scale: 40% 102 | 103 | Again, this is the :math:`F_{\rm o}-F_{\rm c}` omit map contoured at 4σ (where σ is the standard deviation within the mask). 104 | -------------------------------------------------------------------------------- /tests/test_xtal.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import unittest 10 | import numpy 11 | import pandas 12 | import json 13 | import gemmi 14 | import os 15 | import shutil 16 | import sys 17 | import tempfile 18 | import hashlib 19 | from servalcat import utils 20 | from servalcat.xtal import sigmaa 21 | from servalcat.xtal import french_wilson 22 | from servalcat.__main__ import main 23 | 24 | root = os.path.abspath(os.path.dirname(__file__)) 25 | 26 | class XtalTests(unittest.TestCase): 27 | def setUp(self): 28 | self.wd = tempfile.mkdtemp(prefix="servaltest_") 29 | os.chdir(self.wd) 30 | print("In", self.wd) 31 | # setUp() 32 | 33 | def tearDown(self): 34 | os.chdir(root) 35 | shutil.rmtree(self.wd) 36 | # tearDown() 37 | 38 | def test_scale(self): 39 | mtzin = os.path.join(root, "5e5z", "5e5z.mtz.gz") 40 | pdbin = os.path.join(root, "5e5z", "5e5z.pdb.gz") 41 | st = utils.fileio.read_structure(pdbin) 42 | mtz = gemmi.read_mtz_file(mtzin) 43 | hkldata = utils.hkl.hkldata_from_asu_data(mtz.get_float("I"), "I") 44 | 45 | hkldata.df["FC"] = utils.model.calc_fc_fft(st, mtz.resolution_high(), "xray", mott_bethe=False, 46 | miller_array=hkldata.miller_array()) 47 | hkldata.df["IC"] = numpy.abs(hkldata.df["FC"].to_numpy())**2 48 | k, b = hkldata.scale_k_and_b("I", "IC") 49 | self.assertAlmostEqual(k, 0.00665, places=5) 50 | self.assertAlmostEqual(b, -8.6132, places=3) 51 | # test_scale() 52 | 53 | def test_sigmaa(self): 54 | mtzin = os.path.join(root, "5e5z", "5e5z.mtz.gz") 55 | pdbin = os.path.join(root, "5e5z", "5e5z.pdb.gz") 56 | args = sigmaa.parse_args(["--hklin", mtzin, "--model", pdbin, "--D_trans", "exp", "--S_trans", "exp", 57 | "--labin", "FP,SIGFP", "--nbins", "10", "--nbins_ml", "10", "--source", "xray"]) 58 | hkldata = sigmaa.main(args) 59 | os.remove("sigmaa.log") 60 | os.remove("sigmaa.mtz") 61 | 62 | numpy.testing.assert_allclose(hkldata.binned_df["ml"].d_min, 63 | [5.0834, 3.6631, 3.0103, 2.6155, 2.344 , 2.1426, 1.9855, 1.8586, 64 | 1.7532, 1.664 ], 65 | rtol=1e-4) 66 | numpy.testing.assert_allclose(hkldata.binned_df["ml"].D0, 67 | [0.829663, 0.993061, 1.033881, 0.982557, 1.016779, 1.035846, 68 | 1.025311, 1.023788, 0.970901, 0.974236], 69 | rtol=1e-2) 70 | #numpy.testing.assert_allclose(hkldata.binned_df["ml"].D1, 71 | # [2.974164e-01, 5.624910e-08, 2.500540e-10, 3.980775e-10, 72 | # 7.638667e-07, 8.243909e-08, 1.000051e+00, 9.999994e-01, 73 | # 9.999997e-01, 1.000001e+00], 74 | # rtol=1e-5) 75 | numpy.testing.assert_allclose(hkldata.binned_df["ml"].S, 76 | [90.147006, 97.960666, 67.314535, 97.67571 , 72.546305, 77 | 109.631114, 119.247475, 98.844853, 78.992354, 41.845315], 78 | rtol=1e-2) 79 | 80 | # test_sigmaa() 81 | 82 | def test_sigmaa_int(self): 83 | mtzin = os.path.join(root, "5e5z", "5e5z.mtz.gz") 84 | pdbin = os.path.join(root, "5e5z", "5e5z.pdb.gz") 85 | args = sigmaa.parse_args(["--hklin", mtzin, "--model", pdbin, "--D_trans", "splus", "--S_trans", "splus", 86 | "--labin", "I,SIGI", "--nbins", "10", "--nbins_ml", "10", "--source", "xray"]) 87 | hkldata = sigmaa.main(args) 88 | os.remove("sigmaa.mtz") 89 | numpy.testing.assert_allclose(hkldata.binned_df["ml"].D0, 90 | [0.829305, 0.98676 , 1.024622, 0.967299, 0.998059, 91 | 1.014157, 1.000922, 0.995466, 0.93671 , 0.932352], 92 | rtol=1e-2) 93 | numpy.testing.assert_allclose(hkldata.binned_df["ml"].S, 94 | [89.637282, 97.784362, 68.782025, 99.382283, 74.905775, 95 | 112.326708, 126.651614, 106.853499, 83.032016, 42.909208], 96 | rtol=1e-2) 97 | 98 | def test_fw(self): 99 | mtzin = os.path.join(root, "5e5z", "5e5z.mtz.gz") 100 | args = french_wilson.parse_args(["--hklin", mtzin, "--labin", "I,SIGI"]) 101 | B_aniso, hkldata = french_wilson.main(args) 102 | os.remove("5e5z_fw.mtz") 103 | numpy.testing.assert_allclose(B_aniso.elements_pdb(), 104 | [2.640011, 1.679485, -4.319497, 0. ,-1.072883, 0.], 105 | rtol=1e-3) 106 | 107 | @unittest.skipUnless(utils.refmac.check_version(), "refmac unavailable") 108 | def test_refine_cx(self): 109 | mtzin = os.path.join(root, "biotin", "biotin_talos.mtz") 110 | pdbin = os.path.join(root, "biotin", "biotin_talos.pdb") 111 | sys.argv = ["", "refine_cx", "--model", pdbin, 112 | "--hklin", mtzin, "--bref", "iso_then_aniso"] 113 | main() 114 | self.assertTrue(os.path.isfile("refined_2_aniso.mtz")) 115 | r_factor = None 116 | with open("refined_2_aniso.log") as ifs: 117 | for l in ifs: 118 | if l.startswith(" R factor"): 119 | r_factor = float(l.split()[-1]) 120 | self.assertLess(r_factor, 0.185) 121 | # test_refine_cx() 122 | 123 | # class XtalTests 124 | 125 | if __name__ == '__main__': 126 | unittest.main() 127 | 128 | -------------------------------------------------------------------------------- /src/math.hpp: -------------------------------------------------------------------------------- 1 | // Author: "Keitaro Yamashita, Garib N. Murshudov" 2 | // MRC Laboratory of Molecular Biology 3 | 4 | #ifndef SERVALCAT_MATH_HPP_ 5 | #define SERVALCAT_MATH_HPP_ 6 | 7 | #include "lambertw.hpp" 8 | #include // for log_bessel_i0, bessel_i1_over_i0 9 | #include // for log_cosh 10 | #include 11 | 12 | namespace servalcat { 13 | 14 | constexpr double sq(double x) {return x * x;} 15 | 16 | inline double log_i0_or_cosh(double X, int c) { 17 | return c == 1 ? gemmi::log_bessel_i0(2*X) : gemmi::log_cosh(X); 18 | } 19 | 20 | inline double fom(double X, int c) { 21 | return c == 1 ? gemmi::bessel_i1_over_i0(2*X) : std::tanh(X); 22 | } 23 | 24 | inline double fom_der(double m, double X, int c) { 25 | // c=1: d/dX I1(2X)/I0(2X) 26 | // c=2: d/dX tanh(X) 27 | return c == 1 ? 2 - m / X - 2 * sq(m) : 1 - sq(m); 28 | } 29 | 30 | inline double fom_der2(double m, double X, int c) { 31 | // c=1: d^2/dX^2 I1(2X)/I0(2X) 32 | // c=2: d^2/dX^2 tanh(X) 33 | return c == 1 ? m / sq(X) - (4 * m + 1 / X) * (2 - m / X - 2 * sq(m)) : 2 * m * (sq(m) - 1); 34 | } 35 | 36 | inline double x_plus_sqrt_xsq_plus_y(double x, double y) { 37 | // avoid precision loss 38 | const double tmp = std::sqrt(sq(x) + y); 39 | return x < 0 ? y / (tmp - x) : x + tmp; 40 | } 41 | 42 | // solve y - exp(-y) = x for y. 43 | // solution is y = W(exp(-x)) + x 44 | inline double solve_y_minus_exp_minus_y(double x, double prec) { 45 | if (x > 20) return x; 46 | return lambertw::lambertw(std::exp(-x), prec) + x; 47 | } 48 | 49 | struct RootfindResult { 50 | bool success = false; 51 | std::string reason; 52 | int iter = 0; 53 | double x; 54 | }; 55 | 56 | template 57 | RootfindResult newton(Func&& func, Fprime&& fprime, double x0, 58 | int maxiter=50, double tol=1.48e-8) { 59 | RootfindResult ret; 60 | ret.x = x0; 61 | for (int itr = 0; itr < maxiter; ++itr) { 62 | ++ret.iter; 63 | double fval = func(x0); 64 | if (fval == 0) { 65 | ret.success = true; 66 | return ret; 67 | } 68 | double fder = fprime(x0); 69 | if (fder <= 0) { 70 | ret.reason = "fder <= 0"; 71 | return ret; 72 | } 73 | ret.x = x0 - fval / fder; 74 | if (std::abs(ret.x - x0) < tol) { 75 | ret.success = true; 76 | return ret; 77 | } 78 | x0 = ret.x; 79 | } 80 | ret.reason = "maxiter reached"; 81 | return ret; 82 | } 83 | 84 | template 85 | RootfindResult secant(Func&& func, double x0, 86 | int maxiter=50, double tol=1.48e-8) { 87 | const double eps = 1e-1; 88 | RootfindResult ret; 89 | double p = x0, p0 = x0; 90 | double p1 = x0 * (1 + eps); 91 | p1 += p1 >= 0 ? eps : -eps; 92 | double q0 = func(p0); 93 | double q1 = func(p1); 94 | if (std::abs(q1) < std::abs(q0)) { 95 | std::swap(p0, p1); 96 | std::swap(q0, q1); 97 | } 98 | for (int itr = 0; itr < maxiter; ++itr) { 99 | ++ret.iter; 100 | if (q1 == q0) { 101 | ret.x = 0.5 * (p1 + p0); 102 | if (p1 != p0) 103 | return ret; 104 | ret.success = true; 105 | return ret; 106 | } else { 107 | if (std::abs(q1) > std::abs(q0)) 108 | p = (-q0 / q1 * p1 + p0) / (1 - q0 / q1); 109 | else 110 | p = (-q1 / q0 * p0 + p1) / (1 - q1 / q0); 111 | } 112 | ret.x = p; 113 | if (std::abs(p - p1) < tol) { 114 | ret.success = true; 115 | return ret; 116 | } 117 | p0 = p1; 118 | q0 = q1; 119 | p1 = p; 120 | q1 = func(p1); 121 | } 122 | ret.reason = "maxiter reached"; 123 | return ret; 124 | } 125 | 126 | template 127 | RootfindResult newton_or_secant(Func&& func, Fprime&& fprime, double x0, 128 | int maxiter=50, double tol=1.48e-8) { 129 | auto res = newton(func, fprime, x0, maxiter, tol); 130 | if (res.success) 131 | return res; 132 | return secant(func, x0, maxiter, tol); 133 | } 134 | 135 | 136 | template 137 | RootfindResult bisect(Func&& func, double a, double b, 138 | int maxiter=100, double tol=1.48e-8) { 139 | RootfindResult ret; 140 | if (a > b) 141 | std::swap(a, b); 142 | if (func(a) * func(b) >= 0) { 143 | ret.reason = "fa * fb >= 0"; 144 | return ret; 145 | } 146 | for (int itr = 0; itr < maxiter; ++itr) { 147 | ++ret.iter; 148 | ret.x = 0.5 * (a + b); 149 | if (func(ret.x) == 0 || 0.5 * (b - a) < tol) { 150 | ret.success = true; 151 | return ret; 152 | } 153 | if (func(ret.x) * func(a) >= 0) 154 | a = ret.x; 155 | else 156 | b = ret.x; 157 | } 158 | ret.reason = "maxiter reached"; 159 | return ret; 160 | } 161 | 162 | inline double procrust_dist(Eigen::MatrixXd x, Eigen::MatrixXd y) { 163 | if (x.rows() != y.rows() || x.cols() != y.cols() || x.cols() != 3) 164 | throw std::runtime_error("procrust_dist: dimension mismatch"); 165 | if (!x.size()) return NAN; 166 | const Eigen::Vector3d xmean = x.colwise().mean(), ymean = y.colwise().mean(); 167 | x.rowwise() -= xmean.transpose(); 168 | y.rowwise() -= ymean.transpose(); 169 | const Eigen::Matrix3d xty = x.transpose() * y; 170 | const Eigen::JacobiSVD svd(xty); 171 | double dist = -2.0 * svd.singularValues().sum() + x.squaredNorm() + y.squaredNorm(); 172 | dist = std::sqrt(std::max(0., dist) / x.rows()); 173 | return dist; 174 | } 175 | 176 | struct SymMatEig { 177 | SymMatEig(const Eigen::MatrixXd &m) : es(m) {} 178 | double det(bool exclude_zero=false) const { 179 | if (exclude_zero) { 180 | const auto v = es.eigenvalues(); 181 | // not exact zero due to finite precision 182 | return (v.array().abs() < 1e-13).select(1, v).prod(); 183 | } 184 | return es.eigenvalues().prod(); 185 | } 186 | Eigen::MatrixXd inv(double e=1e-8) const { 187 | Eigen::VectorXd eig_inv = es.eigenvalues(); 188 | for (int i = 0; i < eig_inv.size(); ++i) 189 | eig_inv(i) = std::abs(eig_inv(i)) < e ? 1 : (1. / eig_inv(i)); 190 | return es.eigenvectors() * eig_inv.asDiagonal() * es.eigenvectors().adjoint(); 191 | } 192 | Eigen::SelfAdjointEigenSolver es; 193 | }; 194 | 195 | } // namespace servalcat 196 | #endif 197 | -------------------------------------------------------------------------------- /src/refine/cgsolve.hpp: -------------------------------------------------------------------------------- 1 | // Author: "Keitaro Yamashita, Garib N. Murshudov" 2 | // MRC Laboratory of Molecular Biology 3 | 4 | #ifndef SERVALCAT_REFINE_CGSOLVE_HPP_ 5 | #define SERVALCAT_REFINE_CGSOLVE_HPP_ 6 | 7 | #include "ll.hpp" 8 | #include "geom.hpp" 9 | #include 10 | #include // for gemmi::Logger 11 | #include 12 | #include 13 | 14 | namespace servalcat { 15 | 16 | Eigen::SparseMatrix 17 | diagonal_preconditioner(Eigen::SparseMatrix &mat) { 18 | const int n = mat.cols(); 19 | Eigen::SparseMatrix pmat(n, n); 20 | std::vector> data; 21 | for(int j = 0; j < mat.outerSize(); ++j) { 22 | Eigen::SparseMatrix::InnerIterator it(mat, j); 23 | while(it && it.index() != j) ++it; 24 | if(it && it.index() == j && it.value() > 0) 25 | data.emplace_back(j, j, std::sqrt(1. / it.value())); 26 | else 27 | data.emplace_back(j, j, 1); 28 | } 29 | pmat.setFromTriplets(data.begin(), data.end()); 30 | mat = (pmat * mat * pmat).eval(); 31 | // in our case if diagonal is zero, all corresponding non-diagonals are also zero. 32 | // so it's safe to replace diagonal with one. 33 | for(int j = 0; j < mat.outerSize(); ++j) { 34 | Eigen::SparseMatrix::InnerIterator it(mat, j); 35 | while(it && it.index() != j) ++it; 36 | if(it && it.index() == j && it.value() == 0) 37 | it.valueRef() = 1.; 38 | } 39 | return pmat; 40 | } 41 | 42 | struct CgSolve { 43 | const GeomTarget *geom; 44 | const LL *ll; 45 | double gamma = 0.; 46 | double toler = 1.e-4; 47 | int ncycle = 2000; 48 | int max_gamma_cyc = 500; 49 | CgSolve(const GeomTarget *geom, const LL *ll) 50 | : geom(geom), ll(ll) {} 51 | 52 | template 53 | Eigen::VectorXd solve(double weight, const gemmi::Logger& logger){ 54 | Eigen::VectorXd vn = Eigen::VectorXd::Map(geom->vn.data(), geom->vn.size()); 55 | Eigen::SparseMatrix am = geom->make_spmat(); 56 | logger.mesg("diag(geom) min= ", std::to_string(am.diagonal().minCoeff()), 57 | " max= ", std::to_string(am.diagonal().maxCoeff())); 58 | if (ll != nullptr) { 59 | auto ll_mat = ll->make_spmat(); 60 | logger.mesg("diag(data) min= ", std::to_string(ll_mat.diagonal().minCoeff()), 61 | " max= ", std::to_string(ll_mat.diagonal().maxCoeff())); 62 | vn += Eigen::VectorXd::Map(ll->vn.data(), ll->vn.size()) * weight; 63 | am += ll_mat * weight; 64 | } 65 | logger.mesg("diag(all) min= ", std::to_string(am.diagonal().minCoeff()), 66 | " max= ", std::to_string(am.diagonal().maxCoeff())); 67 | const int n = am.cols(); 68 | 69 | // this changes am 70 | Eigen::SparseMatrix pmat = diagonal_preconditioner(am); 71 | vn = (pmat * vn).eval(); 72 | 73 | if (gamma == 0 && max_gamma_cyc == 1) { 74 | Eigen::ConjugateGradient, Eigen::Lower|Eigen::Upper, 75 | Preconditioner> cg; 76 | Eigen::VectorXd dv(n); 77 | cg.setMaxIterations(ncycle); 78 | cg.setTolerance(toler); 79 | cg.compute(am); 80 | dv = cg.solve(vn); 81 | logger.mesg("#iterations: ", cg.iterations(), "\n", 82 | "estimated error: ", std::to_string(cg.error())); 83 | return pmat * dv; 84 | } 85 | 86 | // if Preconditioner is not Identity, gamma cycle should not be used. 87 | Preconditioner precond; 88 | precond.compute(am); 89 | 90 | double gamma_save = 0; 91 | bool gamma_flag = false; 92 | bool conver_flag = false; 93 | Eigen::VectorXd dv(n), dv_save(n); 94 | dv.setZero(); 95 | dv_save.setZero(); 96 | 97 | const double vnorm2 = vn.squaredNorm(); 98 | const double test_lim = std::max(toler * toler * vnorm2, std::numeric_limits::min()); 99 | double step = 0.05; 100 | 101 | for (int gamma_cyc = 0; gamma_cyc < max_gamma_cyc; ++gamma_cyc, gamma+=step) { 102 | logger.mesg("Trying gamma equal ", std::to_string(gamma)); 103 | Eigen::VectorXd r = vn - (am * dv + gamma * dv); 104 | double rnorm2 = r.squaredNorm(); 105 | if (rnorm2 < test_lim) 106 | break; 107 | 108 | Eigen::VectorXd p(n), z(n), tmp(n); 109 | p = precond.solve(r); 110 | double rho0 = r.dot(p); 111 | bool exit_flag = false; 112 | for (int itr = 0; itr < ncycle; ++itr) { 113 | tmp.noalias() = am * p + gamma * p; 114 | double alpha = rho0 / p.dot(tmp); 115 | dv += alpha * p; 116 | r -= alpha * tmp; 117 | rnorm2 = r.squaredNorm(); 118 | if (rnorm2 < test_lim) { 119 | if (!gamma_flag) { 120 | logger.mesg("Convergence reached after ", itr+1, " iterations with no gamma cycles"); 121 | exit_flag = true; 122 | break; 123 | } else if (conver_flag) { 124 | logger.mesg("Convergence reached with gamma equal ", std::to_string(gamma)); 125 | step *= 1.01; 126 | exit_flag = true; 127 | break; 128 | } else { 129 | conver_flag = true; 130 | gamma_save = gamma; 131 | dv_save = dv; 132 | gamma = std::max(0., gamma - step/5.); 133 | step = std::max(step/1.1, 0.0001); 134 | logger.mesg("Gamma decreased to ", std::to_string(gamma)); 135 | exit_flag = true; 136 | break; 137 | } 138 | } 139 | 140 | z = precond.solve(r); 141 | double rho1 = rho0; 142 | rho0 = r.dot(z); 143 | if (rho0 > 4 * rho1) { 144 | logger.mesg("Not converging with gamma equal ", std::to_string(gamma)); 145 | step *= 1.05; 146 | break; 147 | } 148 | double beta = rho0 / rho1; 149 | p = z + beta * p; 150 | } 151 | if (exit_flag) break; 152 | if (max_gamma_cyc == 1) break; // test 153 | gamma_flag = true; 154 | if (!conver_flag) 155 | dv.setZero(); 156 | else { 157 | dv = dv_save; 158 | gamma = gamma_save; 159 | logger.mesg("Back to gamma equal ", std::to_string(gamma)); 160 | } 161 | } 162 | return pmat * dv; 163 | } 164 | }; 165 | 166 | } // namespace servalcat 167 | #endif 168 | -------------------------------------------------------------------------------- /servalcat/refine/spa.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import gemmi 10 | import numpy 11 | import json 12 | import scipy.sparse 13 | from servalcat.utils import logger 14 | from servalcat import utils 15 | from servalcat.spa import fofc 16 | from servalcat.spa import fsc 17 | from servalcat import ext 18 | b_to_u = utils.model.b_to_u 19 | u_to_b = utils.model.u_to_b 20 | 21 | def calc_D_and_S(hkldata, lab_obs): # simplified version of fofc.calc_D_and_S() 22 | bdf = hkldata.binned_df["ml"] 23 | bdf["D"] = 0. 24 | bdf["S"] = 0. 25 | for i_bin, idxes in hkldata.binned("ml"): 26 | Fo = hkldata.df[lab_obs].to_numpy()[idxes] 27 | Fc = hkldata.df.FC.to_numpy()[idxes] 28 | bdf.loc[i_bin, "D"] = numpy.nansum(numpy.real(Fo * numpy.conj(Fc))) / numpy.sum(numpy.abs(Fc)**2) 29 | bdf.loc[i_bin, "S"] = numpy.nanmean(numpy.abs(Fo - bdf.D[i_bin] * Fc)**2) 30 | # calc_D_and_S() 31 | 32 | class LL_SPA: 33 | def __init__(self, hkldata, st, monlib, lab_obs, source="electron", mott_bethe=True): 34 | assert source in ("electron", "xray", "custom") 35 | self.source = source 36 | self.mott_bethe = False if source != "electron" else mott_bethe 37 | self.hkldata = hkldata 38 | self.lab_obs = lab_obs 39 | self.st = st 40 | self.monlib = monlib 41 | self.d_min_max = hkldata.d_min_max() 42 | self.ll = None 43 | self.b_aniso = None 44 | 45 | def refine_id(self): 46 | if self.source in ("electron", "custom"): 47 | # XXX when custom, it's actually unknown.. 48 | return "ELECTRON MICROSCOPY" 49 | return "NON-EM SPA" # does not happen, I guess 50 | 51 | def update_ml_params(self): 52 | # FIXME make sure D > 0 53 | calc_D_and_S(self.hkldata, self.lab_obs) 54 | 55 | def update_fc(self): 56 | if self.st.ncs: 57 | st = self.st.clone() 58 | st.expand_ncs(gemmi.HowToNameCopiedChain.Short, merge_dist=0) 59 | else: 60 | st = self.st 61 | 62 | self.hkldata.df["FC"] = utils.model.calc_fc_fft(st, self.d_min_max[0] - 1e-6, 63 | monlib=self.monlib, 64 | source=self.source, 65 | mott_bethe=self.mott_bethe, 66 | miller_array=self.hkldata.miller_array()) 67 | 68 | def prepare_target(self): 69 | pass 70 | 71 | def overall_scale(self, min_b=0.5): 72 | k, b = self.hkldata.scale_k_and_b(lab_ref=self.lab_obs, lab_scaled="FC") 73 | min_b_iso = self.st[0].calculate_b_aniso_range()[0] # actually min of aniso too 74 | tmp = min_b_iso + b 75 | if tmp < min_b: # perhaps better only adjust b_iso that went too small, but we need to recalculate Fc 76 | logger.writeln("Adjusting overall B to avoid too small value") 77 | b += min_b - tmp 78 | logger.writeln("Applying overall B to model: {:.2f}".format(b)) 79 | utils.model.shift_b(self.st[0], b) 80 | # adjust Fc 81 | k_iso = self.hkldata.debye_waller_factors(b_iso=b) 82 | self.hkldata.df["FC"] *= k_iso 83 | # adjust Fo 84 | self.hkldata.df[self.lab_obs] /= k 85 | # overall_scale() 86 | 87 | def calc_target(self): # -LL target for SPA 88 | ret = 0 89 | for i_bin, idxes in self.hkldata.binned("ml"): 90 | Fo = self.hkldata.df[self.lab_obs].to_numpy()[idxes] 91 | DFc = self.hkldata.df.FC.to_numpy()[idxes] * self.hkldata.binned_df["ml"].D[i_bin] 92 | S = self.hkldata.binned_df["ml"].S[i_bin] 93 | ret += numpy.nansum(numpy.abs(Fo - DFc)**2) / S + numpy.log(S) * len(idxes) 94 | return ret * 2 # friedel mates 95 | # calc_target() 96 | 97 | def calc_stats(self, bin_stats=False): 98 | # ignore bin_stats for now. better stats are calculated after refinement 99 | stats = fsc.calc_fsc_all(self.hkldata, labs_fc=["FC"], lab_f=self.lab_obs) 100 | fsca = fsc.fsc_average(stats.ncoeffs, stats.fsc_FC_full) 101 | logger.writeln("FSCaverage = {:.4f}".format(fsca)) 102 | ret = {"summary": {"FSCaverage": fsca, "-LL": self.calc_target()}} 103 | # XXX in fsc object, _full is misleading - it's not full in cross validation mode 104 | ret["bin_stats"] = stats 105 | ret["ml"] = self.hkldata.binned_df["ml"].copy() 106 | return ret 107 | 108 | def calc_grad(self, refine_params, specs): 109 | dll_dab = numpy.empty_like(self.hkldata.df[self.lab_obs]) 110 | d2ll_dab2 = numpy.zeros(len(self.hkldata.df.index)) 111 | blur = utils.model.determine_blur_for_dencalc(self.st, self.d_min_max[0] / 3) # TODO need more work 112 | logger.writeln("blur for deriv= {:.2f}".format(blur)) 113 | for i_bin, idxes in self.hkldata.binned("ml"): 114 | D = self.hkldata.binned_df["ml"].D[i_bin] 115 | S = self.hkldata.binned_df["ml"].S[i_bin] 116 | Fc = self.hkldata.df.FC.to_numpy()[idxes] 117 | Fo = self.hkldata.df[self.lab_obs].to_numpy()[idxes] 118 | dll_dab[idxes] = -2 * D / S * (Fo - D * Fc)#.conj() 119 | d2ll_dab2[idxes] = 2 * D**2 / S 120 | 121 | if self.mott_bethe: 122 | dll_dab *= self.hkldata.d_spacings()**2 * gemmi.mott_bethe_const() 123 | d2ll_dab2 *= gemmi.mott_bethe_const()**2 124 | 125 | # we need V for Hessian and V**2/n for gradient. 126 | d2ll_dab2 *= self.hkldata.cell.volume 127 | dll_dab_den = self.hkldata.fft_map(data=dll_dab * self.hkldata.debye_waller_factors(b_iso=-blur)) 128 | dll_dab_den.array[:] *= self.hkldata.cell.volume**2 / dll_dab_den.point_count 129 | self.ll = ext.LL(self.st, refine_params, self.mott_bethe) 130 | self.ll.set_ncs([x.tr for x in self.st.ncs if not x.given]) 131 | if self.source == "custom": 132 | self.ll.calc_grad_custom(dll_dab_den, blur) 133 | else: 134 | self.ll.calc_grad_it92(dll_dab_den, blur) 135 | 136 | # second derivative 137 | d2dfw_table = ext.TableS3(*self.hkldata.d_min_max()) 138 | d2dfw_table.make_table(1./self.hkldata.d_spacings(), d2ll_dab2) 139 | if self.source == "custom": 140 | self.ll.make_fisher_table_diag_fast_custom(d2dfw_table, 1.) 141 | self.ll.fisher_diag_from_table_custom() 142 | else: 143 | self.ll.make_fisher_table_diag_fast_it92(d2dfw_table) 144 | self.ll.fisher_diag_from_table_it92() 145 | #json.dump(dict(b=self.ll.table_bs, pp1=self.ll.pp1, bb=self.ll.bb), 146 | # open("ll_fisher.json", "w"), indent=True) 147 | #a, (b,c) = ll.fisher_for_coo() 148 | #json.dump(([float(x) for x in a], ([int(x) for x in b], [int(x) for x in c])), open("fisher.json", "w")) 149 | #logger.writeln("disabling spec_correction in spa target") 150 | if specs is not None: 151 | self.ll.spec_correction(specs, use_rr=False) 152 | -------------------------------------------------------------------------------- /docs/help/refine_spa.txt: -------------------------------------------------------------------------------- 1 | $ servalcat refine_spa_norefmac -h 2 | usage: servalcat refine_spa_norefmac [-h] (--halfmaps HALFMAPS HALFMAPS | --map MAP | --hklin HKLIN) 3 | [--pixel_size PIXEL_SIZE] [--labin LABIN] --model MODEL -d 4 | RESOLUTION [-r MASK_RADIUS] [--padding PADDING] [--no_mask] 5 | [--no_trim] [--mask_soft_edge MASK_SOFT_EDGE] 6 | [--no_sharpen_before_mask] [--b_before_mask B_BEFORE_MASK] 7 | [--blur BLUR] [--monlib MONLIB] [--ligand [LIGAND ...]] 8 | [--newligand_continue] [--hydrogen {all,yes,no}] [--hout] 9 | [--jellybody] [--jellybody_params sigma dmax] [--jellyonly] 10 | [--pg PG] [--twist TWIST] [--rise RISE] 11 | [--center CENTER CENTER CENTER] [--axis1 AXIS1 AXIS1 AXIS1] 12 | [--axis2 AXIS2 AXIS2 AXIS2] [--contacting_only] 13 | [--ignore_symmetry IGNORE_SYMMETRY] [--find_links] 14 | [--no_check_ncs_overlaps] [--no_check_ncs_map] 15 | [--no_check_mask_with_model] [--keywords KEYWORDS [KEYWORDS ...]] 16 | [--keyword_file KEYWORD_FILE [KEYWORD_FILE ...]] 17 | [--randomize RANDOMIZE] [--ncycle NCYCLE] [--weight WEIGHT] 18 | [--no_weight_adjust] 19 | [--target_bond_rmsz_range TARGET_BOND_RMSZ_RANGE TARGET_BOND_RMSZ_RANGE] 20 | [--adpr_weight ADPR_WEIGHT] [--ncsr] [--bfactor BFACTOR] 21 | [--fix_xyz] [--adp {fix,iso,aniso}] [--refine_all_occ] 22 | [--max_dist_for_adp_restraint MAX_DIST_FOR_ADP_RESTRAINT] 23 | [--adp_restraint_power ADP_RESTRAINT_POWER] 24 | [--adp_restraint_exp_fac ADP_RESTRAINT_EXP_FAC] 25 | [--adp_restraint_no_long_range] 26 | [--adp_restraint_mode {diff,kldiv}] [--refine_h] 27 | [--source {electron,xray,neutron}] [-o OUTPUT_PREFIX] 28 | [--cross_validation] 29 | [--mask_for_fofc MASK_FOR_FOFC | --mask_radius_for_fofc MASK_RADIUS_FOR_FOFC] 30 | [--fsc_resolution FSC_RESOLUTION] [--keep_charges] 31 | [--keep_entities] [--write_trajectory] 32 | 33 | EXPERIMENTAL program to refine cryo-EM SPA structures 34 | 35 | options: 36 | -h, --help show this help message and exit 37 | --halfmaps HALFMAPS HALFMAPS 38 | Input half map files 39 | --map MAP Use this only if you really do not have half maps. 40 | --hklin HKLIN Use mtz file. With limited functionality. 41 | --pixel_size PIXEL_SIZE 42 | Override pixel size (A) 43 | --labin LABIN F,PHI for hklin 44 | --model MODEL Input atomic model file 45 | -d RESOLUTION, --resolution RESOLUTION 46 | -r MASK_RADIUS, --mask_radius MASK_RADIUS 47 | mask radius 48 | --padding PADDING Default: 2*mask_radius 49 | --no_mask 50 | --no_trim Keep original box (not recommended) 51 | --mask_soft_edge MASK_SOFT_EDGE 52 | Add soft edge to model mask. Should use with --no_sharpen_before_mask? 53 | --no_sharpen_before_mask 54 | By default half maps are sharpened before masking by std of signal and 55 | unsharpened after masking. This option disables it. 56 | --b_before_mask B_BEFORE_MASK 57 | sharpening B value for sharpen-mask-unsharpen procedure. By default it is 58 | determined automatically. 59 | --blur BLUR Sharpening or blurring B 60 | --monlib MONLIB Monomer library path. Default: $CLIBD_MON 61 | --ligand [LIGAND ...] 62 | restraint dictionary cif file(s) 63 | --newligand_continue Make ad-hoc restraints for unknown ligands (not recommended) 64 | --hydrogen {all,yes,no} 65 | all: (re)generate hydrogen atoms, yes: use hydrogen atoms if present, no: 66 | remove hydrogen atoms in input. Default: all 67 | --hout write hydrogen atoms in the output model 68 | --jellybody Use jelly body restraints 69 | --jellybody_params sigma dmax 70 | Jelly body sigma and dmax (default: [0.01, 4.2]) 71 | --jellyonly Jelly body only (experimental, may not be useful) 72 | --pg PG Point group symbol 73 | --twist TWIST Helical twist (degree) 74 | --rise RISE Helical rise (Angstrom) 75 | --center CENTER CENTER CENTER 76 | Origin of symmetry. Default: center of the box 77 | --axis1 AXIS1 AXIS1 AXIS1 78 | Axis1 (if I: 5-fold, O: 4-fold, T: 3-fold) 79 | --axis2 AXIS2 AXIS2 AXIS2 80 | Axis2 (if I: 5-fold, O: 4-fold, T: 3-fold, Dn: 2-fold) 81 | --contacting_only Filter out non-contacting strict NCS copies 82 | --ignore_symmetry IGNORE_SYMMETRY 83 | Ignore symmetry information (MTRIX/_struct_ncs_oper) in the model file 84 | --find_links Automatically add links 85 | --no_check_ncs_overlaps 86 | Disable model overlap test due to strict NCS 87 | --no_check_ncs_map Disable map symmetry test due to strict NCS 88 | --no_check_mask_with_model 89 | Disable mask test using model 90 | --keywords KEYWORDS [KEYWORDS ...] 91 | refmac keyword(s) 92 | --keyword_file KEYWORD_FILE [KEYWORD_FILE ...] 93 | refmac keyword file(s) 94 | --randomize RANDOMIZE 95 | Shake coordinates with the specified rmsd value 96 | --ncycle NCYCLE number of CG cycles (default: 10) 97 | --weight WEIGHT refinement weight. default: automatic 98 | --no_weight_adjust Do not adjust weight during refinement 99 | --target_bond_rmsz_range TARGET_BOND_RMSZ_RANGE TARGET_BOND_RMSZ_RANGE 100 | Bond rmsz range for weight adjustment (default: [0.5, 1.0]) 101 | --adpr_weight ADPR_WEIGHT 102 | ADP restraint weight (default: 1.000000) 103 | --ncsr Use local NCS restraints 104 | --bfactor BFACTOR reset all atomic B values to the specified value 105 | --fix_xyz Fix atomic coordinates 106 | --adp {fix,iso,aniso} 107 | ADP parameterization 108 | --refine_all_occ 109 | --max_dist_for_adp_restraint MAX_DIST_FOR_ADP_RESTRAINT 110 | --adp_restraint_power ADP_RESTRAINT_POWER 111 | --adp_restraint_exp_fac ADP_RESTRAINT_EXP_FAC 112 | --adp_restraint_no_long_range 113 | --adp_restraint_mode {diff,kldiv} 114 | --refine_h Refine hydrogen against data (default: only restraints apply) 115 | --source {electron,xray,neutron} 116 | -o OUTPUT_PREFIX, --output_prefix OUTPUT_PREFIX 117 | --cross_validation Run cross validation. Only "throughout" mode is available (no "shake" mode) 118 | --mask_for_fofc MASK_FOR_FOFC 119 | Mask file for Fo-Fc map calculation 120 | --mask_radius_for_fofc MASK_RADIUS_FOR_FOFC 121 | Mask radius for Fo-Fc map calculation 122 | --fsc_resolution FSC_RESOLUTION 123 | High resolution limit for FSC calculation. Default: Nyquist 124 | --keep_charges Use scattering factor for charged atoms. Use it with care. 125 | --keep_entities Do not override entities 126 | --write_trajectory Write all output from cycles 127 | -------------------------------------------------------------------------------- /tests/test_refine.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import unittest 10 | import json 11 | import os 12 | import shutil 13 | import tempfile 14 | import sys 15 | import numpy 16 | import test_spa 17 | from servalcat import utils 18 | from servalcat.__main__ import main 19 | 20 | root = os.path.abspath(os.path.dirname(__file__)) 21 | 22 | class TestRefine(unittest.TestCase): 23 | def setUp(self): 24 | self.wd = tempfile.mkdtemp(prefix="servaltest_") 25 | os.chdir(self.wd) 26 | print("In", self.wd) 27 | # setUp() 28 | 29 | def tearDown(self): 30 | os.chdir(root) 31 | shutil.rmtree(self.wd) 32 | # tearDown() 33 | 34 | def test_refine_geom(self): 35 | pdbin = os.path.join(root, "5e5z", "5e5z.pdb.gz") 36 | sys.argv = ["", "refine_geom", "--model", pdbin, "--rand", "0.5"] 37 | main() 38 | with open("5e5z_refined_stats.json") as f: 39 | stats = json.load(f) 40 | self.assertLess(stats[-1]["geom"]["summary"]["r.m.s.d."]["Bond distances, non H"], 0.01) 41 | 42 | def test_refine_xtal_int(self): 43 | mtzin = os.path.join(root, "5e5z", "5e5z.mtz.gz") 44 | pdbin = os.path.join(root, "5e5z", "5e5z.pdb.gz") 45 | sys.argv = ["", "refine_xtal_norefmac", "--model", pdbin, "--rand", "0.5", 46 | "--hklin", mtzin, "-s", "xray", "--labin", "I,SIGI,FREE", "--nbins", "5"] 47 | main() 48 | with open("5e5z_refined_stats.json") as f: 49 | stats = json.load(f) 50 | self.assertGreater(stats[-1]["data"]["summary"]["CCIfreeavg"], 0.70) 51 | self.assertGreater(stats[-1]["data"]["summary"]["CCIworkavg"], 0.91) 52 | 53 | def test_refine_xtal(self): 54 | mtzin = os.path.join(root, "5e5z", "5e5z.mtz.gz") 55 | pdbin = os.path.join(root, "5e5z", "5e5z.pdb.gz") 56 | sys.argv = ["", "refine_xtal_norefmac", "--model", pdbin, "--rand", "0.5", 57 | "--hklin", mtzin, "-s", "xray", "--labin", "FP,SIGFP,FREE"] 58 | main() 59 | with open("5e5z_refined_stats.json") as f: 60 | stats = json.load(f) 61 | self.assertLess(stats[-1]["data"]["summary"]["Rfree"], 0.22) 62 | self.assertLess(stats[-1]["data"]["summary"]["Rwork"], 0.20) 63 | 64 | def test_refine_small_hkl(self): 65 | hklin = os.path.join(root, "biotin", "biotin_talos.hkl") 66 | xyzin = os.path.join(root, "biotin", "biotin_talos.ins") 67 | sys.argv = ["", "refine_xtal_norefmac", "--model", xyzin, 68 | "--hklin", hklin, "-s", "electron", "--unrestrained"] 69 | main() 70 | with open("biotin_talos_refined_stats.json") as f: 71 | stats = json.load(f) 72 | self.assertGreater(stats[-1]["data"]["summary"]["CCIavg"], 0.64) 73 | 74 | def test_refine_small_cif(self): 75 | cifin = os.path.join(root, "biotin", "biotin_talos.cif") 76 | sys.argv = ["", "refine_xtal_norefmac", "--model", cifin, 77 | "--hklin", cifin, "-s", "electron", "--unrestrained"] 78 | main() 79 | with open("biotin_talos_refined_stats.json") as f: 80 | stats = json.load(f) 81 | self.assertGreater(stats[-1]["data"]["summary"]["CCIavg"], 0.64) 82 | 83 | def test_refine_aniso(self): 84 | hklin = os.path.join(root, "biotin", "biotin_talos.hkl") 85 | xyzin = os.path.join(root, "biotin", "biotin_talos.ins") 86 | sys.argv = ["", "refine_xtal_norefmac", "--model", xyzin, 87 | "--hklin", hklin, "-s", "electron", "--unrestrained", 88 | "--adp", "aniso"] 89 | main() 90 | with open("biotin_talos_refined_stats.json") as f: 91 | stats = json.load(f) 92 | self.assertGreater(stats[-1]["data"]["summary"]["CCIavg"], 0.64) 93 | st = utils.fileio.read_structure("biotin_talos_refined.mmcif") 94 | self.assertTrue(all(x.atom.aniso.nonzero() for x in st[0].all())) 95 | 96 | def test_refine_aniso_occ(self): 97 | hklin = os.path.join(root, "biotin", "biotin_talos.hkl") 98 | xyzin = os.path.join(root, "biotin", "biotin_talos.ins") 99 | sys.argv = ["", "refine_xtal_norefmac", "--model", xyzin, 100 | "--hklin", hklin, "-s", "electron", "--unrestrained", 101 | "--adp", "aniso", "--refine_all_occ"] 102 | main() 103 | with open("biotin_talos_refined_stats.json") as f: 104 | stats = json.load(f) 105 | self.assertGreater(stats[-1]["data"]["summary"]["CCIavg"], 0.64) 106 | st = utils.fileio.read_structure("biotin_talos_refined.mmcif") 107 | self.assertTrue(all(x.atom.aniso.nonzero() for x in st[0].all())) 108 | self.assertTrue(sum(x.atom.occ < 1 for x in st[0].all()) > 0.5 * st[0].count_atom_sites()) 109 | 110 | def test_refine_spa(self): 111 | data = test_spa.data 112 | sys.argv = ["", "refine_spa_norefmac", "--halfmaps", data["half1"], data["half2"], 113 | "--model", data["pdb"], 114 | "--resolution", "1.9", "--ncycle", "2", "--write_trajectory"] 115 | main() 116 | self.assertTrue(os.path.isfile("refined_fsc.json")) 117 | self.assertTrue(os.path.isfile("refined.mmcif")) 118 | self.assertTrue(os.path.isfile("refined_maps.mtz")) 119 | self.assertTrue(os.path.isfile("refined_expanded.pdb")) 120 | with open("refined_stats.json") as f: 121 | stats = json.load(f) 122 | self.assertGreater(stats[-1]["data"]["summary"]["FSCaverage"], 0.66) 123 | 124 | def test_refine_group_occ(self): 125 | mtzin = os.path.join(root, "6mw0", "6mw0-sf.cif.gz") 126 | xyzin = os.path.join(root, "6mw0", "6mw0.cif") 127 | sys.argv = ["", "refine_xtal_norefmac", "--model", xyzin, 128 | "--hklin", mtzin, "-s", "xray", "--labin", "IMEAN,SIGIMEAN", 129 | "--bfactor", "5", "--keywords", 130 | "occupancy group id 1 chain A alt A", 131 | "occupancy group id 2 chain A alt B", 132 | "occupancy group alts complete 1 2", 133 | "occupancy refine ncycle 5"] 134 | main() 135 | with open("6mw0_refined_stats.json") as f: 136 | stats = json.load(f) 137 | self.assertLess(stats[-1]["data"]["summary"]["R1"], 0.26) 138 | st = utils.fileio.read_structure("6mw0_refined.pdb") 139 | occ_a = tuple({round(a.occ, 6) for r in st[0]["A"] for a in r if a.altloc == "A"}) 140 | occ_b = tuple({round(a.occ, 6) for r in st[0]["A"] for a in r if a.altloc == "B"}) 141 | self.assertEqual(len(occ_a), 1) 142 | self.assertEqual(len(occ_b), 1) 143 | self.assertGreaterEqual(min(occ_a[0], occ_b[0]), 0.) 144 | self.assertLessEqual(max(occ_a[0], occ_b[0]), 1.) 145 | self.assertAlmostEqual(occ_a[0] + occ_b[0], 1.) 146 | 147 | def test_refine_dfrac(self): 148 | hklin = os.path.join(root, "1v9g", "1v9g-sf.cif.gz") 149 | xyzin = os.path.join(root, "1v9g", "1v9g-spk.cif.gz") 150 | sys.argv = ["", "refine_xtal_norefmac", "--model", xyzin, 151 | "--hklin", hklin, "-s", "neutron", 152 | "--hydr", "yes", "--hout", "--refine_dfrac"] 153 | main() 154 | with open("1v9g-spk_refined_stats.json") as f: 155 | stats = json.load(f) 156 | self.assertGreater(stats[-1]["data"]["summary"]["CCFfreeavg"], 0.52) 157 | st = utils.fileio.read_structure("1v9g-spk_refined.mmcif") 158 | self.assertGreater(numpy.std([x.atom.fraction for x in st[0].all() if x.atom.is_hydrogen()]), 0.3) 159 | 160 | def test_refine_twin(self): 161 | hklin = os.path.join(root, "1l2h", "1l2h.mtz.gz") 162 | xyzin = os.path.join(root, "1l2h", "1l2h.cif.gz") 163 | sys.argv = ["", "refine_xtal_norefmac", "--model", xyzin, 164 | "--hklin", hklin, "-s", "xray", "--twin", 165 | "--ncycle", "5"] 166 | main() 167 | with open("1l2h_refined_stats.json") as f: 168 | stats = json.load(f) 169 | self.assertEqual(list(stats[-1]["twin_alpha"]), ['h,k,l', '-h,k,-l']) 170 | self.assertAlmostEqual(stats[-1]["twin_alpha"]["h,k,l"], 0.66, delta=0.02) 171 | self.assertGreater(stats[-1]["data"]["summary"]["CCIfreeavg"], 0.81) 172 | 173 | if __name__ == '__main__': 174 | unittest.main() 175 | 176 | -------------------------------------------------------------------------------- /docs/spa_examples/chrmine.rst: -------------------------------------------------------------------------------- 1 | Refinement of ChRmine structure 2 | =============================== 3 | 4 | Here we demonstrate the atomic model refinement with *C*\ 3 symmetry using ChRmine (`Kishi et al. 2020 `_, `PDB 7w9w `_, `EMD-32377 `_). 5 | We need the pdb (or mmcif) file, half maps and mask: 6 | :: 7 | 8 | wget https://files.rcsb.org/download/7w9w.pdb 9 | wget https://files.wwpdb.org/pub/emdb/structures/EMD-32377/other/emd_32377_half_map_1.map.gz 10 | wget https://files.wwpdb.org/pub/emdb/structures/EMD-32377/other/emd_32377_half_map_2.map.gz 11 | wget https://files.wwpdb.org/pub/emdb/structures/EMD-32377/masks/emd_32377_msk_1.map 12 | 13 | .. note:: 14 | Half maps should be unsharpened and unweighted. In this example, the half maps are from a RELION Refine3D job. The mask file is only used for map calculation after the refinement and does not affect the refinement itself. 15 | 16 | 17 | **In this example please use at least CCP4 9.0 and Servalcat from it (Servalcat 0.4.72 or newer).** 18 | 19 | Run refinement from command-line 20 | -------------------------------- 21 | Servalcat's refinement pipeline is available through the refine_spa_norefmac subcommand. It is important to run this command in a new directory, as it creates a lot of files with fixed names by default. 22 | 23 | .. code-block:: console 24 | 25 | $ servalcat refine_spa_norefmac \ 26 | --model ../7w9w.pdb \ 27 | --halfmaps ../emd_32377_half_map_1.map.gz ../emd_32377_half_map_2.map.gz \ 28 | --mask_for_fofc ../emd_32377_msk_1.map \ 29 | --pg C3 --resolution 1.95 [--cross_validation] 30 | 31 | * ``--halfmaps`` Provide unsharpened and unweighted half maps. 32 | * ``--mask_for_fofc`` This mask is only used for map calculations after refinement and does not affect the refinement itself. 33 | * ``--cross_validation`` Use this option to run cross-validation with half maps. Half map 1 will be used for refinement, and map 2 for validation. Combine this option with ``--randomize 0.3`` to follow the method described in `Brown et al. 2015 `_. 34 | * ``--pg C3`` This option specifies *C*\ 3 symmetry for the map. When using this option, the ``--model`` argument must be an asymmetric unit. The axis orientation and origin follow `RELION's convention `_: 3-fold is along z-axis through the centre of the box. 35 | * ``--resolution`` is the resolution used in refinement and map calculation. It is always good to specify a bit higher value than the global one (as determined by the FSC=0.143 criterion), because local resolution can be higher. Here the global resolution was 2.02 Å so I put 1.95 Å. 36 | 37 | .. note:: 38 | If the pixel size in map file header is wrong, you can specify the correct pixel size using ``--pixel_size`` option. Note that this affects all input map and mask files, but not for input model. Model should overlap with map with correct the pixel size, and needs to be fixed before refinement if the model is fitted to a map with wrong pixel size. 39 | 40 | In case you want to know what Servalcat does in the pipeline: 41 | 42 | #. Expand model with *C*\ 3 symmetry (written as input_model_expanded.pdb). 43 | #. Create a mask (mask_from_model.ccp4) around the model with 3 Å radius. A radius can be changed using ``--mask_radius``. 44 | #. Trim half maps using the mask and do Fourier transform of the maps after sharpen-mask-unsharpen procedure 45 | #. Perform refinement of coordinates and ADPs in the reciprocal space, with the following features: 46 | * hydrogen atoms are internally generated at riding positions; can be changed using ``--hydrogen`` 47 | * hydrogen atoms are not written to the output model; specify ``--hout`` if you want them 48 | * atomic scattering factors are calculated using Mott-Bethe formula from those for X-ray 49 | * The number of cycles is 10 by default; can be changed using ``--ncycle`` 50 | * The weight is automatically determined by Servalcat using mask volume ratio and effective resolution; use ``--weight`` to change 51 | * Output prefix can be changed by ``-o`` (default: refined) 52 | 53 | #. Expand the final model with symmetry (refined_expanded.pdb) 54 | #. Calculate map-model FSC (half map 1/2 separately, when ``--cross_validation`` is requested) 55 | #. Calculate sharpened and weighted Fo and Fo-Fc maps (refined_diffmap.mtz, refined_diffmap_normalized_fo.mrc, refined_diffmap_normalized_fofc.mrc) 56 | * With ``--cross_validation``, only half map 1 is used for these maps. Half map 2 is only used to derive the weights. 57 | 58 | #. Show final summary 59 | 60 | Final summary is like this: 61 | 62 | .. code-block:: none 63 | 64 | ============================================================================= 65 | * Final Summary * 66 | 67 | Rmsd from ideal 68 | bond lengths: 0.0101 A 69 | bond angles: 1.871 deg 70 | 71 | Map-model FSCaverages (at 1.95 A): 72 | FSCaverage(full) = 0.8522 73 | Run loggraph refined_fsc.log to see plots 74 | 75 | ADP statistics 76 | Chain A (4708 atoms) min= 23.5 median= 57.8 max=146.9 A^2 77 | 78 | Weight used: 7.430e-01 79 | If you want to change the weight, give larger (looser restraints) 80 | or smaller (tighter) value to --weight=. 81 | 82 | Open refined model and refined_diffmap.mtz with COOT: 83 | coot --script refined_coot.py 84 | 85 | List Fo-Fc map peaks in the ASU: 86 | servalcat util map_peaks --map refined_diffmap_normalized_fofc.mrc --model refined.pdb --abs_level 4.0 87 | ============================================================================= 88 | 89 | .. _chrmine-check-fsc: 90 | 91 | Check FSC 92 | ~~~~~~~~~ 93 | Use the loggraph command from CCP4 to see the map-model FSC vs resolution curve. 94 | 95 | .. code-block:: console 96 | 97 | $ loggraph refined_fsc.log 98 | 99 | .. image:: chrmine_figs/refined_fsc_1.png 100 | :align: center 101 | :scale: 40% 102 | 103 | Note 104 | 105 | * Loggraph uses a 1/d^2 scale on the x-axis, while SPA typically uses 1/d. 106 | * Half map FSC (FSC_half) calculations employ sharpened-masked-unsharpened half maps and the mask used during refinement. Phase randomization is currently not performed. 107 | * FSC_full_sqrt estimates correlation between full map and true map: :math:`\sqrt{2{\rm FSC_{half}}/(1+{\rm FSC_{half}})}`. A map-model FSC exceeding this value might indicate overfitting (see `Nicholls et al. 2018 `_). 108 | * FSC curves are calculated up to the Nyquist resolution 109 | 110 | The refined_fsc.json file contains the same data as the plot. To use external programs like R or MS Excel for plotting, you can convert it to a CSV file: 111 | 112 | .. code-block:: console 113 | 114 | $ servalcat util json2csv refined_fsc.json 115 | 116 | Check maps and model 117 | ~~~~~~~~~~~~~~~~~~~~ 118 | Use the following command to open the refined model and maps in COOT with a script: 119 | 120 | .. code-block:: console 121 | 122 | $ coot --script refined_coot.py 123 | 124 | Ignore "rmsd" (sigma) contour levels. In SPA, the sigma level displayed as "rmsd" is not meaningful. The arbitrary box size and zero volumes outside the mask lead to underestimation of sigma. 125 | Since a mask file was provided (``--mask_for_fofc``), these maps are normalised within the mask. Therefore, raw map values can be considered as "sigma levels" in the usual crystallographic sense. However, COOT displays these values with incorrect units (e/A^3 or V). Avoid interpreting sigma based on the "rmsd" unit in SPA. 126 | 127 | The Fo-Fc map might reveal interesting features. The image below shows putative hydrogen densities displayed at a 3 sigma level. Note that the map includes hydrogen contributions by default. Use ``--hydrogen no`` to generate a hydrogen-omit Fo-Fc map, or run the fofc command after refinement. 128 | 129 | .. image:: chrmine_figs/coot_113-fs8.png 130 | :align: center 131 | :scale: 40% 132 | 133 | For other graphics programs like Chimera or PyMOL, open refined_diffmap_normalized_fo.mrc and refined_diffmap_normalized_fofc.mrc for Fo and Fo-Fc maps, respectively. PyMOL by default scales maps by their "sigma”. Before opening MRC files, run ``set normalize_ccp4_maps, off`` to disable this behaviour. 134 | 135 | Run Molprobity 136 | ~~~~~~~~~~~~~~ 137 | To generate a report for your paper with Ramachandran plots, rotamer outliers, and clash scores, run the following command: 138 | 139 | .. code-block:: console 140 | 141 | $ molprobity.molprobity refined_expanded.pdb nqh=false 142 | 143 | This will create molprobity_coot.py. Open it with COOT (from Calculate -> Run Script...) to view a "ToDo list". Remember, outliers may not always be errors; verify them with the density map. 144 | 145 | .. code-block:: console 146 | 147 | $ coot --script refined_coot.py --script molprobity_coot.py 148 | -------------------------------------------------------------------------------- /scripts/refine_xtal_true_phase.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import gemmi 10 | import numpy 11 | import json 12 | import os 13 | import shutil 14 | import argparse 15 | import scipy.optimize 16 | from servalcat.utils import logger 17 | from servalcat import utils 18 | from servalcat.xtal.sigmaa import process_input, calculate_maps, calc_DFc, calc_abs_DFc 19 | from servalcat.refine import refine_xtal 20 | from servalcat.refine.xtal import LL_Xtal 21 | from servalcat.refine.refine import Geom, Refine 22 | b_to_u = utils.model.b_to_u 23 | logger.set_file("refine_xtal_using_true_phase.log") 24 | 25 | # refine_xtal_norefmac with estimation of D and Sigma using true phase information 26 | 27 | def update_ml_params(cls): 28 | assert len(cls.fc_labs) == 2 29 | #use = cls.use_in_est 30 | use = "all" 31 | for i_bin, _ in cls.hkldata.binned(): 32 | if use == "all": 33 | idxes = numpy.concatenate([sel[i] for sel in cls.centric_and_selections[i_bin] for i in (1,2)]) 34 | else: 35 | i = 1 if use == "work" else 2 36 | idxes = numpy.concatenate([sel[i] for sel in cls.centric_and_selections[i_bin]]) 37 | 38 | idxes = idxes[numpy.isfinite(cls.hkldata.df.FP[idxes])] 39 | FC = cls.hkldata.df.FC.to_numpy()[idxes] 40 | FC0 = cls.hkldata.df[cls.fc_labs[0]].to_numpy()[idxes] 41 | FC1 = cls.hkldata.df[cls.fc_labs[1]].to_numpy()[idxes] 42 | FP = cls.hkldata.df.FP.to_numpy()[idxes] 43 | varFP = cls.hkldata.df.SIGFP.to_numpy()[idxes]**2 44 | c = cls.hkldata.df.centric.to_numpy()[idxes] + 1 45 | eps = cls.hkldata.df.epsilon.to_numpy()[idxes] 46 | Phitrue = cls.hkldata.df.PHItrue.to_numpy()[idxes] 47 | Ftrue = FP * numpy.exp(1j * Phitrue) 48 | Phic = numpy.angle(FC) 49 | m_true = numpy.cos(Phic - Phitrue) 50 | logger.writeln("debug: bin {} = {:.4f}".format(i_bin, numpy.nanmean(m_true))) 51 | # Determine D 52 | #re_fc1_fc2 = (FC0 * FC1.conj()).real 53 | #a = numpy.array([[numpy.abs(FC0)**2, re_fc1_fc2], 54 | # [re_fc1_fc2, numpy.abs(FC1)**2]]) / c 55 | #b = m_true * (3 - c) * FP * (numpy.array([FC0, FC1]) * numpy.exp(-1j * Phic)).real / 2 56 | #Ds = numpy.linalg.solve(numpy.nansum(a, axis=-1), numpy.nansum(b, axis=-1)) 57 | def calc_a(Fk, Fj): 58 | ret = numpy.sqrt(numpy.sum(numpy.abs(Fj)**2) * numpy.sum(numpy.abs(Fk)**2)) 59 | ret /= numpy.sum(FP**2) 60 | ret *= numpy.corrcoef(Fj, Fk)[0,1].real 61 | return ret 62 | def calc_b(Fj): 63 | ret = numpy.sqrt(numpy.sum(numpy.abs(Fj)**2)/numpy.sum(FP**2)) 64 | ret *= numpy.corrcoef(Ftrue, Fj)[0,1].real 65 | return ret 66 | a = numpy.array([[calc_a(FC0,FC0), calc_a(FC0,FC1)], 67 | [calc_a(FC0,FC1), calc_a(FC1,FC1)]]) 68 | b = numpy.array([calc_b(FC0), calc_b(FC1)]) 69 | Ds = numpy.linalg.solve(a, b) 70 | 71 | # Determine S 72 | DFc = calc_abs_DFc(Ds, [FC0, FC1]) 73 | S = numpy.nansum(eps * (FP**2 + DFc**2) / c - m_true * (3-c)*eps*FP*DFc - eps * (3-c) * varFP / c) / numpy.nansum(eps**2 / c) 74 | 75 | """ 76 | # Optimize just in case 77 | x0 = [transD_inv(x) for x in Ds] + [transS_inv(S)] 78 | x0 = list(Ds) + [S] 79 | def f(x): 80 | g = numpy.zeros(3) 81 | Sigma = (3-c) * varFP + eps * x[2] 82 | DFc = calc_DFc(x[:2], [FC0, FC1]) 83 | DFc_abs = numpy.abs(DFc) 84 | tmp = (2 / c - m_true * (3 - c) * FP / DFc_abs) / Sigma 85 | g[0] = numpy.nansum(tmp * (FC0 * DFc.conj()).real) 86 | g[1] = numpy.nansum(tmp * (FC1 * DFc.conj()).real) 87 | g[2] = numpy.nansum(eps / c / Sigma - eps * (FP**2 + DFc_abs**2) / Sigma**2 + m_true * (3 - c) * eps * FP * DFc_abs / Sigma**2) 88 | return g 89 | def fprime(x): 90 | g = numpy.zeros(3) 91 | Sigma = (3-c) * varFP + eps * x[2] 92 | DFc = calc_DFc(x[:2], [FC0, FC1]) 93 | DFc_abs = numpy.abs(DFc) 94 | g[0] = numpy.nansum(numpy.abs(FC0)**2 / c / Sigma) 95 | g[1] = numpy.nansum(numpy.abs(FC1)**2 / c / Sigma) 96 | g[2] = numpy.nansum(eps**2 * (-1/c/Sigma**2 + 2*(FP**2+DFc_abs**2)/c/Sigma**3 -2*m_true*(3-c)*FP*DFc_abs/Sigma**3)) 97 | return g 98 | 99 | print(x0, f(x0)) 100 | x = scipy.optimize.newton(func=f, fprime=fprime, x0=x0, maxiter=10000) 101 | #x = scipy.optimize.newton(func=f, x0=x0, maxiter=10000) 102 | print(x, f(x)) 103 | Ds = transD(x[:2]) 104 | S = transS(x[-1]) 105 | """ 106 | for l, d in zip(cls.D_labs, Ds): 107 | cls.hkldata.binned_df.loc[i_bin, l] = d 108 | cls.hkldata.binned_df.loc[i_bin, "S"] = S 109 | 110 | logger.writeln(cls.hkldata.binned_df.to_string()) 111 | for lab in cls.D_labs + ["S"]: 112 | cls.hkldata.binned_df[lab].where(cls.hkldata.binned_df[lab] > 0, 0., inplace=True) # 0 would be ok? 113 | cls.hkldata.binned_df[lab].where(cls.hkldata.binned_df[lab] < numpy.inf, 1, inplace=True) 114 | 115 | 116 | def main(args): 117 | if not args.output_prefix: 118 | args.output_prefix = utils.fileio.splitext(os.path.basename(args.model))[0] + "_refined" 119 | 120 | keywords = [] 121 | if args.keywords or args.keyword_file: 122 | if args.keywords: keywords = sum(args.keywords, []) 123 | if args.keyword_file: keywords.extend(l for f in sum(args.keyword_file, []) for l in open(f)) 124 | 125 | hkldata, sts, fc_labs, centric_and_selections = process_input(hklin=args.hklin, 126 | labin=args.labin, 127 | n_bins=args.nbins, 128 | free=args.free, 129 | xyzins=[args.model], 130 | source=args.source, 131 | d_min=args.d_min) 132 | mtz = gemmi.read_mtz_file(args.hklin_true) 133 | df = utils.hkl.hkldata_from_mtz(mtz, [args.labin_true], newlabels=["PHItrue"], require_types=["P"]).df 134 | hkldata.df = hkldata.df.merge(df) 135 | hkldata.df["PHItrue"] = numpy.deg2rad(hkldata.df.PHItrue) 136 | 137 | st = sts[0] 138 | monlib = utils.restraints.load_monomer_library(st, monomer_dir=args.monlib, cif_files=args.ligand, 139 | stop_for_unknowns=False) 140 | h_change = {"all":gemmi.HydrogenChange.ReAddButWater, 141 | "yes":gemmi.HydrogenChange.NoChange, 142 | "no":gemmi.HydrogenChange.Remove}[args.hydrogen] 143 | try: 144 | topo = utils.restraints.prepare_topology(st, monlib, h_change=h_change, 145 | check_hydrogen=(args.hydrogen=="yes")) 146 | except RuntimeError as e: 147 | raise SystemExit("Error: {}".format(e)) 148 | 149 | # initialize ADP 150 | if args.adp != "fix": 151 | utils.model.reset_adp(st[0], args.bfactor, args.adp == "aniso") 152 | 153 | geom = Geom(st, topo, monlib, shake_rms=args.randomize, sigma_b=args.sigma_b, refmac_keywords=keywords, 154 | jellybody_only=args.jellyonly) 155 | geom.geom.adpr_max_dist = args.max_dist_for_adp_restraint 156 | if args.jellybody or args.jellyonly: 157 | geom.geom.ridge_sigma, geom.geom.ridge_dmax = args.jellybody_params 158 | 159 | ll = LL_Xtal(hkldata, centric_and_selections, args.free, st, monlib, source=args.source, use_solvent=not args.no_solvent) 160 | ll.update_ml_params = lambda : update_ml_params(ll) 161 | 162 | refiner = Refine(st, geom, ll=ll, 163 | refine_xyz=not args.fix_xyz, 164 | adp_mode=dict(fix=0, iso=1, aniso=2)[args.adp], 165 | refine_h=args.refine_h, 166 | unrestrained=args.unrestrained) 167 | 168 | refiner.run_cycles(args.ncycle, weight=args.weight) 169 | utils.fileio.write_model(refiner.st, args.output_prefix, pdb=True, cif=True) 170 | 171 | calculate_maps(ll.hkldata, centric_and_selections, ll.fc_labs, ll.D_labs, args.output_prefix + "_stats.log") 172 | 173 | # Write mtz file 174 | labs = ["FP", "SIGFP", "FOM", "FWT", "DELFWT", "FC"] 175 | if not args.no_solvent: 176 | labs.append("FCbulk") 177 | mtz_out = args.output_prefix+".mtz" 178 | hkldata.write_mtz(mtz_out, labs=labs, types={"FOM": "W", "FP":"F", "SIGFP":"Q"}) 179 | logger.writeln("output mtz: {}".format(mtz_out)) 180 | 181 | if __name__ == "__main__": 182 | parser = argparse.ArgumentParser() 183 | parser.add_argument("--hklin_true", required=True) 184 | parser.add_argument("--labin_true", required=True) 185 | refine_xtal.add_arguments(parser) 186 | args = parser.parse_args() 187 | main(args) 188 | -------------------------------------------------------------------------------- /servalcat/spa/localcc.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import gemmi 10 | import numpy 11 | import pandas 12 | import os 13 | import json 14 | import argparse 15 | from servalcat.utils import logger 16 | from servalcat import utils 17 | 18 | def add_arguments(parser): 19 | parser.description = 'Calculate real space local correlation map from half maps and model' 20 | parser.add_argument("--halfmaps", required=True, nargs=2, 21 | help="Input half map files") 22 | parser.add_argument('--pixel_size', type=float, 23 | help='Override pixel size (A)') 24 | group = parser.add_mutually_exclusive_group(required=True) 25 | group.add_argument("--kernel", type=int, 26 | help="Kernel radius in pixel") 27 | group.add_argument("--kernel_ang", type=float, 28 | help="Kernel radius in Angstrom (hard sphere)") 29 | parser.add_argument('--mask', 30 | help="mask file") 31 | parser.add_argument('--model', 32 | help='Input atomic model file') 33 | parser.add_argument('--resolution', type=float, 34 | help='default: nyquist resolution') 35 | parser.add_argument("-s", "--source", choices=["electron", "xray", "neutron", "custom"], default="electron") 36 | parser.add_argument("--trim", action='store_true', help="Write trimmed map") 37 | parser.add_argument('-o', '--output_prefix', default="ccmap", 38 | help="default: %(default)s") 39 | # add_arguments() 40 | 41 | def parse_args(arg_list): 42 | parser = argparse.ArgumentParser() 43 | add_arguments(parser) 44 | return parser.parse_args(arg_list) 45 | # parse_args() 46 | 47 | def setup_coeffs_for_halfmap_cc(maps, d_min, mask=None, st=None): 48 | hkldata = utils.maps.mask_and_fft_maps(maps, d_min, mask) 49 | hkldata.setup_relion_binning("ml") 50 | utils.maps.calc_noise_var_from_halfmaps(hkldata) 51 | 52 | nref = len(hkldata.df.index) 53 | F1w = numpy.zeros(nref, dtype=complex) 54 | F2w = numpy.zeros(nref, dtype=complex) 55 | F1 = hkldata.df.F_map1.to_numpy() 56 | F2 = hkldata.df.F_map2.to_numpy() 57 | 58 | logger.writeln("Calculating weights for half map correlation.") 59 | logger.writeln(" weight = sqrt(FSChalf / (2*var_noise + var_signal))") 60 | hkldata.binned_df["ml"]["w2_half_varsignal"] = 0. 61 | for i_bin, idxes in hkldata.binned("ml"): 62 | fscfull = hkldata.binned_df["ml"].FSCfull[i_bin] 63 | if fscfull < 0: 64 | break # stop here so that higher resolution are all zero 65 | fsc = fscfull / (2 - fscfull) 66 | var_fo = 2 * hkldata.binned_df["ml"].var_noise[i_bin] + hkldata.binned_df["ml"].var_signal[i_bin] 67 | w = numpy.sqrt(fsc / var_fo) 68 | hkldata.binned_df["ml"].loc[i_bin, "w2_half_varsignal"] = fsc / var_fo * hkldata.binned_df["ml"].var_signal[i_bin] 69 | F1w[idxes] = F1[idxes] * w 70 | F2w[idxes] = F2[idxes] * w 71 | 72 | hkldata.df["F_map1w"] = F1w 73 | hkldata.df["F_map2w"] = F2w 74 | 75 | return hkldata 76 | # setup_coeffs_for_halfmap_cc() 77 | 78 | def add_coeffs_for_model_cc(hkldata, st, source="electron"): 79 | hkldata.df["FC"] = utils.model.calc_fc_fft(st, d_min=hkldata.d_min_max()[0]-1e-6, 80 | source=source, miller_array=hkldata.miller_array()) 81 | nref = len(hkldata.df.index) 82 | FCw = numpy.zeros(nref, dtype=complex) 83 | FPw = numpy.zeros(nref, dtype=complex) 84 | FP = hkldata.df.FP.to_numpy() 85 | FC = hkldata.df.FC.to_numpy() 86 | 87 | logger.writeln("Calculating weights for map-model correlation.") 88 | logger.writeln(" weight for Fo = sqrt(FSCfull / var(Fo))") 89 | logger.writeln(" weight for Fc = sqrt(FSCfull / var(Fc))") 90 | hkldata.binned_df["ml"]["w_mapmodel_c"] = 0. 91 | hkldata.binned_df["ml"]["w_mapmodel_o"] = 0. 92 | hkldata.binned_df["ml"]["var_fc"] = 0. 93 | for i_bin, idxes in hkldata.binned("ml"): 94 | fscfull = hkldata.binned_df["ml"].FSCfull[i_bin] 95 | if fscfull < 0: break 96 | var_fc = numpy.var(FC[idxes]) 97 | wc = numpy.sqrt(fscfull / var_fc) 98 | wo = numpy.sqrt(fscfull / numpy.var(FP[idxes])) 99 | FCw[idxes] = FC[idxes] * wc 100 | FPw[idxes] = FP[idxes] * wo 101 | hkldata.binned_df["ml"].loc[i_bin, "w_mapmodel_c"] = wc 102 | hkldata.binned_df["ml"].loc[i_bin, "w_mapmodel_o"] = wo 103 | hkldata.binned_df["ml"].loc[i_bin, "var_fc"] = var_fc 104 | 105 | hkldata.df["FPw"] = FPw 106 | hkldata.df["FCw"] = FCw 107 | # add_coeffs_for_model_cc() 108 | 109 | def model_stats(st, modelcc_map, halfcc_map, loggraph_out=None, json_out=None): 110 | tmp = dict(chain=[], seqid=[], resn=[], CC_mapmodel=[], CC_halfmap=[]) 111 | for chain in st[0]: 112 | for res in chain: 113 | mm = numpy.mean([modelcc_map.interpolate_value(atom.pos) for atom in res]) 114 | hc = numpy.mean([halfcc_map.interpolate_value(atom.pos) for atom in res]) 115 | tmp["chain"].append(chain.name) 116 | tmp["seqid"].append(str(res.seqid)) 117 | tmp["resn"].append(res.name) 118 | tmp["CC_mapmodel"].append(mm) 119 | tmp["CC_halfmap"].append(hc) 120 | 121 | df = pandas.DataFrame(tmp) 122 | df["sqrt_CC_full"] = numpy.sqrt(2 * df.CC_halfmap / (1 + df.CC_halfmap)) 123 | if loggraph_out is not None: 124 | with open(loggraph_out, "w") as ofs: 125 | for c, g in df.groupby("chain", sort=False): 126 | ofs.write("$TABLE: Chain {} :".format(c)) 127 | ofs.write(""" 128 | $GRAPHS 129 | : average correlations :A:2,4,5,6: 130 | $$ 131 | chain seqid resn CC(map,model) CC_half sqrt(CC_full) 132 | $$ 133 | $$ 134 | """) 135 | ofs.write(g.to_string(header=False, index=False)) 136 | ofs.write("\n\n") 137 | if json_out is not None: 138 | df.to_json(json_out, orient="records", indent=2) 139 | return df 140 | # model_stats() 141 | 142 | def main(args): 143 | maps = utils.fileio.read_halfmaps(args.halfmaps, pixel_size=args.pixel_size) 144 | grid_shape = maps[0][0].shape 145 | if args.mask: 146 | mask = utils.fileio.read_ccp4_map(args.mask)[0] 147 | else: 148 | mask = None 149 | 150 | if args.resolution is None: 151 | d_min = utils.maps.nyquist_resolution(maps[0][0]) 152 | else: 153 | d_min = args.resolution 154 | 155 | hkldata = setup_coeffs_for_halfmap_cc(maps, d_min, mask) 156 | if args.kernel is None: 157 | prefix = "{}_r{}A".format(args.output_prefix, args.kernel_ang) 158 | knl = hkldata.hard_sphere_kernel(r_ang=args.kernel_ang, grid_size=grid_shape) 159 | else: 160 | prefix = "{}_r{}px".format(args.output_prefix, args.kernel) 161 | knl = utils.maps.raised_cosine_kernel(args.kernel) 162 | 163 | halfcc_map = utils.maps.local_cc(hkldata.fft_map("F_map1w", grid_size=grid_shape), 164 | hkldata.fft_map("F_map2w", grid_size=grid_shape), 165 | knl, method="simple" if args.kernel is None else "scipy") 166 | 167 | halfcc_map_in_mask = halfcc_map.array[mask.array>0.5] if mask is not None else halfcc_map 168 | logger.writeln("Half map CC: min/max= {:.4f} {:.4f}".format(numpy.min(halfcc_map_in_mask), numpy.max(halfcc_map_in_mask))) 169 | utils.maps.write_ccp4_map(prefix+"_half.mrc", halfcc_map, hkldata.cell, hkldata.sg, 170 | mask_for_extent=mask if args.trim else None) 171 | 172 | if args.model: 173 | st = utils.fileio.read_structure(args.model) 174 | utils.model.remove_charge([st]) 175 | ccu = utils.model.CustomCoefUtil() 176 | if args.source == "custom": 177 | ccu.read_from_cif(st, args.model) 178 | ccu.show_info() 179 | ccu.set_coeffs(st) 180 | utils.model.expand_ncs(st) 181 | st.cell = hkldata.cell 182 | st.spacegroup_hm = hkldata.sg.xhm() 183 | add_coeffs_for_model_cc(hkldata, st, args.source) 184 | modelcc_map = utils.maps.local_cc(hkldata.fft_map("FPw", grid_size=grid_shape), 185 | hkldata.fft_map("FCw", grid_size=grid_shape), 186 | knl, method="simple" if args.kernel is None else "scipy") 187 | modelcc_map_in_mask = modelcc_map.array[mask.array>0.5] if mask is not None else modelcc_map 188 | logger.writeln("Model-map CC: min/max= {:.4f} {:.4f}".format(numpy.min(modelcc_map_in_mask), numpy.max(modelcc_map_in_mask))) 189 | utils.maps.write_ccp4_map(prefix+"_model.mrc", modelcc_map, hkldata.cell, hkldata.sg, 190 | mask_for_extent=mask if args.trim else None) 191 | model_stats(st, modelcc_map, halfcc_map, loggraph_out=prefix+"_byresidue.log", json_out=prefix+"_byresidue.json") 192 | # main() 193 | 194 | if __name__ == "__main__": 195 | import sys 196 | args = parse_args(sys.argv[1:]) 197 | main(args) 198 | -------------------------------------------------------------------------------- /servalcat/refmac/exte.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import gemmi 10 | import numpy 11 | from servalcat.utils import logger 12 | from servalcat import ext 13 | 14 | """import line_profiler 15 | profile = line_profiler.LineProfiler() 16 | import atexit 17 | atexit.register(profile.print_stats) 18 | @profile""" 19 | def read_external_restraints(params, st, geom): 20 | # default or current values 21 | defs = dict(symall_block=False, exclude_self_block=False, type_default=2, alpha_default=1., 22 | ext_verbose=False, scale_sigma_dist=1., scale_sigma_angl=1., scale_sigma_tors=1., 23 | scale_sigma_chir=1., scale_sigma_plan=1., scale_sigma_inte=1., 24 | sigma_min_loc=0., sigma_max_loc=100., ignore_undefined=False, ignore_hydrogens=True, 25 | dist_max_external=numpy.inf, dist_min_external=-numpy.inf, use_atoms="a", prefix_ch=" ") 26 | #exte = gemmi.ExternalRestraints(st) 27 | extypes = dict(dist=ext.Geometry.Bond, 28 | angl=ext.Geometry.Angle, 29 | chir=ext.Geometry.Chirality, 30 | tors=ext.Geometry.Torsion, 31 | plan=ext.Geometry.Plane, 32 | inte=ext.Geometry.Interval, 33 | harm=ext.Geometry.Harmonic, 34 | spec=ext.Geometry.Special, 35 | stac=ext.Geometry.Stacking) 36 | exlists = dict(dist=geom.bonds, angl=geom.angles, tors=geom.torsions, 37 | chir=geom.chirs, plan=geom.planes, inte=geom.intervals, 38 | stac=geom.stackings, harm=geom.harmonics, spec=geom.specials) 39 | num_org = {x: len(exlists[x]) for x in exlists} 40 | 41 | # XXX There may be duplication (same chain, resi, name, and alt) - we should give error? 42 | lookup = {(cra.chain.name, cra.residue.seqid.num, cra.residue.seqid.icode, 43 | cra.atom.name, cra.atom.altloc) : cra.atom for cra in st[0].all()} 44 | 45 | # TODO main chain / side chain filtering, hydrogen, dist_max_external/dist_min_external 46 | for r in params: 47 | if not r: continue 48 | defs.update(r["defaults"]) 49 | if "rest_type" not in r: continue 50 | if r["rest_type"] not in extypes: 51 | logger.writeln("Warning: unknown external restraint type: {}".format(r["rest_type"])) 52 | continue 53 | 54 | atoms = [] 55 | skip = False 56 | for i, spec in enumerate(r["restr"].get("specs", [])): 57 | if r["rest_type"] == "stac": 58 | atoms.append([]) 59 | if "ifirst" in spec: 60 | for chain in st[0]: 61 | if chain.name != spec["chain"]: continue 62 | for res in chain: 63 | if spec["ifirst"] is not None and res.seqid.num < spec["ifirst"]: continue 64 | if spec["ilast"] is not None and res.seqid.num > spec["ilast"]: continue 65 | atoms.extend([a for a in res if spec.get("atom", "*") == "*" or a.name == spec["atom"]]) 66 | else: 67 | for name in spec["names"]: # only same altloc allowed? 68 | key = (spec["chain"], spec["resi"], spec.get("icode", " "), 69 | name, spec.get("altloc", "\0")) 70 | atom = lookup.get(key) 71 | if atom is None: 72 | if defs["ignore_undefined"]: 73 | logger.writeln("Warning: atom not found: {}".format(key)) 74 | skip = True 75 | continue 76 | raise RuntimeError("Atom not found: {}".format(key)) 77 | if r["rest_type"] == "stac": 78 | atoms[i].append(atom) 79 | else: 80 | atoms.append(atom) 81 | if skip or not atoms: 82 | continue 83 | if r["rest_type"] in ("spec", "harm"): 84 | if r["restr"]["rectype"] == "auto": 85 | assert r["rest_type"] == "spec" 86 | atoms = [cra.atom for cra in st[0].all()] 87 | for atom in atoms: 88 | ex = extypes[r["rest_type"]](atom) 89 | if r["rest_type"] == "spec": 90 | # TODO check if it is on special position. using r["restr"]["toler"] 91 | ex.sigma_t = r["restr"]["sigma_t"] 92 | ex.sigma_u =r["restr"]["sigma_u"] 93 | ex.u_val_incl = r["restr"]["u_val_incl"] 94 | # ex.trans_t = 95 | # ex.mat_u = 96 | else: 97 | ex.sigma = r["restr"]["sigma_t"] 98 | exlists[r["rest_type"]].append(ex) 99 | continue 100 | elif r["rest_type"] == "plan": 101 | ex = extypes[r["rest_type"]](atoms) 102 | else: 103 | ex = extypes[r["rest_type"]](*atoms) 104 | if r["rest_type"] in ("dist", "angl", "chir", "tors"): 105 | value = r["restr"]["value"] 106 | sigma = r["restr"]["sigma_value"] / defs["scale_sigma_{}".format(r["rest_type"])] 107 | if r["rest_type"] == "chir": 108 | ex.value = value 109 | ex.sigma = sigma 110 | else: 111 | if r["rest_type"] == "dist": 112 | sigma = min(max(sigma, defs["sigma_min_loc"]), defs["sigma_max_loc"]) 113 | vals = (value, sigma, value, sigma) # nucleus 114 | elif r["rest_type"] == "tors": 115 | vals = (value, sigma, 1) # period. # Refmac does not seem to read it from instruction 116 | else: 117 | vals = (value, sigma) 118 | ex.values.append(extypes[r["rest_type"]].Value(*vals)) 119 | 120 | if r["rest_type"] == "dist": 121 | if not (defs["dist_min_external"] < r["restr"]["value"] < defs["dist_max_external"]): 122 | continue 123 | ex.alpha = r["restr"].get("alpha_in", defs["alpha_default"]) 124 | ex.type = r["restr"].get("itype_in", defs["type_default"]) 125 | symm1 = any([spec.get("symm") for spec in r["restr"]["specs"]]) # is it the intention? 126 | if r["restr"].get("symm_in", defs["symall_block"]) or symm1: 127 | asu = gemmi.Asu.Different if defs["exclude_self_block"] else gemmi.Asu.Any 128 | ex.set_image(st.cell, asu) 129 | #print("dist=", ex.alpha, ex.type, ex.values[-1].value, ex.values[-1].sigma, ex.sym_idx, ex.pbc_shift, ex.atoms) 130 | elif r["rest_type"] == "angl": 131 | if any(spec.get("symm") for spec in r["restr"]["specs"]): 132 | asus = [gemmi.Asu.Different if r["restr"]["specs"][i].get("symm") else gemmi.Asu.Same 133 | for i in range(3)] 134 | if atoms[0].serial > atoms[2].serial: 135 | asus = asus[::-1] 136 | ex.set_images(st.cell, asus[0], asus[2]) 137 | #print("angl=", ex.values[-1].value, ex.values[-1].sigma, ex.atoms) 138 | elif r["rest_type"] == "tors": 139 | pass 140 | #print("tors=", ex.values[-1].value, ex.values[-1].sigma, ex.atoms) 141 | elif r["rest_type"] == "chir": 142 | #print("chir=", ex.value, ex.sigma, ex.atoms) 143 | ex.sign = gemmi.ChiralityType.Positive if ex.value > 0 else gemmi.ChiralityType.Negative 144 | ex.value = abs(ex.value) 145 | elif r["rest_type"] == "plan": 146 | ex.sigma = r["restr"]["sigma_value"] / defs["scale_sigma_{}".format(r["rest_type"])] 147 | #print("plan=", ex.sigma, ex.atoms) 148 | elif r["rest_type"] == "inte": 149 | dmin, dmax = r["restr"].get("dmin"), r["restr"].get("dmax") 150 | smin, smax = r["restr"].get("smin"), r["restr"].get("smax") 151 | if (smin,smax).count(None) == 2: 152 | smin = smax = 0.05 153 | else: 154 | if smin is None: smin = smax 155 | if smax is None: smax = smin 156 | smin /= defs["scale_sigma_inte"] 157 | smax /= defs["scale_sigma_inte"] 158 | if (dmin,dmax).count(None) == 1: 159 | if dmin is None: dmin = dmax 160 | if dmax is None: dmax = dmin 161 | ex.dmin = dmin 162 | ex.dmax = dmax 163 | ex.smin = smin 164 | ex.smax = smax 165 | symm1 = any(spec.get("symm") for spec in r["restr"]["specs"]) # not tested 166 | if r["restr"].get("symm_in", defs["symall_block"]) or symm1: 167 | asu = gemmi.Asu.Different if defs["exclude_self_block"] else gemmi.Asu.Any 168 | ex.set_image(st.cell, asu) 169 | #print("inte=", ex.dmin, ex.dmax, ex.smin, ex.smax, ex.atoms) 170 | elif r["rest_type"] == "stac": 171 | ex.dist = r["restr"]["dist_id"] 172 | ex.sd_dist = r["restr"]["dist_sd"] 173 | ex.angle = r["restr"].get("angle_id", 0.) 174 | ex.sd_angle = r["restr"]["angle_sd"] 175 | #print("stac=", ex.dist, ex.sd_dist, ex.angle, ex.sd_angle, ex.planes) 176 | 177 | exlists[r["rest_type"]].append(ex) 178 | 179 | logger.writeln("External restraints from Refmac instructions") 180 | labs = dict(dist="distances", angl="angles", tors="torsions", 181 | chir="chirals", plan="planes", inte="intervals", 182 | stac="stackings", harm="harmonics", spec="special positions") 183 | for lab in labs: 184 | logger.writeln(" Number of {:18s} : {}".format(labs[lab], len(exlists[lab]) - num_org[lab])) 185 | logger.writeln("") 186 | # read_external_restraints() 187 | -------------------------------------------------------------------------------- /servalcat/xtal/run_refmac_small.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import gemmi 10 | import numpy 11 | import json 12 | import os 13 | import argparse 14 | from servalcat.utils import logger 15 | from servalcat import utils 16 | 17 | def add_arguments(parser): 18 | parser.description = 'Run REFMAC5 for small molecule crystallography' 19 | parser.add_argument('--exe', default="refmac5", help='refmac5 binary') 20 | parser.add_argument('--cif', help="cif file containing model and data") 21 | parser.add_argument('--sg', help="Space group") 22 | parser.add_argument('--model', 23 | help='Input atomic model file') 24 | parser.add_argument('--hklin', 25 | help='Input reflection file') 26 | parser.add_argument('--hklin_labs', nargs='+', 27 | help='column names to be used') 28 | parser.add_argument('--blur', type=float, 29 | help='Apply B-factor blurring to --hklin') 30 | parser.add_argument('--resolution', 31 | type=float, 32 | help='') 33 | parser.add_argument('--ligand', nargs="*", action="append", 34 | help="restraint dictionary cif file(s)") 35 | parser.add_argument('--ncycle', type=int, default=10) 36 | #parser.add_argument('--jellybody', action='store_true') 37 | #parser.add_argument('--jellybody_params', nargs=2, type=float, 38 | # metavar=("sigma", "dmax"), default=[0.01, 4.2]) 39 | parser.add_argument('--hydrogen', default="all", choices=["all", "yes", "no"], 40 | help="all: add riding hydrogen atoms, yes: use hydrogen atoms if present, no: remove hydrogen atoms in input") 41 | parser.add_argument('--no_hout', action='store_true', help="do not write hydrogen atoms in the output model") 42 | 43 | group = parser.add_mutually_exclusive_group() 44 | group.add_argument('--weight_auto_scale', type=float, 45 | help="'weight auto' scale value. automatically determined from resolution and mask/box volume ratio if unspecified") 46 | group.add_argument('--weight_matrix', type=float, 47 | help="weight matrix value") 48 | parser.add_argument('--invert', action='store_true', help="invert handednes") 49 | parser.add_argument('--bref', choices=["aniso","iso","iso_then_aniso"], default="aniso") 50 | parser.add_argument('--unrestrained', action='store_true') 51 | parser.add_argument('--bulk_solvent', action='store_true') 52 | parser.add_argument('-s', '--source', choices=["electron", "xray", "neutron"], default="electron") #FIXME 53 | parser.add_argument('--keywords', nargs='+', action="append", 54 | help="refmac keyword(s)") 55 | parser.add_argument('--keyword_file', nargs='+', action="append", 56 | help="refmac keyword file(s)") 57 | parser.add_argument('--external_restraints_json') 58 | parser.add_argument('--show_refmac_log', action='store_true') 59 | parser.add_argument('--output_prefix', default="refined", 60 | help='output file name prefix') 61 | parser.add_argument("--monlib", 62 | help="Monomer library path. Default: $CLIBD_MON") 63 | # add_arguments() 64 | 65 | def parse_args(arg_list): 66 | parser = argparse.ArgumentParser() 67 | add_arguments(parser) 68 | return parser.parse_args(arg_list) 69 | # parse_args() 70 | 71 | def make_invert_tr(sg, cell): 72 | ops = sg.operations() 73 | coh = sg.change_of_hand_op() 74 | ops.change_basis_forward(sg.change_of_hand_op()) 75 | new_sg = gemmi.find_spacegroup_by_ops(ops) 76 | tr = cell.op_as_transform(coh) 77 | return new_sg, tr 78 | # make_invert_tr() 79 | 80 | def merge_anomalous(mtz): 81 | dlabs = utils.hkl.mtz_find_data_columns(mtz) 82 | if dlabs["J"] or dlabs["F"]: 83 | return # no need to merge 84 | if not dlabs["K"] and not dlabs["G"]: 85 | return # nothing can be done 86 | data = mtz.array.copy() 87 | for typ in ("K", "G"): 88 | if dlabs[typ] and len(dlabs[typ][0]) == 4: 89 | idxes = [mtz.column_with_label(x).idx for x in dlabs[typ][0]] 90 | mean = numpy.nanmean(mtz.array[:,[idxes[0],idxes[2]]], axis=1) 91 | sig_mean = numpy.sqrt(numpy.nanmean(mtz.array[:,[idxes[1],idxes[3]]]**2, axis=1)) 92 | data = numpy.hstack([data, mean.reshape(-1,1), sig_mean.reshape(-1,1)]) 93 | if typ == "K": 94 | mtz.add_column("IMEAN", "J") 95 | mtz.add_column("SIGIMEAN", "Q") 96 | else: 97 | mtz.add_column("FP", "F") 98 | mtz.add_column("SIGFP", "Q") 99 | mtz.set_data(data) 100 | # merge_anomalous() 101 | 102 | def main(args): 103 | if not args.cif and not (args.model and args.hklin): 104 | raise SystemExit("Give [--model and --hklin] or --cif") 105 | 106 | if args.sg: 107 | try: 108 | sg_user = gemmi.SpaceGroup(args.sg) 109 | logger.writeln("User-specified space group: {}".format(sg_user.xhm())) 110 | except ValueError: 111 | raise SystemExit("Error: Unknown space group '{}'".format(args.sg)) 112 | else: 113 | sg_user = None 114 | 115 | if args.cif: 116 | mtz, ss, info = utils.fileio.read_smcif_shelx(args.cif) 117 | st = utils.model.cx_to_mx(ss) 118 | else: 119 | st = utils.fileio.read_structure(args.model) 120 | if utils.fileio.is_mmhkl_file(args.hklin): # TODO may be unmerged mtz 121 | mtz = utils.fileio.read_mmhkl(args.hklin) 122 | elif args.hklin.endswith(".hkl"): 123 | mtz = utils.fileio.read_smcif_hkl(args.hklin, st.cell, st.find_spacegroup()) 124 | else: 125 | raise SystemExit("Error: unsupported hkl file: {}".format(args.hklin)) 126 | 127 | mtz_in = "input.mtz" # always write this file as an input for Refmac 128 | sg_st = st.find_spacegroup() 129 | if not mtz.cell.approx(st.cell, 1e-3): 130 | logger.writeln(" Warning: unit cell mismatch!") 131 | if sg_user: 132 | if not mtz.cell.is_compatible_with_spacegroup(sg_user): 133 | raise SystemExit("Error: Specified space group {} is incompatible with the unit cell parameters {}".format(sg_user.xhm(), 134 | mtz.cell.parameters)) 135 | mtz.spacegroup = sg_user 136 | logger.writeln(" Writing {} as space group {}".format(mtz_in, sg_user.xhm())) 137 | elif mtz.spacegroup != sg_st: 138 | if st.cell.is_crystal() and sg_st and sg_st.laue_str() != mtz.spacegroup.laue_str(): 139 | raise RuntimeError("Crystal symmetry mismatch between model and data") 140 | logger.writeln(" Warning: space group mismatch between model and mtz") 141 | if sg_st and sg_st.laue_str() == mtz.spacegroup.laue_str(): 142 | logger.writeln(" using space group from model") 143 | mtz.spacegroup = sg_st 144 | else: 145 | logger.writeln(" using space group from mtz") 146 | 147 | if args.hklin_labs: 148 | try: mtz = utils.hkl.mtz_selected(mtz, args.hklin_labs) 149 | except RuntimeError as e: 150 | raise SystemExit("Error: {}".format(e)) 151 | if args.blur is not None: utils.hkl.blur_mtz(mtz, args.blur) 152 | merge_anomalous(mtz) 153 | mtz.write_to_file(mtz_in) 154 | st.cell = mtz.cell 155 | st.spacegroup_hm = mtz.spacegroup.xhm() 156 | 157 | if args.invert: 158 | logger.writeln("Inversion of structure is requested.") 159 | old_sg = st.find_spacegroup() 160 | new_sg, tr = make_invert_tr(old_sg, st.cell) 161 | logger.writeln(" new space group = {} (no. {})".format(new_sg.xhm(), new_sg.number)) 162 | st[0].transform_pos_and_adp(tr) 163 | if old_sg != new_sg: 164 | st.spacegroup_hm = new_sg.xhm() 165 | # overwrite mtz 166 | mtz = gemmi.read_mtz_file(mtz_in) 167 | mtz.spacegroup = new_sg 168 | mtz.write_to_file(mtz_in) 169 | 170 | if args.keyword_file: 171 | args.keyword_file = sum(args.keyword_file, []) 172 | for f in args.keyword_file: 173 | logger.writeln("Keyword file: {}".format(f)) 174 | assert os.path.exists(f) 175 | else: 176 | args.keyword_file = [] 177 | 178 | if args.keywords: 179 | args.keywords = sum(args.keywords, []) 180 | else: 181 | args.keywords = [] 182 | 183 | for m in st: 184 | for chain in m: 185 | # Fix if they are blank TODO if more than one chain/residue? 186 | if chain.name == "": chain.name = "A" 187 | for res in chain: 188 | if res.name == "": res.name = "00" 189 | 190 | # FIXME in some cases mtz space group should be modified. 191 | utils.fileio.write_model(st, prefix="input", pdb=True, cif=True) 192 | 193 | if args.ligand: args.ligand = sum(args.ligand, []) 194 | 195 | prefix = "refined" 196 | if args.bref == "aniso": 197 | args.keywords.append("refi bref aniso") 198 | elif args.bref == "iso_then_aniso": 199 | prefix = "refined_1_iso" 200 | 201 | if args.unrestrained: 202 | args.keywords.append("refi type unre") 203 | args.no_hout = False 204 | else: 205 | monlib = utils.restraints.load_monomer_library(st, monomer_dir=args.monlib, cif_files=args.ligand, 206 | stop_for_unknowns=False) 207 | 208 | # no bulk solvent by default 209 | if not args.bulk_solvent: 210 | args.keywords.append("solvent no") 211 | 212 | # Run Refmac 213 | refmac = utils.refmac.Refmac(prefix=prefix, global_mode="cx", 214 | exe=args.exe, 215 | source=args.source, 216 | monlib_path=args.monlib, 217 | xyzin="input.mmcif", 218 | hklin=mtz_in, 219 | ncycle=args.ncycle, 220 | weight_matrix=args.weight_matrix, 221 | weight_auto_scale=args.weight_auto_scale, 222 | hydrogen=args.hydrogen, 223 | hout=not args.no_hout, 224 | resolution=args.resolution, 225 | keyword_files=args.keyword_file, 226 | keywords=args.keywords) 227 | refmac.set_libin(args.ligand) 228 | refmac_summary = refmac.run_refmac() 229 | 230 | if args.bref == "iso_then_aniso": 231 | refmac2 = refmac.copy(xyzin=prefix+".mmcif", 232 | prefix="refined_2_aniso") 233 | refmac2.keywords.append("refi bref aniso") 234 | refmac_summary = refmac2.run_refmac() 235 | 236 | if __name__ == "__main__": 237 | import sys 238 | args = parse_args(sys.argv[1:]) 239 | main(args) 240 | 241 | -------------------------------------------------------------------------------- /servalcat/xtal/twin.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import argparse 10 | import gemmi 11 | import numpy 12 | import pandas 13 | import scipy.optimize 14 | from servalcat.utils import logger 15 | from servalcat import utils 16 | from servalcat import ext 17 | 18 | def calculate_obliquity(gv, twin_op): 19 | """ 20 | Beforehand, the following calculation must be done 21 | gv = gemmi.GruberVector(cell, sg.centring_type(), True) 22 | gv.niggli_reduce() 23 | """ 24 | def get_axis(r): 25 | eigenvalues, eigenvectors = numpy.linalg.eig(r) 26 | idx = numpy.argmin(numpy.abs(eigenvalues - 1)) 27 | return eigenvectors[:,idx].real 28 | reduced_cell = gv.get_cell() 29 | orth = numpy.array(reduced_cell.orth.mat.tolist()) 30 | frac = numpy.array(reduced_cell.frac.mat.tolist()) 31 | op = gv.change_of_basis.inverse().combine(twin_op.as_xyz()).combine(gv.change_of_basis) 32 | r = numpy.array(op.rot) / op.DEN 33 | u = get_axis(r) 34 | h = get_axis(numpy.linalg.inv(r.transpose())) 35 | tau = h.dot(frac) 36 | t = orth.dot(u) 37 | obl_deg = numpy.rad2deg(numpy.arccos(numpy.clip(t.dot(tau) / numpy.linalg.norm(t) / numpy.linalg.norm(tau), -1, 1))) 38 | return obl_deg 39 | # calculate_obliquity() 40 | 41 | def find_twin_domains_from_data(hkldata, max_oblique=5, min_cc=0.2): 42 | logger.writeln("Finding possible twin operators from data") 43 | ops = gemmi.find_twin_laws(hkldata.cell, hkldata.sg, max_oblique, False) 44 | logger.writeln(f" {len(ops)} possible twin operator(s) found") 45 | #for op in ops: 46 | # logger.writeln(f" {op.triplet()}") 47 | if not ops: 48 | logger.writeln("") 49 | return None, None 50 | gv = gemmi.GruberVector(hkldata.cell, hkldata.sg.centring_type(), True) 51 | gv.niggli_reduce() 52 | twin_data = ext.TwinData() 53 | twin_data.setup(hkldata.miller_array(), hkldata.df.bin_ml, hkldata.sg, hkldata.cell, ops) 54 | if "I" in hkldata.df: 55 | Io = hkldata.df.I.to_numpy() 56 | else: 57 | Io = hkldata.df.FP.to_numpy()**2 58 | ccs, nums = [], [] 59 | tmp = [] 60 | for i_bin, bin_idxes in hkldata.binned("ml"): 61 | ccs.append([]) 62 | nums.append([]) 63 | rs = [] 64 | for i_op, op in enumerate(ops): 65 | cc = r = numpy.nan 66 | ii = numpy.array(twin_data.pairs(i_op, i_bin)) 67 | val = numpy.all(numpy.isfinite(Io[ii]), axis=1) if ii.size != 0 else [] 68 | if numpy.sum(val) != 0: 69 | cc = numpy.corrcoef(Io[ii][val].T)[0,1] 70 | r = numpy.sum(numpy.abs(Io[ii][val, 0] - Io[ii][val, 1])) / numpy.sum(Io[ii][val]) 71 | ccs[-1].append(cc) 72 | rs.append(r) 73 | nums[-1].append(len(val)) 74 | tmp.append(rs + ccs[-1] + nums[-1]) 75 | df = pandas.DataFrame(tmp, columns=[f"{n}_op{i+1}" for n in ("R", "CC", "num") for i in range(len(ops))]) 76 | with logger.with_prefix(" "): 77 | logger.writeln(df.to_string(float_format="%.4f")) 78 | ccs = numpy.array(ccs) 79 | nums = numpy.array(nums) 80 | tmp = [{"Operator": "h,k,l", 81 | "Obliquity": 0, 82 | "R_twin_obs": 0, 83 | "CC_mean": 1}] 84 | for i_op, op in enumerate(ops): 85 | ii = numpy.array(twin_data.pairs(i_op)) 86 | val = numpy.all(numpy.isfinite(Io[ii]), axis=1) 87 | if numpy.sum(val) == 0: 88 | r_obs = numpy.nan 89 | else: 90 | r_obs = numpy.sum(numpy.abs(Io[ii][val, 0] - Io[ii][val, 1])) / numpy.sum(Io[ii][val]) 91 | good = numpy.isfinite(ccs[:,i_op]) 92 | cc = numpy.sum(nums[good,i_op] * ccs[good,i_op]) / numpy.sum(nums[good,i_op]) 93 | tmp.append({"Operator": op.as_hkl().triplet(), 94 | "Obliquity": calculate_obliquity(gv, op), 95 | "CC_mean": cc, 96 | "R_twin_obs": r_obs, 97 | }) 98 | df = pandas.DataFrame(tmp) 99 | with logger.with_prefix(" "): 100 | logger.writeln(df.to_string(float_format="%.2f")) 101 | 102 | sel = df["CC_mean"].to_numpy() > min_cc 103 | if sel[1:].sum() == 0: 104 | logger.writeln(" No possible twinning detected\n") 105 | return None, None 106 | 107 | if 0:#not sel.all(): 108 | ops = [ops[i] for i in range(len(ops)) if sel[i+1]] 109 | logger.writeln(f"\n Twin operators after filtering small correlations (<= {min_cc})") 110 | df = df[sel] 111 | with logger.with_prefix(" "): 112 | logger.writeln(df.to_string(float_format="%.2f")) 113 | twin_data = ext.TwinData() 114 | twin_data.setup(hkldata.miller_array(), hkldata.df.bin_ml, hkldata.sg, hkldata.cell, ops) 115 | twin_data.alphas = [1. / len(twin_data.alphas) for _ in range(len(twin_data.alphas)) ] 116 | if "I" not in hkldata.df: 117 | logger.writeln('Generating "observed" intensities for twin refinement: Io = Fo**2, SigIo = 2*F*SigFo') 118 | hkldata.df["I"] = hkldata.df.FP**2 119 | hkldata.df["SIGI"] = 2 * hkldata.df.FP * hkldata.df.SIGFP 120 | logger.writeln("") 121 | return twin_data, df 122 | 123 | # find_twin_domains_from_data() 124 | 125 | def estimate_twin_fractions_from_model(twin_data, hkldata, min_alpha=0.02): 126 | logger.writeln("Estimating twin fractions") 127 | Ic = numpy.abs(twin_data.f_calc.sum(axis=1))**2 128 | idx_all = twin_data.twin_related(hkldata.sg) 129 | Ic_all = Ic[idx_all] 130 | Ic_all[(idx_all < 0).any(axis=1)] = numpy.nan 131 | rr = twin_data.obs_related_asu() 132 | tmp = [] 133 | P_list, cc_oc_list, weight_list = [], [], [] 134 | n_ops = len(twin_data.ops) + 1 135 | tidxes = numpy.triu_indices(n_ops, 1) 136 | if "CC*" in hkldata.binned_df["ml"]: 137 | logger.writeln(" data-correlations are corrected using CC*") 138 | for i_bin, bin_idxes in hkldata.binned("ml"): # XXX 139 | i_tmp = Ic_all[numpy.asarray(twin_data.bin)==i_bin,:] 140 | i_tmp = i_tmp[numpy.isfinite(i_tmp).all(axis=1)] 141 | P = numpy.corrcoef(i_tmp.T) 142 | iobs = hkldata.df.I.to_numpy()[bin_idxes] 143 | ic_bin = Ic[rr[bin_idxes,:]] 144 | val = numpy.isfinite(iobs) & numpy.isfinite(ic_bin).all(axis=1) & numpy.all(rr[bin_idxes,:]>=0, axis=1) 145 | iobs, ic_bin = iobs[val], ic_bin[val,:] 146 | cc_star = hkldata.binned_df["ml"]["CC*"][i_bin] if "CC*" in hkldata.binned_df["ml"] else 1 147 | if cc_star < 0.5: 148 | break 149 | cc_oc = [numpy.corrcoef(iobs, ic_bin[:,i])[0,1] / cc_star for i in range(n_ops)] 150 | P_list.append(P) 151 | cc_oc_list.append(cc_oc) 152 | weight_list.append(numpy.sum(val)) 153 | frac_est = numpy.dot(numpy.linalg.pinv(P), cc_oc) 154 | frac_est /= frac_est.sum() 155 | tmp.append(P[tidxes].tolist() + cc_oc + [weight_list[-1]] + frac_est.tolist()) 156 | 157 | good = numpy.logical_and(numpy.isfinite(P_list).any(axis=(1,2)), numpy.isfinite(cc_oc_list).any(axis=1)) 158 | P_list = numpy.array(P_list)[good] 159 | cc_oc_list = numpy.array(cc_oc_list)[good] 160 | weight_list = numpy.array(weight_list)[good] 161 | P = numpy.average(P_list, axis=0, weights=weight_list) 162 | cc_oc = numpy.average(cc_oc_list, axis=0, weights=weight_list) 163 | frac_est = numpy.dot(numpy.linalg.pinv(P), cc_oc) 164 | frac_est = numpy.maximum(0, frac_est) 165 | frac_est /= frac_est.sum() 166 | df = pandas.DataFrame(tmp, columns=[f"cc_{i+1}_{j+1}" for i, j in zip(*tidxes)] + 167 | [f"cc_o_{i+1}" for i in range(n_ops)] + 168 | ["nref"] + [f"raw_est_{i+1}" for i in range(n_ops)]) 169 | with logger.with_prefix(" "): 170 | logger.writeln(df.to_string(float_format="%.4f")) 171 | logger.write(" Final twin fraction estimate: ") 172 | logger.writeln(" ".join("%.4f"%x for x in frac_est)) 173 | twin_data.alphas = frac_est 174 | 175 | if numpy.logical_and(0 < frac_est, frac_est < min_alpha).any(): 176 | frac_est[frac_est < min_alpha] = 0. 177 | frac_est /= frac_est.sum() 178 | logger.write(" Small fraction removed: ") 179 | logger.writeln(" ".join("%.4f"%x for x in frac_est)) 180 | twin_data.alphas = frac_est 181 | 182 | return df 183 | 184 | def mlopt_twin_fractions(hkldata, twin_data, b_aniso): 185 | k_ani2_inv = 1 / hkldata.debye_waller_factors(b_cart=b_aniso)**2 186 | Io = hkldata.df.I.to_numpy(copy=True) * k_ani2_inv 187 | sigIo = hkldata.df.SIGI.to_numpy(copy=True) * k_ani2_inv 188 | def fun(x): 189 | twin_data.alphas = x 190 | twin_data.est_f_true(Io, sigIo, 10) 191 | ret = twin_data.ll(Io, sigIo) 192 | return ret 193 | def grad(x): 194 | twin_data.alphas = x 195 | twin_data.est_f_true(Io, sigIo, 10) 196 | return twin_data.ll_der_alpha(Io, sigIo, True) 197 | if 0: 198 | bak = [_ for _ in twin_data.alphas] 199 | with open("alpha_ll.csv", "w") as ofs: 200 | ofs.write("a,ll,ll_new,der1,der2,der_new1,der_new2\n") 201 | for a in numpy.linspace(0., 1.0, 100): 202 | x = [a, 1-a] 203 | twin_data.alphas = x 204 | twin_data.est_f_true(Io, sigIo, 100) 205 | f_new = twin_data.ll(Io, sigIo) 206 | f = twin_data.ll_rice() 207 | der = twin_data.ll_der_alpha(Io, sigIo, False) 208 | #der = [x - der[-1] for x in der[:-1]] 209 | der_new = twin_data.ll_der_alpha(Io, sigIo, True) 210 | #der_new = [x - der_new[-1] for x in der_new[:-1]] 211 | ofs.write(f"{a},{f},{f_new},{der[0]},{der[1]},{der_new[0]},{der_new[1]}\n") 212 | ofs.write("\n") 213 | twin_data.alphas = bak 214 | if 0: 215 | x0 = [x for x in twin_data.alphas] 216 | f0 = fun(x0) 217 | ader = grad(x0) 218 | 219 | print(f"{ader=}") 220 | for e in (1e-2, 1e-3, 1e-4, 1e-5): 221 | nder = [] 222 | for i in range(len(x0)): 223 | x = [_ for _ in x0] 224 | x[i] += e 225 | f1 = fun(x) 226 | nder.append((f1 - f0) / e) 227 | print(f"{e=} {nder=}") 228 | 229 | logger.writeln("ML twin fraction refinement..") 230 | num_params = len(twin_data.alphas) 231 | A = numpy.ones((1, num_params)) 232 | linear_constraint = scipy.optimize.LinearConstraint(A, [1.0], [1.0]) 233 | bounds = scipy.optimize.Bounds(numpy.zeros(num_params), numpy.ones(num_params)) 234 | logger.writeln(" starting with " + " ".join("%.4f"%x for x in twin_data.alphas)) 235 | logger.writeln(f" f0= {fun(twin_data.alphas)}") 236 | res = scipy.optimize.minimize(fun=fun, x0=twin_data.alphas, 237 | bounds=bounds, 238 | constraints=[linear_constraint], 239 | jac=grad, 240 | #callback=lambda *x: logger.writeln(f"callback {x}"), 241 | ) 242 | logger.writeln(" finished in {} iterations ({} evaluations)".format(res.nit, res.nfev)) 243 | logger.writeln(f" f = {res.fun}") 244 | # ensure constraints 245 | alphas = numpy.clip(res.x, 0, 1) 246 | twin_data.alphas = list(alphas / alphas.sum()) 247 | logger.write(" ML twin fraction estimate: ") 248 | logger.writeln(" ".join("%.4f"%x for x in twin_data.alphas)) 249 | -------------------------------------------------------------------------------- /servalcat/utils/generate_operators.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: "Keitaro Yamashita, Garib N. Murshudov" 3 | MRC Laboratory of Molecular Biology 4 | 5 | This software is released under the 6 | Mozilla Public License, version 2.0; see LICENSE. 7 | """ 8 | from __future__ import absolute_import, division, print_function, generators 9 | import numpy 10 | import copy 11 | eps_l = 1.0e-5 12 | 13 | def generate_all_elements(axis1, order1, axis2=None, order2=0, toler=1.0e-6, toler2=1.e-3): 14 | # 15 | # Generate all group elements. Output will be as a list of cyclic groups. 16 | # 17 | if axis2 is None: axis2 = numpy.array([0.0,0.0,1.0]) 18 | grp_out = [] 19 | order_out = [] 20 | axes_out = [] 21 | if order2 == 0: 22 | grp_out = generate_cyclic(axis1, order1) 23 | order_out.append(order1) 24 | axes_out.append(axis1) 25 | return order_out, axes_out, grp_out 26 | 27 | grp_out = generate_cyclic(axis1,order1) 28 | order_out.append(order1) 29 | axes_out.append(axis1) 30 | grp1 = generate_cyclic(axis2,order2) 31 | grp_out = add_groups_together(grp_out,grp1,toler2) 32 | order_out.append(order2) 33 | axes_out.append(axis2) 34 | 35 | grp_out_new = copy.copy(grp_out) 36 | things_to_do = True 37 | while things_to_do: 38 | things_to_do = False 39 | for i, ri in enumerate(grp_out): 40 | for j, rj in enumerate(grp_out): 41 | if i !=0 and j!=0 and i != j: 42 | r3 = numpy.dot(ri,rj) 43 | if not is_in_the_list_rotation(r3, grp_out, toler2): 44 | things_to_do = True 45 | order3, axis3 = find_order(r3, toler2) 46 | grp3 = generate_cyclic(axis3, order3) 47 | grp_out_new = add_groups_together(grp_out_new, grp3,toler2) 48 | order_out.append(order3) 49 | axes_out.append(axis3) 50 | r3 = numpy.dot(rj, ri) 51 | if not is_in_the_list_rotation(r3, grp_out, toler2): 52 | things_to_do = True 53 | order3, axis3 = find_order(r3, toler2) 54 | grp3 = generate_cyclic(axis3,order3) 55 | grp_out_new = add_groups_together(grp_out_new, grp3,toler2) 56 | order_out.append(order3) 57 | axes_out.append(axis3) 58 | grp_out = copy.copy(grp_out_new) 59 | # 60 | # Filter out axes (if they are parallel to each other then select one, for the order we should take 61 | # the highest order). In our case it should happen only for the group O. We may have 2 and four fold symmetries 62 | # with the same axis 63 | axes_out_new = [] 64 | order_out_new = [] 65 | for i, axisi in enumerate(axes_out): 66 | order_cp = order_out[i] 67 | for j, axisj in enumerate(axes_out): 68 | if i < j: 69 | cangle = numpy.dot(axisi,axisj)/(numpy.linalg.norm(axisi)*numpy.linalg.norm(axisj)) 70 | if numpy.abs(cangle-1.0) < toler or numpy.abs(cangle+1) < toler: 71 | # same axis 72 | if order_cp <= order_out[j]: 73 | order_cp= 0 74 | break 75 | if order_cp > 0: 76 | axes_out_new.append(axisi) 77 | order_out_new.append(order_cp) 78 | 79 | return order_out_new, axes_out_new, grp_out 80 | # generate_all_elements() 81 | 82 | def find_order(r, toler=1.0e-3): 83 | order = 1 84 | r_id = numpy.identity(3) 85 | r3 = numpy.copy(r_id) 86 | things_to_do = True 87 | A = r_id 88 | while things_to_do and order < 100: 89 | things_to_do = False 90 | r3 = numpy.dot(r3, r) 91 | if numpy.sum(numpy.abs(r3-r_id)) > toler: 92 | A = A + r3 93 | things_to_do = True 94 | order += 1 95 | 96 | if order >= 100: 97 | raise RuntimeError("The order of the group is too high: order > 100") 98 | A = A/order 99 | axis_l = find_axis(A) 100 | 101 | return order, axis_l 102 | # find_order() 103 | 104 | def add_groups_together(grp_in, grp_add, toler=1.e-3): 105 | grp_out = copy.copy(grp_in) 106 | grp_out.extend(filter(lambda r: not is_in_the_list_rotation(r, grp_out, toler), grp_add)) 107 | #for r in grp_add: 108 | # if not is_in_the_list_rotation(r, grp_out): 109 | # grp_out.append(r) 110 | return grp_out 111 | # add_groups_together() 112 | 113 | def generate_cyclic(axis, order): 114 | # 115 | # This function generates all cyclic group elements using axis and order of the group 116 | if order <=0 or numpy.sum(numpy.abs(axis)) < eps_l: 117 | raise RuntimeError("Either order or axis is zero. order= {} axis= {}".format(order, axis)) 118 | gout = [] 119 | id_matr = numpy.identity(3) 120 | gout.append(id_matr) 121 | angle = 2.0*numpy.pi/order 122 | axis = axis/numpy.linalg.norm(axis) 123 | exp_matr = numpy.array([[0, -axis[2], axis[1]], 124 | [axis[2], 0, -axis[0]], 125 | [-axis[1], axis[0], 0]]) 126 | axis_outer = numpy.outer(axis, axis) 127 | m_int = id_matr - axis_outer 128 | for i in range(order-1): 129 | angle_l = angle*(i+1) 130 | stheta = numpy.sin(angle_l) 131 | ctheta = numpy.cos(angle_l) 132 | m_l = exp_matr*stheta + m_int*ctheta +axis_outer 133 | m_l = numpy.where(numpy.abs(m_l) < 1e-9, 0, m_l) 134 | gout.append(m_l) 135 | 136 | return gout 137 | # generate_cyclic() 138 | 139 | def AngleAxis2rotatin(axis, angle): 140 | # 141 | # Convert axis and ange to a rotation matrix. Here we use a mtrix form of the relatiionship 142 | # IT may not be the moost efficient algorithm, but it should work (it is more elegant) 143 | if numpy.sum(numpy.abs(axis)) < eps_l: 144 | raise RuntimeError("Axis is zero. axis= {} angle= {}".format(axis, angle)) 145 | id_matr = numpy.identity(3) 146 | axis = axis/numpy.sqrt(numpy.dot(axis,axis)) 147 | exp_matr = numpy.array([[0, -axis[2], axis[1]], 148 | [axis[2], 0, -axis[0]], 149 | [-axis[1], axis[0], 0]]) 150 | axis_outer = numpy.outer(axis, axis) 151 | m_int = id_matr - axis_outer 152 | stheta = numpy.sin(angle) 153 | ctheta = numpy.cos(angle) 154 | m_l = exp_matr*stheta + m_int*ctheta +axis_outer 155 | m_l = numpy.where(numpy.abs(m_l) < 1e-9, 0, m_l) 156 | return m_l 157 | # AngleAxis2rotatin() 158 | 159 | def Rotation2AxisAngle_cyclic(m_in, eps_l=1.0e-5): 160 | # 161 | # Here we assume that rotation matrix is an element of a cyclic group 162 | # This routine gives the smallest angle for this cyclic group. 163 | # To find axis of the rotation we use the fact that if we define 164 | # A = 1/n sum_i-0^(n-1) (R^i) then this operator is a projector to the axis of rotation 165 | # i.e. for Ax will be on the axis for any x. IT could be equal 0, in this case we select another x 166 | A = m_in 167 | m1 = m_in 168 | id_matr = numpy.identity(3) 169 | cycle_number = 1 170 | ended = False 171 | while not ended and cycle_number < 200: 172 | if numpy.sum(numpy.abs(m1-id_matr)) < eps_l: 173 | ended = True 174 | break 175 | m1 = numpy.dot(m1,m_in) 176 | A = A + m1 177 | cycle_number = cycle_number + 1 178 | # take a ranom vector 179 | if cycle_number >= 150 : 180 | print("matrix ",m_in) 181 | print("Try to change the tolerance: eps_l = XXX") 182 | raise RuntimeError("The matrix does not seem to be producing a finite cyclic group") 183 | A = A/cycle_number 184 | axis = numpy.zeros(3) 185 | for xin in ((0,0,1.), (0,1.,0), (1.,0,0,)): 186 | axis = numpy.dot(A,xin) 187 | if numpy.dot(axis,axis) > eps_l: 188 | axis = axis/numpy.sqrt(numpy.dot(axis,axis)) 189 | if numpy.dot(axis,axis) >= eps_l: 190 | break 191 | 192 | if axis[2] < 0.0: 193 | axis = -axis 194 | elif axis[2] == 0.0 and axis[1] < 0.0: 195 | axis = -axis 196 | angle = 2.0*numpy.pi/cycle_number 197 | axis[axis==0.]=0. 198 | return axis,angle,cycle_number 199 | # Rotation2AxisAngle_cyclic() 200 | 201 | def Rotation2AxisAngle_general(m_in, eps_l=1.0e-5): 202 | # 203 | # This routine should work for any rotation matrix 204 | axis = numpy.array([1, 0.0, 0.0]) 205 | angle = numpy.arccos(max(-1.0, numpy.min((numpy.trace(m_in)-1)/2.0))) 206 | if numpy.sum(numpy.abs(m_in-numpy.transpose(m_in))) < eps_l: 207 | # 208 | # It is a symmetric matrix. so I and m_in form a cyclic group 209 | A = (numpy.identity(3) + m_in)/2.0 210 | axis = numpy.zeros(3) 211 | for a in ((0,0,1.), (0,1.,0), (1.,0,0,)): 212 | axis = numpy.dot(A, a) 213 | if numpy.linalg.norm(axis) >= eps_l: break 214 | else: 215 | axis[0] = m_in[1,2] - m_in[2,1] 216 | axis[1] = m_in[0,2] - m_in[2,0] 217 | axis[2] = m_in[0,1] - m_in[1,0] 218 | if axis[2] < 0.0: 219 | axis = -axis 220 | angle = 2.0*numpy.pi - angle 221 | elif axis[2] < eps_l and axis[1] < 0.0: 222 | axis = -axis 223 | angle = 2.0*numpy.pi - angle 224 | axis = axis/numpy.linalg.norm(axis) 225 | axis[axis==0.]=0. 226 | return axis, angle 227 | # Rotation2AxisAngle_general() 228 | 229 | def is_in_the_list_rotation(m_in, m_list, toler = 1.0e-3): 230 | id_matr = numpy.identity(3) 231 | return numpy.any(numpy.abs(numpy.trace(numpy.dot(numpy.transpose(m_in), m_list)-id_matr[:,None], axis1=0,axis2=2)) < toler) 232 | # is_in_the_list_rotation() 233 | 234 | def closest_rotation(m_in, m_list): 235 | id_matr = numpy.identity(3) 236 | return min(numpy.abs(numpy.trace(numpy.dot(numpy.transpose(m_in), m_list)-id_matr[:,None], axis1=0,axis2=2))) 237 | # closest_rotation() 238 | 239 | def find_axis(amatr): 240 | # 241 | # We assume that amatr is a projector. I.e. y = amatr x is on the the symmetry axis. 242 | # To avoid problem of 0 vector we try several times to make sure that 0 vector is not generated 243 | axis1 = numpy.zeros(3) 244 | for a in ((0,0,1.), (0,1.,0), (1.,0,0,)): 245 | axis1 = numpy.dot(amatr, a) 246 | if numpy.linalg.norm(axis1) >= 0.001: 247 | break 248 | 249 | axis1 /= numpy.linalg.norm(axis1) 250 | axis1 = numpy.around(axis1, 10) 251 | axis1 /= numpy.linalg.norm(axis1) 252 | # Remove annoying negative signs 253 | axis1[axis1==0.]=0. 254 | 255 | if axis1[2] < 0.0: 256 | axis1 *= -1 257 | elif axis1[2] == 0.0 and axis1[1] < 0.0: 258 | axis1 *= -1 259 | 260 | return axis1 261 | # find_axis() 262 | 263 | def rotate_group_elements(Rg, matrices): 264 | # 265 | # assume an input list and return a list of matrices 266 | # 267 | mm_out = [] 268 | Rg_t = numpy.transpose(Rg) 269 | for i, mm in enumerate(matrices): 270 | m1 = numpy.dot(numpy.dot(Rg_t, mm), Rg) 271 | mm_out.append(m1) 272 | return(mm_out) 273 | # rotate_group_elements() 274 | 275 | if __name__ == "__main__": 276 | import sys 277 | from servalcat.utils import symmetry 278 | symbol = sys.argv[1] 279 | order, axes, grp = symmetry.operators_from_symbol(symbol) 280 | #print(order) 281 | #print(axes) 282 | #print(grp) 283 | #quit() 284 | rgs = symmetry.get_matrices_using_relion(symbol) 285 | all_ok = True 286 | max_diff = 0 287 | for i, m in enumerate(grp): 288 | #print("Op", i) 289 | #print(m) 290 | #ok = is_in_the_list_rotation(m, rgs, toler=1e-4) 291 | diff = closest_rotation(m, rgs) 292 | ok = diff < 1e-4 293 | #print("match? {} {:.1e}".format(ok, diff)) 294 | if not ok: all_ok = False 295 | if diff > max_diff: max_diff = diff 296 | print("Final=", all_ok, max_diff) 297 | --------------------------------------------------------------------------------