├── MANIFEST.in ├── pyproject.toml ├── doc ├── project.md └── Rekordbox Fields.md ├── test ├── ddb1.csv └── test_rkbeets.py ├── beetsplug └── rkbeets │ ├── rkbeets-fields.csv │ └── __init__.py ├── LICENSE ├── README.md └── .gitignore /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include beetsplug/rkbeets/rkbeets-fields.csv 2 | 3 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "rkbeets" 7 | version = "0.1.0" 8 | dependencies = [ 9 | "beets>=1.6.0", 10 | "pandas", 11 | "pyrekordbox@git+https://github.com/voigtjr/pyrekordbox@master", 12 | "tqdm", 13 | ] 14 | 15 | [tool.setuptools.package-data] 16 | beetsplug = ["*.csv"] 17 | -------------------------------------------------------------------------------- /doc/project.md: -------------------------------------------------------------------------------- 1 | * A flag to dump export xml to stdout for command line composition 2 | * A report on differences in filesystem path case for paths in libraries vs actual paths on disk 3 | * Preserve NAs in sync command instead of using default values, but this has a bit more intense matching logic. 4 | * Figure out the minimal thing to do to sync beets metadata instead of calling `try_sync` unless that is, in fact, idiomatic 5 | -------------------------------------------------------------------------------- /test/ddb1.csv: -------------------------------------------------------------------------------- 1 | rkb_field,rkb_type,beets_field,beets_type,no_export,convert_to_rkb,sync 2 | r01,string,b01,string,,,true 3 | r02,float64,b02,float64,,,true 4 | r03,float64,b03,Float64,,,true 5 | r04,float64,b04,int32,,,true 6 | r05,float64,b05,Int32,,,true 7 | r06,float64,b06,int64,,, 8 | r07,float64,b07,Int64,,, 9 | r08,int32,b08,int32,,, 10 | r09,int32,b09,Int32,,, 11 | r10,int32,b10,int64,,, 12 | r11,int32,b11,Int64,,, 13 | r12,int64,b12,int64,,, 14 | r13,int64,b13,Int64,,, 15 | r14,string,b14,bytes,,, 16 | r15,str,b15,str,,, 17 | ,,b0,string,,, 18 | r0,string,,,,, 19 | r90,string,b90,string,true,, 20 | r91,string,,,true,, 21 | ,,b92,string,true,, -------------------------------------------------------------------------------- /beetsplug/rkbeets/rkbeets-fields.csv: -------------------------------------------------------------------------------- 1 | rkb_field,rkb_type,beets_field,beets_type,no_export,convert_to_rkb,sync 2 | Album,string,album,string,,, 3 | Artist,string,artist,string,,, 4 | AverageBpm,float64,rkb_AverageBpm,float64,,,true 5 | BitRate,int32,bitrate,int32,,, 6 | Colour,string,rkb_Colour,string,,,true 7 | Comments,string,comments,string,,, 8 | Composer,string,composer,string,,, 9 | DateAdded,string,rkb_DateAdded,string,,,true 10 | DateModified,string,rkb_DateModified,string,,,true 11 | DiscNumber,int32,disc,int32,,, 12 | Genre,string,genre,string,,, 13 | Grouping,string,grouping,string,,, 14 | Kind,string,format,string,,format_to_kind, 15 | Label,string,label,string,,, 16 | LastPlayed,string,rkb_LastPlayed,string,,,true 17 | Location,string,path,bytes,,, 18 | Mix,string,rkb_Mix,string,,,true 19 | Name,string,title,string,,, 20 | PlayCount,int32,rkb_PlayCount,Int32,,,true 21 | Rating,int32,rkb_Rating,Int32,,,true 22 | Remixer,string,remixer,string,,, 23 | SampleRate,float64,samplerate,int32,,, 24 | Size,int64,filesize,int64,,, 25 | Tonality,string,rkb_Tonality,string,,,true 26 | TotalTime,float64,length,float64,,, 27 | TrackID,int64,rkb_TrackID,Int64,true,,true 28 | TrackNumber,int32,track,int32,,, 29 | Year,int32,year,int32,,, 30 | ,,id,int32,true,, -------------------------------------------------------------------------------- /doc/Rekordbox Fields.md: -------------------------------------------------------------------------------- 1 | # Rekordbox Fields 2 | 3 | * [Rekordbox database format](https://pyrekordbox.readthedocs.io/en/latest/formats/xml.html) 4 | 5 | ## Primary key: `Location` 6 | 7 | * `Location`: Forced to lowercase because the filesystem I'm testing this on is in lowercase. 8 | * Rekordbox also strips the root `/` which we add back in. 9 | * Making this correctly portable will be some amount of work. 10 | 11 | ## beets source of truth 12 | 13 | These should be controlled by beets and written to rekordbox 14 | 15 | * `Name` 16 | * `Artist` 17 | * `Composer` 18 | * `Album` 19 | * `Grouping` 20 | * `Genre` 21 | * `DiscNumber` 22 | * `TrackNumber` 23 | * `Year` 24 | * `Comments` 25 | * `Label` 26 | 27 | ## rekordbox additional metadata 28 | 29 | New fields to sync by default into beets from rekordbox. 30 | 31 | * `AverageBpm` 32 | * `Colour`: User-set color identifiers 33 | * `DateAdded`: Date track added to rekordbox, useful to preserve 34 | * `DateModified` 35 | * `LastPlayed` 36 | * `Mix`: Track metadata that doesn't have a key in beets 37 | * `PlayCount`: How many times it has been played in Rekordbox 38 | * `Rating`: Default 0 for no rating 39 | * `Tonality`: Key encoding, computed from analysis 40 | * `TrackID`: Rekordbox internal track identifier number, note that I think this is not viewable anymore in the GUI and is back-end only 41 | 42 | # Rekordbox analyzed metadata 43 | 44 | * `Kind`: `['MP3 File', 'M4A File', 'WAV File']` unclear if there are more 45 | * `Size` 46 | * `TotalTime` 47 | * `BitRate` 48 | * `SampleRate` -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2022, Jonathan Voigt 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products derived from 17 | this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /test/test_rkbeets.py: -------------------------------------------------------------------------------- 1 | import pandas 2 | 3 | import beetsplug.rkbeets as rkb 4 | 5 | def test_default_ddb(): 6 | """Make sure it is roughly functional without inspecting it too much.""" 7 | # I don't understand why pytest can't see this resource. 8 | ddb = rkb.DimensionsDB('src/beetsplug/rkbeets-fields.csv') 9 | 10 | for row in ddb.get_beets_cols(): 11 | assert pandas.Series(data=None, dtype=row.dtype) is not None 12 | 13 | for row in ddb.get_rkb_cols(): 14 | assert pandas.Series(data=None, dtype=row.dtype) is not None 15 | 16 | ei = ddb.get_export_conversion_info() 17 | assert ei is not None 18 | assert ei.drop_fields is not None 19 | assert ei.export_fields is not None 20 | 21 | for ef in ei.export_fields: 22 | assert ef.beets is not None 23 | assert ef.rkb is not None 24 | 25 | count = 0 26 | for pair in ddb.get_sync_pairs(): 27 | count += 1 28 | assert pair.beets is not None 29 | assert pair.rkb is not None 30 | assert count > 0 31 | 32 | def test_ddb_cols(): 33 | ddb = rkb.DimensionsDB('test/ddb1.csv') 34 | assert ddb.num_beets_cols() == 18 35 | assert ddb.num_rkb_cols() == 18 36 | 37 | for row in ddb.get_beets_cols(): 38 | assert row.field.startswith('b') 39 | assert pandas.Series(data=None, dtype=row.dtype) is not None 40 | 41 | for row in ddb.get_rkb_cols(): 42 | assert row.field.startswith('r') 43 | assert pandas.Series(data=None, dtype=row.dtype) is not None 44 | 45 | def test_ddb_export(): 46 | ddb = rkb.DimensionsDB('test/ddb1.csv') 47 | 48 | ei = ddb.get_export_conversion_info() 49 | assert list(ei._asdict().keys()) == ['drop_fields', 'export_fields'] 50 | assert ei.drop_fields == ['b90', 'b92'] 51 | for ef in ei.export_fields: 52 | assert list(ef._asdict().keys()) == ['beets', 'rkb', 'func'] 53 | assert ef.func is None 54 | assert ef.beets[1:] == ef.rkb[1:] 55 | 56 | def test_ddb_sync(): 57 | ddb = rkb.DimensionsDB('test/ddb1.csv') 58 | 59 | count = 0 60 | for pair in ddb.get_sync_pairs(): 61 | assert list(pair._asdict().keys()) == ['beets', 'rkb'] 62 | count += 1 63 | assert pair.beets[1:] == pair.rkb[1:] 64 | assert count == 5 65 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rkbeets 2 | 3 | A [beets](https://beets.io/) plugin for simple integration with rekordbox. 4 | 5 | Python 3.10. Be careful - **use a copy of your database** - there are no confirmation flows. 6 | 7 | ## Quick tour 8 | 9 | _Tested only on recent macos, case-insensitive fs with beets library on a vfat drive._ 10 | 11 | ### Data input and output 12 | 13 | * The plugin will export data for import into rekordbox using the option `--export-file`. 14 | * Some commands use data from rekordbox, provided to beets via the option `--rekordbox-file`. 15 | * Setting these in the configuration file cleans up the command line: 16 | 17 | `.config/beets/config.yaml`: 18 | ```yaml 19 | rkbeets: 20 | export-file: ~/Documents/rekordbox/rekordbox.xml 21 | rekordbox-file: ~/Documents/rekordbox/export.xml 22 | ``` 23 | 24 | ### Export beets to rekordbox with `rkb-export` 25 | 26 | Export your beets library for import into rekordbox with `rkb-export`: 27 | 28 | ```sh 29 | # beets library -exported-to-> rekordbox 30 | beet rkb-export 31 | 32 | # only export files missing from rekordbox, requires `rekordbox-file` 33 | beet rkb-export --missing 34 | 35 | # export only missing files further filtered by a query 36 | beet rkb-export artist:radiohead --missing 37 | ``` 38 | 39 | ### Inspect beets and rekordbox differences with `rkb-diff` 40 | 41 | Inspect how many tracks are shareed between the two libraries (and a list of those that aren't) with `rkb-diff`: 42 | 43 | ```sh 44 | # rekordbox exported xml -compared-to-> beets library 45 | beet rkb-diff 46 | ``` 47 | 48 | Tracks are matched between the two only by using file paths. 49 | 50 | ### Copy metadata from rekordbox into beets with `rkb-sync` 51 | 52 | The `rkb-sync` command lets you pull metadata from rekordbox into beets. 53 | 54 | ```sh 55 | # rekordbox metadata -written-to-> beets library 56 | beet rkb-sync 57 | 58 | # dump to console instead of updating 59 | beet rkb-sync --dry-run 60 | 61 | # only consider shared tracks that satisfy a query 62 | beet rkb-sync artist:radiohead 63 | ``` 64 | 65 | Currently implemented for these rekordbox fields: 66 | 67 | * `AverageBpm` 68 | * `Colour` 69 | * `DateAdded` 70 | * `DateModified` 71 | * `LastPlayed` 72 | * `Mix` 73 | * `PlayCount` 74 | * `Rating` 75 | * `Tonality` 76 | * `TrackID`: This is rekordbox internal and isn't exported but we save it anyway. 77 | 78 | ### Importing from rekordbox to beets 79 | 80 | Import files using `beets import`. 81 | 82 | ## Installation 83 | 84 | Since it is under development, use source: 85 | 86 | * Clone [beets](https://github.com/beetbox/beets) and [rkbeets](https://github.com/voigtjr/rkbeets) next to each other 87 | * Create a virtual environment or similar in `beets` repo: `python -m venv .venv` 88 | * Install `beets` into that environment 89 | * `pip install -e ../rkbeets` 90 | * `beet rkb-diff` and friends should work 91 | 92 | ## License 93 | 94 | See [LICENSE](LICENSE). 95 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | share/python-wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .nox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | *.py,cover 52 | .hypothesis/ 53 | .pytest_cache/ 54 | cover/ 55 | 56 | # Translations 57 | *.mo 58 | *.pot 59 | 60 | # Django stuff: 61 | *.log 62 | local_settings.py 63 | db.sqlite3 64 | db.sqlite3-journal 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | .pybuilder/ 78 | target/ 79 | 80 | # Jupyter Notebook 81 | .ipynb_checkpoints 82 | 83 | # IPython 84 | profile_default/ 85 | ipython_config.py 86 | 87 | # pyenv 88 | # For a library or package, you might want to ignore these files since the code is 89 | # intended to run in multiple environments; otherwise, check them in: 90 | # .python-version 91 | 92 | # pipenv 93 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 94 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 95 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 96 | # install all needed dependencies. 97 | #Pipfile.lock 98 | 99 | # poetry 100 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 101 | # This is especially recommended for binary packages to ensure reproducibility, and is more 102 | # commonly ignored for libraries. 103 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 104 | #poetry.lock 105 | 106 | # pdm 107 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 108 | #pdm.lock 109 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 110 | # in version control. 111 | # https://pdm.fming.dev/#use-with-ide 112 | .pdm.toml 113 | 114 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 115 | __pypackages__/ 116 | 117 | # Celery stuff 118 | celerybeat-schedule 119 | celerybeat.pid 120 | 121 | # SageMath parsed files 122 | *.sage.py 123 | 124 | # Environments 125 | .env 126 | .venv 127 | env/ 128 | venv/ 129 | ENV/ 130 | env.bak/ 131 | venv.bak/ 132 | 133 | # Spyder project settings 134 | .spyderproject 135 | .spyproject 136 | 137 | # Rope project settings 138 | .ropeproject 139 | 140 | # mkdocs documentation 141 | /site 142 | 143 | # mypy 144 | .mypy_cache/ 145 | .dmypy.json 146 | dmypy.json 147 | 148 | # Pyre type checker 149 | .pyre/ 150 | 151 | # pytype static type analyzer 152 | .pytype/ 153 | 154 | # Cython debug symbols 155 | cython_debug/ 156 | 157 | # PyCharm 158 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 159 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 160 | # and can be added to the global gitignore or merged into this file. For a more nuclear 161 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 162 | #.idea/ 163 | -------------------------------------------------------------------------------- /beetsplug/rkbeets/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, Jonathan Voigt 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions are met: 5 | # 6 | # 1. Redistributions of source code must retain the above copyright notice, this 7 | # list of conditions and the following disclaimer. 8 | # 9 | # 2. Redistributions in binary form must reproduce the above copyright notice, 10 | # this list of conditions and the following disclaimer in the documentation 11 | # and/or other materials provided with the distribution. 12 | # 13 | # 3. Neither the name of the copyright holder nor the names of its 14 | # contributors may be used to endorse or promote products derived from 15 | # this software without specific prior written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 21 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | from collections import namedtuple 29 | from functools import cached_property, reduce 30 | from importlib import resources 31 | import logging 32 | import operator 33 | from pathlib import Path 34 | from typing import Any, Callable, Iterable 35 | 36 | from beets import config 37 | from beets import plugins 38 | from beets.dbcore import db, types 39 | from beets import ui 40 | from beets import library 41 | import pandas 42 | from tqdm import tqdm 43 | 44 | import beetsplug 45 | import beetsplug.rkbeets 46 | 47 | # pyrekordbox is chatty about missing/broken rekordbox configuration files 48 | previous_level = logging.root.manager.disable 49 | logging.disable(logging.CRITICAL) 50 | try: 51 | import pyrekordbox.xml as pxml # type: ignore 52 | finally: 53 | logging.disable(previous_level) 54 | 55 | class DimensionsDB(): 56 | """ 57 | Manages metadata about beets and rekordbox fields and their relationships to 58 | one another. 59 | 60 | Parameters 61 | ---------- 62 | csv: Path, optional 63 | Use this csv file instead of the default. 64 | """ 65 | 66 | _df: pandas.DataFrame 67 | 68 | def __init__(self, csv_path: Path | None = None): 69 | path = csv_path if csv_path is not None else resources.path( 70 | beetsplug.rkbeets, 'rkbeets-fields.csv') 71 | self._df = pandas.read_csv(path) 72 | if self._df.index.has_duplicates: 73 | # TODO test this works 74 | raise RuntimeError("rkbeets-fields contains duplicate columns between library types") 75 | 76 | def to_pickle(self, dir: Path) -> None: 77 | """ 78 | Pickle the DataFrame to `ddb.pkl` in the given directory. 79 | 80 | Parameters 81 | ---------- 82 | dir: Path 83 | Directory to write 'ddb.pkl' 84 | """ 85 | 86 | self._df.to_pickle(dir / Path('ddb.pkl')) 87 | 88 | def num_beets_cols(self) -> int: 89 | """The number of beets fields.""" 90 | 91 | return self._df['beets_field'].dropna().size 92 | 93 | def get_beets_cols(self) -> Iterable[tuple[str, str]]: 94 | """Returns a namedtuple of beets `field`s and their corresponding `dtype`s.""" 95 | 96 | df = self._df[['beets_field', 'beets_type']].dropna() 97 | mapping = { 98 | 'beets_field': 'field', 99 | 'beets_type': 'dtype', 100 | } 101 | return df.rename(columns=mapping).itertuples(name='FieldInfo', index=False) 102 | 103 | def num_rkb_cols(self) -> int: 104 | """The number of rekordbox fields.""" 105 | 106 | return self._df['rkb_field'].dropna().size 107 | 108 | def get_rkb_cols(self) -> Iterable[tuple[str, str]]: 109 | """Returns a namedtuple of rekordbox `field`s and their corresponding `dtype`s.""" 110 | 111 | df = self._df[['rkb_field', 'rkb_type']].dropna() 112 | mapping = { 113 | 'rkb_field': 'field', 114 | 'rkb_type': 'dtype', 115 | } 116 | return df.rename(columns=mapping).itertuples(name='FieldInfo', index=False) 117 | 118 | def get_export_conversion_info( 119 | self 120 | ) -> tuple[list[str], tuple[str, str, Callable[[Any], Any] | None]]: 121 | """ 122 | Get the information required to transform beets metadata to rekordbox 123 | metadata to export from beets to import into rekordbox. 124 | 125 | Returns 126 | ------- 127 | drop_fields : list 128 | List of beets fields to drop as they are not exported to rekordbox 129 | export_fields: iterator 130 | A named tuple `ExportFields` with the `beets` field to rename to 131 | `rkb` field name, and an optional function `func` to use to 132 | transform the values. 133 | """ 134 | 135 | no_export = self._df['no_export'].fillna(False) 136 | drop_fields = self._df[no_export]['beets_field'].dropna().tolist() 137 | 138 | def format_to_kind(format: str) -> str: 139 | mapping = { 140 | 'AAC': 'M4A File', 141 | 'MP3': 'MP3 File', 142 | 'WAV': 'WAV File', 143 | # Unclear if there are more types... 144 | } 145 | kind = mapping.get(format) 146 | return kind if kind is not None else format 147 | 148 | xform = { 149 | 'format_to_kind': format_to_kind 150 | } 151 | 152 | df_export = self._df[~no_export][ 153 | ['beets_field', 'rkb_field', 'convert_to_rkb'] 154 | ].dropna(subset=['beets_field', 'rkb_field']) 155 | 156 | ef = namedtuple("ExportFields", ['beets', 'rkb', 'func']) 157 | export_fields = ( 158 | ef._make([row.beets_field, row.rkb_field, xform.get(row.convert_to_rkb)]) 159 | for row in df_export.itertuples(index=False)) 160 | 161 | ei = namedtuple("ExportInfo", ['drop_fields', 'export_fields']) 162 | return ei._make([drop_fields, export_fields]) 163 | 164 | def get_sync_pairs(self) -> Iterable[tuple[str, str]]: 165 | """ 166 | Get corresponding beets and rekordbox fields. 167 | 168 | Returns 169 | ------- 170 | FieldPairs : iterator 171 | namedtuples with `beets` and corresponding `rkb` fields. 172 | 173 | """ 174 | df = self._df[self._df['sync'].fillna(False)] 175 | df = df[['beets_field', 'rkb_field']].rename(columns={ 176 | 'beets_field': 'beets', 177 | 'rkb_field': 'rkb', 178 | }) 179 | return df.itertuples(name='FieldPairs', index=False) 180 | 181 | 182 | ComputedLibraries: tuple[ 183 | pandas.DataFrame, pandas.Index, pandas.Index 184 | ] = namedtuple('ComputedLibraries', ['df_common', 'only_beets', 'only_rbxml']) 185 | 186 | class Libraries(): 187 | """ 188 | Manages beets and rekordbox libraries and the operations between them 189 | delegating a ton of actual work to dataframes. 190 | 191 | Loads the libraries on demand, so, if something isn't configured correctly, 192 | it won't error out until it is used. 193 | 194 | Parameters 195 | ---------- 196 | lib: Library 197 | Beets library. 198 | query: beets command line query, optional 199 | The query from the command line arguments, usually from `ui.decargs`. 200 | xml_path: Path 201 | Path to the exported rekordbox xml library that will be read for various operations. 202 | ddb_csv_path: Path 203 | Override data for the dimensions db, for testing. 204 | """ 205 | 206 | _ddb: DimensionsDB 207 | _items: db.Results 208 | _xml_path = Path 209 | 210 | def __init__( 211 | self, lib: library.Library, 212 | query: str | list | tuple, 213 | xml_path: Path = None, 214 | ddb_csv_path: Path = None, 215 | ): 216 | self._ddb = DimensionsDB(csv_path=ddb_csv_path) 217 | self._items = lib.items(query) 218 | self._xml_path = xml_path 219 | 220 | @cached_property 221 | def _df_beets(self) -> pandas.DataFrame: 222 | print("Loading beets library metadata...") 223 | with tqdm(total=self._ddb.num_beets_cols(), unit='columns') as pbar: 224 | def get_series(cols): 225 | series = pandas.Series( 226 | data=[i.get(cols.field) for i in self._items], 227 | dtype=cols.dtype 228 | ) 229 | pbar.update() 230 | return series 231 | 232 | series_data = { 233 | cols.field: get_series(cols) 234 | for cols in self._ddb.get_beets_cols() 235 | } 236 | 237 | df = pandas.DataFrame(data=series_data) 238 | index = df['path'].str.decode('utf-8').str.normalize('NFD').str.lower() 239 | return df.set_index(index) 240 | 241 | def beets_track_count(self) -> int: 242 | """ 243 | Returns the number of beets track loaded, subject to the query if any. 244 | """ 245 | 246 | return self._df_beets.index.size 247 | 248 | @cached_property 249 | def _df_rbxml(self) -> pandas.DataFrame: 250 | xml = pxml.RekordboxXml(self._xml_path) 251 | tracks = xml.get_tracks() 252 | 253 | print("Loading rekordbox xml...") 254 | with tqdm(total=self._ddb.num_rkb_cols(), unit='columns') as pbar: 255 | def get_series(cols): 256 | series = pandas.Series( 257 | data=[t[cols.field] for t in tracks], 258 | dtype=cols.dtype 259 | ) 260 | pbar.update() 261 | return series 262 | 263 | series_data = { 264 | cols.field: get_series(cols) 265 | for cols in self._ddb.get_rkb_cols() 266 | } 267 | 268 | df = pandas.DataFrame(data=series_data) 269 | 270 | # Prepend a slash to the paths, Rekordbox removes this 271 | df['Location'] = '/' + df['Location'] 272 | 273 | index = df['Location'].str.normalize('NFD').str.lower() 274 | return df.set_index(index) 275 | 276 | def to_pickle(self, dir: Path) -> None: 277 | """ 278 | Pickle the beets and rekordbox `DataFrame`s to `df_beets.pkl` and 279 | `df_rbxml.pkl` in the given directory. Call `to_pickle` on ddb. 280 | 281 | Parameters 282 | ---------- 283 | dir: Path 284 | Directory to write pickle files. 285 | """ 286 | 287 | self._df_beets.to_pickle(dir / Path('df_beets.pkl')) 288 | self._ddb.to_pickle(dir) 289 | if self._df_rbxml is not None: 290 | self._df_rbxml.to_pickle(dir / Path('df_rbxml.pkl')) 291 | 292 | def crop(self, music_directory: str | None = None) -> ComputedLibraries: 293 | """ 294 | Compare the two libraries using only filesystem paths. If a 295 | `music_directory` is given, only consider files to be missing from the 296 | beets library if they are present in that tree. Join all the 297 | fields and return that, as well as lists of which files are only in 298 | each. 299 | 300 | Parameters 301 | ---------- 302 | music_directory: str, optional 303 | The configured music directory for beets files, usually straight 304 | from config. 305 | 306 | Returns 307 | ------- 308 | df_common: pandas.DataFrame 309 | All files common to both libraries with all fields in both 310 | repositories. 311 | only_beets: pandas.Index 312 | Paths only in beets library. 313 | only_rbxml: pandas.Index 314 | Paths only in rekordbox library. 315 | """ 316 | 317 | df_r = self._df_rbxml 318 | if music_directory: 319 | # Filter tracks outside of music directory 320 | i = self._df_rbxml.index.str.startswith(music_directory.lower()) 321 | df_r = self._df_rbxml[i] 322 | 323 | only_rbxml = df_r.index.difference(self._df_beets.index) 324 | only_beets = self._df_beets.index.difference(df_r.index) 325 | 326 | intersection = df_r.index.intersection(self._df_beets.index) 327 | df_common = self._df_beets.loc[intersection].join( 328 | df_r.loc[intersection] 329 | ) 330 | 331 | return ComputedLibraries(df_common=df_common, only_beets=only_beets, only_rbxml=only_rbxml) 332 | 333 | def get_export_df(self, index: pandas.Index | None = None) -> pandas.DataFrame: 334 | """ 335 | Get the dataframe of tracks to export to rekordbox. Renames and converts 336 | field values using the dimensions db. 337 | 338 | Parameters 339 | ---------- 340 | index: pandas.Index, optional 341 | If present, filter using this index of file paths instead of 342 | considering the entire library. 343 | 344 | Returns 345 | ------- 346 | df: pandas.DataFrame 347 | The tracks indexed by paths including all field columns with 348 | converted values ready for pyrekordbox xml api. 349 | """ 350 | df_beets = self._df_beets if index is None else self._df_beets.loc[index] 351 | 352 | export_info = self._ddb.get_export_conversion_info() 353 | df = df_beets.drop(columns=export_info.drop_fields) 354 | df = df.rename(columns={ 355 | row.beets: row.rkb 356 | for row in export_info.export_fields 357 | }, errors='raise') 358 | 359 | # Use the type's default value to fill the nulls 360 | for field, value in df.dtypes.items(): 361 | if value.type() is None: 362 | continue 363 | df[field] = df[field].fillna(value=value.type()) 364 | 365 | # Required conversions 366 | for row in export_info.export_fields: 367 | if row.func is not None: 368 | df[row.rkb] = df[row.rkb].transform(row.func) 369 | 370 | return df 371 | 372 | def get_sync_changed(self, df_common: pandas.DataFrame) -> pandas.DataFrame: 373 | """ 374 | Compare the columns that are marked to sync and include them in the 375 | returned data if they are changed. 376 | 377 | Parameters 378 | ---------- 379 | df_common: pandas.DataFrame 380 | Output from crop, potentially filtered. 381 | 382 | Returns 383 | ------- 384 | df_changed: pandas.DataFrame 385 | The changed items to sync, indexed by beets `id` field, with NAs 386 | filled with default values. 387 | """ 388 | def ne(l, r): 389 | return l.fillna(l.dtype.type()) != r 390 | 391 | compares = ( 392 | ne(df_common[cols.beets], df_common[cols.rkb]) 393 | for cols in self._ddb.get_sync_pairs() 394 | ) 395 | mask = reduce(operator.or_, compares) 396 | df_changed = df_common[mask] 397 | df_changed = df_changed.set_index('id') 398 | 399 | def transform_column(cols): 400 | default = df_common[cols.beets].dtype.type() 401 | return df_changed[cols.rkb].fillna(default) 402 | 403 | return pandas.DataFrame(data={ 404 | cols.beets: transform_column(cols) 405 | for cols in self._ddb.get_sync_pairs() 406 | }) 407 | 408 | 409 | def export_df(xml_path: Path, df: pandas.DataFrame) -> None: 410 | """ 411 | Convert a dataframe filled with rekordbox metadata into xml. 412 | 413 | Parameters 414 | ---------- 415 | xml_path: Path 416 | Output xml file path. 417 | df: pandas.DataFrame 418 | Input data. 419 | """ 420 | outxml = pxml.RekordboxXml( 421 | name='rekordbox', version='5.4.3', company='Pioneer DJ' 422 | ) 423 | 424 | # Strip leading slash because rekordbox doesn't like it 425 | locations = df['Location'].str.slice(1) 426 | 427 | # Simplify loop by extracting this now 428 | df = df.drop(columns=['Location']) 429 | 430 | print("Rendering {} tracks...".format(df.index.size)) 431 | with tqdm(total=df.index.size, unit='tracks') as pbar: 432 | for row, location in zip(df.itertuples(index=False), locations): 433 | outxml.add_track( 434 | location=location, 435 | # Filter empty strings 436 | **{ k: v for k, v in row._asdict().items() if v != '' } 437 | ) 438 | pbar.update() 439 | 440 | print("Writing {}...".format(xml_path)) 441 | outxml.save(path=xml_path) 442 | 443 | 444 | class RkBeetsPlugin(plugins.BeetsPlugin): 445 | """ 446 | Integrate beets and rekordbox using rekordbox exported xml library data. 447 | 448 | Configuration 449 | ------------- 450 | rkbeets.export_file: Path 451 | The plugin will export data for import into rekordbox using an xml file 452 | written to this path. 453 | rkbeets.rekordbox_file: Path 454 | Some commands use data exported from rekordbox into an xml file at this 455 | path. 456 | """ 457 | 458 | # beets plugin interface to declare flexible attr types 459 | item_types: dict[str, types.Type] = { 460 | 'rkb_AverageBpm': types.FLOAT, 461 | 'rkb_Colour': types.STRING, 462 | 'rkb_DateAdded': types.STRING, 463 | 'rkb_DateModified': types.STRING, 464 | 'rkb_LastPlayed': types.STRING, 465 | 'rkb_Mix': types.STRING, 466 | 'rkb_PlayCount': types.INTEGER, 467 | 'rkb_Rating': types.INTEGER, 468 | 'rkb_Tonality': types.STRING, 469 | 'rkb_TrackID': types.INTEGER, 470 | } 471 | 472 | def __init__(self): 473 | super().__init__() 474 | 475 | self.config.add({ 476 | 'export_file': None, 477 | 'rekordbox_file': None, 478 | }) 479 | 480 | def commands(self) -> list[Callable[[library.Library, Any, Any], Any]]: 481 | """ 482 | Returns a small set of commands, all prefixed with `rkb-`, for addition 483 | to the beets cli. 484 | """ 485 | 486 | def rkb_export_func(lib: library.Library, opts, args): 487 | """export beets library for import into rekordbox""" 488 | 489 | self.config.set_args(opts) 490 | export_path = self.config['export_file'].get() 491 | 492 | libs = Libraries( 493 | lib, query=ui.decargs(args), 494 | xml_path = self.config['rekordbox_file'].get() 495 | ) 496 | 497 | index = None 498 | if opts.missing: 499 | cl = libs.crop(config['directory'].get()) 500 | 501 | if cl.only_beets.empty: 502 | print("nothing to do: no tracks are missing from rekordbox") 503 | return 504 | 505 | index = cl.only_beets 506 | 507 | df_export = libs.get_export_df(index) 508 | 509 | export_df(export_path, df_export) 510 | 511 | rkb_export_cmd = ui.Subcommand( 512 | 'rkb-export', 513 | help=rkb_export_func.__doc__ 514 | ) 515 | rkb_export_cmd.func = rkb_export_func 516 | rkb_export_cmd.parser.add_option( 517 | '-e', '--export-file', dest='export_file', 518 | help="target file for beets data exported for rekordbox" 519 | ) 520 | rkb_export_cmd.parser.add_option( 521 | '-r', '--rekordbox-file', dest='rekordbox_file', 522 | help="rekordbox xml library" 523 | ) 524 | rkb_export_cmd.parser.add_option( 525 | '-m', '--missing', dest='missing', action='store_true', default=False, 526 | help="only consider files not already in rekordbox library" 527 | ) 528 | 529 | def rkb_diff_func(lib: library.Library, opts, args): 530 | """show information and differences between the rekordbox and beets libraries""" 531 | 532 | self.config.set_args(opts) 533 | 534 | libs = Libraries( 535 | lib, query=ui.decargs(args), 536 | xml_path = self.config['rekordbox_file'].get() 537 | ) 538 | 539 | if opts.pickle: 540 | print("Writing dataframes to {}".format(opts.pickle)) 541 | libs.to_pickle(opts.pickle) 542 | 543 | cl = libs.crop(config['directory'].get()) 544 | 545 | print("{:>6d} tracks in rekordbox library (in beets directory)".format( 546 | cl.df_common.index.size + cl.only_rbxml.size)) 547 | print("{:>6d} tracks in beets library (subject to query if any)".format(libs.beets_track_count())) 548 | print("{:>6d} shared tracks in both".format(cl.df_common.index.size)) 549 | 550 | if not cl.only_rbxml.empty: 551 | print("Only in Rekordbox:") 552 | for path in cl.only_rbxml: 553 | print(" ", path) 554 | 555 | if not cl.only_beets.empty: 556 | print("Only in beets:") 557 | for path in cl.only_beets: 558 | print(" ", path) 559 | 560 | rkb_diff_cmd = ui.Subcommand( 561 | 'rkb-diff', 562 | help=rkb_diff_func.__doc__ 563 | ) 564 | rkb_diff_cmd.func = rkb_diff_func 565 | rkb_diff_cmd.parser.add_option( 566 | '-r', '--rekordbox-file', dest='rekordbox_file', 567 | help="rekordbox xml library" 568 | ) 569 | rkb_diff_cmd.parser.add_option( 570 | '--pickle', 571 | help="export dataframes to given directory" 572 | ) 573 | 574 | def rkb_sync_func(lib: library.Library, opts, args): 575 | """sync metadata from rekordbox xml to beets database""" 576 | 577 | self.config.set_args(opts) 578 | 579 | libs = Libraries( 580 | lib, query=ui.decargs(args), 581 | xml_path = self.config['rekordbox_file'].get() 582 | ) 583 | 584 | cl = libs.crop(config['directory'].get()) 585 | 586 | df_sync_changed = libs.get_sync_changed(cl.df_common) 587 | 588 | if df_sync_changed.empty: 589 | print("nothing to update") 590 | return 591 | 592 | print("Updating {} tracks...".format(df_sync_changed.index.size)) 593 | with tqdm(total=df_sync_changed.index.size, unit='tracks') as pbar: 594 | for row in df_sync_changed.itertuples(): 595 | data = row._asdict() 596 | id = data.pop('Index') 597 | item = lib.get_item(id) 598 | item.update(data) 599 | 600 | if opts.dry_run: 601 | print("{} --> {}".format(item.get('path').decode('utf-8'), data)) 602 | else: 603 | item.try_sync(False, False) 604 | pbar.update() 605 | 606 | rkb_sync_cmd = ui.Subcommand( 607 | 'rkb-sync', 608 | help=rkb_sync_func.__doc__ 609 | ) 610 | rkb_sync_cmd.func = rkb_sync_func 611 | rkb_sync_cmd.parser.add_option( 612 | '-r', '--rekordbox-file', dest='rekordbox_file', 613 | help="rekordbox xml library" 614 | ) 615 | rkb_sync_cmd.parser.add_option( 616 | '-n', '--dry-run', dest='dry_run', action='store_true', default=False, 617 | help="print the changes instead of committing them" 618 | ) 619 | 620 | return [rkb_export_cmd, rkb_diff_cmd, rkb_sync_cmd] 621 | --------------------------------------------------------------------------------