├── .gitignore ├── README.md ├── kill_playlist_dupes └── kill_dupes /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .DS_Store 3 | .project 4 | .pydevproject 5 | credentials.py -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Google Music Dupe Killer 2 | ======================== 3 | 4 | Really small scripts designed to remove pesky duplicates from Google Music. 5 | 6 | ** The duplicates must have all correct meta data as this evaluates their title and album for detection. 7 | 8 | Thanks to [simon weber](https://github.com/simon-weber) for a GREAT client library that made this super easy to code. 9 | 10 | ## Usage 11 | These are incredibly simple scripts, but do require a few small configurations. 12 | 13 | ### Install [Unofficial Google Music Api](https://github.com/simon-weber/Unofficial-Google-Music-API) 14 | * For most environments with Python 3 already installed: 15 | 16 | ```$ pip install gmusicapi``` 17 | * If you do not have pip or are running windows, please see [Unofficial Google Music API usage](http://unofficial-google-music-api.readthedocs.org/en/latest/usage.html) 18 | 19 | ### Change login credentials 20 | * Near line 12 in the script, change the 'username' and 'password' to your Google account credentials. 21 | * Alternatively, create an empty file called 'credentials.py' in the script directory and set the username and password variables there. 22 | 23 | ```python 24 | # credentials.py 25 | username = 'changeme@gmail.com' 26 | password = 'changeme' 27 | #android_id = 'deadbeefc0decafe' 28 | ``` 29 | 30 | * NOTE: Users with 2-step authentication enabled will have to create an App Specific Key/Password. 31 | Login into your Google account and head to https://security.google.com/settings/security/apppasswords, there you will be able to manually generate an App Specific password. 32 | After creating the Key/Password, just use it to login into this App, together with your usual Google username/email. 33 | 34 | * NOTE: If you see the error message "a valid MAC could not be determined." 35 | you are running into a [known issue](https://github.com/simon-weber/gmusicapi/issues/408) with the Google Music API. 36 | To workaround, set a new 16 digit hexadecimal number as your android_id on line 17 or in credentials.py. 37 | 38 | ### Run kill_dupes 39 | * The script will automatically detect and remove duplicates on any songs in your library. 40 | 41 | ### Run kill_playlist_dupes 42 | * This script works on one playlist at a time and removes the second through nth duplicate of a track in a playlist. 43 | 44 | 45 | Thanks! 46 | -------------------------------------------------------------------------------- /kill_playlist_dupes: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | 4 | from gmusicapi import Mobileclient 5 | 6 | try: 7 | import credentials 8 | # credentials.py is in .gitignore and can be shared with other scripts 9 | username = credentials.username 10 | password = credentials.password 11 | except ImportError: 12 | username = 'username' 13 | password = 'password' 14 | 15 | try: 16 | # optionally hardcode id to workaround https://github.com/simon-weber/gmusicapi/issues/408 17 | android_id = credentials.android_id 18 | except (NameError, AttributeError): 19 | android_id = Mobileclient.FROM_MAC_ADDRESS 20 | 21 | def get_dupes_in_playlist(playlist): 22 | duplicate_Ids = [] 23 | unique_trackIds = [] 24 | 25 | for track in playlist['tracks']: 26 | if track['trackId'] not in unique_trackIds: 27 | unique_trackIds.append(track['trackId']) 28 | 29 | for atrack in unique_trackIds: 30 | count = 0 31 | for btrack in playlist['tracks']: 32 | #print(atrack + " b " + btrack['trackId'] ) 33 | if atrack == btrack['trackId'] : 34 | count += 1 35 | if count > 1: 36 | duplicate_Ids.append(btrack['id']) 37 | 38 | return duplicate_Ids 39 | 40 | def query_yes_no(question, default="yes"): 41 | """Ask a yes/no question via input() and return their answer. 42 | 43 | "question" is a string that is presented to the user. 44 | "default" is the presumed answer if the user just hits . 45 | It must be "yes" (the default), "no" or None (meaning 46 | an answer is required of the user). 47 | 48 | The "answer" return value is one of "yes" or "no". 49 | """ 50 | valid = {"yes":True, "y":True, "ye":True, 51 | "no":False, "n":False} 52 | if default == None: 53 | prompt = " [y/n] " 54 | elif default == "yes": 55 | prompt = " [Y/n] " 56 | elif default == "no": 57 | prompt = " [y/N] " 58 | else: 59 | raise ValueError("invalid default answer: '%s'" % default) 60 | 61 | while True: 62 | sys.stdout.write(question + prompt) 63 | if sys.version_info[0] < 3: 64 | choice = raw_input().lower() 65 | else: 66 | choice = input().lower() 67 | if default is not None and choice == '': 68 | return valid[default] 69 | elif choice in valid: 70 | return valid[choice] 71 | else: 72 | sys.stdout.write("Please respond with 'yes' or 'no' "\ 73 | "(or 'y' or 'n').\n") 74 | 75 | 76 | api = Mobileclient() 77 | logged_in = api.login(username, password, android_id) 78 | 79 | if logged_in: 80 | print("Successfully logged in. Beginning duplicate detection process.") 81 | 82 | all_playlists = api.get_all_user_playlist_contents() 83 | 84 | for playlist in all_playlists: 85 | #if playlist['name'] == 'aaaDuplicateTest': 86 | if 1==1: 87 | duplicate_tracks = get_dupes_in_playlist(playlist) 88 | if len(duplicate_tracks) > 0: 89 | if query_yes_no("Found " + str(len(duplicate_tracks)) + " duplicate tracks in playlist " + playlist['name'] + ". Delete duplicates?", "yes"): 90 | deleted_track_ids = [] 91 | deleted_track_ids += api.remove_entries_from_playlist(duplicate_tracks) 92 | print("Successfully deleted " + str(len(deleted_track_ids)) + " tracks of " + playlist['name'] ) 93 | else: 94 | print("Skipping " + playlist['name']) 95 | 96 | print("Thank you!") 97 | -------------------------------------------------------------------------------- /kill_dupes: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | 4 | from gmusicapi import Mobileclient 5 | 6 | try: 7 | import credentials 8 | # credentials.py is in .gitignore and can be shared with other scripts 9 | username = credentials.username 10 | password = credentials.password 11 | except ImportError: 12 | username = 'username' 13 | password = 'password' 14 | 15 | try: 16 | # optionally hardcode id to workaround https://github.com/simon-weber/gmusicapi/issues/408 17 | android_id = credentials.android_id 18 | except (NameError, AttributeError): 19 | android_id = Mobileclient.FROM_MAC_ADDRESS 20 | 21 | 22 | def map_track_duplication(tracks): 23 | album_track_duplicate_map = {} 24 | for track in tracks: 25 | albumNorm = track['album'].lower() 26 | titleNorm = track['title'].lower() 27 | try: 28 | trackNumberNorm = track['trackNumber'] 29 | except: 30 | trackNumberNorm = 0 31 | try: 32 | discNumberNorm = track['discNumber'] 33 | except: 34 | discNumberNorm = 0 35 | if albumNorm not in album_track_duplicate_map: 36 | album_track_duplicate_map.update({albumNorm: {}}) 37 | if titleNorm not in album_track_duplicate_map[albumNorm]: 38 | album_track_duplicate_map[albumNorm].update({titleNorm: {}}) 39 | if trackNumberNorm not in album_track_duplicate_map[albumNorm][titleNorm]: 40 | album_track_duplicate_map[albumNorm][titleNorm].update( 41 | {trackNumberNorm: {}}) 42 | if discNumberNorm in album_track_duplicate_map[albumNorm][titleNorm][trackNumberNorm]: 43 | album_track_duplicate_map[albumNorm][titleNorm][trackNumberNorm][discNumberNorm] += 1 44 | else: 45 | album_track_duplicate_map[albumNorm][titleNorm][trackNumberNorm][discNumberNorm] = 1 46 | return album_track_duplicate_map 47 | 48 | 49 | def sort_tracks_by_album(tracks): 50 | tracks_by_album = {} 51 | for track in tracks: 52 | albumNorm = track['album'].lower() 53 | if albumNorm not in tracks_by_album: 54 | tracks_by_album[albumNorm] = [] 55 | tracks_by_album[albumNorm].append(track) 56 | return tracks_by_album 57 | 58 | 59 | def get_duplicate_tracks(all_tracks_by_album, album_track_duplicate_map): 60 | duplicate_tracks = [] 61 | for album_title in album_track_duplicate_map: 62 | for track_title in album_track_duplicate_map[album_title]: 63 | for track_number in album_track_duplicate_map[album_title][track_title]: 64 | for disc_number in album_track_duplicate_map[album_title][track_title][track_number]: 65 | # As one is always added 66 | duplicates = album_track_duplicate_map[album_title][track_title][track_number][disc_number] - 1 67 | if duplicates > 0: 68 | for album in all_tracks_by_album: 69 | if album_title == album.lower(): 70 | for track in all_tracks_by_album[album]: 71 | titleNorm = track['title'].lower() 72 | if titleNorm == track_title: 73 | try: 74 | trackNumberNorm = track['trackNumber'] 75 | except: 76 | trackNumberNorm = 0 77 | if trackNumberNorm == track_number: 78 | try: 79 | discNumberNorm = track['discNumber'] 80 | except: 81 | discNumberNorm = 0 82 | 83 | if discNumberNorm == disc_number and duplicates > 0: 84 | duplicate_tracks.append(track) 85 | # PR Encode output in utf-8 to avoid errors when printing special characters not supported by default codeset 86 | print( 87 | ( 88 | "Queuing for removal: '" 89 | + track.get('title', 'No Track Title') 90 | + "' from album '" 91 | + track.get('album', 'None') 92 | + "' by '" 93 | + track.get('artist', 'None') 94 | ).encode('utf8') 95 | ) 96 | all_tracks.remove(track) 97 | duplicates -= 1 98 | return duplicate_tracks 99 | 100 | 101 | def query_yes_no(question, default="yes"): 102 | """Ask a yes/no question via input() and return their answer. 103 | 104 | "question" is a string that is presented to the user. 105 | "default" is the presumed answer if the user just hits . 106 | It must be "yes" (the default), "no" or None (meaning 107 | an answer is required of the user). 108 | 109 | The "answer" return value is one of "yes" or "no". 110 | """ 111 | valid = {"yes": True, "y": True, "ye": True, 112 | "no": False, "n": False} 113 | if default == None: 114 | prompt = " [y/n] " 115 | elif default == "yes": 116 | prompt = " [Y/n] " 117 | elif default == "no": 118 | prompt = " [y/N] " 119 | else: 120 | raise ValueError("invalid default answer: '%s'" % default) 121 | 122 | while True: 123 | sys.stdout.write(question + prompt) 124 | if sys.version_info[0] < 3: 125 | choice = raw_input().lower() 126 | else: 127 | choice = input().lower() 128 | 129 | if default is not None and choice == '': 130 | return valid[default] 131 | elif choice in valid: 132 | return valid[choice] 133 | else: 134 | sys.stdout.write("Please respond with 'yes' or 'no' " 135 | "(or 'y' or 'n').\n") 136 | 137 | 138 | def get_track_ids(tracks): 139 | track_ids = [] 140 | for track in tracks: 141 | track_ids.append(track['id']) 142 | return track_ids 143 | 144 | 145 | api = Mobileclient() 146 | api.__init__(True, True, True) 147 | oauth_credentials = api.perform_oauth(username, password) 148 | logged_in = api.oauth_login(api.FROM_MAC_ADDRESS, oauth_credentials) 149 | 150 | if logged_in: 151 | print("Successfully logged in. Beginning duplicate detection process.") 152 | all_tracks = api.get_all_songs() 153 | album_track_duplicate_map = map_track_duplication(all_tracks) 154 | all_tracks_by_album = sort_tracks_by_album(all_tracks) 155 | duplicate_tracks = get_duplicate_tracks( 156 | all_tracks_by_album, album_track_duplicate_map) 157 | duplicate_track_ids = get_track_ids(duplicate_tracks) 158 | if len(duplicate_track_ids) > 0: 159 | if query_yes_no("Found " + str(len(duplicate_track_ids)) + " duplicate tracks. Delete duplicates?", "no"): 160 | deleted_track_ids = [] 161 | for track in duplicate_track_ids: 162 | deleted_track_ids += api.delete_songs(track) 163 | print("Successfully deleted " + str(len(deleted_track_ids)) + " of " + 164 | str(len(duplicate_track_ids)) + " queued songs for removal.") 165 | else: 166 | print("I didn't find any duplicate tracks.") 167 | print("Thank you!") 168 | --------------------------------------------------------------------------------